diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1c8a44a --- /dev/null +++ b/.gitignore @@ -0,0 +1,211 @@ + +# Created by https://www.gitignore.io/api/linux,python,pycharm+all +# Edit at https://www.gitignore.io/?templates=linux,python,pycharm+all + +### Linux ### +*~ + +# temporary files which can be created if a process still has a handle open of a deleted file +.fuse_hidden* + +# KDE directory preferences +.directory + +# Linux trash folder which might appear on any partition or disk +.Trash-* + +# .nfs files are created when an open file is removed but is still being accessed +.nfs* + +### PyCharm+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### PyCharm+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +# Sonarlint plugin +.idea/sonarlint + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# End of https://www.gitignore.io/api/linux,python,pycharm+all diff --git a/README.md b/README.md index a750be4..9d5e9ba 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,155 @@ -# FSFF +# FI(le) SY(stem) - FUZZer + + This is the full file system fuzzing framework that I presented at the Hack in the Box 2020 Lockdown Edition conference in April. + + * [Conference Talk](https://youtu.be/VNzKVOsn5qQ?t=17032) + * [Conference Material](https://github.com/0xricksanchez/HITB2020_FSFUZZER) + + +![Framework overview](assets/framework.png) + + +## Goal + +The goal of this framework is to uncover kernel security bugs on UNIX systems with a strong focus on BSD systems. +It was developed and heavily tested against FreeBSD, OpenBSD, and NetBSD but also already has some minor support for Linux based Hosts. +We were able to successfully uncover over 100 unique kernel bugs for UFS and EXT based file systems while also getting +a lot of insight into the recent addition of ZFS. + +### Test case generator + +The `makeFS2.py` can be used as a standalone utility to generate different valid file systems. +The usage is explained in detail in the [conference material repo](https://github.com/0xricksanchez/HITB2020_FSFUZZER) + +## Requirements + +The framework was solely tested on Ubuntu 18.04. +It relies on KVM, QEMU and libvirt. +The `Requirements.sh` sets up all necessary dependencies. +The framework may be fully functional on the latest Ubuntu 20.04 release. +Other non `apt` based host system should be easily supported and only requires minor changes in the `Requirements.sh`. +Once you're done with the requirements you can continue with the setup steps! + +## Setup + +Refer to the [SETUP.md](SETUP.md). If some steps are unclear please reach out! + +## Fuzz it! + +Starting the fuzzer only requires executing: `python3 run.py`. +Depending on your setup you may require `sudo` privileges. +When everything started successfully you can attach to the tmux fuzzing session via: + +``` +(sudo) tmux attach-session -t fsfuzzer +``` + +### Config + +The framework can be configured with the script `src/config/fuzzing_config.py`: + +```python + +# [fuzzing task specs] +# List of dictionaries specifying each fuzzing instance +fuzzer = [ + { + "name": "fuzz1", # Some name for internal bookkeeping + "fs_creator_vm": "genBox", # Name as specified in libvirt for the VM handling the file system generation, can be the same across all instances + "fuzzing_vm": "fuzzBox_0", # Name as specified in libvirt for the VM handling the file system generation + "mutation_engine": "radamsa, 0", # Mutation Engine that is to be used, and size of mutation (radamsa takes no size argument) + "target_fs": "ufs2", # Target file system + "target_size": 15, # Max file system size in Megabyte + "populate_with_files": 10, # Amount of file that will be generated + "max_file_size": 1024, # Maximum file size in bytes for each generated file + "enable_dyn_scaling": False, # Dynamic scaling will increase the filesystem size periodically + }, +] + +# [credentials] +# Credentials for the root user for the VMs +# It is expected that these are the same across all instances, but not necessarily root +user = "root" +pw = "root" +``` + +The available mutation engines are: + + - radamsa + - byte_flip_seq + - byte_flip_rnd + - metadata + +Dynamic scaling was initially implemented to test whether the size of a file system affects the possible crashes. +I was not able to identify a trigger value for file system size where crashes change, so this flag can stay disabled. +This also prevents suffering from a performance drop on longer runs as larger file systems take longer to mutate. + +The remaining config parameters should be self explanatory. + +### PoC + +The video below shows a quick demo where two fuzzing instances both targeting FreeBSD with a radamsa mutated UFS2 and a random bytes flipped EXT file system. +The mutated UFS file system directly leads to a crash showcasing how quickly you can crash a kernel. + +[![asciicast](https://asciinema.org/a/ABgNGFLU3TrUe4n7gOcduTrGk.svg)](https://asciinema.org/a/ABgNGFLU3TrUe4n7gOcduTrGk) + + +![fuzz_example](assets/stdout.png) + +## Feature set + +- [X] Full Support for FFS, UFS, EXT, and ZFS file systems +- [X] Full support for FreeBSD, NetBSD, and OpenBSD + - [X] Experimental support for Ubuntu (and probably other Debian based derivatives) +- [X] Mutations via + - [X] radamsa + - [X] global random byte flips + - [X] global random byte sequence changes + - [X] superblock only changes +- [X] Fully randomized user emulation to access/change broken but mounted file system +- [X] Crash database +- [X] Crash Verification +- [X] VM reset via snapshots +- [X] Fully automated fuzzing process +- [X] Reasonable user interface on the commandline + +## Future Work + +- [ ] Support more/different mutation engines +- [ ] Automated VM setup +- [ ] Check for available snapshots and if missing take some before fuzzing +- [ ] Support for macOS +- [ ] Performance tweaks + - [ ] Make framework async + - [ ] Continuous sample creation w/o waiting for fuzzing run + - [ ] Use same sample with different mutations before generating a new one + - [ ] Rework algorithms/interactions for speedup +- [ ] Code cleanup & refactoring + - [ ] Make logging verbosity togglable + + +## [Trophies](https://www.freshbsd.org/search?q=Christopher+Krah&sort=commit_date) + +With this setup I was able to find > 100 hundred unique kernel bugs in FreeBSD, NetBSD and OpenBSD for UFS and EXT based file systems. +Among these crashes I had a bunch of very interesting ones: + +- Out of bounds reads +- Out of bounds writes +- Double Fault in EXT +- Triple Fault in UFS +- Non-deterministic kernel bug with > 6 unique core dumps + +The majority of crashes were kernel DoS. +Furthermore, the majority of encountered crashes are still not fixed as of today (May 2020). +So have a go at finding kernel bugs in file system implementations :). + +## Disclaimer + +This whole thing was built with 'trust driven development', meaning that it grew way too large way too quick from a sole PoC idea. +Hence, there are no tests either and most likely a few bugs. +I'm sorry if you run into crashes (that are not kernel panic related) but feel free to make a PR or hit me up and I'll fix things ASAP! + +## Contact + +Twitter: [@0xricksanchez](https://twitter.com/0xricksanchez) diff --git a/SETUP.md b/SETUP.md new file mode 100644 index 0000000..894027a --- /dev/null +++ b/SETUP.md @@ -0,0 +1,83 @@ +Currently, the framework requires that the user sets up all necessary VMs with a fully functional `root:root` user by hand. + + +### VM installation +After installing all necessary dependencies this can be done as follows: + +``` +virt-install --name "my_vm" --memory 2047 --vcpus 2 --disk size=20 --cdrom ~/OS.iso --os-variant OS +``` + +Once the VM is installed you can verify it is available: + +``` +virsh list --all +``` + +To find out the IP you can: + +``` +virsh domifaddr my_vm +``` + + +To access a VNC windows of the running VM: + +``` +virt-viewer my_vm +``` + +### VM prep work - General + +Once you have a VM up and running access it either via VNC/SSH and prepare it for fuzzing. +Anyhow make sure *all* VMs are accessible via SSH and the `root:root` user +The basic framework setup currently expects one dedicated VM instance that handles the file system creation. + +### VM prep work - FS creator +This VM is needed for the sole reason of providing fresh new random file system samples for mutation on the Host. +To speed up the writing to disk I recommend having a TMPFS in place for this particular instance, e.g.: + +```bash +#!/usr/bin/env sh + +pkg install -y vim python2 e2fsprogs p7zip +kldload ext1fs +# Needs to be run as root on the FreeBSD VMs since we require root privs via ssh +sed -i -e 's/#PermitRootLogin no/PermitRootLogin yes/g' /etc/ssh/sshd_config +sed -i -e 's/#PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config +sed -i -e 's/#PermitEmptyPasswords no/PermitEmptyPasswords yes/g' /etc/ssh/sshd_config +/etc/rc.d/sshd restart + +echo "tmpfs_load='YES'" >> /boot/loader.conf +echo "tmpfs /tmp tmpfs rw,mode=1776 0 0" >> /etc/fstab +``` + +### VM prep work - Fuzzing instance +Now for the fuzzing instances the ssh requirement from above still holds. +The `scripts` folder contains all config parameters to tweak a fuzzing FreeBSD instance. +For proper fuzzing we extend the FreeBSD kernel with debugging functionality and also tweak some other parameters. +The procedure is pretty analogous for NetBSD and OpenBSD. +Creating a custom kernel and setting the correct flags under Linux requires a little bit more work and is not documented here. + +Once you fully prepared all the VMs you need to feed some information into the `config/fuzzing_config.py`. +There is already a fully documented example, which you can replace. + +**Note: ** You really only need to set up a single custom fuzzing instance as you can fully clone it via *libvirt* where it gets a new name and a new IP as well. + +``` +virt-clone --original current_instance --name new_instance --auto-clone +``` + + +### Finishing touches - Important! + +Once you have all the KVM VMs up and running and fully customized with new kernels and ssh access you have to take a snapshot of it. +The reason being that in case of the VM fully panicking beyond repair the framework will automatically try to reset the VM to +the latest available snapshot. + +``` +virsh snapshot-create-as --domain my_vm --name my_vm_snapshot +``` + + +When you have done all this you can start configuring the fuzzer framework in `src/config/fuzzing_config.py` and start fuzzing. \ No newline at end of file diff --git a/assets/334433.cast b/assets/334433.cast new file mode 100644 index 0000000..9ac58d2 --- /dev/null +++ b/assets/334433.cast @@ -0,0 +1,111 @@ +{"version": 2, "width": 115, "height": 61, "timestamp": 1590599289, "env": {"SHELL": "/usr/bin/zsh", "TERM": "xterm"}} +[0.169318, "o", "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r\u001b]2;dev@dev: ~/fisy-fuzz/src\u0007\u001b]1;~/fisy-fuzz/src\u0007"] +[0.182045, "o", "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J\u001b[01;32mdev@dev \u001b[34m[07:08:09 PM] \u001b[00m\u001b[37m[~/fisy-fuzz/src]\u001b[00m \u001b[32m[master \u001b[31m*\u001b[32m]\u001b[00m\r\n\u001b[34m->\u001b[01;34m %\u001b[00m \u001b[K\u001b[?1h\u001b="] +[0.182146, "o", "\u001b[?2004h"] +[2.318476, "o", "s"] +[2.41199, "o", "\bsu"] +[2.50355, "o", "d"] +[2.647733, "o", "o"] +[3.27065, "o", " "] +[4.435704, "o", "p"] +[5.24657, "o", "ython3 run.py && sudo tmux attach-session -t fsfuzzer"] +[7.573557, "o", "\u001b[?1l\u001b>"] +[7.573842, "o", "\u001b[?2004l\r\r\n"] +[7.574712, "o", "\u001b]2;sudo python3 run.py && sudo tmux attach-session -t fsfuzzer\u0007\u001b]1;python3\u0007"] +[7.581332, "o", "[sudo] password for dev: "] +[10.066939, "o", "\r\n"] +[10.115497, "o", "error connecting to /tmp/tmux-0/default (No such file or directory)\r\n"] +[10.115835, "o", "python3 Fuzzer/Fuzzer.py fuzz1 hitb_manager hitb_fuzzer1 'radamsa, 0' ufs2 15 10 1024 False\r\n"] +[10.12091, "o", "python3 Fuzzer/Fuzzer.py fuzz2 hitb_manager hitbfuzzer0 'byte_flip_rnd, 10' ext2 15 10 1024 False\r\n"] +[10.121956, "o", "\u001b[6;30;42mAttach to tmux session via \"tmux attach-session -t fsfuzzer\"!\u001b[0m\r\n\u001b[6;30;42mCtrl+C to kill tmux session!\u001b[0m\r\n"] +[10.136289, "o", "\u001b[?1049h\u001b[22;0;0t\u001b[?1h\u001b=\u001b[H\u001b[2J\u001b[?12l\u001b[?25h\u001b[?1000l\u001b[?1002l\u001b[?1006l\u001b[?1005l\u001b[c\u001b(B\u001b[m\u001b[?12;25h\u001b[?12l\u001b[?25h\u001b[?1003l\u001b[?1006l\u001b[?2004l\u001b[1;1H\u001b[1;61r\u001b]112\u0007\u001b[2;1H"] +[10.136817, "o", "\u001b[?25l\u001b[Hpython3 Fuzzer/Fuzzer.py fuzz2 hitb_manager hitbfuzzer0 'byte_flip_rnd, 10' ext2 15 10 1024 False\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[30m\u001b[42m[fsfuzzer]0:bash- 1:bash* \"dev\" 19:08 27-May-20\u001b(B\u001b[m\u001b[2;1H\u001b[?12l\u001b[?25h"] +[10.136867, "o", "\u001b(B\u001b[m\u001b[?12;25h\u001b[?12l\u001b[?25h\u001b[?1003l\u001b[?1006l\u001b[?2004l\u001b[1;1H\u001b[1;61r\u001b[2;1H"] +[10.137387, "o", "\u001b[?25l\u001b[Hpython3 Fuzzer/Fuzzer.py fuzz2 hitb_manager hitbfuzzer0 'byte_flip_rnd, 10' ext2 15 10 1024 False\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[30m\u001b[42m[fsfuzzer]0:bash- 1:bash* \"dev\" 19:08 27-May-20\u001b(B\u001b[m\u001b[2;1H\u001b[?12l\u001b[?25h"] +[10.162321, "o", "root@dev:~/fisy-fuzz/src# "] +[10.162523, "o", "python3 Fuzzer/Fuzzer.py fuzz2 hitb_manager hitbfuzzer0 'byte_flip_rnd, 10' ext2 15 10 102\r24 False\r\n"] +[14.617284, "o", "\u001b[?25l\u001b[61d\u001b[30m\u001b[42m[fsfuzzer]0:python3- 1:python3* \"dev\" 19:08 27-May-20\u001b(B\u001b[m\u001b[4;1H\u001b[?12l\u001b[?25h"] +[16.862576, "o", "\u001b[1;60r\u001b[2;60r\u001b[59S\u001b[1;1H\u001b[K\u001b[1;61r\u001b[1;1H"] +[16.866343, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[2;1HStart date: 2020-05-27 19:08:20.08 | Runtime: 0:00:00.01 | OS : None | Mutation engine: byte_flip_rnd\r\nFilesystem type: ext2 | Filesystem size: 15MB \r\nIteration: 0 | Last iteration time: 0s | Avg. iteration time: 0s\r\n#Crashes: 0 | #New crashes: 0 | Last panic: | Last new crash (iter): 0\r\nSuccessful mounts: 0 (0%) | 0/0 (0%) Commands executed\r\n"] +[16.868912, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[8;1H"] +[17.478383, "o", "\u001b[31m[!] Mounting failed!\r\n\u001b(B\u001b[m"] +[17.602113, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[10;1H"] +[18.618083, "o", "\u001b[?25l\u001b[H───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[2;1HStart date: 2020-05-27 19:08:20.08 | Runtime: 0:00:00.01 | OS : None | Mutation engine: radamsa\u001b[K\r\nFilesystem type: ufs2 | Filesystem size: 15MB\u001b[K\r\nIteration: 0 | Last iteration time: 0s | Avg. iteration time: 0s\u001b[K\r\n#Crashes: 0 | #New crashes: 0 | Last panic: | Last new crash (iter): 0\u001b[K\r\nSuccessful mounts: 0 (0%) | 0/0 (0%) Commands executed\u001b[K\r\n───────────────────────────────────────────────────────────────────────────────────────────────────────"] +[18.618273, "o", "────────────\u001b[8;1H\u001b[31m[!] Mounting failed!\u001b[39m\u001b[K\r\n───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[10;1H\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[30m\u001b[42m[fsfuzzer]0:python3* 1:python3- \"dev\" 19:08 27-May-20\u001b(B\u001b[m\u001b[10;1H\u001b[?12l\u001b[?25h"] +[19.33766, "o", "\u001b[1;60r\u001b[2;60r\u001b[59S\u001b[1;1H\u001b[K\u001b[1;61r\u001b[1;1H"] +[19.340421, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[2;1HStart date: 2020-05-27 19:08:20.08 | Runtime: 0:00:04.75 | OS : freebsd | Mutation engine: radamsa\r\nFilesystem type: ufs2 | Filesystem size: 15MB \r\nIteration: 1 | Last iteration time: 4.63s | Avg. iteration time: 4.63s\r\n#Crashes: 0 | #New crashes: 0 | Last panic: | Last new crash (iter): 0\r\nSuccessful mounts: 0 (0.0%) | 0/0 (0%) Commands executed\r\n"] +[19.342673, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[8;1H"] +[19.920455, "o", "\u001b[32m[+] VM status: OK\r\n[+] Mounting successful!\r\n\u001b(B\u001b[m"] +[19.920622, "o", "\u001b[8C\u001b[33m\u001b[1m[*] Accessing & modifying mounted filesystem: /mnt/radamsa_fuzz1_ufs2_15MB\r\n\u001b(B\u001b[m"] +[20.284001, "o", "\u001b[8C > Failed to perform a file traversal on mounted file system\u001b[12;9H > Starting backup user emulation that only attempts to write to disk...\r\n"] +[33.389233, "o", "\u001b[31m[!] VM status: Not Responding\r\n\u001b(B\u001b[m"] +[33.38938, "o", "\u001b[33m\u001b[1m[*] Completed 0/12 program calls\r\n\u001b(B\u001b[m"] +[33.575239, "o", "\u001b[33m\u001b[1m[*] Checking for crash dump..!\r\n\u001b(B\u001b[m"] +[33.576477, "o", "\u001b[8C\u001b[33m\u001b[1m[*] Please stand by..\u001b(B\u001b[m"] +[34.57778, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[35.578814, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[36.579819, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[37.580818, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[38.581833, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[39.582874, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[40.584021, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[41.584956, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[42.585997, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[43.417584, "o", "\u001b[?25l\u001b[H───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[2;1HStart date: 2020-05-27 19:08:20.08 | Runtime: 0:00:26.16 | OS : freebsd | Mutation engine: byte_flip_rnd\u001b[K\r\nFilesystem type: ext2 | Filesystem size: 15MB\u001b[K\r\nIteration: 4 | Last iteration time: 5.94s | Avg. iteration time: 6.44s\u001b[K\r\n#Crashes: 0 | #New crashes: 0 | Last panic: | Last new crash (iter): 0\u001b[K\r\nSuccessful mounts: 0 (0.0%) | 0/0 (0%) Commands executed\u001b[K\r\n─────────────────────────────────────────────────────────────────────────────────────────────────"] +[43.417652, "o", "──────────────────\u001b[8;1H\u001b[31m[!] Mounting failed!\u001b[39m\u001b[K\r\n───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[10;1H\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[30m\u001b[42m[fsfuzzer]0:python3- 1:python3* \"dev\" 19:08 27-May-20\u001b(B\u001b[m\u001b[10;1H\u001b[?12l\u001b[?25h"] +[44.545871, "o", "\u001b[?25l\u001b[H───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[2;1HStart date: 2020-05-27 19:08:20.08 | Runtime: 0:00:04.75 | OS : freebsd | Mutation engine: radamsa\u001b[K\r\nFilesystem type: ufs2 | Filesystem size: 15MB\u001b[K\r\nIteration: 1 | Last iteration time: 4.63s | Avg. iteration time: 4.63s\u001b[K\r\n#Crashes: 0 | #New crashes: 0 | Last panic: | Last new crash (iter): 0\u001b[K\r\nSuccessful mounts: 0 (0.0%) | 0/0 (0%) Commands executed\u001b[K\r\n───────────────────────────────────────────────────────────────────────────────────────────────────"] +[44.546024, "o", "────────────────\u001b[8;1H\u001b[32m[+] VM status: OK\u001b[39m\u001b[K\r\n\u001b[32m[+] Mounting successful!\u001b[39m\u001b[K\r\n \u001b[33m\u001b[1m[*] Accessing & modifying mounted filesystem: /mnt/radamsa_fuzz1_ufs2_15MB\u001b(B\u001b[m\u001b[K\r\n > Failed to perform a file traversal on mounted file system\u001b[K\r\n > Starting backup user emulation that only attempts to write to disk...\u001b[K\r\n\u001b[31m[!] VM status: Not Responding\u001b[39m\u001b[K\r\n\u001b[33m\u001b[1m[*] Completed 0/12 program calls\u001b(B\u001b[m\u001b[K\r\n\u001b[33m\u001b[1m[*] Checking for crash dump..!\u001b(B\u001b[m\u001b[K\r\n \u001b[33m\u001b[1m[*] Please stand by............\u001b(B\u001b[m\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[30m\u001b[42m[fsfuzzer]0:python3* 1:python3- \"dev\" 19:08 27-May-20\u001b(B\u001b[m\u001b[16;40H\u001b[?12l\u001b[?25h"] +[44.588034, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[45.589231, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[46.589808, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[47.590828, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[48.59209, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[49.592898, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[50.59385, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[51.594932, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[52.595075, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[53.596119, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[54.597228, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[55.13557, "o", "\u001b[?25l\u001b[61;1H\u001b[30m\u001b[42m[fsfuzzer]0:python3* 1:python3- \"dev\" 19:09 27-May-20\u001b(B\u001b[m\u001b[16;51H\u001b[?12l\u001b[?25h"] +[55.598208, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[56.599349, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[57.600399, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[58.014886, "o", "\u001b[?25l\u001b[H───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[2;1HStart date: 2020-05-27 19:08:20.08 | Runtime: 0:00:38.12 | OS : freebsd | Mutation engine: byte_flip_rnd\u001b[K\r\nFilesystem type: ext2 | Filesystem size: 15MB\u001b[K\r\nIteration: 6 | Last iteration time: 5.76s | Avg. iteration time: 6.25s\u001b[K\r\n#Crashes: 0 | #New crashes: 0 | Last panic: | Last new crash (iter): 0\u001b[K\r\nSuccessful mounts: 0 (0.0%) | 0/0 (0%) Commands executed\u001b[K\r\n─────────────────────────────────────────────────────────────────────────────────────────────────"] +[58.015112, "o", "──────────────────\u001b[8;1H\u001b[31m[!] Mounting failed!\u001b[39m\u001b[K\r\n───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[10;1H\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[30m\u001b[42m[fsfuzzer]0:python3- 1:python3* \"dev\" 19:09 27-May-20\u001b(B\u001b[m\u001b[10;1H\u001b[?12l\u001b[?25h"] +[60.132365, "o", "\u001b[1;60r\u001b[2;60r\u001b[59S\u001b[1;1H\u001b[K\u001b[1;61r\u001b[1;1H"] +[60.134927, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[2;1HStart date: 2020-05-27 19:08:20.08 | Runtime: 0:00:44.29 | OS : freebsd | Mutation engine: byte_flip_rnd\r\nFilesystem type: ext2 | Filesystem size: 15MB \r\nIteration: 7 | Last iteration time: 6.07s | Avg. iteration time: 6.22s\r\n#Crashes: 0 | #New crashes: 0 | Last panic: | Last new crash (iter): 0\r\nSuccessful mounts: 0 (0.0%) | 0/0 (0%) Commands executed\r\n"] +[60.137464, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[8;1H"] +[60.722648, "o", "\u001b[31m[!] Mounting failed!\r\n\u001b(B\u001b[m"] +[60.837507, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[10;1H"] +[61.447128, "o", "\u001b[?25l\u001b[H───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[2;1HStart date: 2020-05-27 19:08:20.08 | Runtime: 0:00:04.75 | OS : freebsd | Mutation engine: radamsa\u001b[K\r\nFilesystem type: ufs2 | Filesystem size: 15MB\u001b[K\r\nIteration: 1 | Last iteration time: 4.63s | Avg. iteration time: 4.63s\u001b[K\r\n#Crashes: 0 | #New crashes: 0 | Last panic: | Last new crash (iter): 0\u001b[K\r\nSuccessful mounts: 0 (0.0%) | 0/0 (0%) Commands executed\u001b[K\r\n───────────────────────────────────────────────────────────────────────────────────────────────────"] +[61.447291, "o", "────────────────\u001b[8;1H\u001b[32m[+] VM status: OK\u001b[39m\u001b[K\r\n\u001b[32m[+] Mounting successful!\u001b[39m\u001b[K\r\n \u001b[33m\u001b[1m[*] Accessing & modifying mounted filesystem: /mnt/radamsa_fuzz1_ufs2_15MB\u001b(B\u001b[m\u001b[K\r\n > Failed to perform a file traversal on mounted file system\u001b[K\r\n > Starting backup user emulation that only attempts to write to disk...\u001b[K\r\n\u001b[31m[!] VM status: Not Responding\u001b[39m\u001b[K\r\n\u001b[33m\u001b[1m[*] Completed 0/12 program calls\u001b(B\u001b[m\u001b[K\r\n\u001b[33m\u001b[1m[*] Checking for crash dump..!\u001b(B\u001b[m\u001b[K\r\n \u001b[33m\u001b[1m[*] Please stand by.............................\u001b(B\u001b[m\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[30m\u001b[42m[fsfuzzer]0:python3* 1:python3- \"dev\" 19:09 27-May-20\u001b(B\u001b[m\u001b[16;57H\u001b[?12l\u001b[?25h"] +[61.60453, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[62.605669, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[63.606733, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[64.607785, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[65.608805, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[66.609917, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[67.610851, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[68.612011, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[69.61308, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[70.614132, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[71.615242, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[72.616308, "o", "\u001b[33m\u001b[1m.\u001b(B\u001b[m"] +[73.621209, "o", "\r\n"] +[82.383551, "o", "\u001b[8C\u001b[33m\u001b[1m[*] Found core file\r\n\u001b(B\u001b[m"] +[82.607434, "o", "\u001b[36m[+] New unseen crash found: d2f866e87424d46871c0df72b78f7b3bc6934123b24b2036f5f3f0101f613db8!\r\n\u001b(B\u001b[m"] +[85.581697, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[20;1H"] +[89.424818, "o", "\u001b[1;60r\u001b[2;60r\u001b[59S\u001b[1;1H\u001b[K\u001b[1;61r\u001b[1;1H"] +[89.427779, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[2;1H"] +[89.427912, "o", "Start date: 2020-05-27 19:08:20.08 | Runtime: 0:01:14.92 | OS : freebsd | Mutation engine: radamsa\r\nFilesystem type: ufs2 | Filesystem size: 15MB \r\nIteration: 2 | Last iteration time: 70.16s | Avg. iteration time: 37.39s\r\n#Crashes: 1 | #New crashes: 1 | Last panic: ufs_dirbad | Last new crash (iter): 1\r\nSuccessful mounts: 1 (50.0%) | 0/12 (0.0%) Commands executed\r\n"] +[89.430231, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[8;1H"] +[90.057182, "o", "\u001b[31m[!] Mounting failed!\r\n\u001b(B\u001b[m"] +[90.178817, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[10;1H"] +[91.472826, "o", "\u001b[1;61r\u001b(B\u001b[m\u001b[?1l\u001b>\u001b[H\u001b[2J\u001b]112\u0007"] +[91.472978, "o", "\u001b[?12l\u001b[?25h\u001b[?1000l\u001b[?1002l\u001b[?1006l\u001b[?1005l\u001b[?1049l\u001b[23;0;0t[detached (from session fsfuzzer)]\r\n"] +[91.473899, "o", "\u001b[1m\u001b[7m%\u001b[27m\u001b[1m\u001b[0m \r \r"] +[91.474017, "o", "\u001b]2;dev@dev: ~/fisy-fuzz/src\u0007"] +[91.474466, "o", "\u001b]1;~/fisy-fuzz/src\u0007"] +[91.483022, "o", "\r\u001b[0m\u001b[27m\u001b[24m\u001b[J\u001b[01;32mdev@dev \u001b[34m[07:09:40 PM] \u001b[00m\u001b[37m[~/fisy-fuzz/src]\u001b[00m \u001b[32m[master \u001b[31m*\u001b[32m]\u001b[00m\r\n\u001b[34m->\u001b[01;34m %\u001b[00m \u001b[K"] +[91.483175, "o", "\u001b[?1h\u001b=\u001b[?2004h"] +[94.061336, "o", "\u001b[?2004l\r\r\n"] diff --git a/assets/framework.png b/assets/framework.png new file mode 100644 index 0000000..193fcff Binary files /dev/null and b/assets/framework.png differ diff --git a/assets/stdout.png b/assets/stdout.png new file mode 100644 index 0000000..7fdee2d Binary files /dev/null and b/assets/stdout.png differ diff --git a/assets/stdout_old.png b/assets/stdout_old.png new file mode 100644 index 0000000..a6fa835 Binary files /dev/null and b/assets/stdout_old.png differ diff --git a/flake8.conf b/flake8.conf new file mode 100644 index 0000000..6c7d210 --- /dev/null +++ b/flake8.conf @@ -0,0 +1,14 @@ +[flake8] +exclude = + .git, + __pycache__, + src/UserEmultion/_deprecated, + src/config/ + src/scripts + src/utility/ +ignore = + E501, + E402, + F405, + W503, + E203 diff --git a/src/Fuzzer/Fuzzer.py b/src/Fuzzer/Fuzzer.py new file mode 100644 index 0000000..80ca103 --- /dev/null +++ b/src/Fuzzer/Fuzzer.py @@ -0,0 +1,644 @@ +import datetime +import json +import logging +import os +import pathlib +import random +import re +import signal +import socket +import subprocess +import sys +import time +import zipfile +from distutils.util import strtobool +import colorama as clr +import paramiko + + +from byte_flipper import ByteFlipper +from radamsa import Radamsa +from metadata import MetaMutation + + +THIS_FILE = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(str(pathlib.Path(THIS_FILE).parent)) +sys.path.append(str(pathlib.Path(THIS_FILE))) +from Manager.Manager_FreeBSD import FreeBSD +from Manager.Manager_NetBSD import NetBSD +from Manager.Manager_OpenBSD import OpenBSD +from Manager.Manager_Ubuntu import Ubuntu +from Manager.Manager import VmManager, get_basename, get_parent_path, create_directory + +from config import fuzzing_config + +from UserEmulation.UE_Generic import GenericUserEmulation +from UserEmulation.UE_FreeBSD import FreebsdUserEmulation +from UserEmulation.UE_NetBSD import NetbsdUserEmulation +from UserEmulation.UE_OpenBSD import OpenbsdUserEmulation +from UserEmulation.UE_Ubuntu import UbuntuUserEmulation + +from utility import extract_core_features + + +def get_random_list_entry(file_list): + return random.choice(list(filter(None, file_list))).rstrip(",").strip() + + +class Fuzzer: + def __init__(self): + self.name = None # internal name for this Fuzzer object + self.vm_name = None # Name of the libvirt VM + self.rmount = None # full remote path where file system is mounted + self.new_crash_dir = None # local directory where crash is saved + self.vm_object = None # Manager object instance + self.lpath_mfs = None # full path to the current mutated fs + self.syscall_log = None # full path to the log for the executed syscalls + self.mutation_engine = None + self.mutation_size = None + self.mfs_type = "ufs" # file system type that is currently used + self.mfs_size = 20 # size of said file system + self.mfs_files = 20 # amount of files on the filesystem + self.mfs_max_file_size = 1000 # max size of each file on the filesystem + self.iter = 0 # current iteration + self.crashes = 0 # amount of crashes in current cycle + self.ucrashes = 0 # amount unique crashes in current cycle + self.last_panic = "" # displays the type of the last panic + self.start = datetime.datetime.now() # datetime object when fuzzing started + self.end = None # datetime object when fuzzing ended + self.runtime = 0 # essentially datetime.now() - self.start + self.success_mounts = 0 # number of times mounting succeeded + self.last_crash_iter = 0 # used to deduct whether a system reset is reasonable + self.last_unique = 0 + self.start_iter = 0 # used for timing iterations + self.end_iter = 0 # used to time iterations + self.all_iter_time = 0 # used for timing iterations + self.avg_iter_time = 0 # used for timing iterations + self.max_exec = 0 # used for producing stats about possible executed syscalls + self.actual_exec = 0 # used for producing stats about possible executed syscalls + self.radamsa_seed = None # generated seed by radamsa + self.fs_log = None # logs the created filesystem in a json serializable format + self.dyn_scaling = True # if enabled increases the fs size after 15k iterations of not finding a unique crash + self.target_os = None + self.host_os = None + signal.signal(signal.SIGINT, self.signal_handler) + + def __setup__(self, **kwargs): + if "name" in kwargs: + self.name = kwargs["name"] + if "vm_name" in kwargs: + self.vm_name = kwargs["vm_name"] + if "mfs_type" in kwargs: + self.mfs_type = kwargs["mfs_type"] + if "mfs_size" in kwargs: + self.mfs_size = kwargs["mfs_size"] + if "mfs_files" in kwargs: + self.mfs_files = kwargs["mfs_files"] + if "mfs_max_file_size" in kwargs: + self.mfs_max_file_size = kwargs["mfs_max_file_size"] + if "mutation_engine" in kwargs: + self.mutation_engine = kwargs["mutation_engine"][0] + self.mutation_size = int(kwargs["mutation_engine"][1]) + if "vm_object" in kwargs: + self.vm_object = kwargs["vm_object"] + if "dyn_scaling" in kwargs: + try: + self.dyn_scaling = bool(strtobool(kwargs["dyn_scaling"])) + except ValueError: + self.dyn_scaling = False + + def signal_handler(self, sig, frame): + print("Observed Ctrl+C! Exiting...") + self._save_stats() + sys.exit(1) + + def get_guest_os_kernel(self): + self.host_os = self.vm_object.exec_cmd_quiet("uname").lower() + + def set_target(self, rpath): + self.get_guest_os_kernel() + if self.host_os == "freebsd": + self.target_os = FreeBSD(mount_at=self.rmount, rfile=rpath, vm_object=self.vm_object) + elif self.host_os == "openbsd": + self.target_os = OpenBSD(mount_at=self.rmount, rfile=rpath, vm_object=self.vm_object) + elif self.host_os == "netbsd": + self.target_os = NetBSD(mount_at=self.rmount, rfile=rpath, vm_object=self.vm_object) + elif self.host_os == "linux": + self.target_os = Ubuntu(mount_at=self.rmount, rfile=rpath, vm_object=self.vm_object) + else: + logging.error("Unknown or not implemented target kernel: {}".format(self.host_os)) + sys.exit(1) + + def mutation_radamsa( + self, file_system, preserve_magic=True, preserve_uberblock=False, determinism=True, + ): + self.radamsa_seed, self.lpath_mfs = Radamsa(file_system).mutation(preserve_magic, preserve_uberblock, determinism) + + def mutation_byte_flip_seq(self, fs_path, n_bytes=1): + self.lpath_mfs = ByteFlipper(fs_path, n_bytes, mode="seq").mutation_seq() + + def mutation_byte_flip_rnd(self, fs_path, n_bytes=1): + self.lpath_mfs = ByteFlipper(fs_path, n_bytes, mode="rnd").mutation_rnd() + + def mutation_metadata(self, fs_path, n_bytes=3): + self.lpath_mfs = MetaMutation(fs_path, n_bytes, mode="sb_meta").mutation() + + def mutation_metablock(self, fs_path): + pass + + def _get_two_distinct_list_elements(self, list_one, list_two): + _file1 = get_random_list_entry(list_one) + _file2 = get_random_list_entry(list_two) + try: + if _file1 != _file2: + return _file1, _file2 + else: + self._get_two_distinct_list_elements(list_one, list_two) + except RecursionError: + return None, None + + @staticmethod + def _flush_write(open_file_handle): + open_file_handle.flush() + os.fsync(open_file_handle.fileno()) + + def user_interaction_emulation(self, syscall_log): + print(clr.Fore.LIGHTYELLOW_EX + "\t[*] Accessing & modifying mounted filesystem: {}".format(self.rmount) + clr.Fore.RESET) + copy_scripts_to_fuzzer(self.vm_object) + exec_cmds = 0 + try: + total_cmds = self._set_user_emulation() + except IndexError: + return 1 + self.max_exec += len(total_cmds) + for cmd in total_cmds: + if any(x in cmd for x in ["cp", "mv"]): + cmd = self.dynamic_resolving_of_cp_and_mv_command(cmd, syscall_log) + ret_cmd = self.vm_object.exec_cmd_quiet(cmd) + logging.debug("RET VAL FOR {} IS: {}".format(cmd, ret_cmd)) + if ret_cmd == 2 and not self.vm_object.check_vm_state(): + self.print_successful_executed_commands(exec_cmds, total_cmds) + return self._flush_write_crash_syscall_log(cmd, syscall_log, exec_cmds) + if ( + any(x in cmd for x in ["dd", "find", "readlink", "getfacl", "ls", "stat", "tar", "du", "wc"]) + and type(ret_cmd) == str + ): + if "tar" in cmd and "Error" in ret_cmd: + syscall_log.write("[-] {}\n".format(cmd)) + elif "getfacl" in cmd and "stat() failed" in ret_cmd: + syscall_log.write("[-] {}\n".format(cmd)) + elif "No such" in ret_cmd: + syscall_log.write("[-] {}\n".format(cmd)) + else: + exec_cmds = self._write_success_log(cmd, syscall_log, exec_cmds) + elif ( + not any(x in cmd for x in ["dd", "find", "readlink", "getfacl", "ls", "stat", "tar", "du", "wc"]) and not ret_cmd + ): + exec_cmds = self._write_success_log(cmd, syscall_log, exec_cmds) + else: + syscall_log.write("[-] {}\n>>{}\n".format(cmd, ret_cmd)) + self._flush_write(syscall_log) + self.print_successful_executed_commands(exec_cmds, total_cmds) + self.actual_exec += exec_cmds + return 1 + + def _set_user_emulation(self): + # return UserEmulation(vm_object=self.vm_object, rpath=self.rmount).set_user_emulation() + if self.host_os == "freebsd": + user_emulation_command_list = FreebsdUserEmulation( + vm_object=self.vm_object, rpath=self.rmount + ).set_freebsd_user_emulation() + elif self.host_os == "openbsd": + user_emulation_command_list = OpenbsdUserEmulation( + vm_object=self.vm_object, rpath=self.rmount + ).set_openbsd_user_emulation() + elif self.host_os == "netbsd": + user_emulation_command_list = NetbsdUserEmulation( + vm_object=self.vm_object, rpath=self.rmount + ).set_netbsd_user_emulation() + elif self.host_os == "linux": + user_emulation_command_list = UbuntuUserEmulation( + vm_object=self.vm_object, rpath=self.rmount + ).set_ubuntu_user_emulation() + else: + logging.error("unknown target kernel/os!") + sys.exit(1) + return user_emulation_command_list + + @staticmethod + def print_successful_executed_commands(exec_cmds, total_cmds): + print(clr.Fore.LIGHTYELLOW_EX + "[*] Completed {}/{} program calls".format(exec_cmds, len(total_cmds)) + clr.Fore.RESET) + + def dynamic_resolving_of_cp_and_mv_command(self, cmd, syscall_log): + try: + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rmount) + _file, _dir = self._get_two_distinct_list_elements( + gen_user.get_files_of_mounted_file_system(param="files"), gen_user.get_files_of_mounted_file_system(param="dir"), + ) + cmd = cmd.format(_file, _dir) + except (AttributeError, TypeError): + syscall_log.write("[-] {}\n".format(cmd)) + return cmd + + def _write_success_log(self, cmd, log_handler, ctr_exec_cmd): + log_handler.write("[+] {}\n".format(cmd)) + self._flush_write(log_handler) + ctr_exec_cmd += 1 + return ctr_exec_cmd + + def _flush_write_crash_syscall_log(self, cmd, fd, success): + fd.write("[!] {}\n".format(cmd)) + self._flush_write(fd) + self.actual_exec += success + self.check_if_crash_sample() + return None + + def _compress_vmcore(self): + vmcore = [os.path.join(self.new_crash_dir, f) for f in os.listdir(self.new_crash_dir) if f.startswith("vmcore")][0] + zip_path = os.path.join(self.new_crash_dir, get_basename(vmcore)) + with zipfile.ZipFile(zip_path + ".zip", "w", compression=zipfile.ZIP_DEFLATED) as myzip: + myzip.write(vmcore, arcname=get_basename(vmcore)) + os.remove(vmcore) + + def save_fs_dict_to_disk(self): + try: + _path = os.path.join(self.new_crash_dir, "fs.json") + self.fs_log = re.sub(r"[\a-zA-Z0-9]*?{\"fs", '{"fs', self.fs_log) + if self.fs_log.endswith("%"): + self.fs_log = self.fs_log[-1] + self.fs_log = json.loads(self.fs_log) + self.fs_log["crash_meta_data"] = {} + self.fs_log["crash_meta_data"]["seed"] = self.radamsa_seed + self.fs_log["crash_meta_data"]["panic"] = self.last_panic + with open(_path, "w") as f: + f.write(json.dumps(self.fs_log, indent=4)) + except TypeError: + pass + + def check_if_crash_sample(self): + self.last_crash_iter = self.iter + try: + self.new_crash_dir = self.vm_object.crash_handler() + if self.new_crash_dir: + self._backup_samples() + self._check_if_crash_is_yet_unknown() + self.save_fs_dict_to_disk() + self.vm_object.exec_cmd_quiet("/bin/rm -rf /var/crash/*") + self._compress_vmcore() + self._save_stats() + except socket.timeout as e: + logging.error("SOCKET TIMEOUT: {}".format(e)) + self.vm_object.reset_vm() + if self.vm_object.silent_vm_state(): + self.check_if_crash_sample() + else: + self.vm_object.restore_snapshot(self.vm_object.get_current_snapshot()) + self.vm_object.quick_boot(vm_name=self.vm_object.name) + + def _save_stats(self): + statsp = os.path.join(os.getcwd(), "stats") + create_directory(statsp) + with open(os.path.join(statsp, str(self.start)[:-4] + "_" + get_basename(self.lpath_mfs)) + ".txt", "w",) as s: + s.write("> Start date: {}\n".format(str(self.start))) + s.write("> End date: {}\n".format(str(datetime.datetime.now()))) + s.write("> Engine: {}\n".format(str(get_basename(self.lpath_mfs)).split("_"))[-5].strip()) + s.write("> Runtime: {}\n".format(str(self.runtime))) + s.write("> File system name: {}\n".format(str(self.lpath_mfs))) + s.write("> File system type: {}\n".format(str(self.mfs_type))) + s.write("> File system size: {}MB\n".format(str(self.mfs_size))) + s.write("> #Files in initial file system: {}\n".format(str(self.mfs_files))) + s.write("> #Max_size of files: {}KB\n".format(str(self.mfs_max_file_size))) + s.write("> Iterations: {}\n".format(str(self.iter))) + s.write("> Avg Iteration time: {}s\n".format(str(self.avg_iter_time))) + s.write("> #Crashes: {}\n".format(str(self.crashes))) + s.write("> #Unique_Crashes: {}\n".format(str(self.ucrashes))) + s.write( + "> #Successful_Mounts: {}({}%)\n".format( + str(self.success_mounts), str(self._get_percentage(self.success_mounts, self.iter)), + ) + ) + s.write("> #Unsuccessful_Mounts {}\n".format(str(int(self.iter) - int(self.success_mounts)))) + s.write( + "> {}/{} ({}%) Commands executed\n".format( + str(self.actual_exec), str(self.max_exec), str(self._get_percentage(self.actual_exec, self.max_exec)), + ) + ) + + def automate(self, rpath_mfs, mount_at): + self.rmount = mount_at + self._print_statistics_output_to_tty() + self.syscall_log = os.path.join(os.getcwd(), "file_system_storage/{}_syscall.log".format(self.name)) + with open(self.syscall_log, "w") as syscall_log: + self.set_target(rpath_mfs) + mount_ret = self.target_os.mount_file_system() + if mount_ret == 1 and self.vm_object.check_vm_state(): + print(clr.Fore.GREEN + "[+] Mounting successful!" + clr.Fore.RESET) + self.success_mounts += 1 + if self.user_interaction_emulation(syscall_log): + self.unmount_file_system_on_remote() + else: + print(clr.Fore.RED + "[!] Mounting failed!" + clr.Fore.RESET) + if mount_ret == 0 and not self.target_os.destroy_bdev() and self.vm_object.silent_vm_state(): + pass + else: + syscall_log.write("[!] mount\n") + self.check_if_crash_sample() + self.iter += 1 + self._print_separator() + self.end_iter = round(time.time() - self.start_iter, 2) + self.all_iter_time += self.end_iter + self.avg_iter_time = round(self.all_iter_time / self.iter, 2) + + def unmount_file_system_on_remote(self): + if self.target_os.unmount_file_system() and self.vm_object.silent_vm_state(): + print(clr.Fore.GREEN + "[+] Unmounted {} successfully".format(self.rmount) + clr.Fore.RESET) + else: + self.check_if_crash_sample() + + def _print_statistics_output_to_tty(self): + os.system("clear") + self._print_separator() + print( + "Start date: {} | Runtime: {} | OS: {} | Mutation engine: {}\n" + "Filesystem type: {} | Filesystem size: {}MB \n" + "Iteration: {} | Last iteration time: {}s | Avg. iteration time: {}s\n" + "# Crashes: {} | # New crashes: {} | Last panic: {} | Last new crash (iter): {}\n" + "Successful mounts: {} ({}%) | {}/{} ({}%) Commands executed".format( + str(self.start)[:-4], + self.runtime, + self.host_os, + self.mutation_engine, + self.mfs_type, + self.mfs_size, + self.iter, + self.end_iter, + self.avg_iter_time, + self.crashes, + self.ucrashes, + self.last_panic, + self.last_unique, + self.success_mounts, + self._get_percentage(self.success_mounts, self.iter), + self.actual_exec, + self.max_exec, + self._get_percentage(self.actual_exec, self.max_exec), + ) + ) + self._print_separator() + + @staticmethod + def _print_separator(): + line_separator = "─" + print(line_separator * int(subprocess.check_output(["stty", "size"], encoding="utf-8").split()[1])) + + def _get_core_details(self, file): + with open(file, "rb") as f: + data = f.read() + self.last_panic = extract_core_features.get_panic_name(data) + return extract_core_features.get_core_details(data) + + @staticmethod + def _write_sha256sum_txt(core_txt, sha256_trace): + sum_txt = str(pathlib.Path(core_txt).parent) + "/shasum256.txt" + with open(sum_txt, "w") as g: + g.write(sha256_trace) + g.close() + + def _check_if_crash_is_yet_unknown(self): + core_txt = [os.path.join(self.new_crash_dir, f) for f in os.listdir(self.new_crash_dir) if f.startswith("core.txt")][0] + sanitized_bt = self._get_core_details(core_txt) + if len(self.last_panic) <= 2: + return 0 + self.crashes += 1 + sha256_trace = extract_core_features.get_sha256_sum(sanitized_bt) + self._write_sha256sum_txt(core_txt, sha256_trace) + crashdb = os.path.join(os.getcwd(), "crash_dumps/crash.db") + pathlib.Path(self.new_crash_dir).rename(self.new_crash_dir + "_" + str(self.last_panic)) + self.new_crash_dir = self.new_crash_dir + "_" + self.last_panic + with open(crashdb, "a+") as f: + f.seek(0) + db_contents = f.read() + if sha256_trace not in db_contents: + self.ucrashes += 1 + self.last_unique = self.iter + print(clr.Fore.CYAN + "[+] New unseen crash found: {}!".format(sha256_trace) + clr.Fore.RESET) + mfs_meta = str(get_basename(self.lpath_mfs)).split("_") + engine = mfs_meta[0].strip() + if "radamsa" in engine and self.radamsa_seed: + engine = engine + " (seed: {})".format(self.radamsa_seed) + else: + engine = "None" + fs = mfs_meta[-2].strip() + size = mfs_meta[-1].strip() + entry = ( + str(self.name) + + "; " + + str(self.vm_name) + + "; " + + fs + + "; " + + size + + "; " + + engine + + "; " + + str(self.last_panic) + + "; " + + sha256_trace + + "; " + + str(self.new_crash_dir) + + "; " + + str(self.runtime) + + "; " + + str(self.iter) + + "\n" + ) + f.write(entry) + f.close() + + def _backup_samples(self): + files = [ + self.lpath_mfs, + self.syscall_log, + os.path.join( + str(get_parent_path(self.lpath_mfs)), "_".join(x for x in str(get_basename(self.lpath_mfs)).split("_")[1:]), + ), + ] + logging.debug("BACKUP FILES: {}".format(files)) + archive = os.path.join(self.new_crash_dir, "sample.zip") + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as myzip: + for f in list(filter(None, files)): + myzip.write(f, pathlib.Path(f).name) + myzip.close() + + def _iter_reset(self): + if self.iter % 150 == 0 and self.iter - self.last_crash_iter > 50: + logging.warning("Automatic VM reset in progress...") + cur_snap = self.vm_object.get_current_snapshot() + try: + self.vm_object.restore_snapshot(cur_snap) + self.vm_object.new_rshell() + except socket.timeout as e: + logging.debug("Socket timed out during snapshot restoring: {}".format(e)) + time.sleep(2) + if self.vm_object.check_vm_state(): + self.vm_object.new_rshell() + else: + self.vm_object.crash_handler() + + @staticmethod + def _get_percentage(part, whole): + try: + return round(100 * float(part) / float(whole), 2) + except ZeroDivisionError: + return 0 + + @staticmethod + def _remove_iteration_leftovers_on_target(fs_maker_vm, fs_name): + fs_maker_vm.exec_cmd_quiet("/bin/rm -rf {}".format(os.path.join("/tmp/", fs_name))) + fs_maker_vm.exec_cmd_quiet("/bin/rm -rf {}".format(os.path.join("/mnt", fs_name))) + + def copy_generated_file_system_to_host(self, fs_maker_vm, fs_name): + fs_maker_vm.cp_to_host( + save_files_at=os.path.join(os.getcwd() + "/file_system_storage"), + get_files_from="/tmp/", + list_of_files_to_copy=fs_name, + ) + self._remove_iteration_leftovers_on_target(fs_maker_vm, "fs_" + fs_name) + + def fuzz(self, fuzzy_vm, fs_maker_vm): + create_directory(os.getcwd() + "/file_system_storage") + while True: + if self.dyn_scaling: + self._change_fs_parameters() + try: + self.start_iter = time.time() + self.runtime = str(datetime.datetime.now() - self.start)[:-4] + self._iter_reset() + fs_name = "{}_{}_{}MB".format(self.name, self.mfs_type, self.mfs_size) + cmd = ( + "python3 /tmp/makeFS2.py -fs {} -m 1" + ' -n "{}"' + " -s {}" + " -p {}" + " -ps {}" + " -o {}".format(self.mfs_type, fs_name, self.mfs_size, self.mfs_files, self.mfs_max_file_size, "/tmp/",) + ) + if fs_maker_vm.silent_vm_state(): + self.fs_log = fs_maker_vm.exec_cmd_quiet(cmd) + if "ERROR" in self.fs_log: + print("Failed FS creation: {}".format(self.fs_log)) + sys.exit(1) + else: + fs_maker_vm.restore_snapshot(fs_maker_vm.get_current_snapshot()) + fs_maker_vm.quick_boot(vm_name=fs_maker_vm.name) + self.fs_log = fs_maker_vm.exec_cmd_quiet(cmd) + if not self.fs_log: + logging.error("Failed to fetch fs sample log.. Exiting..!\n") + sys.exit(1) + self.copy_generated_file_system_to_host(fs_maker_vm, fs_name) + self._make_mutation(fs_name) + if not self.lpath_mfs: + continue + fuzzy_vm.cp_to_guest( + get_files_from="file_system_storage/", + list_of_files_to_copy=get_basename(self.lpath_mfs), + save_files_at="/tmp", + ) + if "zfs" in self.mfs_type: + mnt_path = "pool_" + "_".join(x for x in get_basename(self.lpath_mfs).split("_")[1:]) + else: + mnt_path = get_basename(self.lpath_mfs) + self.automate( + rpath_mfs="/tmp/{}".format(get_basename(self.lpath_mfs)), mount_at="/mnt/{}".format(mnt_path), + ) + self.vm_object.exec_cmd_quiet("rm -rf {}".format(os.path.join("/tmp", self.lpath_mfs))) + except (paramiko.ssh_exception.SSHException, paramiko.ssh_exception.NoValidConnectionsError, socket.timeout,) as e: + logging.error("SSH/socket exception: {}. Resetting VM".format(e)) + self.check_if_crash_sample() + except (ValueError, OSError) as e: + logging.error("Ran into a problem during fuzzing: {}".format(e)) + except (EOFError, AttributeError) as e: + logging.error(e) + logging.error("VM may be down..? Resetting") + self.vm_object.restore_snapshot(snap_name=self.vm_object.get_current_snapshot()) + self.vm_object.reset_vm() + + def _make_mutation(self, fs_name): + try: + fs = os.path.join(os.getcwd() + "/file_system_storage/", fs_name) + if self.mutation_engine == "radamsa": + self.mutation_radamsa(fs) + elif self.mutation_engine == "byte_flip_seq": + self.mutation_byte_flip_seq(fs, self.mutation_size) + elif self.mutation_engine == "byte_flip_rnd": + self.mutation_byte_flip_rnd(fs, self.mutation_size) + elif self.mutation_engine == "metadata": + self.mutation_metadata(fs, self.mutation_size) + else: + logging.error("Unknown mutation engine specified! Exiting...") + sys.exit(1) + except (AttributeError, IndexError): + logging.error("Ran into a problem during mutation. Exiting..!") + sys.exit(1) + + def _change_fs_parameters(self): + if self.mfs_size >= 750: + if self.mfs_type == "zfs": + self.mfs_size = 65 + self.mfs_files = 20 + self.mfs_max_file_size = 2048 + else: + self.mfs_size = 15 + self.mfs_files = 10 + self.mfs_max_file_size = 1024 + return 0 + if (int(self.iter) - int(self.last_unique)) >= 15000: + self.last_unique += 15000 # workaround so the new fs size is fuzzing for another 15k iters + self.mfs_size += 50 + if random.randint(0, 1) == 0: + self.mfs_files = int(((self.mfs_size << 10) - 3000) / self.mfs_max_file_size) + else: + self.mfs_max_file_size = int(((self.mfs_size << 10) - 3000) / self.mfs_files) + + +def to_dict(input_ordered_dict): + return json.loads(json.dumps(input_ordered_dict)) + + +def copy_scripts_to_fuzzer(fuzzy_vm): + if not int(fuzzy_vm.exec_cmd_quiet("[ -f /tmp/get_users_and_groups.py ] && echo 1 || echo 0 | /usr/bin/head -n1")): + fuzzy_vm.cp_to_guest( + get_files_from="utility/", list_of_files_to_copy="get_users_and_groups.py", save_files_at="/tmp", + ) + if not int(fuzzy_vm.exec_cmd_quiet("[ -f /tmp/file_traversal.py ] && echo 1 || echo 0 | /usr/bin/head -n1")): + fuzzy_vm.cp_to_guest( + get_files_from="utility/", list_of_files_to_copy="file_traversal.py", save_files_at="/tmp", + ) + + +def main(): + fs_generator = VmManager() + fs_generator.setup(vm_user=fuzzing_config.user, vm_password=fuzzing_config.pw, name=sys.argv[2]) + fs_generator.quick_boot(vm_name=sys.argv[2]) + if not int(fs_generator.exec_cmd_quiet("[ -f /tmp/makeFS2.py ] && echo 1 || echo 0 | head -n1")): + fs_generator.cp_to_guest(get_files_from=".", list_of_files_to_copy="makeFS2.py", save_files_at="/tmp/") + fuzzer = Fuzzer() + fuzzer.__setup__( + name=sys.argv[1], + vm_name=sys.argv[3], + mutation_engine=tuple(x for x in sys.argv[4].split(", ")), + mfs_type=sys.argv[5], + mfs_size=int(sys.argv[6]), + mfs_files=int(sys.argv[7]), + mfs_max_file_size=int(sys.argv[8]), + dyn_scaling=sys.argv[9], + ) + fuzz_vm = VmManager() + fuzz_vm.setup(vm_user=fuzzing_config.user, vm_password=fuzzing_config.pw, name=fuzzer.name) + fuzz_vm.quick_boot(vm_name=fuzzer.vm_name) + fuzzer.vm_object = fuzz_vm + fuzzer.fuzz(fuzz_vm, fs_generator) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/Fuzzer/__init__.py b/src/Fuzzer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/Fuzzer/byte_flipper.py b/src/Fuzzer/byte_flipper.py new file mode 100644 index 0000000..5d3b0a8 --- /dev/null +++ b/src/Fuzzer/byte_flipper.py @@ -0,0 +1,44 @@ +import logging +import random +import struct + +from file_system_magic.fs_util import get_bytearray, write_to_file, get_mutated_bytes + + +class ByteFlipper: + def __init__(self, fs, nbytes, mode=None): + self.nbytes = nbytes + self.fs = fs + self.mfs = None + self.mime = None + self.mode = mode + self.rnd = random.Random() + self.rnd.seed(random.getrandbits(1024)) + + def mutation_seq(self): + byte_array = get_bytearray(self.fs) + try: + rnd_pos = random.randint(0, len(byte_array) - self.nbytes) + muta_byte_seq = get_mutated_bytes(self.nbytes) + ctr = 0 + while ctr <= self.nbytes: + byte_array[rnd_pos + ctr] = struct.pack("B", muta_byte_seq[ctr]) + ctr += 1 + except IndexError as e: + logging.error(e) + return None + return write_to_file(self.fs, self.nbytes, self.mode, byte_array) + + def mutation_rnd(self): + ctr = 0 + byte_array = get_bytearray(self.fs) + while ctr < self.nbytes: + try: + rnd_pos = random.randint(0, len(byte_array)) + muta_byte_seq = get_mutated_bytes(self.nbytes) + byte_array[rnd_pos] = muta_byte_seq + ctr += 1 + except IndexError as e: + logging.error(e) + return None + return write_to_file(self.fs, self.nbytes, self.mode, byte_array) diff --git a/src/Fuzzer/file_system_magic/__init__.py b/src/Fuzzer/file_system_magic/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/Fuzzer/file_system_magic/ext_superblock_parser.py b/src/Fuzzer/file_system_magic/ext_superblock_parser.py new file mode 100644 index 0000000..656c407 --- /dev/null +++ b/src/Fuzzer/file_system_magic/ext_superblock_parser.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 + +import argparse +import os +import pathlib +import pprint as pp +import re +import sys +from collections import OrderedDict +from ctypes import Structure, sizeof, c_uint8, c_char, c_uint32, c_uint64, c_uint16 +from .fs_util import get_int + +# xxd EXT_FS | 'ef53' +# at offset 1080 +EXT_MAGIC = b"\x53\xef" + +SBLOCK_EXT2 = 1024 # First 1024 bytes are unused, block group 0 starts with a superblock @ offset 1024d +MAGIC_BYTES_OFF = 56 + +EXT_SB = [ + ("e2fs_icount", c_uint32), + ("e2fs_bcount", c_uint32), + ("e2fs_rbcount", c_uint32), + ("e2fs_fbcount", c_uint32), + ("e2fs_ficount", c_uint32), + ("e2fs_first_dblock", c_uint32), + ("e2fs_log_bsize", c_uint32), + ("e2fs_log_fsize", c_uint32), + ("e2fs_bpg", c_uint32), + ("e2fs_fpg", c_uint32), + ("e2fs_ipg", c_uint32), + ("e2fs_mtime", c_uint32), + ("e2fs_wtime", c_uint32), + ("e2fs_mnt_count", c_uint16), + ("e2fs_max_mnt_count", c_uint16), + ("e2fs_magic", c_uint16), + ("e2fs_state", c_uint16), + ("e2fs_beh", c_uint16), + ("e2fs_minrev", c_uint16), + ("e2fs_lastfsck", c_uint32), + ("e2fs_fsckintv", c_uint32), + ("e2fs_creator", c_uint32), + ("e2fs_rev", c_uint32), + ("e2fs_ruid", c_uint16), + ("e2fs_rgid", c_uint16), + ("e2fs_first_ino", c_uint32), + ("e2fs_inode_size", c_uint16), + ("e2fs_block_group_nr", c_uint16), + ("e2fs_features_compat", c_uint32), + ("e2fs_features_incompat", c_uint32), + ("e2fs_features_rocompat", c_uint32), + ("e2fs_uuid", c_uint8 * 16), # arr[16], at offset 104 + ("e2fs_vname", c_char * 16), # arr[16] + ("e2fs_fsmnt", c_char * 64), # arr[64] + ("e2fs_algo", c_uint32), + ("e2fs_prealloc", c_uint8), + ("e2fs_dir_prealloc", c_uint8), + ("e2fs_reserved_ngdb", c_uint16), + ("e3fs_journal_uuid", c_char * 16), # arr[16] + ("e3fs_journal_inum", c_uint32), + ("e3fs_journal_dev", c_uint32), + ("e3fs_last_orphan", c_uint32), + ("e3fs_hash_seed", c_uint32 * 4), # arr[4] + ("e3fs_def_hash_version", c_char), + ("e3fs_jnl_backup_type", c_char), + ("e3fs_desc_size", c_uint16), + ("e3fs_default_mount_opts", c_uint32), + ("e3fs_first_meta_bg", c_uint32), + ("e3fs_mkfs_time", c_uint32), + ("e3fs_jnl_blks", c_uint32), + ("e4fs_bcount_hi", c_uint32), + ("e4fs_rbcount_hi", c_uint32), + ("e4fs_fbcount_hi", c_uint32), + ("e4fs_min_extra_isize", c_uint16), + ("e4fs_want_extra_isize", c_uint16), + ("e4fs_flags", c_uint32), + ("e4fs_raid_stride", c_uint16), + ("e4fs_mmpintv", c_uint16), + ("e4fs_mmpblk", c_uint64), + ("e4fs_raid_stripe_wid", c_uint32), + ("e4fs_log_gpf", c_uint8), + ("e4fs_chksum_type", c_uint8), + ("e4fs_encrypt", c_uint8), + ("e4fs_reserved_pad", c_uint8), + ("e4fs_kbytes_written", c_uint64), + ("e4fs_snapinum", c_uint32), + ("e4fs_snapid", c_uint32), + ("e4fs_snaprbcount", c_uint64), + ("e4fs_snaplist", c_uint32), + ("e4fs_errcount", c_uint32), + ("e4fs_first_errtime", c_uint32), + ("e4fs_first_errino", c_uint32), + ("e4fs_first_errblk", c_uint64), + ("e4fs_first_errfunc", c_uint8 * 32), # arr[32] + ("e4fs_first_errline", c_uint32), + ("e4fs_last_errtime", c_uint32), + ("e4fs_last_errino", c_uint32), + ("e4fs_last_errline", c_uint32), + ("e4fs_last_errblk", c_uint64), + ("e4fs_last_errfunc", c_uint8 * 32), # arr[32] + ("e4fs_mount_opts", c_uint8 * 64), # arr[64] + ("e4fs_usrquota_inum", c_uint32), + ("e4fs_grpquota_inum", c_uint32), + ("e4fs_overhead_clusters", c_uint32), + ("e4fs_backup_bgs", c_uint32 * 2), # arr[2] + ("e4fs_encrypt_algos", c_uint8 * 4), # arr[4] + ("e4fs_encrypt_pw_salt", c_uint8 * 16), # arr[16] + ("e4fs_lpf_ino", c_uint32), + ("e4fs_proj_quota_inum", c_uint32), + ("e4fs_chksum_seed", c_uint32), + ("e4fs_reserved", c_uint32 * 98), # arr[98] + ("e4fs_sbchksum", c_uint32), +] + + +class EXT(Structure): + def __init__(self, fs, fst): + super(Structure).__init__() + self.sb = OrderedDict() + self.sb_expected_len = 960 + self.fs = fs + self.fst = fst + self.sb_locs = [] + self.sb_locs = [] + self.fields_sb = EXT_SB + + def _sanity_check(self): + res_sb = 0 + for _, v in self.fields_sb: + res_sb += sizeof(v) + assert res_sb == self.sb_expected_len + + @staticmethod + def get_offset_in_sb(fn): + off = 0 + sb = EXT_SB + for i, v in sb: + if i == fn: + return off, sizeof(v) + off += sizeof(v) + return None, None + + def read_superblock_in_dict(self, loc=SBLOCK_EXT2): + with open(self.fs, "rb") as f: + f.seek(loc) + for field in self.fields_sb: + self.sb[field[0]] = f.read(sizeof(field[1])) + + def find_all_superblocks(self): + self.read_superblock_in_dict() + with open(self.fs, "rb") as f: + f.seek(0) + data = f.read() + # Using uuid because the EXT2 magic is too short to yield good results + matches = re.finditer(self.sb["e2fs_uuid"], data) + for m in matches: + bytearr = bytearray() + sb = m.span()[0] - 104 + bytearr.append(data[sb + MAGIC_BYTES_OFF]) + bytearr.append(data[sb + MAGIC_BYTES_OFF + 1]) + if bytearr == EXT_MAGIC: + self.sb_locs.append(sb) + return self.sb_locs + + def find_all_cylinder_groups(self): + self.cg_locs = [] + + def print_superblock(self): + tmp = OrderedDict() + for key, value in self.sb.items(): + if key in ["e3fs_def_hash_version", "e3fs_jnl_backup_type", "e3fs_journal_uuid", "e2fs_fsmnt", "e2fs_vname"]: + tmp[key] = hex(get_int(value, signed=False)) + else: + tmp[key] = hex(get_int(value, signed=False)) + pp.pprint(tmp) + + def dump_superblock(self, n=SBLOCK_EXT2): + self.read_superblock_in_dict(loc=n) + p = str(pathlib.Path(self.fs).parent) + c = str(pathlib.Path(self.fs).name) + fp = os.path.join(p, f"superblock_{hex(n)}_" + c + ".dump") + with open(fp, "wb") as f: + for _, value in self.sb.items(): + f.write(value) + print(f"[+] Dumped {fp}") + + def dump_all_superblocks(self): + self.find_all_superblocks() + for i in self.sb_locs: + self.dump_superblock(n=i) + + +def main(): + parser = argparse.ArgumentParser(description="EXT file system parser") + parser.add_argument( + "--dump", "-d", action="store_true", default=False, dest="dump", help="Dumps the first superblock to disk" + ) + parser.add_argument( + "--dump_all", "-da", action="store_true", default=False, dest="dump_all", help="Dumps all superblocks to disk" + ) + parser.add_argument( + "--print_superblock", + "-ps", + type=int, + default=-1, + dest="print_sb", + help="Print the n-th superblock to stdout. Default: %(default)s", + ) + parser.add_argument( + "--find_all", + "-fa", + action="store_true", + default=False, + dest="find_all", + help="Finds all superblock locations and prints them to stdout", + ) + parser.add_argument("--file_system", "-f", required=True, type=pathlib.Path, help="UFS Filesystem") + + args = parser.parse_args() + + ext = EXT(args.file_system, "ext") + if args.dump: + ext.dump_superblock() + if args.dump_all: + ext.dump_all_superblocks() + if args.find_all: + ext.find_all_superblocks() + res = ", ".join(hex(e) for e in ext.sb_locs) + print(f"[+] Found superblock offsets: {res}") + if args.print_sb >= 0: + ext.find_all_superblocks() + ext.read_superblock_in_dict(ext.sb_locs[args.print_sb]) + ext.print_superblock() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/Fuzzer/file_system_magic/fs_util.py b/src/Fuzzer/file_system_magic/fs_util.py new file mode 100644 index 0000000..0813764 --- /dev/null +++ b/src/Fuzzer/file_system_magic/fs_util.py @@ -0,0 +1,87 @@ +import os +import re +import magic +import pathlib +from ctypes import sizeof +from datetime import datetime +from secrets import token_bytes + + +def get_int(n, signed=False): + return int.from_bytes(n, byteorder="little", signed=signed) + + +def get_time(n): + return datetime.fromtimestamp(n).strftime("%c") + + +def get_hstr(hex_str, inv=False): + if len(hex_str[2:]) % 16 != 0: + hex_str = "0" + hex_str[2:] + else: + hex_str = hex_str[2:] + if inv: + return bytes.fromhex(hex_str[::-1]).decode("ASCII") + else: + return bytes.fromhex(hex_str).decode("ASCII") + + +def make_zero(size): + return b"\x00" * size + + +def make_ff(size): + return b"\xFF" * size + + +def make_rnd(size): + return token_bytes(size) + + +def get_bytearray(fs): + with open(fs, "rb") as f: + byte_array = re.findall(b".", f.read()) + return byte_array + + +def set_mime(fs): + file_mime = magic.from_file(fs) + if "Unix Fast" in file_mime and "[v1]" in file_mime: + return "ufs1" + elif "Unix Fast" in file_mime and "[v2]" in file_mime: + return "ufs2" + elif re.findall(r"ext[2-4]", file_mime): + return "ext" + elif "data" in file_mime: + return "zfs" + + +def write_to_file(fs, nbytes, mode, byte_array): + name = pathlib.Path(fs).name + _path = pathlib.Path(fs).parent + mfs = os.path.join(_path, "{}b_{}_".format(nbytes, mode) + name) + with open(mfs, "wb") as g: + g.write(b"".join(x for x in byte_array)) + return mfs + + +def get_mutated_bytes(nbytes, mode=None): + if mode == "ff": + return make_ff(nbytes) + elif mode == "00": + return make_zero(nbytes) + else: + return make_rnd(nbytes) + + +def get_offset_in_sb(fsp, fn): + off = 0 + for i, v in fsp.fields_sb: + if i == fn: + return off, sizeof(v) + off += sizeof(v) + return None, None + + +if __name__ == "__main__": + pass diff --git a/src/Fuzzer/file_system_magic/ufs_superblock_parser.py b/src/Fuzzer/file_system_magic/ufs_superblock_parser.py new file mode 100644 index 0000000..af6e95b --- /dev/null +++ b/src/Fuzzer/file_system_magic/ufs_superblock_parser.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 + +import argparse +import os +import pathlib +import pprint as pp +import re +from collections import OrderedDict +from ctypes import Structure, sizeof, c_int32, c_int64, c_uint8, c_char, c_int8, c_uint32, c_int16, c_void_p, c_uint64, c_size_t +from .fs_util import get_int + +# xxd UFS_FS | grep '1901 5419' +# multiple offsets +UFS_MAGIC = b"\x19\x01\x54\x19" +CG_MAGIC = b"\x55\x02\x09" +SBLOCK_PIGGY = 262144 +SBLOCKSIZE = 8192 +MAXMNTLEN = 468 +MAXVOLLEN = 32 +FSMAXSNAP = 20 +NOCSPTRS = int(128 / (sizeof(c_void_p)) - 4) +MAXFRAG = 8 +SBLOCK_UFS1 = 8192 +SBLOCK_UFS2 = 65536 + +ufs_time_t = c_int64 +ufs2_daddr_t = c_int64 + +UFS_SB = [ + ("fs_firstfield", c_int32), + ("fs_unused_1", c_int32), + ("fs_sblkno", c_int32), + ("fs_cblkno", c_int32), + ("fs_iblkno", c_int32), + ("fs_dblkno", c_int32), + ("fs_old_cgoffset", c_int32), + ("fs_old_cgmask", c_int32), + ("fs_old_time", c_int32), + ("fs_old_size", c_int32), + ("fs_old_dsize", c_int32), + ("fs_ncg", c_uint32), + ("fs_bsize", c_int32), + ("fs_fsize", c_int32), + ("fs_frag", c_int32), + ("fs_minfree", c_int32), + ("fs_old_rotdelay", c_int32), + ("fs_old_rps", c_int32), + ("fs_bmask", c_int32), + ("fs_fmask", c_int32), + ("fs_bshift", c_int32), + ("fs_fshift", c_int32), + ("fs_maxcontig", c_int32), + ("fs_maxbpg", c_int32), + ("fs_fragshift", c_int32), + ("fs_fsbtodb", c_int32), + ("fs_sbsize", c_int32), + ("fs_spare1", c_int32 * 2), # arr[2] + ("fs_nindir", c_int32), + ("fs_inopb", c_uint32), + ("fs_old_nspf", c_int32), + ("fs_optim", c_int32), + ("fs_old_npsect", c_int32), + ("fs_old_interleave", c_int32), + ("fs_old_trackskew", c_int32), + ("fs_id", c_int32 * 2), # arr[2] + ("fs_old_csaddr", c_int32), + ("fs_cssize", c_int32), + ("fs_cgsize", c_int32), + ("fs_spare2", c_int32), + ("fs_old_nsect", c_int32), + ("fs_old_spc", c_int32), + ("fs_old_ncyl", c_int32), + ("fs_old_cpg", c_int32), + ("fs_ipg", c_uint32), + ("fs_fpg", c_int32), + ("fs_old_cstotal__cs_ndir", c_int32), + ("fs_old_cstotal__cs_nbfree", c_int32), + ("fs_old_cstotal__cs_nifree", c_int32), + ("fs_old_cstotal__cs_nffree", c_int32), + # ('fs_old_cstotal', c_int32 * 4), # struct csum + ("fs_fmod", c_int8), + ("fs_clean", c_int8), + ("fs_ronly", c_int8), + ("fs_old_flags", c_int8), + ("fs_fsmnt", c_char * MAXMNTLEN), + ("fs_volname", c_char * MAXVOLLEN), + ("fs_swuid", c_uint64), + ("fs_pad", c_int32), + ("fs_cgrotor", c_int32), + ("*fs_ocsp", c_void_p * NOCSPTRS), # void *fs_ocsp[NOCSPTRS] + ("*fs_contigdirs", c_size_t), # *fs_contigdirs + ("*fs_csp", c_size_t), # struct csum *fs_csp + ("*fs_maxcluster", c_size_t), + ("*fs_active", c_uint64), + ("fs_old_cpc", c_int32), + ("fs_maxbsize", c_int32), + ("fs_unrefs", c_int64), + ("fs_providersize", c_int64), + ("fs_metaspace", c_int64), + ("fs_sparecon64", c_int64 * 13), # arr[13] + ("fs_sblockactualloc", c_int64), + ("fs_sblockloc", c_int64), + ("fs_cstotal__cs_ndir", c_int64), + ("fs_cstotal__cs_nbfree", c_int64), + ("fs_cstotal__cs_nifree", c_int64), + ("fs_cstotal__cs_nffree", c_int64), + ("fs_cstotal__cs_numclusters", c_int64), + ("fs_cstotal__cs_spare", c_int64 * 3), + # ('fs_cstotal', c_size_t * 8), # struct csum_total + ("fs_time", ufs_time_t), + ("fs_size", c_int64), + ("fs_dsize", c_int64), + ("fs_csaddr", ufs2_daddr_t), + ("fs_pendingblocks", c_int64), + ("fs_pendinginodes", c_uint32), + ("fs_snapinum", c_uint32 * FSMAXSNAP), + ("fs_avgfilesize", c_uint32), + ("fs_avgfpdir", c_uint32), + ("fs_save_cgsize", c_int32), + ("fs_mtime", ufs_time_t), + ("fs_sujfree", c_int32), + ("fs_sparecon32", c_int32 * 21), # arr[21] + ("fs_ckhash", c_uint32), + ("fs_metackhash", c_uint32), + ("fs_flags", c_int32), + ("fs_contigsumsize", c_int32), + ("fs_maxsymlinklen", c_int32), + ("fs_old_inodefmt", c_int32), + ("fs_maxfilesize", c_uint64), + ("fs_qbmask", c_int64), + ("fs_qfmask", c_int64), + ("fs_state", c_int32), + ("fs_old_postblformat", c_int32), + ("fs_old_nrpos", c_int32), + ("fs_spare5", c_int32 * 2), # arr[2] + ("fs_magic", c_int32), +] + +UFS_CG = [ + ("cg_firstfield", c_int32), + ("cg_magic", c_int32), + ("cg_old_time", c_int32), + ("cg_cgx", c_uint32), + ("cg_old_nyl", c_int16), + ("cg_old_niblk", c_int16), + ("cg_ndblk", c_uint32), + ("cg_cs__cs_ndir", c_int32), + ("cg_cs__cs_nbfree", c_int32), + ("cg_cs__cs_nifree", c_int32), + ("cg_cs__cs_nffree", c_int32), + ("cg_rotor", c_uint32), + ("cg_frotor", c_uint32), + ("cg_irotor", c_uint32), + ("cg_frsum", c_uint32 * MAXFRAG), # arr[MAXFRAG] + ("cg_old_btotoff", c_int32), + ("cg_old_boff", c_int32), + ("cg_iusedoff", c_uint32), + ("cg_freeoff", c_uint32), + ("cg_nextfreeoff", c_uint32), + ("cg_clustersumoff", c_uint32), + ("cg_clusteroff", c_uint32), + ("cg_nclusterblks", c_uint32), + ("cg_niblk", c_uint32), + ("cg_initediblk", c_uint32), + ("cg_unrefs", c_uint32), + ("cg_sparecon32", c_int32), + ("cg_ckhash", c_uint32), + ("cg_time", ufs_time_t), + ("cg_sparecon64", c_uint64 * 3), # arr[3] + ("cg_space", c_uint8), +] + + +class UFS(Structure): + def __init__(self, fs, fst): + super(Structure).__init__() + self.sb = OrderedDict() + self.cg = OrderedDict() + self.sb_expected_len = 1376 + self.cg_expected_len = 169 + self.fs = fs + self.fst = fst + if fst == "ufs2": + self.sbo = SBLOCK_UFS2 + else: + self.sbo = SBLOCK_UFS1 + self.sb_locs = [] + self.fields_sb = UFS_SB + self.cg_locs = [] + self.fields_cg = UFS_CG + self._sanity_check() + + def _sanity_check(self): + res_sb = 0 + res_cg = 0 + for _, v in self.fields_sb: + res_sb += sizeof(v) + for _, v in self.fields_cg: + res_cg += sizeof(v) + assert res_sb == self.sb_expected_len + assert res_cg == self.cg_expected_len + + def get_superblock(self, n=0): + self.find_all_superblocks() + self._read_superblock_in_dict(self.sb_locs[n]) + return self.sb + + def get_cylinder_group(self, n=0): + self.find_all_cylinder_groups() + self._read_cylinder_group_in_dict(self.cg_locs[n]) + return self.cg + + def _read_superblock_in_dict(self, loc=SBLOCK_UFS2): + + with open(self.fs, "rb") as f: + f.seek(loc) + for field in self.fields_sb: + self.sb[field[0]] = f.read(sizeof(field[1])) + + def _read_cylinder_group_in_dict(self, loc=None): + with open(self.fs, "rb") as f: + f.seek(loc) + for field in self.fields_cg: + self.cg[field[0]] = f.read(sizeof(field[1])) + + def find_all_superblocks(self): + with open(self.fs, "rb") as f: + data = f.read() + matches = re.finditer(UFS_MAGIC, data) + for m in matches: + sb = m.span()[0] - (self.sb_expected_len - 4) + self.sb_locs.append(sb) + self.sb_locs = self.sb_locs[1:] + if (not self.sb_locs or SBLOCK_UFS2 not in self.sb_locs) and self.fst == "ufs2": + self.sb_locs = [SBLOCK_UFS2] + self.sb_locs + elif (not self.sb_locs or SBLOCK_UFS1 not in self.sb_locs) and self.fst == "ufs1": + self.sb_locs = [SBLOCK_UFS1] + self.sb_locs + return self.sb_locs + + def find_all_cylinder_groups(self): + with open(self.fs, "rb") as f: + data = f.read() + matches = re.finditer(CG_MAGIC, data) + for m in matches: + cg = m.span()[0] - 4 + self.cg_locs.append(cg) + return self.cg_locs + + def print_superblock(self): + tmp = OrderedDict() + for key, value in self.sb.items(): + if key in [ + "fs_maxfilesize", + "fs_metackhash", + "fs_ckhash", + "fs_avgfpdir", + "fs_avgfilesize", + "fs_snapinum", + "fs_pendinginodes", + "*fs_active", + "fs_swuid", + "fs_ipg", + "fs_inopb", + "fs_ncg", + ]: + tmp[key] = hex(get_int(value, signed=False)) + else: + tmp[key] = hex(get_int(value)) + pp.pprint(tmp) + + def print_cylinder_group(self): + tmp = OrderedDict() + for key, value in self.cg.items(): + if key in [ + "cg_firstfield", + "cg_magic", + "cg_old_time", + "cg_old_ncyl", + "cg_old_niblk", + "cg_old_btotoff", + "cg_old_boff", + "cg_sparecon32", + "cg_time", + "cg_sparecon64", + "cg_cs__cs_ndir", + "cg_cs__cs_nbfree", + "cg_cs__cs_nifree", + "cg_cs__cs_nffree", + ]: + tmp[key] = hex(get_int(value, signed=True)) + else: + tmp[key] = hex(get_int(value)) + pp.pprint(tmp) + + def dump_superblock(self, n=0): + if not self.sb_locs: + self.find_all_superblocks() + self._read_superblock_in_dict(loc=self.sb_locs[n]) + p = str(pathlib.Path(self.fs).parent) + c = str(pathlib.Path(self.fs).name) + fp = os.path.join(p, f"superblock_{hex(n)}_" + c + ".dump") + with open(fp, "wb") as f: + for _, value in self.sb.items(): + f.write(value) + print(f"[+] Dumped {fp}") + + def dump_all_superblocks(self): + self.find_all_superblocks() + for i, _ in enumerate(self.sb_locs): + self.dump_superblock(n=i) + + +def main(): + parser = argparse.ArgumentParser(description="UFS file system parser") + parser.add_argument( + "--dump", "-d", action="store_true", default=False, dest="dump", help="Dumps the first superblock to disk" + ) + parser.add_argument( + "--dump_all", "-da", action="store_true", default=False, dest="dump_all", help="Dumps all superblocks to disk" + ) + parser.add_argument( + "--print_superblock", + "-ps", + type=int, + default=-1, + dest="print_sb", + help="Print the n-th superblock to stdout. Default: %(default)s", + ) + parser.add_argument( + "--print_cylinder_groups", + "-pcg", + type=int, + help="Print the n-th cylinder group to stdout. Default: %(default)s", + default=-1, + dest="print_cg", + ) + parser.add_argument( + "--find_all", + "-fa", + action="store_true", + default=False, + dest="find_all", + help="Finds all superblock locations and prints them to stdout. Default: %(default)s", + ) + parser.add_argument("--file_system", "-f", required=True, type=pathlib.Path, help="UFS Filesystem") + parser.add_argument( + "--file_system_type", "-ft", type=str, default="ufs2", dest="fst", help="[ufs1, ufs2]. Default: %(default)s" + ) + + args = parser.parse_args() + + ufs = UFS(args.file_system, args.fst) + if args.dump: + ufs.dump_superblock() + if args.dump_all: + ufs.dump_all_superblocks() + if args.find_all: + ufs.find_all_superblocks() + ufs.find_all_cylinder_groups() + res = ", ".join(hex(e) for e in ufs.sb_locs) + print(f"[+] Found superblock offsets: {res}") + res = ", ".join(hex(e) for e in ufs.cg_locs) + print(f"[+] Found cylinder group offsets: {res}") + if args.print_sb >= 0: + ufs.find_all_superblocks() + if not ufs.sb_locs and args.fst == "ufs2": + ufs.sb_locs.append(SBLOCK_UFS2) + elif not ufs.sb_locs and args.fst == "ufs1": + ufs.sb_locs.append(SBLOCK_UFS1) + ufs._read_superblock_in_dict(ufs.sb_locs[args.print_sb]) + ufs.print_superblock() + if args.print_cg >= 0: + ufs.find_all_cylinder_groups() + ufs._read_cylinder_group_in_dict(ufs.cg_locs[args.print_cg]) + ufs.print_cylinder_group() + + +if __name__ == "__main__": + main() diff --git a/src/Fuzzer/file_system_magic/zfs_uberblock_parser.py b/src/Fuzzer/file_system_magic/zfs_uberblock_parser.py new file mode 100644 index 0000000..54db621 --- /dev/null +++ b/src/Fuzzer/file_system_magic/zfs_uberblock_parser.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 + +import argparse +import os +import pathlib +import re +from collections import OrderedDict +from ctypes import Structure, sizeof, c_uint64, c_uint8, c_uint16 + +# xxd ZFS_FS | grep '0cb1 ba00' +# multiple offsets +ZFS_MAGIC = b"\x0c\xb1\xba\x00\x00\x00\x00\x00" +# xxd test_zfs | grep '117a 0cb1 7ada 1002' +ZBT_MAGIC = b"\x11\x7a\x0c\xb1\x7a\xda\x10\x02" +# xxf test_zfs | grep '11ea 1ca1 0000 0000' +MMP_MAGIC = b"\x11\xea\x1c\xa1\x00\x00\x00\x00" + +# TODO: Finish ZFS implementation https://github.com/freebsd/freebsd/blob/master/sys/cddl/boot/zfs/zfsimpl.h +ZFS_UB = [ + ("ub_magic", c_uint64), + ("ub_version", c_uint64), + ("ub_txg", c_uint64), + ("ub_guid_sum", c_uint64), + ("ub_timestamp", c_uint64), + # ("blk_dva0", c_uint64), + # ("blk_dva1", c_uint64), + # ("blk_prop", c_uint64), + # ("blk_pad0", c_uint64), + # ("blk_pad1", c_uint64), + # ("blk_phys_birth", c_uint64), + # ("blk_birth", c_uint64), + # ("blk_fill", c_uint64), + # ("blk_cksum0", c_uint64), + # ("blk_cksum1", c_uint64), + # ("blk_cksum2", c_uint64), + # ("blk_cksum3", c_uint64), + ("TODO_resolve_data_fields", c_uint64 * 117), + ("ub_software_version", c_uint64), + ("ub_mmp_magic", c_uint64), + ("ub_mmp_delay", c_uint64), + ("ub_mmp_config", c_uint64), + ("ub_mmp_config_VALID", c_uint8), + ("ub_mmp_config_write_interval", c_uint8 * 3), + ("ub_mmp_config_seq", c_uint16), + ("ub_mmp_config_fail_intervals", c_uint16), + ("ub_checkpoint_txg", c_uint64), +] + + +class ZFS(Structure): + def __init__(self, fs, fst): + super(Structure).__init__() + self.sb = OrderedDict() + self.sb_expected_len = 1024 + self.fs = fs + self.fst = fst + self.sb_locs = [] + self.fields_sb = ZFS_UB + + def _sanity_check(self): + res_ub = 0 + for _, v in self.fields_sb: + res_ub += sizeof(v) + assert res_ub == self.sb_expected_len + + def get_superblock(self, n=0): + self.find_all_superblocks() + self._read_superblock_in_dict(self.sb_locs[n]) + return self.sb + + def _read_superblock_in_dict(self, loc=None): + with open(self.fs, "rb") as f: + f.seek(loc) + for field in self.fields_sb: + self.sb[field[0]] = f.read(sizeof(field[1])) + + def find_all_superblocks(self): + with open(self.fs, "rb") as f: + data = f.read() + matches = re.finditer(ZFS_MAGIC, data) + for m in matches: + self.sb_locs.append(m.span()[0]) + return self.sb_locs + + def dump_superblock(self, n=0): + if not self.sb_locs: + self.find_all_superblocks() + self._read_superblock_in_dict(loc=self.sb_locs[n]) + p = str(pathlib.Path(self.fs).parent) + c = str(pathlib.Path(self.fs).name) + fp = os.path.join(p, f"superblock_{hex(n)}_" + c + ".dump") + with open(fp, "wb") as f: + for _, value in self.sb.items(): + f.write(value) + print(f"[+] Dumped {fp}") + + def dump_all_superblocks(self): + self.find_all_superblocks() + for i, _ in enumerate(self.sb_locs): + self.dump_superblock(n=i) + + +def main(): + parser = argparse.ArgumentParser(description="UFS file system parser") + parser.add_argument( + "--dump", "-d", action="store_true", default=False, dest="dump", help="Dumps the first superblock to disk" + ) + parser.add_argument( + "--dump_all", "-da", action="store_true", default=False, dest="dump_all", help="Dumps all superblocks to disk" + ) + parser.add_argument( + "--find_all", + "-fa", + action="store_true", + default=False, + dest="find_all", + help="Finds all superblock locations and prints them to stdout. Default: %(default)s", + ) + parser.add_argument("--file_system", "-f", required=True, type=pathlib.Path, help="UFS Filesystem") + parser.add_argument( + "--file_system_type", "-ft", type=str, default="zfs2", dest="fst", help="[zfs1, zfs2]. Default: %(default)s" + ) + + args = parser.parse_args() + + zfs = ZFS(args.file_system, args.fst) + if args.dump: + zfs.dump_superblock() + if args.dump_all: + zfs.dump_all_superblocks() + if args.find_all: + zfs.find_all_superblocks() + res = ", ".join(hex(e) for e in zfs.sb_locs) + print(f"[+] Found superblock offsets: {res}") + + +if __name__ == "__main__": + main() diff --git a/src/Fuzzer/metadata.py b/src/Fuzzer/metadata.py new file mode 100644 index 0000000..53ca38b --- /dev/null +++ b/src/Fuzzer/metadata.py @@ -0,0 +1,52 @@ +import logging +import random +import sys + +from file_system_magic.ext_superblock_parser import EXT +from file_system_magic.fs_util import get_bytearray, set_mime, write_to_file, get_mutated_bytes +from file_system_magic.ufs_superblock_parser import UFS +from file_system_magic.zfs_uberblock_parser import ZFS + + +class MetaMutation: + def __init__(self, fs, nbytes=5, restore=True, mode=None): + self.nbytes = nbytes + self.fs = fs + self.mfs = None + self.mime = None + self.mode = mode + self.s_locs = None + self.restore = restore + self.rnd = random.Random() + self.rnd.seed(random.getrandbits(1024)) + + def mutation(self): + barray = get_bytearray(self.fs) + self.mime = set_mime(self.fs) + if "ufs" in self.mime: + fs_p = UFS(fs=self.fs, fst=self.mime) + elif self.mime == "ext": + fs_p = EXT(fs=self.fs, fst=self.mime) + elif self.mime == "zfs": + fs_p = ZFS(fs=self.fs, fst=self.mime) + else: + logging.error("Could not detect file system type correctly") + sys.exit(1) + self.s_locs = fs_p.find_all_superblocks() + + good_locs = [] + for i in self.s_locs: + for j in range(fs_p.sb_expected_len): + good_locs += i + j + + ctr = 0 + while ctr < self.nbytes: + try: + rnd_pos = random.choice(good_locs) + muta_byte_seq = get_mutated_bytes(1) + barray[rnd_pos] = muta_byte_seq + ctr += 1 + except IndexError as e: + logging.error(e) + return None + return write_to_file(self.fs, self.nbytes, self.mode, barray) diff --git a/src/Fuzzer/radamsa.py b/src/Fuzzer/radamsa.py new file mode 100644 index 0000000..4a66211 --- /dev/null +++ b/src/Fuzzer/radamsa.py @@ -0,0 +1,132 @@ +import logging +import os +import pathlib +import random +import re +import subprocess +import sys + +from file_system_magic.ufs_superblock_parser import UFS, UFS_MAGIC +from file_system_magic.ext_superblock_parser import EXT, EXT_MAGIC +from file_system_magic.zfs_uberblock_parser import ZFS, ZFS_MAGIC +from file_system_magic.fs_util import get_offset_in_sb, set_mime + + +class Radamsa: + def __init__(self, path_to_file_system): + self.radamsa_seed = None + self.path_to_file_system = path_to_file_system + self.path_to_mutated_file_system = None + self.mime = None + + @staticmethod + def _get_ufs_zfs_magic_pos(path_to_file_system, file_system_type=None): + with open(path_to_file_system, "rb") as f: + data = f.read() + magic_positions = [] + if file_system_type == "ufs": + magic_sequence = UFS_MAGIC + elif file_system_type == "zfs": + magic_sequence = ZFS_MAGIC + else: + return False + matches = re.finditer(magic_sequence, data) + for m in matches: + magic_positions.append(m.span()[0]) + return magic_positions + + @staticmethod + def _get_ext_magic_pos(path_to_file_system): + with open(path_to_file_system, "rb") as f: + data = f.read() + match = data.find(EXT_MAGIC) + return [match] + + def _set_magic(self, mgc_offs, mime=None): + # logging.error("[*] Restoring magic bytes in {}".format(self.path_to_mutated_file_system)) + if "ext" in mime: + mgc_seq = EXT_MAGIC + elif "ufs" in mime: + mgc_seq = UFS_MAGIC + elif mime == "zfs": + mgc_seq = ZFS_MAGIC + else: + print("[!] Unknown mime type - Cannot restore magic bytes - radamsa") + sys.exit(1) + with open(self.path_to_mutated_file_system, "rb+") as f: + for m in mgc_offs: + f.seek(m) + f.write(mgc_seq) + + def _restore_magic_bytes(self): + if "ufs" in self.mime: + fs_p = UFS(fs=self.path_to_file_system, fst=self.mime) + fn = "fs_magic" + elif self.mime == "ext": + fs_p = EXT(fs=self.path_to_file_system, fst=self.mime) + fn = "ext2fs_magic" + elif self.mime == "zfs": + fs_p = ZFS(fs=self.path_to_file_system, fst=self.mime) + fn = "ub_magic" + else: + logging.error("Could not detect file system type correctly") + sys.exit(1) + sb_locs = self._get_magic_offs_lst(fs_p, fn) + self._set_magic(sb_locs, self.mime) + + @staticmethod + def _get_magic_offs_lst(fs_p, mgc_n): + sb_locs = fs_p.find_all_superblocks() + magic_off, _ = get_offset_in_sb(fs_p, mgc_n) + for i, _ in enumerate(sb_locs): + sb_locs[i] = sb_locs[i] + magic_off + return sb_locs + + def _restore_uberblock(self): + sbs = [] + if "ufs" in self.mime: + fsp = UFS(fs=self.path_to_file_system, fst=self.mime) + elif "ext" in self.mime: + fsp = EXT(fs=self.path_to_file_system, fst=self.mime) + elif self.mime == "zfs": + fsp = ZFS(fs=self.path_to_file_system, fst=self.mime) + else: + logging.error("Could not detect file system type correctly") + return 0 + sb_locs = fsp.find_all_superblocks() + with open(self.path_to_file_system, "rb") as f: + for loc in sb_locs: + f.seek(loc) + sbs += f.read(fsp.expected_sb_len) + with open(self.path_to_mutated_file_system, "wb") as f: + for i, loc in enumerate(sb_locs): + f.seek(loc) + f.write(sbs[i]) + + def mutation(self, preserve_magic=True, preserve_uberblock=False, determinism=True): + self.mime = set_mime(self.path_to_file_system) + name = pathlib.Path(self.path_to_file_system).name + _path = pathlib.Path(self.path_to_file_system).parent + self.path_to_mutated_file_system = os.path.join(_path, "radamsa_" + name) + if determinism: + self.radamsa_seed = random.getrandbits(100) + cmd = "radamsa {} -s {} > {}".format(self.path_to_file_system, self.radamsa_seed, self.path_to_mutated_file_system,) + else: + cmd = "radamsa {} > {}".format(self.path_to_file_system, self.path_to_mutated_file_system) + if preserve_uberblock: + preserve_magic = False + subprocess.call(cmd, shell=True) + if preserve_magic: + self._restore_magic_bytes() + if preserve_uberblock: + self._restore_uberblock() + return self.radamsa_seed, self.path_to_mutated_file_system + + +def main(): + rad = Radamsa(sys.argv[1]) + rad.mutation(preserve_magic=True) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/Manager/Manager.py b/src/Manager/Manager.py new file mode 100644 index 0000000..d6d7928 --- /dev/null +++ b/src/Manager/Manager.py @@ -0,0 +1,925 @@ +import datetime +import getpass +import ipaddress +import logging +import lzma +import os +import pathlib +import socket +import subprocess +import sys +import time +import zipfile +from io import BytesIO + +import colorama as clr +import libvirt +import magic +import paramiko +import wget +from PIL import Image, ImageFile + +from SnapshotTemplate import snapshot + + +def get_absolute_path(_path): + if _path[0] == "~" and not os.path.exists(_path): + _path = os.path.expanduser(_path) + abs_path = os.path.abspath(_path) + return abs_path + + +def check_if_file_exists(_path): + if pathlib.Path(_path).exists(): + return True + else: + return False + + +def create_directory(_path): + pathlib.Path(_path).mkdir(parents=True, exist_ok=True) + return _path + + +def create_empty_file(_path, fn): + pathlib.Path(os.path.join(_path, fn)).touch() + + +def get_basename(_path): + return str(pathlib.Path(str(_path)).name) + + +def get_parent_path(_path): + return str(pathlib.Path(str(_path)).parent) + + +def kill_process_by_pid(process): + # don't send the signal unless it seems it is necessary + if process.poll() is None: + try: + process.kill() + except OSError: + pass + + +class VmManager: + def setup(self, **kwargs): + if "name" in kwargs: + self.name = kwargs["name"] + if "vm_arch" in kwargs: + self.vm_arch = kwargs["vm_arch"] + if "port" in kwargs: + self.port = kwargs["port"] + if "enable_kvm" in kwargs: + if True or False in kwargs["enable_kvm"]: + self.enable_kvm = kwargs["enable_kvm"] + else: + pass + if "url" in kwargs: + self.url = kwargs["url"] + if "save_dl_to" in kwargs: + self.save_dl_to = get_absolute_path(kwargs["save_dl_to"]) + if "save_unpck_to" in kwargs: + self.save_unpck_to = get_absolute_path(kwargs["save_unpck_to"]) + if "save_unpck_as" in kwargs: + self.save_unpck_as = kwargs["save_unpck_as"] + if "unpckd_url_cntnt" in kwargs: + if check_if_file_exists(get_absolute_path(kwargs["unpckd_url_cntnt"])) != 0: + self.path_to_iso_or_qcow = kwargs["unpckd_url_cntnt"] + else: + print("{} does not exist!".format(kwargs["unpckd_url_cntnt"])) + sys.exit(1) + if "path_to_archive" in kwargs: + self.p_archive = get_absolute_path(kwargs["path_to_archive"]) + if "copy_files_to" in kwargs: + self.copy_files_to = get_absolute_path(kwargs["copy_files_to"]) + if "vm_memory" in kwargs: + self.vm_memory = kwargs["vm_memory"] + if "vm_cpus" in kwargs: + self.vm_cpus = kwargs["vm_cpus"] + if "vm_hdd" in kwargs: + self.vm_hdd = kwargs["vm_hdd"] + if "vm_user" in kwargs: + self.vm_user = kwargs["vm_user"] + if "vm_password" in kwargs: + self.vm_password = kwargs["vm_password"] + + def __init__(self): + logging.basicConfig(level=logging.INFO) + logging.getLogger("paramiko").setLevel(logging.CRITICAL) + self.name = "" + self.vm_arch = "x86" # The vm_type,enable_kvm flags are only relevant + self.port = 22 # without having it installed + self.enable_kvm = True # enables hardware virtualization features + self.url = None # iso/qcow download URL + self.save_dl_to = "/tmp" # save download to + self.save_dl_as = None # save download file as + self.save_unpck_to = "/tmp" # save unpacked file to + self.save_unpck_as = None # save unpacked file as + self.path_to_iso_or_qcow = None # unzipped .xz image + self.p_archive = None # path specified by self.save_url_cntnt_to + self.save_url_cntnt_as + self.copy_files_to = None # path can set for copies between host<->guest if path shall stay static + self.vm_user = None # saved username for booted VM + self.vm_password = None # saved passwd for booted VM + self.vm_ip = "127.0.0.1" # saved IPv4 for booted VM + self.vm_memory = 4096 # RAM of booted VM + self.vm_cpus = 2 # Number of cores of VM + self.vm_hdd = 10 # VM HDD size in GB + self.rshell = None # remote shell object for host<-> operations + self.curr_crash_dir = None # is set to path new directory path for current crash + self.conn_err_ctr = 0 # counter for connerr that might be a hint towards a broken VM + + def __exit__(self): + return 1 + + ###################################################################################################### + # BASIC VM OPERATIONS VIA PARAMIKO SSH # + ###################################################################################################### + + def _get_basic_ssh_conn(self): + self.get_vm_credentials() + ssh_conn = paramiko.SSHClient() + ssh_conn.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_conn.connect( + hostname=self.vm_ip, + port=self.port, + username=self.vm_user, + password=self.vm_password, + look_for_keys=False, + allow_agent=False, + timeout=15, + ) + return ssh_conn + + def get_vm_credentials(self): + if self.vm_user is None or self.vm_password is None: + self.vm_user = str(input('Username for "{}": '.format(self.name))) + self.vm_password = getpass.getpass("Password: ") + else: + logging.debug("Reusing stored vm credentials.") + + def invoke_remote_ssh_shell(self): + if not self.silent_vm_state(): + self.reset_vm() + ssh_conn = self._get_basic_ssh_conn() + ssh_conn.get_transport().set_keepalive(200) + ssh_conn.get_transport().open_session() + ssh_conn.invoke_shell() + self.rshell = ssh_conn + return ssh_conn + + def _exec(self, cmd, timeout=10): + if not self.rshell: + self.invoke_remote_ssh_shell() + try: + # get_pty=True combines stdout/stderr + _, stdout, _ = self.rshell.exec_command(cmd, get_pty=True, timeout=timeout) + stdout_decoded = stdout.read().decode().strip() + if stdout_decoded != "": + return stdout_decoded + else: + return None + except (paramiko.ssh_exception.SSHException, socket.timeout, paramiko.ssh_exception.NoValidConnectionsError,) as e: + logging.debug("_EXEC ERROR: {}".format(e)) + return 2 + except UnicodeDecodeError: + return 1 + + def exec_cmd_quiet(self, cmd): + stdout = self._exec(cmd) + return stdout + + def exec_cmd(self, cmd): + stdout = self._exec(cmd) + print(">> {}".format(stdout)) + return stdout + + def exec_get_return_code(self, cmd): + res = str(self.exec_cmd_quiet(cmd)) + if int(res[-1]) != 0: + return False + else: + return True + + def interactive_shell(self): + print(clr.Fore.RED + 'Exit remote shell via "exit"' + clr.Fore.RESET) + while True: + command = input("$> ") + if str.lower(command.strip()) == "exit": + self.__exit__() + self._exec(command) + + def zip_files(self, path_to_save_archive_in, files, archive_path): + file_list = " ".join(os.path.join(path_to_save_archive_in, f) for f in files) + logging.debug("Packing {} in {}".format(file_list, archive_path)) + return self.exec_get_return_code("tar -jcvf {} {}; echo $?".format(archive_path, file_list)) + + def mkdir(self, rpath): + return self.exec_get_return_code("/bin/mkdir -p {}; echo $?".format(rpath)) + + def rm_files(self, rpath): + return self.exec_get_return_code("/bin/rm -rf {}; echo $?".format(rpath)) + + def vm_ls(self, rpath): + return self._exec("/bin/ls -lah {}".format(rpath)) + + @staticmethod + def _prep_file_list(list_of_files_to_copy, files_to_copy, save_files_at): + if isinstance(files_to_copy, str): + list_of_files_to_copy.append(os.path.join(save_files_at, files_to_copy)) + else: + list_of_files_to_copy.extend(files_to_copy) + list_of_files_to_copy[:] = [os.path.join(save_files_at, x) for x in list_of_files_to_copy] + return list_of_files_to_copy + + def _get_files(self, list_of_files_to_copy, save_files_at): + try: + if not self.rshell: + self.invoke_remote_ssh_shell() + ftp_client = self.rshell.open_sftp() + for f in list_of_files_to_copy: + ftp_client.get(f, os.path.join(save_files_at, get_basename(f))) + ftp_client.close() + except paramiko.ssh_exception.SSHException as e: + logging.error("SSHException during get files: {}\nResetting and trying to invoke new rshell".format(e)) + self.reset_vm() + self.new_rshell() + self._get_files(list_of_files_to_copy, save_files_at) + + def cp_to_host(self, save_files_at, get_files_from, list_of_files_to_copy, zipped=False): + sanitized_list_of_files_to_copy = [] + try: + save_files_at, get_files_from = ( + get_absolute_path(save_files_at), + get_absolute_path(get_files_from), + ) + self.mkdir(save_files_at) + if zipped: + archive_name = str(datetime.datetime.today().strftime("%d_%m_%Y")) + ".bzip2" + archive_path = os.path.join("/tmp", archive_name) + if self.zip_files(get_files_from, list_of_files_to_copy, archive_path=archive_path): + sanitized_list_of_files_to_copy.append(archive_path) + else: + logging.error("Failed to compress: {}/{}".format(get_files_from, list_of_files_to_copy)) + sanitized_list_of_files_to_copy = self._prep_file_list( + sanitized_list_of_files_to_copy, list_of_files_to_copy, get_files_from, + ) + else: + sanitized_list_of_files_to_copy = self._prep_file_list( + sanitized_list_of_files_to_copy, list_of_files_to_copy, get_files_from, + ) + self._get_files(sanitized_list_of_files_to_copy, save_files_at) + return 1 + # except (TypeError, OSError) as e: + # logging.error('Encountered an error while fetching files: {}'.format(e)) + # return 0 + except FileNotFoundError: + logging.error("To host: One or more files of {} not found".format(list_of_files_to_copy)) + return None + + def _send_files(self, list_of_files_to_copy, save_files_at): + try: + if not self.rshell: + self.invoke_remote_ssh_shell() + ftp_client = self.rshell.open_sftp() + for f in list_of_files_to_copy: + ftp_client.put(f, os.path.join(save_files_at, get_basename(f))) + ftp_client.close() + except paramiko.ssh_exception.SSHException as e: + logging.error("SSHException during send files: {}\n Trying to invoke new rshell".format(e)) + self.new_rshell() + self._send_files(list_of_files_to_copy, save_files_at) + + def cp_to_guest(self, get_files_from, list_of_files_to_copy, save_files_at, zipped=False): + try: + get_files_from, save_files_at = ( + get_absolute_path(get_files_from), + get_absolute_path(save_files_at), + ) + self.mkdir(save_files_at) + sanitized_list_of_files_to_copy = [] + if zipped: + file_list = " ".join(os.path.join(get_files_from, f) for f in list_of_files_to_copy) + with zipfile.ZipFile("/tmp/files.zip", "w", compression=zipfile.ZIP_DEFLATED) as myzip: + for f in file_list: + # arcname removes dirpath to actual file and just takes the file itself + myzip.write(f, arcname=pathlib.Path(f).name) + sanitized_list_of_files_to_copy.append("/tmp/files.zip") + else: + sanitized_list_of_files_to_copy = self._prep_file_list( + sanitized_list_of_files_to_copy, list_of_files_to_copy, get_files_from, + ) + self._send_files(sanitized_list_of_files_to_copy, save_files_at) + # except (TypeError, OSError) as e: + # logging.error('Encountered an error while fetching files: {}'.format(e)) + # return 0 + except FileNotFoundError: + logging.error("To guest: One or more files of {} not found".format(list_of_files_to_copy)) + return None + + def get_last_changed_file_in_dir(self, search_path): + cmd = "/bin/ls -t {} | head -n1".format(search_path) + try: + latest_file = self._exec(cmd) + logging.debug(latest_file) + return search_path, latest_file + except AttributeError as e: + logging.debug(e) + + def _get_latest_core(self): + try: + core_files = [] + find_latest_core_file_cmd = '/usr/bin/find /var/crash -name "core*" -print0' + latest_core_file = self._exec(find_latest_core_file_cmd) + if not latest_core_file: + return None + else: + find_latest_core_file_cmd = ( + '/usr/bin/find /var/crash -name "core*" -print0 | /usr/bin/xargs -0 ls -t | /usr/bin/head -n1' + ) + latest_core_file = self._exec(find_latest_core_file_cmd) + find_latest_vmcore_cmd = ( + '/usr/bin/find /var/crash -name "vmcore*" -print0 | /usr/bin/xargs -0 ls -t | /usr/bin/head -n1' + ) + latest_vmcore_file = self._exec(find_latest_vmcore_cmd) + get_core_file_size_cmd = '/usr/bin/stat {} | /usr/bin/cut -d" " -f8'.format(latest_core_file) + if latest_core_file and latest_vmcore_file and int(self._exec(get_core_file_size_cmd).strip()) > 1: + core_files.append(os.path.join("/var/crash", latest_core_file.strip())) + core_files.append(os.path.join("/var/crash", latest_vmcore_file.strip())) + return core_files + else: + return None + except ValueError: + return None + + def fetch_latest_core_file(self): + self.curr_crash_dir = None + try: + latest_core_files = self._get_latest_core() + logging.debug("LATEST CORES: {}".format(latest_core_files)) + if len(latest_core_files) == 2: + _timestamp_dir = os.path.join("crash_dumps", self._get_timestamp()) + self.curr_crash_dir = os.path.join(os.getcwd(), _timestamp_dir) + create_directory(self.curr_crash_dir) + for list_entry in latest_core_files: + if "core.txt" in list_entry: + zipped_enable = False + else: + zipped_enable = False + self.cp_to_host( + save_files_at=self.curr_crash_dir, + get_files_from=get_parent_path(list_entry), + list_of_files_to_copy=get_basename(list_entry), + zipped=zipped_enable, + ) + return 1 + else: + logging.error("No new core files found!") + return 0 + except TypeError: + logging.error("No core file(s) found. Continuing!") + return 0 + + def fetch_all_core_files(self): + get_all_core_files_cmd = '/bin/ls /var/crash/ | grep "core.txt"' + # gets a list of files without any empty string entries + core_files = list(filter(None, self._exec(get_all_core_files_cmd).split("\n"))) + if core_files: + all_cores = " ".join(f for f in core_files) + self.cp_to_host( + save_files_at="crash_dumps/", get_files_from="/var/crash/", list_of_files_to_copy=all_cores, + ) + else: + logging.info("No core files found!") + + def check_vm_state(self): + try: + if not int( + subprocess.check_output( + "timeout 3 nc -z {} 22; echo $?".format(self.vm_ip), shell=True, encoding="utf-8", + ).strip() + ): + print(clr.Fore.GREEN + "[+] VM status: {}".format("OK" + clr.Fore.RESET)) + return 1 + else: + print(clr.Fore.RED + "[!] VM status: {}".format("Not Responding" + clr.Fore.RESET)) + return 0 + except subprocess.TimeoutExpired: + print(clr.Fore.RED + "[!] VM status: {}".format("Not Responding" + clr.Fore.RESET)) + return 0 + + def silent_vm_state(self): + if not int( + subprocess.check_output("timeout 3 nc -z {} 22; echo $?".format(self.vm_ip), shell=True, encoding="utf-8",).strip() + ): + # ret code was 0, VM reachable + return 1 + else: + return 0 + + def crash_handler(self): + print(clr.Fore.LIGHTYELLOW_EX + "[*] Checking for crash dump..!" + clr.Fore.RESET) + try: + self.reset_vm() + return self._return_core_path_if_present() + except paramiko.ssh_exception.NoValidConnectionsError as e: + logging.error("NoValidConnErr in Manager crash handler: {}\n self.cerr is {}".format(e, self.conn_err_ctr)) + self.conn_err_ctr += 1 + if self.conn_err_ctr == 2: + self.conn_err_ctr = 0 + # Probably stuck in a boot loop at this point because of a broken system + # ext2: fsck /dev/ad0p2: Segmentation fault + # Unknown error1: help! + self.restore_snapshot(self.get_current_snapshot()) + self.new_rshell() + else: + self.crash_handler() + + def _return_core_path_if_present(self): + self.new_rshell() + if self.fetch_latest_core_file(): + print(clr.Fore.LIGHTYELLOW_EX + "\t[*] Found core file" + clr.Fore.RESET) + self.conn_err_ctr = 0 + return self.curr_crash_dir + else: + self.conn_err_ctr = 0 + return None + + def new_rshell(self): + self.rshell = None + self.invoke_remote_ssh_shell() + + ###################################################################################################### + # LIBVIRT VM OPERATIONS # + ###################################################################################################### + + def test_install(self): + if subprocess.run("virsh list --all".split(), stdout=subprocess.DEVNULL) and subprocess.run( + "qemu-system-{} --help".format(self.vm_arch).split(), stdout=subprocess.DEVNULL, + ): + logging.info("Tests passed!") + else: + logging.error("QEMU and/or libvirt not properly installed!") + sys.exit(1) + + def _boot_sleep(self, sleep_timer_in_seconds): + sys.stdout.write(clr.Fore.LIGHTYELLOW_EX + "\t[*] Please stand by." + clr.Fore.RESET) + for i in range(sleep_timer_in_seconds): + sys.stdout.write(clr.Fore.LIGHTYELLOW_EX + "." + clr.Fore.RESET) + sys.stdout.flush() + time.sleep(1) + sys.stdout.write("\033[F") # back to previous line + + @staticmethod + def get_open_libvirt_connection(): + conn = libvirt.open() + if conn is None: + logging.error("Failed to open connection to hypervisor") + sys.exit(1) + else: + return conn + + def get_domain_object(self): + conn = self.get_open_libvirt_connection() + dom = conn.lookupByName(self.name) + return conn, dom + + def set_vcpus(self, vcpus): + conn, dom = self.get_domain_object() + try: + dom.setVcpus(vcpus) + except libvirt.libvirtError: + logging.error("Failed to set new max amount of vcpus for {}!".format(dom.name())) + finally: + conn.close() + + def set_memory(self, memory): + conn, dom = self.get_domain_object() + try: + dom.setMemory(memory) + except libvirt.libvirtError: + logging.error("Failed to set new max amount of RAM for {}!".format(dom.name())) + finally: + conn.close() + + def list_installed_vms(self): + conn = self.get_open_libvirt_connection() + hosts = conn.listAllDomains(0) + print("[*] Found vms: [" + ", ".join(h.name() for h in hosts) + "]") + conn.close() + + def get_ip_of_vm(self): + # TODO: Fix by utilizing libvirts python API + iface = subprocess.check_output("virsh domifaddr {}".format(self.name).split(), encoding="utf-8") + ipv4 = iface.split()[-1].split("/")[0].strip() + try: + if ipaddress.ip_address(ipv4): + self.vm_ip = ipv4 + except ValueError as e: + logging.error("[!] Failed to fetch an IPv4 address of {}".format(self.name)) + logging.error("Expected: xxy.xxy.xxy.xyz") + logging.error("Got : {}".format(e)) + logging.error("[*] Trying to restore from snapshot") + cur_snap = self.get_current_snapshot() + if cur_snap: + self.restore_snapshot(cur_snap) + self.reset_vm() + self.quick_boot(self.name) + else: + logging.error("[!] No snapshot found. Trying to reset VM instead") + self.reset_vm() + self.get_ip_of_vm() + + def quick_boot(self, vm_name): + try: + self.name = vm_name + conn, dom = self.get_domain_object() + self.get_vm_credentials() + if not dom.isActive(): + dom.create() + # needs around 60 seconds until booting seq started network ifaces + self._boot_sleep(60) + self.get_ip_of_vm() + print("\n[+] VM started @ {}!".format(self.vm_ip)) + else: + self.get_ip_of_vm() + conn.close() + except libvirt.libvirtError: + pass + + def boot_vm(self): + if self.list_installed_vms() != "": + print("[*] Which VM do you want to start?") + self.name = input(">> ") + conn = self.get_open_libvirt_connection() + dom = conn.lookupByName(self.name) + active = dom.isActive() + if active == 1: + self.get_ip_of_vm() + else: + dom.create() + self._boot_sleep(60) + self.get_ip_of_vm() + print("[+] VM started @ {}!".format(self.vm_ip)) + conn.close() + else: + print("[!] No VMs found!") + sys.exit(1) + + def shutdown_vm(self): + conn, dom = self.get_domain_object() + dom.shutdown() + conn.close() + + def suspend_vm(self): + conn, dom = self.get_domain_object() + dom.suspend() + conn.close() + + def resume_vm(self): + conn, dom = self.get_domain_object() + self._boot_sleep(20) + dom.resume() + conn.close() + + def reset_vm(self): + # Reset emulates the power reset button on a machine, where all + # hardware sees the RST line set and re-initializes internal state + conn, dom = self.get_domain_object() + dom.reset() + self._boot_sleep(40) + timer = 40 + while True: + if timer > 120: + logging.error("\nCould not fully boot in {} seconds. Resetting VM again!".format(timer)) + self.restore_snapshot(self.get_current_snapshot()) + break + elif not self.silent_vm_state(): + sys.stdout.write(".....") + sys.stdout.flush() + time.sleep(5) + timer += 5 + else: + break + print("\n") + conn.close() + + def reboot_vm(self): + # The hypervisor will choose the method of shutdown it considers best + conn, dom = self.get_domain_object() + dom.reboot() + self._boot_sleep(60) + conn.close() + + def force_stop_vm(self): + conn, dom = self.get_domain_object() + dom.destroy() + conn.close() + + def delete_vm(self, vm_name): + conn = self.get_open_libvirt_connection() + try: + pool = conn.storagePoolLookupByName("default") + dom = conn.lookupByName(vm_name) + dom.undefineFlags(libvirt.VIR_DOMAIN_UNDEFINE_NVRAM) + stgvol = pool.storageVolLookupByName("{}.qcow2".format(self.name)).path() + if stgvol: + stgvol.wipe(0) + stgvol.delete(0) + except libvirt.libvirtError as e: + logging.error("Failed to properly delete: {}".format(e)) + finally: + conn.close() + + def rename_vm(self, new_name): + conn, dom = self.get_domain_object() + dom.rename(new_name) + conn.close() + + def clone_vm(self, clone_name): + # TODO: Fix by utilizing libvirts python API + conn, dom = self.get_domain_object() + domname = dom.name() + cmd = "virt-clone --original {} --name {} --auto-clone".format(domname, clone_name) + subprocess.call(cmd.split()) + + def _png_writer(self, stream, data, buffer): + # Writes screenshot to disk + ImageFile.LOAD_TRUNCATED_IMAGES = True + buffer.write(data) + buffer.seek(0) + image = Image.open(buffer) + image.save(os.path.join("vm_screenshots", "{}_{}.png".format(self.name, self._get_timestamp()))) + + @staticmethod + def _get_timestamp(): + time_stamp = datetime.datetime.now().isoformat(timespec="seconds") + time_stamp = str(time_stamp).replace(":", "_").replace(" ", "_").rsplit(".")[0] + return time_stamp + + def take_screenshot(self): + conn = self.get_open_libvirt_connection() + create_directory(os.path.join(os.getcwd(), "vm_screenshots")) + dom = conn.lookupByName(self.name) + if dom.isActive(): + stream = conn.newStream() + dom.screenshot(stream=stream, screen=0) + buffer = BytesIO() + stream.recvAll(self._png_writer, buffer) + stream.finish() + conn.close() + + @staticmethod + def get_snapshot_xml_representation(dom, snap_name=None): + raw_xml = dom.XMLDesc(0) + if snap_name: + return snapshot.xml_head_name.format(snap_name) + raw_xml + snapshot.xml_tail + else: + return snapshot.xml_head.format(snap_name) + raw_xml + snapshot.xml_tail + + def create_snapshot(self): + conn, dom = self.get_domain_object() + try: + xml_descr = self.get_snapshot_xml_representation(dom) + dom.snapshotCreateXML(xml_descr) + except libvirt.libvirtError: + logging.error("Failed to create snapshot for {}".format(self.name)) + finally: + conn.close() + + def create_snapshot_with_name(self, snap_name): + conn, dom = self.get_domain_object() + try: + xml_descr = self.get_snapshot_xml_representation(dom, snap_name) + dom.snapshotCreateXML(xml_descr) + except libvirt.libvirtError as e: + logging.error("{} - Failed to create snapshot: {}".format(dom, e)) + finally: + conn.close() + + def delete_snapshot(self, snap_name): + conn, dom = self.get_domain_object() + snap_list = dom.snapshotListNames() + if snap_name in snap_list: + try: + _snapshot = dom.snapshotLookupByName(snap_name) + _snapshot.delete() + except libvirt.libvirtError as e: + logging.error("{} - Failed to delete snapshot: {}".format(snap_name, e)) + conn.close() + + def list_snapshots(self): + conn, dom = self.get_domain_object() + snapshots = dom.listAllSnapshots() + if snapshots: + print("[*] Found snapshots: " + ", ".join(x.getName() for x in snapshots)) + return snapshots + else: + logging.error("Failed to find any snapshots for {}!".format(self.name)) + conn.close() + + def restore_snapshot(self, snap_name): + conn, dom = self.get_domain_object() + try: + snap = dom.snapshotLookupByName(snap_name) + dom.revertToSnapshot(snap) + self.get_vm_credentials() + if not dom.isActive(): + dom.create() + # needs around 60 seconds until booting seq started network ifaces + self._boot_sleep(60) + conn.close() + logging.warning("Successfully reset {} to snapshot: {}, VM status: {}".format(dom.name(), snap_name, dom.state())) + except libvirt.libvirtError: + logging.error("Failed to reset {} to snapshot: {}".format(dom.name(), snap_name)) + + def get_current_snapshot(self): + conn, dom = self.get_domain_object() + try: + cur = dom.snapshotCurrent().getName() + return cur + except libvirt.libvirtError: + logging.error("Failed to find current snapshots for {}!".format(dom.name())) + return None + finally: + conn.close() + + ###################################################################################################### + # VM IMAGE FETCHING AND INSTALLATION # + ###################################################################################################### + + def get_img(self, url, save_to): + """ + Attempts to download a(n) (packen) iso + :param url: Full URL to image + :param save_to: lpath to save to + :return: path to downloaded file + """ + if self.p_archive: + print("[!] Archive specified!\nAborting image fetch") + if not url: + print("[!] Requires url!\nAborting...") + sys.exit(1) + else: + create_directory(save_to) + url_split_archive_name_only = url.split("/")[-1] + save_img = get_absolute_path(os.path.join(save_to, url_split_archive_name_only)) + if check_if_file_exists(save_img): + print("[!] File {} already exists.".format(save_img)) + self.p_archive = save_img + else: + print("[*] Fetching {}".format(url)) + if wget.download(url, save_img): + self.p_archive = save_img + if "QEMU QCOW Image" in magic.from_file(self.p_archive): + self.path_to_iso_or_qcow = self.p_archive + return self.p_archive + + def _unpack_lzma(self, save_as): + """ + Unpacks a lzma compatible archive (e.g ISO.xz) + :param save_as: save ISO as + :return: path to unpacked content + """ + with lzma.open(self.p_archive) as f, open(save_as, "wb") as fout: + logging.info("Decompressing....") + file_cntnt = f.read() + fout.write(file_cntnt) + fout.close() + self.path_to_iso_or_qcow = save_as + + def _set_unpckd_path(self, save_as, save_to): + """ + Some path demangling + :param save_as: Name of unpacked iso.xz + :param save_to: Path to save unpacked iso.xz to + :return: + """ + img_n = "" + archive_no_ext, _ = os.path.splitext(self.p_archive) + save_to = get_absolute_path(save_to) + if save_as and save_to: + img_n = os.path.join(save_to, save_as) + elif save_as and save_to is None: + abs_path_without_archive = pathlib.Path(self.p_archive).parent + img_n = os.path.join(abs_path_without_archive, save_as) + elif save_as is None and save_to: + base_name = pathlib.Path(archive_no_ext).name + img_n = os.path.join(save_to, base_name) + elif save_as is None and save_to is None: + img_n = archive_no_ext + return get_absolute_path(img_n) + + def unpack(self, save_as, save_to): + """ + Unpack downloaded ISO.xz as save_as to save_to + :param save_as: Name of unpacked iso.xz + :param save_to: Path to save unpacked iso.xz to + """ + create_directory(save_to) + if check_if_file_exists(self.p_archive): + unpkd_img_name = self._set_unpckd_path(save_as, save_to) + logging.debug("[*] Trying to decompress {} as {}".format(self.p_archive, unpkd_img_name)) + if not pathlib.Path(unpkd_img_name).exists(): + self._unpack_lzma(unpkd_img_name) + else: + logging.debug("[!] Error unpacking {}".format(self.p_archive)) + + def _install(self, iso): + abs_iso = get_absolute_path(iso) + cmd = "virt-install --name {}\ + --memory {}\ + --vcpus {}\ + --disk size={}\ + --cdrom {}\ + --os-variant freebsd8".format( + self.name, self.vm_memory, self.vm_cpus, self.vm_hdd, abs_iso + ) + subprocess.run(cmd.split()) + + def install_vm( + self, iso=None, url_iso=None, url_zip=None, save_dl_to="~/Downloads", save_unpckd_to=None, save_unpckd_as=None, + ): + """ + Setups a VM install that varies on the supplied parameters + :param iso: ISO.iso VM image + :param url_iso: Full URL to a ISO.iso image + :param url_zip: Full URL to a iso.iso.xz image + :param save_dl_to: Path to save the downloaded content to + :param save_unpckd_to: Path to save the unpacked downloaded content to + :param save_unpckd_as: Full Path that specifies the name of the unpacked content + """ + if iso: + self._install(iso) + if url_iso: + self.get_img(url_iso, save_to=save_dl_to) + self._install(self.p_archive) + if url_zip: + self.get_img(url=url_zip, save_to=save_dl_to) + self.unpack(save_to=save_unpckd_to, save_as=save_unpckd_as) + self._install(self.path_to_iso_or_qcow) + + +###################################################################################################### +# QUICK QEMU BOOT WITHOUT INSTALLATION # +###################################################################################################### + + +class QEMU(VmManager): + def __init__( + self, name, architecture, path_to_hdd, enable_kvm=True, port=22, ip="127.0.0.1", memory=2048, + ): + super(QEMU, self).__init__() + logging.basicConfig(level=logging.INFO) + logging.getLogger("paramiko").setLevel(logging.CRITICAL) + self.name = name + self.vm_arch = architecture + self.port = port + self.enable_kvm = enable_kvm + self.qemu_process = None + self.vm_memory = memory + self.ip = ip + self.path_to_iso_or_qcow = path_to_hdd + + def qemu_boot_qcow(self): + """ + Requires a proper prior VmManager.setup(...) call to set all the necessary parameters + """ + if self.enable_kvm: + qemu_cmd = ( + "qemu-system-{} -m {} --enable-kvm" + " -netdev user,id=mynet0,hostfwd=tcp:{}:{}-:22" + " -device e1000,netdev=mynet0 {}".format( + self.vm_arch, self.vm_memory, self.vm_ip, self.port, self.path_to_iso_or_qcow, + ) + ) + else: + qemu_cmd = ( + "qemu-system-{} -m {}" + " -netdev user,id=mynet0,hostfwd=tcp:{}:{}-:22" + " -device e1000,netdev=mynet0 {}".format( + self.vm_arch, self.vm_memory, self.vm_ip, self.port, self.path_to_iso_or_qcow, + ) + ) + logging.debug(qemu_cmd) + try: + active_machine = subprocess.Popen(qemu_cmd.split(), stdout=subprocess.PIPE, preexec_fn=os.setsid) + self.qemu_process = active_machine + if self.qemu_process.poll() is None: + print("[*] Sleeping for 60s to wait for boot...") + self._boot_sleep(60) + else: + sys.exit(1) + return active_machine + except RuntimeError as e: + logging.debug(e) + + +def main(): + vm = VmManager() + vm.setup(name="fuzzbox") + vm.install_vm(iso="~/Downloads/FreeBSD-11.2-RELEASE-amd64-dvd1.iso") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/Manager/Manager_FreeBSD.py b/src/Manager/Manager_FreeBSD.py new file mode 100644 index 0000000..5f4ca72 --- /dev/null +++ b/src/Manager/Manager_FreeBSD.py @@ -0,0 +1,120 @@ +import logging +import os +import re + +import paramiko + + +class FreeBSD: + def __init__(self, rfile, mount_at, vm_object): + logging.basicConfig(level=logging.DEBUG) + self.bdev = None + self.fs_type = None + self.rfile = rfile + self.mount_at = mount_at + self.vm_object = vm_object + self.pool = None + + def make_block_device(self): + logging.debug("CREATING BLKDEV FOR: {}".format(self.rfile)) + cmd = "/sbin/mdconfig -a -t vnode -f {}".format(self.rfile) + logging.debug(cmd) + self.bdev = os.path.join("/dev", self.vm_object.exec_cmd_quiet(cmd)) + + def destroy_bdev(self): + cmd = "/sbin/mdconfig -d -u {}".format(self.bdev) + logging.debug(cmd) + return self.vm_object.exec_cmd_quiet(cmd) + + def _determine_fs_type(self): + if self.vm_object.silent_vm_state(): + out = self.vm_object.exec_cmd_quiet("/usr/bin/file {}".format(self.rfile)) + match = re.search(r"ext[1-4] filesystem data", out) + if match: + self.fs_type = match.group(0).split()[0] + elif "Unix Fast File system" in out: + self.fs_type = "ufs" + elif "data" in out: + self.fs_type = "zfs" + + def _clean_mount_dir(self): + self.vm_object.rm_files(self.mount_at) + self.vm_object.mkdir(self.mount_at) + + def _get_mount_switch(self): + if any(x == self.fs_type for x in ["ext2", "ext3", "ext4"]): + flag = "ext2fs" + elif self.fs_type == "ufs": + flag = "ufs" + else: + logging.debug("Malformed file system") + logging.debug('Trying mount -t "auto" ...') + flag = "auto" + return flag + + def _mount_ext_ufs(self): + cmd = '/sbin/mount -t "{}" {} {}'.format(self._get_mount_switch(), self.bdev, self.mount_at) + logging.debug(cmd) + if not self.vm_object.exec_cmd_quiet(cmd): + return 1 # Success + else: + logging.debug("Mounting of {} failed".format(self.bdev)) # Failed + return 0 + + def _mount_zfs(self): + res = self.vm_object.exec_cmd_quiet("zpool import") + logging.debug("ZPOOL IMPORT STDOUT: <<{}>>".format(res)) + if res and len(str(res)) > 2: + self.pool = res.split()[1] + if not self.vm_object.exec_cmd_quiet("zpool import {} -f".format(self.pool)): + return 1 # Success + else: + return 0 + else: + logging.debug("No zpool to import found") + return 0 + + def mount_file_system(self): + try: + self._clean_mount_dir() + self._determine_fs_type() + self.make_block_device() + if any(mime in self.fs_type for mime in ["ext2", "ext3", "ext4", "ufs"]) and self._mount_ext_ufs(): + return 1 + # ugly hack for zfs + elif "data" in self.fs_type and self._mount_zfs(): + return 1 + else: + if self.vm_object.silent_vm_state(): + return 0 + else: + return 2 + except TypeError: + return None + + def _unmount_ext_ufs(self): + cmd_mount = "/sbin/umount -f {}".format(self.mount_at) + logging.debug(cmd_mount) + if not self.vm_object.exec_cmd_quiet(cmd_mount) and not self.destroy_bdev(): + return 1 # Success + else: + logging.debug("Failed to properly umount {}".format(self.mount_at)) + return 0 + + def _unmount_zfs(self): + cmd_zpool_export = "zpool export {}".format(self.pool) + logging.debug(cmd_zpool_export) + if not self.vm_object.exec_cmd_quiet(cmd_zpool_export) and not self.destroy_bdev(): + return 1 # Success + else: + logging.debug("Failed to export pool".format(self.pool)) + return 0 + + def unmount_file_system(self): + try: + if any(ext in self.fs_type for ext in ["ext2", "ext3", "ext4", "ufs"]): + return self._unmount_ext_ufs() + elif "data" in self.fs_type: + return self._unmount_zfs() + except (paramiko.ssh_exception.SSHException, paramiko.ssh_exception.NoValidConnectionsError,) as e: + logging.debug(e) diff --git a/src/Manager/Manager_NetBSD.py b/src/Manager/Manager_NetBSD.py new file mode 100644 index 0000000..9142964 --- /dev/null +++ b/src/Manager/Manager_NetBSD.py @@ -0,0 +1,87 @@ +import logging +import re + +import paramiko + + +class NetBSD: + def __init__(self, rfile, mount_at, vm_object): + logging.basicConfig(level=logging.DEBUG) + self.bdev = None + self.fs_type = None + self.rfile = rfile + self.mount_at = mount_at + self.vm_object = vm_object + self.pool = None + + def make_block_device(self): + cmd = "/usr/sbin/vndconfig vnd0 {}".format(self.rfile) + self.vm_object.exec_cmd_quiet(cmd) + self.bdev = "/dev/vnd0" + cmd_disklabel = "/sbin/disklabel {}".format(self.bdev) + self.vm_object.exec_cmd_quiet(cmd_disklabel) + self.bdev = "/dev/rvnd0" + logging.debug("block device {} created".format(self.bdev)) + + def destroy_block_device(self): + cmd = "/usr/sbin/vndconfig -u {}".format(self.bdev.split("/")[-1]) + logging.debug(cmd) + return self.vm_object.exec_cmd_quiet(cmd) + + def _determine_fs_type(self): + if self.vm_object.silent_vm_state(): + file_output = self.vm_object.exec_cmd_quiet("/usr/bin/file {}".format(self.rfile)) + match = re.search(r"ext[1-4] filesystem data", file_output) + if match: + self.fs_type = match.group(0).split()[0] + elif "Unix Fast File system" in file_output or "4.3bsd" in file_output: + self.fs_type = "ufs" + + def _clean_mount_dir(self): + self.vm_object.rm_files(self.mount_at) + self.vm_object.mkdir(self.mount_at) + + def _get_mount_switch(self): + flag = "" + self.bdev = self.bdev.translate({ord(c): None for c in "r"}) + if self.fs_type == "ext2": + flag = "ext2fs" + elif "ufs" in self.fs_type: + flag = "ufs" + return flag + + def _mount_ext_ufs(self): + cmd = "/sbin/mount -t {} {} {}".format(self._get_mount_switch(), self.bdev, self.mount_at) + logging.debug(cmd) + if not self.vm_object.exec_cmd_quiet(cmd): + return 1 # Success + else: + logging.debug("Mounting of {} failed".format(self.bdev)) # Failed + return 0 + + def mount_file_system(self): + self._clean_mount_dir() + self._determine_fs_type() + self.make_block_device() + cmd_mount = "/sbin/mount -t {} {} {}".format(self._get_mount_switch(), self.bdev, self.mount_at) + logging.debug(cmd_mount) + if not self.vm_object.exec_cmd_quiet(cmd_mount): + return 1 + else: + logging.debug("Mounting of {} failed".format(self.bdev)) # Failed + return 0 + + def _unmount_ext_ufs(self): + cmd_mount = "/sbin/umount -f {}".format(self.mount_at) + logging.debug(cmd_mount) + if not self.vm_object.exec_cmd_quiet(cmd_mount) and not self.destroy_block_device(): + return 1 # Success + else: + logging.debug("Failed to properly umount {}".format(self.mount_at)) + return 0 + + def unmount_file_system(self): + try: + self._unmount_ext_ufs() + except (paramiko.ssh_exception.SSHException, paramiko.ssh_exception.NoValidConnectionsError,) as e: + logging.debug(e) diff --git a/src/Manager/Manager_OpenBSD.py b/src/Manager/Manager_OpenBSD.py new file mode 100644 index 0000000..5de906f --- /dev/null +++ b/src/Manager/Manager_OpenBSD.py @@ -0,0 +1,93 @@ +import logging +import re + +import paramiko + + +class OpenBSD: + def __init__(self, rfile, mount_at, vm_object): + logging.basicConfig(level=logging.DEBUG) + self.bdev = None + self.fs_type = None + self.rfile = rfile + self.mount_at = mount_at + self.vm_object = vm_object + self.pool = None + + def make_block_device(self): + cmd = "/sbin/vnconfig vnd0 {}".format(self.rfile) + self.vm_object.exec_cmd_quiet(cmd) + self.bdev = "vnd0" + cmd_disklabel = "/sbin/disklabel -A {}".format(self.bdev) + self.bdev = ( + self.vm_object.exec_cmd_quiet(cmd_disklabel) + .split("\n")[0] + .replace("# ", "") + .replace("r", "") + .replace(":", "") + .strip() + ) + logging.debug("block device {} created".format(self.bdev)) + + def destroy_block_device(self): + cmd = "/sbin/vnconfig -u {}".format(self.bdev) + logging.debug(cmd) + return self.vm_object.exec_cmd_quiet(cmd) + + def _determine_fs_type(self): + if self.vm_object.silent_vm_state(): + file_output = self.vm_object.exec_cmd_quiet("/usr/bin/file {}".format(self.rfile)) + match = re.search(r"ext[1-4] filesystem data", file_output) + if match: + self.fs_type = match.group(0).split()[0] + elif "Unix Fast File system" in file_output or "4.3bsd" in file_output: + self.fs_type = "ufs" + + def _clean_mount_dir(self): + self.vm_object.rm_files(self.mount_at) + self.vm_object.mkdir(self.mount_at) + + def _get_mount_switch(self): + flag = "" + self.bdev = self.bdev.translate({ord(c): None for c in "r"}) + if self.fs_type == "ext2": + flag = "ext2fs" + elif "ufs" in self.fs_type: + flag = "ffs" + return flag + + def _mount_ext_ufs(self): + cmd = "/sbin/mount -t {} {} {}".format(self._get_mount_switch(), self.bdev, self.mount_at) + logging.debug(cmd) + if not self.vm_object.exec_cmd_quiet(cmd): + return 1 # Success + else: + logging.debug("Mounting of {} failed".format(self.bdev)) # Failed + return 0 + + def mount_file_system(self): + self._clean_mount_dir() + self._determine_fs_type() + self.make_block_device() + cmd_mount = "/sbin/mount -t {} {} {}".format(self._get_mount_switch(), self.bdev, self.mount_at) + logging.debug(cmd_mount) + if not self.vm_object.exec_cmd_quiet(cmd_mount): + return 1 + else: + logging.debug("Mounting of {} failed".format(self.bdev)) # Failed + return 0 + + def _unmount_ext_ufs(self): + cmd_mount = "/sbin/umount -f {}".format(self.mount_at) + logging.debug(cmd_mount) + if not self.vm_object.exec_cmd_quiet(cmd_mount) and not self.destroy_block_device(): + return 1 # Success + else: + logging.debug("Failed to properly umount {}".format(self.mount_at)) + return 0 + + def unmount_file_system(self): + try: + self._unmount_ext_ufs() + except (paramiko.ssh_exception.SSHException, paramiko.ssh_exception.NoValidConnectionsError,) as e: + logging.debug(e) diff --git a/src/Manager/Manager_Ubuntu.py b/src/Manager/Manager_Ubuntu.py new file mode 100644 index 0000000..4d0f0a6 --- /dev/null +++ b/src/Manager/Manager_Ubuntu.py @@ -0,0 +1,86 @@ +import logging +import re + +import paramiko + + +class Ubuntu: + def __init__(self, rfile, mount_at, vm_object): + logging.basicConfig(level=logging.DEBUG) + self.bdev = None + self.fs_type = None + self.rfile = rfile + self.mount_at = mount_at + self.vm_object = vm_object + + def make_block_device(self): + cmd_find_loopdev = "losetup -f" + self.bdev = self.vm_object.exec_cmd_quiet(cmd_find_loopdev) + cmd = "losetup {} {}".format(self.bdev, self.rfile) + self.vm_object.exec_cmd_quiet(cmd) + logging.debug("block device {} created".format(self.bdev)) + + def destroy_block_device(self): + cmd = "losetup -d {}".format(self.bdev) + logging.debug(cmd) + return self.vm_object.exec_cmd_quiet(cmd) + + def _determine_fs_type(self): + if self.vm_object.silent_vm_state(): + file_output = self.vm_object.exec_cmd_quiet("/usr/bin/file {}".format(self.rfile)) + match = re.search(r"ext[1-4] filesystem data", file_output) + if match: + self.fs_type = match.group(0).split()[0] + + def _clean_mount_dir(self): + self.vm_object.rm_files(self.mount_at) + self.vm_object.mkdir(self.mount_at) + + def _get_mount_switch(self): + if any(x == self.fs_type for x in ["ext2", "ext3", "ext4"]): + flag = self.fs_type + else: + logging.debug("Malformed file system") + logging.debug('Trying mount -t "auto" ...') + flag = "auto" + return flag + + def _mount_ext(self): + cmd = '/bin/mount -t "{}" {} {}'.format(self._get_mount_switch(), self.bdev, self.mount_at) + logging.debug(cmd) + if not self.vm_object.exec_cmd_quiet(cmd): + return 1 # Success + else: + logging.debug("Mounting of {} failed".format(self.bdev)) # Failed + return 0 + + def mount_file_system(self): + try: + self._clean_mount_dir() + self._determine_fs_type() + self.make_block_device() + if any(mime in self.fs_type for mime in ["ext2", "ext3", "ext4", "ufs"]) and self._mount_ext(): + return 1 + else: + if self.vm_object.silent_vm_state(): + return 0 + else: + return 2 + except TypeError: + return None + + def _unmount_ext_ufs(self): + cmd_mount = "/bin/umount -f {}".format(self.mount_at) + logging.debug(cmd_mount) + if not self.vm_object.exec_cmd_quiet(cmd_mount) and not self.destroy_block_device(): + return 1 # Success + else: + logging.debug("Failed to properly umount {}".format(self.mount_at)) + return 0 + + def unmount_file_system(self): + try: + if any(ext in self.fs_type for ext in ["ext2", "ext3", "ext4"]): + return self._unmount_ext_ufs() + except (paramiko.ssh_exception.SSHException, paramiko.ssh_exception.NoValidConnectionsError,) as e: + logging.debug(e) diff --git a/src/Manager/__init__.py b/src/Manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/README.md b/src/README.md new file mode 100644 index 0000000..c3dfa77 --- /dev/null +++ b/src/README.md @@ -0,0 +1,196 @@ +# makeFS + +This script produces a valid ext2/ext3/ext4/ufs/zfs filesystem of size 'X'. +Naming can be specified, otherwise a random UID is created. +It also can be populated with with 'N' files of max size 'M' if specified. +This includes, files made from `/dev/urandom`, directories, sym- and hardlinks. +The file and directory structure is created at random. + +e.g.: `python3 makeFS2.py -fs "ext2" -n "ext20MB" -s 20 -p 15 -ps 1000` + +# VmManager +This script is able to install and manage multiple VMs via libvirt and its python bindings + +## VM Setup and Install: +It's expected to create an instance of VMManager first: +``` +vmm = VMMANAGER() +``` + +You can tweak hardware details of your VM by calling `setup()`: +``` +@param name= Name for the VM +@param vm_memory= RAM in MB. Default=4096 +@param vm_cpus= #CPU_CORES. Default=2 +@param vm_hdd= HDD size in GB. Default=10 +vmm.setup(name='', vm_memory=, vm_cpus=, vm_hdd=) + +``` + +#### Install VM from local .ISO +``` +@param iso= Path to the .iso image file +vmm.install_vm(iso='') +``` + +#### Install VM from web .ISO: +``` +@param url_iso= URL from where to wget the .iso image +@param save_dl_to= Path on the local disk. Default='/tmp' +vmm.install_vm(url_iso='', save_dl_to='') +``` + +#### Install VM from zipped web .ISO: +``` +@param url_zip= URL from where to wget the .iso.xz image +@param save_dl_to= Path on the local disk. Default='/tmp' +@param save_unpckd_to= Path on the local disk. Default='/tmp' +@param save_unpckd_as= Filename for the unpacked .iso. Default=Tries to use Archive name without extension +vmm.install_vm(url_zip='', + save_dl_to='', + save_unpckd_to='', + save_unpckd_as='') +``` + + + +#### Boot directly from a QCOW2 VM hdd +##### 1. If fetching QCOW from web: +``` +@param name= Name for the VM +@param vm_type= A fitting representation of the image from 'qemu-system-XXX' +@param url= URL from where to wget the .qcow2 image +@param save_dl_to= Path on the local disk to save download to. Default='/tmp' +@param save_dl_as= Filename for download. Default=Tries to use Archive name without extension +@param port= Port for port forwarding between Host<->Guest. Default=22 +vmm.setup(name='', vm_type='', url='', save_url_cntnt_to='', save_url_cntnt_as='', fwdPort=) +vmm.get_img() +machine = vmm.qemu_boot_qcow() +``` + +If the QCOW is zipped as .xz you can call `vmm.unpack()` prior to `vmm.run_qemu()`. +`vmm.unpack()` + + +##### 2. If QCOW already on disk: +``` +@param name= Name for the VM +@param vm_type= A fitting representation of the image from 'qemu-system-XXX' +@param unpckd_url_cntnt= Path to and including the local .qcow2 image +@param port= Port for port forwarding between Host<->Guest. Default=22 +vmm.setup(name='', vm_type='', unpckd_url_cntnt='', port=) +machine = vmm.qemu_boot_qcow() +``` + + + +## VM user interaction: +Note: relative paths should always be resolved automatically + +#### Copy files to guest: +``` +@param lfile= Path to a file on the host system +@param path_on_guest= Path (dir!) on the guest to save the lfile to +vmm.cp_files_to_guest(lfile='', path_on_guest='') +``` +If path does not exist the path gets created + +#### Copy files to host: +``` +@param rfile= Path to a file on the guest system +@param path_on_host= Path (dir!) on the guest to save the lfile to +@param filename= Name for the file on the host system +vmm.cp_files_to_host(rfile='', path_on_host='', filename='') +``` + +#### Tar n files on guest: +Tars the files given in `lst_of_files` in `PATH_REMOTE_DIR` on guest +``` +@param rpath= The parent directory on where the files are located +@param archive_name= Name for the archive +@param lst_of_files= List of files @ location rpath +rpath = vmm.qemu_exec_tar_files(rpath='', archive_name='', lst_of_files=['', '', ..., '']) +``` + +#### Tar a dir on guest: +Tars the whole dir specified +``` +@param rpath= The parent directory on where the files are located +@param archive_name= Name for the archive +rpath = vm_exec_tar_dir(rpath='', archive_name='') +``` + +#### Exec any cmd on guest from cmdline: +``` +Executes the CMD given on the commandline on the guest system +vmm.vm_exec_cmd(CMD) +``` + +#### Exec makedir on guest: +``` +Creates a directory including all necessary parents supplied +vmm.vm_exec_mkdir(PATH) +``` + +#### Exec remove on guest: +``` +Does a recursive remove to be able to handle directory and single files +vmm.vm_exec_rm(PATH) +``` + +#### Get latest crash from guests `/var/crash/:` +``` +@param lpath= Path on the host system to save the `latest_crash.tar` archive to +vmm.fetch_latest_crash(lpath=) +``` + +#### Get latest unknown crash from guests `/var/crash/`: +``` +@param lpath= Path on the host system to save the `latest_unknown_crash.tar` archive to +vmm.fetch_latest_unknown_crash(lpath=) +``` + +#### Get all crashes from guests `/var/crash/`: +``` +@param lpath= Path on the host system to save the `allcrashes.tar` archive to +vmm.fetch_all_crashes(lpath=LOCAL_PATH_WHERE_TO_SAVE) +``` + +## VM Management + +On top of providing the above user interface to the VM for interaction several management convenience functions were added too. + +#### Boot VMs +This will list the the installed/found VMs managed by libvirt and prompts the user for a box to start + +* `vmm.boot_vm()` + +#### Manage VMs +All the functions below take an optional `vm_name=` argument for convenient management of other existing VMs. +* `vmm.shutdown_vm()` +* `vmm.suspend_vm()` +* `vmm.resume_vm()` +* `vmm.reset_vm()` +* `vmm.force_stop_vm()` +* `vmm.delete_vm()` +* `vmm.check_vm_state()` +* `vmm.quick_boot()` +* `vmm.reboot_vm_if_crashed()` +* `vmm.reset_vm_if_crashed()` +* `vmm.restore_latest_snap_if_crashed()` + +#### Snapshots + +Furthermore a snapshot interface was added. These can take the same `vm_name` optional argument as well. + +* `vmm.create_snapshot()` +* `vmm.create_snapshot_as(snapshot_name)` +* `vmm.delete_snapshot(snapshot_name)` +* `vmm.restore_snapshot(snapshot_name)` +* `vmm.get_current_snapshot()` + +#### Others + +* `vmm.create_core_dump()` +* `vmm.take_screenshot()` +* `vmm.check_vm_state()` diff --git a/src/Requirements.sh b/src/Requirements.sh new file mode 100755 index 0000000..4763f25 --- /dev/null +++ b/src/Requirements.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -e + +echo "[*] Installing system dependencies..." +sudo apt-get install -y python3-dev python3-pip qemu-kvm libvirt-clients libvirt-dev libvirt-daemon-system gcc \ + libsdl1.2-dev zlib1g-dev libasound2-dev linux-kernel-headers pkg-config libgnutls28-dev \ + libpci-dev libglib2.0-dev libfdt-dev libpixman-1-dev net-tools virtinst git libnl-3-dev \ + libnl-route-3-dev libxml2-dev libpciaccess-dev libyajl-dev xsltproc libdevmapper-dev \ + uuid-dev qemu qemu-block-extra qemu-guest-agent qemu-system qemu-system-common libvirt-bin \ + qemu-utils qemu-user qemu-efi openbios-ppc sgabios systemtap pm-utils open-iscsi debootstrap \ + zfsutils-linux file tmux + + +mkdir -p ~/git + +echo "[*] Installing radamsa" +git clone https://gitlab.com/akihe/radamsa.git ~/git/radamsa/ +cd ~/git/radamsa +make +sudo make install + +echo "[*] Installing needed python packages..." +sudo -EH python3 -m pip install libvirt-python wget paramiko pprint scp python-magic Pillow colorama seaborn + +echo "[*] Setting up users..." +sudo usermod -aG libvirt $USER +sudo usermod -aG libvirt-qemu $USER +sudo usermod -aG kvm $USER + +echo "[*] Testing install..." +sudo systemctl enable libvirtd +sudo systemctl start libvirtd +virsh list --all >> /dev/null +if [[ $? == 0 ]]; then + echo "[+] libvirt successfully set up!" +else + echo "[-] libvirt failed to install!" +fi + +kvm-ok >> /dev/null +if [[ $? == 0 ]]; then + echo "[+] kvm support successfully set up!" + cpu_check=$(cat /proc/cpuinfo | grep "model name" | uniq | grep -oh ": [a-zA-Z]*" | cut -c 3-) + if [[ ${cpu_check} == "Intel" ]]; then + modprobe kvm_intel + elif [[ ${cpu_check} == "AMD" ]]; then + modprobe kvm_amd + else + echo "[*] Unknown CPU, skipping modprobe" + fi +else + echo "[-] kvm support failed to install!" +fi + +installed_qemu_packages=$(dpkg -l | grep '^ii' | grep -o ' qemu-[a-zA-Z0-9-]*') +echo "[*] Installed QEMU modules:"${installed_qemu_packages} diff --git a/src/SnapshotTemplate/__init__.py b/src/SnapshotTemplate/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/SnapshotTemplate/snapshot.py b/src/SnapshotTemplate/snapshot.py new file mode 100644 index 0000000..818eafa --- /dev/null +++ b/src/SnapshotTemplate/snapshot.py @@ -0,0 +1,9 @@ +xml_head_name = """ {} + running + """ + +xml_head = """ + running + """ + +xml_tail = "" diff --git a/src/UserEmulation/UE_FreeBSD.py b/src/UserEmulation/UE_FreeBSD.py new file mode 100644 index 0000000..6df2c9d --- /dev/null +++ b/src/UserEmulation/UE_FreeBSD.py @@ -0,0 +1,156 @@ +import os +import random + +from UserEmulation.UE_Generic import GenericUserEmulation + + +def get_random_list_entry(file_list): + return random.choice(list(filter(None, file_list))).rstrip(",").strip() + + +class FreebsdUserEmulation: + def __init__(self, vm_object, rpath): + self.vm_object = vm_object + self.rpath = rpath + + def prepare_freebsd_user_emulation(self): + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + users, groups = gen_user.get_users_and_groups_of_target_os() + chflags_modes = [ + "arch", + "nodump", + "opaque", + "sappnd", + "schg", + "snapshot", + "sunlnk", + "uappnd", + "uarch", + "uchg", + "hidden", + ] + # hardcoded path should probably be replaced with "$(which CMD) flags flag_args" + free_bsd_user_emulation = [ + "/usr/bin/find {}/*".format(self.rpath), # yes + "/bin/ls -lah {}/*".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), # yes + "/usr/bin/touch {}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "TOUCHED",) + ), # no + "/bin/mkdir -p {}/a/b/c".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), # no + "/bin/dd if=/dev/urandom of={} bs={} count={}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "DATA",), + 1 << 20, + random.randint(1, 5), + ), # yes + "/bin/ln {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + os.path.join(self.rpath, "HARDLINK"), + ), # no + "/bin/ln -s {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + os.path.join(self.rpath, "SOFTLINK"), + ), # no + "/usr/bin/file {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), # yes + "/usr/bin/readlink {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="link"))), # yes + "/usr/bin/stat {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), # yes + "/bin/cp -R {} {}/COPIED", # no + "/sbin/mknod {}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "NODDED",) + ), # no + '/usr/bin/tar -jcvf {} "{} {}"'.format( + os.path.join(self.rpath, "archive.bzip2"), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + ), # yes + "/bin/chmod {} {}".format( + gen_user.get_random_chmod_mode(), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), # no + "/usr/sbin/chown {}:{} {}".format( + get_random_list_entry(users), + get_random_list_entry(groups), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), # no + "/usr/bin/chgrp {} {}".format( + get_random_list_entry(groups), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), # no + "/bin/mv {} {}", # no + "/bin/echo APPENDED >> {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")) + ), # no + "chdir {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), # no + "/bin/rm -rf {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")[:-1])), # no + "/bin/chflags {} {}".format( + get_random_list_entry(chflags_modes), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), # no + "/bin/getfacl {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), # yes + "/usr/bin/split {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), + ), + "/usr/bin/du {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/wc {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/truncate -s {} {}".format( + random.randint(1, 5), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + ), + "/usr/bin/dirname {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/basename {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + ] + # '/bin/setfacl -m {}:{} {}'.format(get_random_list_entry(users), + # gen_user.get_random_chmod_mode(), + # get_random_list_entry( + # gen_user.get_files_of_mounted_file_system( + # param='files'))) + # ] needs to be explicitly enabled during mount -o acls + random.shuffle(free_bsd_user_emulation) + return free_bsd_user_emulation + + def get_freebsd_backup_user_emulation(self): + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + + freebsd_user_emulation = [ + "/usr/bin/find {}/*".format(self.rpath), + "/bin/ls -lah {}/*".format(self.rpath), + "/usr/bin/touch {}".format(os.path.join(self.rpath, "TOUCHED")), + "/bin/mkdir -p {}/a/b/c".format(self.rpath), + "/bin/dd if=/dev/urandom of={} bs={} count={}".format( + os.path.join(self.rpath, "DATA"), 1 << 20, random.randint(1, 5) + ), + "/sbin/mknod {}".format(os.path.join(self.rpath, "NODDED")), + "/bin/echo APPENDED >> {}".format(os.path.join(self.rpath, "ECHOED")), + ] + random.shuffle(freebsd_user_emulation) + for entry in [ + "/bin/ln {} {}".format(os.path.join(self.rpath, "DATA"), "HARDLINK"), + "/bin/ln -s {} {}".format(os.path.join(self.rpath, "DATA"), "SOFTLINK"), + "/usr/bin/readlink {}".format(os.path.join(self.rpath, get_random_list_entry(["SOFTLINK", "HARDLINK"]))), + "/bin/chmod {} {}".format(gen_user.get_random_chmod_mode(), os.path.join(self.rpath, "DATA")), + "/bin/rm -rf {}/*".format(self.rpath), + ]: + freebsd_user_emulation.append(entry) + return freebsd_user_emulation + + def set_freebsd_user_emulation(self): + try: + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + res = gen_user.get_files_of_mounted_file_system("files") + if any("Traceback" in pos for pos in res): + user_emulation_command_list = self._failed_trav_user_emul() + else: + user_emulation_command_list = self.prepare_freebsd_user_emulation() + except AttributeError: + user_emulation_command_list = self._failed_trav_user_emul() + return user_emulation_command_list + + def _failed_trav_user_emul(self): + print("\t > Failed to perform a file traversal on mounted file system") + print("\t > Starting backup user emulation that only attempts to write to disk...") + user_emulation_command_list = self.get_freebsd_backup_user_emulation() + return user_emulation_command_list + + @staticmethod + def randomize_emulation_list_length(emu_list, multiplier): + new_emu_list = [] + for i in range(multiplier): + new_emu_list += random.sample(emu_list, random.randint(0, len(emu_list))) + return new_emu_list diff --git a/src/UserEmulation/UE_Generic.py b/src/UserEmulation/UE_Generic.py new file mode 100644 index 0000000..0f0e081 --- /dev/null +++ b/src/UserEmulation/UE_Generic.py @@ -0,0 +1,36 @@ +import random + + +class GenericUserEmulation: + def __init__(self, vm_object, remote_mount_path): + self.vm_object = vm_object + self.rpath = remote_mount_path + + def get_users_and_groups_of_target_os(self): + res = self.vm_object.exec_cmd_quiet("python3 /tmp/get_users_and_groups.py").split("") + users = res[0].split() + groups = res[1].split() + return users, groups + + def get_files_of_mounted_file_system(self, param="all"): + res = self.vm_object.exec_cmd_quiet("python3 /tmp/file_traversal.py {} {}".format(self.rpath, param)).split("") + if param == "all": + list_of_all_directories = res[0].split() + list_of_all_files = res[1].split() + list_of_files_and_links = res[2].split() + list_of_all_links = res[3].split() + return ( + list_of_all_directories, + list_of_all_files, + list_of_files_and_links, + list_of_all_links, + ) + else: + return res[0].split(",") + + @staticmethod + def get_random_chmod_mode(): + mod = "" + for i in range(3): + mod += str(random.randint(0, 7)) + return int(mod) diff --git a/src/UserEmulation/UE_NetBSD.py b/src/UserEmulation/UE_NetBSD.py new file mode 100644 index 0000000..f273c22 --- /dev/null +++ b/src/UserEmulation/UE_NetBSD.py @@ -0,0 +1,143 @@ +import os +import random + +from UserEmulation.UE_Generic import GenericUserEmulation + + +def get_random_list_entry(file_list): + return random.choice(list(filter(None, file_list))).rstrip(",").strip() + + +class NetbsdUserEmulation: + def __init__(self, vm_object, rpath): + self.vm_object = vm_object + self.rpath = rpath + + def prepare_netbsd_user_emulation(self): + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + users, groups = gen_user.get_users_and_groups_of_target_os() + chflags_modes = [ + "arch", + "nodump", + "opaque", + "sappnd", + "schg", + "snapshot", + "sunlnk", + "uappnd", + "uarch", + "uchg", + "hidden", + ] + + netbsd_user_emulation = [ + "/usr/bin/find {}/*".format(self.rpath), + "/bin/ls -lah {}/*".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), + "/usr/bin/touch {}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "TOUCHED",) + ), + "/bin/mkdir -p {}/a/b/c".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), + "/bin/dd if=/dev/urandom of={} bs={} count={}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "DATA",), + 1 << 20, + random.randint(1, 5), + ), + "/bin/ln {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + os.path.join(self.rpath, "HARDLINK"), + ), + "/bin/ln -s {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + os.path.join(self.rpath, "SOFTLINK"), + ), + "/usr/bin/file {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/readlink {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="link"))), + "/usr/bin/stat {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), + "/bin/cp -R {} {}/COPIED", # no + "/sbin/mknod {}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "NODDED",) + ), # no + '/bin/tar -jcvf {} "{} {}"'.format( + os.path.join(self.rpath, "archive.bzip2"), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + ), + "/bin/chmod {} {}".format( + gen_user.get_random_chmod_mode(), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "/sbin/chown {}:{} {}".format( + get_random_list_entry(users), + get_random_list_entry(groups), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "/bin/chgrp {} {}".format( + get_random_list_entry(groups), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "/bin/mv {} {}", + "/bin/echo APPENDED >> {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file"))), + "/bin/rm -rf {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")[:-1])), + "/usr/bin/chflags {} {}".format( + get_random_list_entry(chflags_modes), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "/usr/bin/split {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), + ), + "/usr/bin/du {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/wc {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/dirname {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/basename {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + ] + random.shuffle(netbsd_user_emulation) + return netbsd_user_emulation + + def get_netbsd_backup_user_emulation(self): + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + + netbsd_user_emulation = [ + "/usr/bin/find {}/*".format(self.rpath), + "/bin/ls -lah {}/*".format(self.rpath), + "/usr/bin/touch {}".format(os.path.join(self.rpath, "TOUCHED")), + "/bin/mkdir -p {}/a/b/c".format(self.rpath), + "/bin/dd if=/dev/urandom of={} bs={} count={}".format( + os.path.join(self.rpath, "DATA"), 1 << 20, random.randint(1, 5) + ), + "/sbin/mknod {}".format(os.path.join(self.rpath, "NODDED")), + "/bin/echo APPENDED >> {}".format(os.path.join(self.rpath, "ECHOED")), + ] + random.shuffle(netbsd_user_emulation) + for entry in [ + "/bin/ln {} {}".format(os.path.join(self.rpath, "DATA"), "HARDLINK"), + "/bin/ln -s {} {}".format(os.path.join(self.rpath, "DATA"), "SOFTLINK"), + "/usr/bin/readlink {}".format(os.path.join(self.rpath, get_random_list_entry(["SOFTLINK", "HARDLINK"]))), + "/bin/chmod {} {}".format(gen_user.get_random_chmod_mode(), os.path.join(self.rpath, "DATA")), + "/bin/rm -rf {}/*".format(self.rpath), + ]: + netbsd_user_emulation.append(entry) + return netbsd_user_emulation + + def set_netbsd_user_emulation(self): + try: + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + res = gen_user.get_files_of_mounted_file_system("files") + if any("Traceback" in pos for pos in res): + user_emulation_command_list = self._failed_trav_user_emul() + else: + user_emulation_command_list = self.prepare_netbsd_user_emulation() + except AttributeError: + user_emulation_command_list = self._failed_trav_user_emul() + return user_emulation_command_list + + def _failed_trav_user_emul(self): + print("\t > Failed to perform a file traversal on mounted file system") + print("\t > Starting backup user emulation that only attempts to write to disk...") + user_emulation_command_list = self.get_netbsd_backup_user_emulation() + return user_emulation_command_list + + @staticmethod + def randomize_emulation_list_length(emu_list, multiplier): + new_emu_list = [] + for i in range(multiplier): + new_emu_list += random.sample(emu_list, random.randint(0, len(emu_list))) + return new_emu_list diff --git a/src/UserEmulation/UE_OpenBSD.py b/src/UserEmulation/UE_OpenBSD.py new file mode 100644 index 0000000..892d0f2 --- /dev/null +++ b/src/UserEmulation/UE_OpenBSD.py @@ -0,0 +1,143 @@ +import os +import random + +from UserEmulation.UE_Generic import GenericUserEmulation + + +def get_random_list_entry(file_list): + return random.choice(list(filter(None, file_list))).rstrip(",").strip() + + +class OpenbsdUserEmulation: + def __init__(self, vm_object, rpath): + self.vm_object = vm_object + self.rpath = rpath + + def prepare_openbsd_user_emulation(self): + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + users, groups = gen_user.get_users_and_groups_of_target_os() + chflags_modes = [ + "arch", + "nodump", + "opaque", + "sappnd", + "schg", + "snapshot", + "sunlnk", + "uappnd", + "uarch", + "uchg", + "hidden", + ] + + openbsd_user_emulation = [ + "/usr/bin/find {}/*".format(self.rpath), + "/bin/ls -lah {}/*".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), + "/usr/bin/touch {}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "TOUCHED",) + ), + "/bin/mkdir -p {}/a/b/c".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), + "/bin/dd if=/dev/urandom of={} bs={} count={}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "DATA",), + 1 << 20, + random.randint(1, 5), + ), + "/bin/ln {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + os.path.join(self.rpath, "HARDLINK"), + ), + "/bin/ln -s {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + os.path.join(self.rpath, "SOFTLINK"), + ), + "/usr/bin/file {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/readlink {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="link"))), + "/usr/bin/stat {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), + "/bin/cp -R {} {}/COPIED", # no + "/sbin/mknod {}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "NODDED",) + ), # no + '/bin/tar -jcvf {} "{} {}"'.format( + os.path.join(self.rpath, "archive.bzip2"), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + ), + "/bin/chmod {} {}".format( + gen_user.get_random_chmod_mode(), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "/sbin/chown {}:{} {}".format( + get_random_list_entry(users), + get_random_list_entry(groups), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "/usr/sbin/chgrp {} {}".format( + get_random_list_entry(groups), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "/bin/mv {} {}", + "/bin/echo APPENDED >> {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file"))), + "/bin/rm -rf {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")[:-1])), + "/usr/bin/chflags {} {}".format( + get_random_list_entry(chflags_modes), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "/usr/bin/split {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), + ), + "/usr/bin/du {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/wc {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/dirname {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/basename {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + ] + random.shuffle(openbsd_user_emulation) + return openbsd_user_emulation + + def get_openbsd_backup_user_emulation(self): + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + + openbsd_user_emulation = [ + "/usr/bin/find {}/*".format(self.rpath), + "/bin/ls -lah {}/*".format(self.rpath), + "/usr/bin/touch {}".format(os.path.join(self.rpath, "TOUCHED")), + "/bin/mkdir -p {}/a/b/c".format(self.rpath), + "/bin/dd if=/dev/urandom of={} bs={} count={}".format( + os.path.join(self.rpath, "DATA"), 1 << 20, random.randint(1, 5) + ), + "/sbin/mknod {}".format(os.path.join(self.rpath, "NODDED")), + "/bin/echo APPENDED >> {}".format(os.path.join(self.rpath, "ECHOED")), + ] + random.shuffle(openbsd_user_emulation) + for entry in [ + "/bin/ln {} {}".format(os.path.join(self.rpath, "DATA"), "HARDLINK"), + "/bin/ln -s {} {}".format(os.path.join(self.rpath, "DATA"), "SOFTLINK"), + "/usr/bin/readlink {}".format(os.path.join(self.rpath, get_random_list_entry(["SOFTLINK", "HARDLINK"]))), + "/bin/chmod {} {}".format(gen_user.get_random_chmod_mode(), os.path.join(self.rpath, "DATA")), + "/bin/rm -rf {}/*".format(self.rpath), + ]: + openbsd_user_emulation.append(entry) + return openbsd_user_emulation + + def set_openbsd_user_emulation(self): + try: + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + res = gen_user.get_files_of_mounted_file_system("files") + if any("Traceback" in pos for pos in res): + user_emulation_command_list = self._failed_trav_user_emul() + else: + user_emulation_command_list = self.prepare_openbsd_user_emulation() + except AttributeError: + user_emulation_command_list = self._failed_trav_user_emul() + return user_emulation_command_list + + def _failed_trav_user_emul(self): + print("\t > Failed to perform a file traversal on mounted file system") + print("\t > Starting backup user emulation that only attempts to write to disk...") + user_emulation_command_list = self.get_openbsd_backup_user_emulation() + return user_emulation_command_list + + @staticmethod + def randomize_emulation_list_length(emu_list, multiplier): + new_emu_list = [] + for i in range(multiplier): + new_emu_list += random.sample(emu_list, random.randint(0, len(emu_list))) + return new_emu_list diff --git a/src/UserEmulation/UE_Ubuntu.py b/src/UserEmulation/UE_Ubuntu.py new file mode 100644 index 0000000..6cbcf67 --- /dev/null +++ b/src/UserEmulation/UE_Ubuntu.py @@ -0,0 +1,136 @@ +import os +import random + +from UserEmulation.UE_Generic import GenericUserEmulation + + +def get_random_list_entry(file_list): + return random.choice(list(filter(None, file_list))).rstrip(",").strip() + + +class UbuntuUserEmulation: + def __init__(self, vm_object, rpath): + self.vm_object = vm_object + self.rapth = rpath + + def prepare_ubuntu_user_emulation(self): + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rapth) + users, groups = gen_user.get_users_and_groups_of_target_os() + # chflags_modes = ['arch', 'nodump', 'opaque', 'sappnd', 'schg', 'snapshot', + # 'sunlnk', 'uappnd', 'uarch', 'uchg', 'hidden'] + + ubuntu_user_emulation = [ + "/usr/bin/find {}/*".format(self.rapth), # yes + "/bin/ls -lah {}/*".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), # yes + "/usr/bin/touch {}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "TOUCHED",) + ), # no + "/bin/mkdir -p {}/a/b/c".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), # no + "/bin/dd if=/dev/urandom of={} bs={} count={}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "DATA",), + 1 << 20, + random.randint(1, 5), + ), # yes + "/bin/ln {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + os.path.join(self.rapth, "HARDLINK"), + ), # no + "/bin/ln -s {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + os.path.join(self.rapth, "SOFTLINK"), + ), # no + "/usr/bin/file {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), # yes + "/bin/readlink {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="link"))), # yes + "/usr/bin/stat {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), # yes + "/bin/cp -R {} {}/COPIED", # no + "/bin/mknod {}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "NODDED",) + ), # no + '/bin/tar -jcvf {} "{} {}"'.format( + os.path.join(self.rapth, "archive.bzip2"), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + ), # yes + "/bin/chmod {} {}".format( + gen_user.get_random_chmod_mode(), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), # no + "/bin/chown {}:{} {}".format( + get_random_list_entry(users), + get_random_list_entry(groups), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), # no + "/bin/chgrp {} {}".format( + get_random_list_entry(groups), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), # no + "/bin/mv {} {}", # no + "/bin/echo APPENDED >> {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")) + ), # no + "chdir {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), # no + "/bin/rm -rf {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")[:-1])), # no + "/usr/bin/getfacl {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), # yes + "/usr/bin/split {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), + ), + "/usr/bin/du {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/wc {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/truncate -s {} {}".format( + random.randint(1, 5), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + ), + "/usr/bin/dirname {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "/usr/bin/basename {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + ] + + random.shuffle(ubuntu_user_emulation) + return ubuntu_user_emulation + + def get_ubuntu_backup_user_emulation(self): + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rapth) + + ubuntu_user_emulation = [ + "/usr/bin/find {}/*".format(self.rapth), + "/bin/ls -lah {}/*".format(self.rapth), + "/usr/bin/touch {}".format(os.path.join(self.rapth, "TOUCHED")), + "/bin/mkdir -p {}/a/b/c".format(self.rapth), + "/bin/dd if=/dev/urandom of={} bs={} count={}".format( + os.path.join(self.rapth, "DATA"), 1 << 20, random.randint(1, 5) + ), + "/bin/mknod {}".format(os.path.join(self.rapth, "NODDED")), + "/bin/echo APPENDED >> {}".format(os.path.join(self.rapth, "ECHOED")), + ] + random.shuffle(ubuntu_user_emulation) + for entry in [ + "/bin/ln {} {}".format(os.path.join(self.rapth, "DATA"), "HARDLINK"), + "/bin/ln -s {} {}".format(os.path.join(self.rapth, "DATA"), "SOFTLINK"), + "/bin/readlink {}".format(os.path.join(self.rapth, get_random_list_entry(["SOFTLINK", "HARDLINK"]))), + "/bin/chmod {} {}".format(gen_user.get_random_chmod_mode(), os.path.join(self.rapth, "DATA")), + "/bin/rm -rf {}/*".format(self.rapth), + ]: + ubuntu_user_emulation.append(entry) + return ubuntu_user_emulation + + def set_ubuntu_user_emulation(self): + try: + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rapth) + res = gen_user.get_files_of_mounted_file_system("files") + if any("Traceback" in pos for pos in res): + user_emulation_command_list = self._failed_trav_user_emul() + else: + user_emulation_command_list = self.prepare_ubuntu_user_emulation() + except AttributeError: + user_emulation_command_list = self._failed_trav_user_emul() + return user_emulation_command_list + + def _failed_trav_user_emul(self): + print("\t > Failed to perform a file traversal on mounted file system") + print("\t > Starting backup user emulation that only attempts to write to disk...") + user_emulation_command_list = self.get_ubuntu_backup_user_emulation() + return user_emulation_command_list + + @staticmethod + def randomize_emulation_list_length(emu_list, multiplier): + new_emu_list = [] + for i in range(multiplier): + new_emu_list += random.sample(emu_list, random.randint(0, len(emu_list))) + return new_emu_list diff --git a/src/UserEmulation/UE_all.py b/src/UserEmulation/UE_all.py new file mode 100644 index 0000000..514b24a --- /dev/null +++ b/src/UserEmulation/UE_all.py @@ -0,0 +1,154 @@ +import os +import random + +from UserEmulation.UE_Generic import GenericUserEmulation + + +def get_random_list_entry(file_list): + return random.choice(list(filter(None, file_list))).rstrip(",").strip() + + +class UserEmulation: + def __init__(self, vm_object, rpath): + self.vm_object = vm_object + self.rpath = rpath + + def prepare_user_emulation(self): + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + users, groups = gen_user.get_users_and_groups_of_target_os() + chflags_modes = [ + "arch", + "nodump", + "opaque", + "sappnd", + "schg", + "snapshot", + "sunlnk", + "uappnd", + "uarch", + "uchg", + "hidden", + ] + + user_emulation = [ + "$(which find) {}/*".format(self.rpath), + "$(which ls) -lah {}/*".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), + "$(which touch) {}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "TOUCHED",) + ), + "$(which mkdir) -p {}/a/b/c".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), + "$(which dd) if=/dev/urandom of={} bs={} count={}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "DATA",), + 1 << 20, + random.randint(1, 5), + ), + "$(which ln) {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + os.path.join(self.rpath, "HARDLINK"), + ), + "$(which ln) -s {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + os.path.join(self.rpath, "SOFTLINK"), + ), + "$(which file) {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "$(which readlink) {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="link"))), + "$(which stat) {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), + "$(which cp) -R {} {}/COPIED", + "$(which mknod) {}".format( + os.path.join(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), "NODDED",) + ), + '$(which tar) -jcvf {} "{} {}"'.format( + os.path.join(self.rpath, "archive.bzip2"), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + ), + "$(which chmod) {} {}".format( + gen_user.get_random_chmod_mode(), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "$(which chown) {}:{} {}".format( + get_random_list_entry(users), + get_random_list_entry(groups), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "$(which chgrp) {} {}".format( + get_random_list_entry(groups), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "$(which mv) {} {}", + "$(which echo) APPENDED >> {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file"))), + "$(which chdir) {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir"))), + "$(which rm) -rf {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")[:-1])), + "$(which chflags) {} {}".format( + get_random_list_entry(chflags_modes), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + ), + "$(which getfacl) {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "$(which split) {} {}".format( + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files")), + get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="dir")), + ), + "$(which du) {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "$(which wc) {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "$(which truncate) -s {} {}".format( + random.randint(1, 5), get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="file")), + ), + "$(which dirname) {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + "$(which basename) {}".format(get_random_list_entry(gen_user.get_files_of_mounted_file_system(param="files"))), + ] + # '$(which setfacl) -m {}:{} {}'.format(get_random_list_entry(users), + # gen_user.get_random_chmod_mode(), + # get_random_list_entry( + # gen_user.get_files_of_mounted_file_system( + # param='files'))) + # ] needs to be explicitly enabled during mount -o acls + random.shuffle(user_emulation) + return user_emulation + + def get_backup_user_emulation(self): + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + + user_emulation = [ + "$(which find) {}/*".format(self.rpath), + "$(which ls) -lah {}/*".format(self.rpath), + "$(touch) {}".format(os.path.join(self.rpath, "TOUCHED")), + "$(mkdir) -p {}/a/b/c".format(self.rpath), + "$(which dd) if=/dev/urandom of={} bs={} count={}".format( + os.path.join(self.rpath, "DATA"), 1 << 20, random.randint(1, 5) + ), + "$(which mknod) {}".format(os.path.join(self.rpath, "NODDED")), + "$(which echo) APPENDED >> {}".format(os.path.join(self.rpath, "ECHOED")), + ] + random.shuffle(user_emulation) + for entry in [ + "$(which ln) {} {}".format(os.path.join(self.rpath, "DATA"), "HARDLINK"), + "$(which ln) -s {} {}".format(os.path.join(self.rpath, "DATA"), "SOFTLINK"), + "$(which readlink) {}".format(os.path.join(self.rpath, get_random_list_entry(["SOFTLINK", "HARDLINK"]))), + "$(which chmod) {} {}".format(gen_user.get_random_chmod_mode(), os.path.join(self.rpath, "DATA")), + "$(which rm) -rf {}/*".format(self.rpath), + ]: + user_emulation.append(entry) + return user_emulation + + def set_user_emulation(self): + try: + gen_user = GenericUserEmulation(vm_object=self.vm_object, remote_mount_path=self.rpath) + res = gen_user.get_files_of_mounted_file_system("files") + if any("Traceback" in pos for pos in res): + ue_cmd_lst = self._failed_trav_user_emul() + else: + ue_cmd_lst = self.prepare_user_emulation() + except AttributeError: + ue_cmd_lst = self._failed_trav_user_emul() + return ue_cmd_lst + + def _failed_trav_user_emul(self): + print("\t > Failed to perform a file traversal on mounted file system") + print("\t > Starting backup user emulation that only attempts to write to disk...\n") + ue_cmd_lst = self.get_backup_user_emulation() + return ue_cmd_lst + + @staticmethod + def rnd_e_lst_len(emu_list, multiplier): + new_emu_list = [] + for i in range(multiplier): + new_emu_list += random.sample(emu_list, random.randint(0, len(emu_list))) + return new_emu_list diff --git a/src/UserEmulation/__init__.py b/src/UserEmulation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/Verifier/Crash_Verifier.py b/src/Verifier/Crash_Verifier.py new file mode 100644 index 0000000..27bb044 --- /dev/null +++ b/src/Verifier/Crash_Verifier.py @@ -0,0 +1,222 @@ +import logging +import os +import pathlib +import re +import sys +import threading +import time +import zipfile +from collections import deque + +THIS_FILE = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(str(pathlib.Path(THIS_FILE).parent)) +sys.path.append(str(pathlib.Path(THIS_FILE))) +from Manager.Manager_FreeBSD import FreeBSD +from Manager.Manager import VmManager +from utility import extract_core_features +from config import fuzzing_config + + +class CrashVerifier(object): + def __init__(self, crash_db, target_os, interval=60): + self.interval = interval + self.crash_fifo = None + self.last_crash_line = 0 + self.path_crash_db = crash_db + self.checked_crashes = [] + self.target = target_os + self.vm_object = None + self.reprod_crash_file = None + self.orig_sha_sum = None + time.sleep(self.interval * 15) + self.init_crash_fifo() + + thread = threading.Thread(target=self.run, name="crash_verifier", args=()) + thread.daemon = True + thread.start() + thread.join() + + @staticmethod + def get_vm_object(): + verifier_vm = VmManager() + verifier_vm.setup(vm_user=fuzzing_config.user, vm_password=fuzzing_config.pw, name="verifier") + verifier_vm.quick_boot(vm_name="verifier") + return verifier_vm + + def run(self): + if len(list(self.crash_fifo)) >= 1: + self.verify() + self.run() + else: + time.sleep(self.interval * 15) + self.init_crash_fifo() + self.run() + + def init_crash_fifo(self): + self.crash_fifo = deque() + crash_db_contents = self.read_crash_db() + self._fill_crash_fifo(crash_db_contents) + + def _fill_crash_fifo(self, crash_db_contents): + for line in crash_db_contents: + if line.split(";")[-3].strip() not in self.checked_crashes: + crash_path = str(line.split("; ")[-3].strip()) + origin_sha_sum = line.split("; ")[-4].strip() + self.crash_fifo.append((crash_path, origin_sha_sum)) + + def verify(self): + next_unverified_crash_tuple = self.crash_fifo.popleft() + next_unverified_crash = next_unverified_crash_tuple[0] + self.orig_sha_sum = next_unverified_crash_tuple[1] + sample = os.path.join(next_unverified_crash, "sample.zip") + syscall_log, sample_file_system = self.get_file_system_and_log_if_present(next_unverified_crash, sample) + command_chain = self.get_command_chain(syscall_log) + if self.target == "freebsd": + self._freebsd_verifier(command_chain, next_unverified_crash, sample_file_system) + self.checked_crashes.append(next_unverified_crash) + self._remove_archive_contents(next_unverified_crash, sample_file_system, syscall_log) + self.vm_object.restore_snapshot(self.vm_object.get_current_snapshot()) + + def _freebsd_verifier(self, command_chain, next_unverified_crash, sample_file_system): + self.vm_object = self.get_vm_object() + self.vm_object.cp_to_guest( + get_files_from=next_unverified_crash, list_of_files_to_copy=sample_file_system, save_files_at="/tmp/", + ) + freebsd_target = FreeBSD( + mount_at=os.path.join("/mnt/", sample_file_system), + rfile=os.path.join("/tmp", sample_file_system), + vm_object=self.vm_object, + ) + if command_chain and freebsd_target.mount_file_system() and self.vm_object.silent_vm_state(): + self._execute_command_chain(command_chain, next_unverified_crash) + elif not command_chain and freebsd_target.mount_file_system() == 2: + with open(os.path.join(next_unverified_crash, "reprod.1"), "w") as f: + f.write("System crashed as expected during mount!\n") + else: + with open(os.path.join(next_unverified_crash, "reprod.0"), "w") as f: + f.write("Command chain was empty. Expected the system to crash during mount!\n") + + def _reset_verifier_vm(self): + self.vm_object.reset_vm() + self.vm_object.new_rshell() + + @staticmethod + def _remove_archive_contents(next_unverified_crash, sample_file_system, syscall_log): + pathlib.Path(syscall_log).unlink() + pathlib.Path(os.path.join(next_unverified_crash, sample_file_system)).unlink() + pathlib.Path(os.path.join(next_unverified_crash, "_".join(x for x in sample_file_system.split("_")[1:]),)).unlink() + + def _execute_command_chain(self, command_chain, next_unverified_crash): + for command in command_chain: + self.vm_object.exec_cmd_quiet(command) + if command != command_chain[-1] and self.vm_object.silent_vm_state(): + continue + elif command != command_chain[-1] and not self.vm_object.silent_vm_state(): + with open(os.path.join(next_unverified_crash, "reprod.2"), "w") as f: + f.write("Command chain mismatch. Manual review necessary!\n") + f.write("Originally crashed at: {}\n".format(command_chain[-1])) + f.write("Now crashed at: {}".format(command)) + self._reset_verifier_vm() + self._fetch_latest_core_file(next_unverified_crash) + elif command == command_chain[-1] and self.vm_object.silent_vm_state(): + with open(os.path.join(next_unverified_crash, "reprod.0"), "w") as f: + f.write("Could not verify crash with loaded command chain: \n{}".format(command_chain)) + elif command == command_chain[-1] and not self.vm_object.silent_vm_state(): + self._reset_verifier_vm() + new_sha_sum = self.get_shasum_of_repro_crash(next_unverified_crash) + if self.orig_sha_sum == new_sha_sum: + with open(os.path.join(next_unverified_crash, "reprod.1"), "w") as f: + f.write("System crashed after executing the same command chain!\n") + f.write("sha256 sums are a match!") + else: + with open(os.path.join(next_unverified_crash, "reprod.2"), "w") as f: + f.write("System crashed after executing the same command chain!\n") + f.write("sha256 sums are a mismatch:\n") + f.write("> Original crash: {}\n".format(self.orig_sha_sum)) + f.write("> Reproduced crash: {}\n".format(new_sha_sum)) + f.write("Manual review necessary!") + + @staticmethod + def get_command_chain(syscall_log): + command_chain = [] + with open(syscall_log, "r") as f: + data = f.readlines() + for line in data: + if line.startswith("[+]") or line.startswith("[!]") and not line.startswith("[!] mount"): + command_chain.append(line.split("] ")[1].strip()) + return command_chain + + def get_file_system_and_log_if_present(self, next_unverified_crash, sample): + if pathlib.Path(sample).is_file(): + with zipfile.ZipFile(sample, "r") as zip_ref: + zip_ref.extractall(next_unverified_crash) + syscall_log, sample_fs = None, None + for file in pathlib.Path(next_unverified_crash).iterdir(): + log_match = re.search(r".*fuzz[0-9]{1,2}_syscall.log", str(file)) + fs_match = re.search(r".*_fuzz[0-9]_[a-zA-Z0-9]+_[0-9]+MB", str(file)) + if log_match: + syscall_log = log_match.group(0) + if fs_match: + sample_fs = fs_match.group(0) + if syscall_log and sample_fs: + return syscall_log, str(pathlib.Path(sample_fs).name) + else: + logging.error("No sample.zip found. Continuing!") + self.verify() + + def read_crash_db(self): + with open(self.path_crash_db, "r") as f: + contents = f.readlines() + return contents + + def _get_latest_core(self): + try: + find_latest_core_file_cmd = ( + '/usr/bin/find /var/crash -name "core*" -print0 | /usr/bin/xargs -0 ls -t | /usr/bin/head -n1' + ) + latest_core_file = self.vm_object.exec_cmd_quiet(find_latest_core_file_cmd) + get_core_file_size_cmd = '/usr/bin/stat {} | /usr/bin/cut -d" " -f8'.format(latest_core_file) + if latest_core_file and int(str(self.vm_object.exec_cmd_quiet(get_core_file_size_cmd)).strip()) > 1: + self.vm_object.exec_cmd_quiet("mv {} /var/crash/core.txt.reprod".format(latest_core_file)) + self.reprod_crash_file = "/var/crash/core.txt.reprod" + else: + return None + except ValueError: + return None + + def _fetch_latest_core_file(self, save_to): + try: + self._get_latest_core() + logging.error("LATEST CORES: {}".format(self.reprod_crash_file)) + if self.reprod_crash_file: + self.vm_object.cp_to_host( + save_files_at=save_to, + get_files_from=str(pathlib.Path(self.reprod_crash_file).parent), + list_of_files_to_copy=str(pathlib.Path(self.reprod_crash_file).name), + ) + self.reprod_crash_file = os.path.join(save_to, str(pathlib.Path(self.reprod_crash_file).name)) + return 1 + else: + logging.error("No new core files found!") + return 0 + except TypeError: + logging.error("No core file(s) found. Continuing!") + return 0 + + def get_shasum_of_repro_crash(self, save_to): + if self._fetch_latest_core_file(save_to): + with open(self.reprod_crash_file, "rb") as f: + data = f.read() + return extract_core_features.get_sha256_sum(extract_core_features.get_core_details(data)) + else: + return "NONE" + + +def main(): + CrashVerifier( + crash_db="/home/dev/git/bsdfuzz/fsfuzz/src/crash_dumps/crash.db", target_os="freebsd", + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/Verifier/__init__.py b/src/Verifier/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/_deprecated/__init__.py b/src/_deprecated/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/_deprecated/makeFS.py b/src/_deprecated/makeFS.py new file mode 100644 index 0000000..120fa3a --- /dev/null +++ b/src/_deprecated/makeFS.py @@ -0,0 +1,940 @@ +import argparse +import json +import logging +import os +import pathlib +import platform +import random +import string +import subprocess +import sys +import uuid + +CHARSET_EASY = string.ascii_letters + string.digits # excluding special characters due to parsing difficulties + +SUPPORTED_FILE_SYSTEMS = { + "freebsd": ["ufs1", "ufs2", "zfs", "ext2", "ext3", "ext4"], + "netbsd": ["4.3bsd", "ufs1", "ufs2", "ext2"], + "openbsd": ["4.3bsd", "ufs1", "ufs2", "ext2"], + "darwin": [], + "linux": ["ext2", "ext3", "ext4", "zfs"], +} + + +def get_os_platform(): + return platform.system().lower() + + +def create_directory(_path): + try: + pathlib.Path(_path).mkdir(parents=True, exist_ok=True) + except OSError as e: + logging.error(e) + + +def get_all_directories_in_path(dir_path): + return [x[0] for x in os.walk(dir_path)] + + +def get_all_files_in_path(dir_path): + list_of_file_paths = [] + for (_dir, _, file_names) in os.walk(dir_path): + list_of_file_paths += [os.path.join(_dir, file) for file in file_names] + for (_dir, dir_names, _) in os.walk(dir_path): + list_of_file_paths += [os.path.join(_dir, d) for d in dir_names] + return list_of_file_paths + + +def get_only_files_in_path(dir_path): + list_of_file_paths = [] + for (_dir, _, file_names) in os.walk(dir_path): + list_of_file_paths += [os.path.join(_dir, file) for file in file_names] + return list_of_file_paths + + +class FileSystemCreator: + def setup(self, **kwargs): + if "file_system_name" in kwargs: + self.file_system_name = kwargs["file_system_name"] + if "file_system_type" in kwargs: + self.file_system_type = kwargs["file_system_type"] + if "file_system_size" in kwargs: + self.file_system_size = kwargs["file_system_size"] + if "generate_amount_of_files" in kwargs: + self.generate_amount_of_files = kwargs["generate_amount_of_files"] + if "max_file_size" in kwargs: + self.max_file_size = kwargs["max_file_size"] + if "mount_at" in kwargs: + self.mount_at = kwargs["mount_at"] + if "save_file_system_at" in kwargs: + self.save_file_system_at = kwargs["save_file_system_at"] + if "mode" in kwargs: + self.mode = kwargs["mode"] + else: + pass + + def __init__(self): + logging.basicConfig(level=logging.DEBUG) + self.file_system_name = None + self.file_system_type = None + self.file_system_size = None + self.generate_amount_of_files = None + self.max_file_size = None + self.mount_at = "/mnt" + self.save_file_system_at = "/tmp/" + self.block_device = None + self.path_to_file_system = None + self.seed = None + self.file_system_logger = {} + self.mode = None + self.class_random_generator = random.Random() # Class bound number generator + self.target_maker = None + + def get_random_string_of_size(self, size, chars=CHARSET_EASY): + self.class_random_generator.seed(self.seed) + generated_string = "".join(self.class_random_generator.choice(chars) for x in range(size)) + return generated_string + + def create_new_path_at_random_location(self, list_of_dirs, ignore_sys_folders=False): + self.class_random_generator.seed(self.seed) + random_index_in_dir_list = self.class_random_generator.randint(0, len(list_of_dirs) - 1) + name_length = self.class_random_generator.randint(1, 255) + if ignore_sys_folders: + if list_of_dirs[random_index_in_dir_list] not in [ + os.path.join(self.mount_at, "lost+found"), + os.path.join(self.mount_at, ".snap"), + ]: + return os.path.join(list_of_dirs[random_index_in_dir_list], self.get_random_string_of_size(size=name_length),) + else: + logging.debug("lost+found or .snap reached, recalling method...") + return self.create_new_path_at_random_location(list_of_dirs) + else: + return os.path.join(list_of_dirs[random_index_in_dir_list], self.get_random_string_of_size(size=name_length),) + + def init_file_system_creation(self): + host_os = get_os_platform() + if not any(x == self.file_system_type for x in SUPPORTED_FILE_SYSTEMS[host_os]): + print("Requested file system not supported on current host os: {}".format(host_os)) + sys.exit(1) + self.create_raw_disk_image() + if host_os == "freebsd": + self.target_maker = FreeBSD( + self.file_system_type, + self.file_system_size, + self.file_system_name, + self.path_to_file_system, + self.mount_at, + self.generate_amount_of_files, + self.max_file_size, + self.mode, + self.save_file_system_at, + ) + elif host_os == "netbsd": + self.target_maker = NetBSD( + self.file_system_type, + self.file_system_size, + self.file_system_name, + self.path_to_file_system, + self.mount_at, + self.generate_amount_of_files, + self.max_file_size, + self.mode, + self.save_file_system_at, + ) + elif host_os == "openbsd": + self.target_maker = OpenBSD( + self.file_system_type, + self.file_system_size, + self.file_system_name, + self.path_to_file_system, + self.mount_at, + self.generate_amount_of_files, + self.max_file_size, + self.mode, + self.save_file_system_at, + ) + elif host_os == "linux": + self.target_maker = Ubuntu( + self.file_system_type, + self.file_system_size, + self.file_system_name, + self.path_to_file_system, + self.mount_at, + self.generate_amount_of_files, + self.max_file_size, + self.mode, + self.save_file_system_at, + ) + self.target_maker.create_file_system() + + def create_raw_disk_image(self): + if not self.file_system_name: + self.file_system_name = "fs_" + str(uuid.uuid4()) + logging.debug(self.file_system_name) + self.path_to_file_system = os.path.join(self.save_file_system_at, self.file_system_name) + logging.debug(self.path_to_file_system) + with open(self.path_to_file_system, "wb") as file: + file.write(b"0" * self.file_system_size) + + def _set_serialize_file_data(self, ctr, full_path): + self.file_system_logger["files"]["seed_{}".format(ctr)]["file_name"] = str(pathlib.Path(full_path).name) + self.file_system_logger["files"]["seed_{}".format(ctr)]["file_path"] = str(pathlib.Path(full_path).parent) + self.file_system_logger["files"]["seed_{}".format(ctr)]["full_path"] = str(full_path) + + def create_new_hardlink_in_current_file_system_structure(self, list_of_files, list_of_only_directories, ctr): + try: + src = self.class_random_generator.choice(list_of_files) + dst = self.create_new_path_at_random_location(list_of_only_directories) + os.link(src, dst) + self.file_system_logger["files"]["seed_{}".format(ctr)]["file_type"] = "HARD_LINK" + self.file_system_logger["files"]["seed_{}".format(ctr)]["source"] = str(src) + self._set_serialize_file_data(ctr, dst) + except OSError: + pass + + def create_new_symlink_in_current_file_system_structure(self, list_of_files, list_of_only_directories, ctr): + try: + src = self.class_random_generator.choice(list_of_files) + dst = self.create_new_path_at_random_location(list_of_only_directories) + os.symlink(src, dst) + self.file_system_logger["files"]["seed_{}".format(ctr)]["file_type"] = "SYM_LINK" + self.file_system_logger["files"]["seed_{}".format(ctr)]["source"] = str(src) + self._set_serialize_file_data(ctr, dst) + except OSError: + pass + + def create_new_file_in_current_file_system_structure(self, location, ctr): + try: + file_size = self.class_random_generator.randrange(0.25 * self.max_file_size, self.max_file_size, 50) + self.file_system_logger["files"]["seed_{}".format(ctr)]["file_type"] = "FILE" + self.file_system_logger["files"]["seed_{}".format(ctr)]["file_size"] = file_size + self._set_serialize_file_data(ctr, location) + with open(location, "wb") as file: + file.write(os.urandom(file_size)) + except OSError: + pass + + def create_new_directory_in_current_file_system_structure(self, dir_path, ctr): + if not os.path.exists(dir_path): + try: + pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True) + self.file_system_logger["files"]["seed_{}".format(ctr)]["file_type"] = "DIR" + self._set_serialize_file_data(ctr, dir_path) + except (OSError, BlockingIOError): + self.create_new_directory_in_current_file_system_structure(dir_path[-3], ctr) + + def _setup_serialize(self): + self.file_system_logger["fs_name"] = self.file_system_name + self.file_system_logger["fs_type"] = self.file_system_type + self.file_system_logger["fs_size (MB)"] = str(int(self.file_system_size) >> 20) + self.file_system_logger["amount_files"] = self.generate_amount_of_files + self.file_system_logger["max_file_size (MB)"] = str(int(self.max_file_size) >> 20) + self.file_system_logger["files"] = {} + self.file_system_logger["files"]["init_files"] = {} + + def _set_seed(self): + if self.mode: + self.seed = self.class_random_generator.getrandbits(random.randint(1, 1024)) + else: + self.seed = None + self.class_random_generator.seed(self.seed) + + def populate_file_system(self): + self._setup_serialize() + try: + self._prepare_file_system_with_dummy_files() + except OSError: + pass + for file_ctr in range(self.generate_amount_of_files): + self._set_seed() + self.file_system_logger["files"]["seed_{}".format(file_ctr)] = {} + self.file_system_logger["files"]["seed_{}".format(file_ctr)]["seed_value"] = self.seed + coin_toss = self.class_random_generator.randint(0, 7) + all_dirs = get_all_directories_in_path(self.mount_at) + if coin_toss in range(0, 4): + self.file_system_logger["files"]["seed_{}".format(file_ctr)]["file_type"] = "FILE" + self.create_new_file_in_current_file_system_structure(self.create_new_path_at_random_location(all_dirs), file_ctr) + if coin_toss in range(4, 6): + self.file_system_logger["files"]["seed_{}".format(file_ctr)]["file_type"] = "DIR" + self.create_new_directory_in_current_file_system_structure( + self.create_new_path_at_random_location(all_dirs), file_ctr + ) + if coin_toss == 6: + self.file_system_logger["files"]["seed_{}".format(file_ctr)]["file_type"] = "SYM_LINK" + all_files = get_all_files_in_path(self.mount_at) + self.create_new_symlink_in_current_file_system_structure(all_files, all_dirs, file_ctr) + if coin_toss == 7: + self.file_system_logger["files"]["seed_{}".format(file_ctr)]["file_type"] = "HARD_LINK" + only_files_no_dirs = get_only_files_in_path(self.mount_at) + self.create_new_hardlink_in_current_file_system_structure(only_files_no_dirs, all_dirs, file_ctr) + print(json.dumps(self.file_system_logger, separators=(",", ":"))) + + def _get_random_name_for_dummy_file(self): + self._set_seed() + name_length = self.class_random_generator.randint(1, 255) + _name = self.get_random_string_of_size(size=name_length) + return _name + + def _prepare_file_system_with_dummy_files(self): + for i, v in list(enumerate(["FILE", "SYM_LINK", "DIR"])): + _name = self._get_random_name_for_dummy_file() + _path = os.path.join(self.mount_at, _name) + if "FILE" in v: + _touch_fn = _name + pathlib.Path(_path).touch() + elif "SYM_LINK" in v: + lnk_path = os.path.join(self.mount_at, _name) + os.symlink(os.path.join(self.mount_at, _touch_fn), lnk_path) + else: + pathlib.Path(_path).mkdir(parents=True, exist_ok=True) + self._set_serialize_init_files_data(_name, _path, i, v) + + def _set_serialize_init_files_data(self, _name, _path, i, v): + self.file_system_logger["files"]["init_files"]["init_{}".format(i)] = {} + self.file_system_logger["files"]["init_files"]["init_{}".format(i)]["seed"] = self.seed + self.file_system_logger["files"]["init_files"]["init_{}".format(i)]["file_type"] = v + self.file_system_logger["files"]["init_files"]["init_{}".format(i)]["name"] = _name + self.file_system_logger["files"]["init_files"]["init_{}".format(i)]["path"] = self.mount_at + self.file_system_logger["files"]["init_files"]["init_{}".format(i)]["full_path"] = _path + if v == "SYM_LINK": + self.file_system_logger["files"]["init_files"]["init_{}".format(i)]["source"] = self.file_system_logger["files"][ + "init_files" + ]["init_0"]["full_path"] + + def parse_arguments(self): + parser = argparse.ArgumentParser() + parser.add_argument("-fs", "--filesystem", type=str, help="ext2, ext3, ext4, ufs, zfs") + parser.add_argument( + "-s", "--size", type=int, default=1024, help="Specify the size in MB of the newly created file system, default: 1MB", + ) + parser.add_argument( + "-n", "--name", type=str, help="custom name you want to give the file system, default: fs_rndstr", + ) + parser.add_argument( + "-o", "--output_dir", type=str, help="Path to store the newly created file system, default: /tmp/", + ) + parser.add_argument( + "-p", "--populate", type=int, help="Number of files/directories that should be created", + ) + parser.add_argument( + "-ps", "--populate_size", type=int, help="Upper file size limit in KB for -p option", + ) + parser.add_argument( + "-mnt", "--mount", type=str, help="path to mount the filesystem for populating it", + ) + parser.add_argument( + "-m", + "--mode", + type=int, + default=1, + help="1 for determinism, or 0 for random (random does not use seeds and does no logging)", + ) + parser.add_argument("-fsls", "--fileSystemLogFile_and_size", action="append", nargs=2) + args = parser.parse_args() + if args.fileSystemLogFile_and_size: + with open(args.fileSystemLogFile_and_size[0][0]) as f: + log_data = json.loads(f.read()) + + shaper = FreeBSDShaper( + log_data=log_data, + fs_name=log_data["fs_name"], + fs_type=log_data["fs_type"], + fs_size=int(args.fileSystemLogFile_and_size[0][1]), + amount_files=int(log_data["amount_files"]), + max_file_size=int(log_data["max_file_size (MB)"]), + path_to_file_system=None, + mode=1, + ) + shaper.init_file_system_creation() + sys.exit(1) + if not args.size or not args.filesystem: + parser.print_help() + sys.exit(1) + if args.size < 64 and args.filesystem == "zfs": + parser.error("ZFS needs at least 64MB of disk size") + sys.exit(1) + elif args.size < 2 and args.filesystem == "ext3": + parser.error("EXT3 needs at least 2MB of disk size") + sys.exit(1) + if (args.populate and not args.populate_size) or (args.populate_size and not args.populate): + parser.error("-p and -ps depend on each other. Set both or neither of them!") + sys.exit(1) + elif args.populate_size and args.populate: + args.populate_size = args.populate_size << 10 # shift bytes into Megabytes + args.size = args.size << 20 + if args.populate and args.populate_size and (args.populate * args.populate_size > args.size): + parser.error("New file system does not hold enough free space to write all requested files!") + sys.exit(1) + if args.output_dir: + create_directory(args.output_dir) + else: + create_directory(self.save_file_system_at) + if args.mount: + create_directory(args.mount) + else: + create_directory(self.mount_at) + if args.mode: + self.mode = 1 + else: + self.mode = 0 + return args + + +####################################################################################################################### +####################################################################################################################### +####################################################################################################################### + + +class Ubuntu(FileSystemCreator): + def __init__( + self, fs_type, fs_size, fs_name, path_to_file_system, mount_at, amount_files, max_file_size, mode, save_file_system_at, + ): + super(Ubuntu, self).__init__() + self.file_system_type = fs_type + self.file_system_size = fs_size + self.file_system_name = fs_name + self.path_to_file_system = path_to_file_system + self.block_device = None + self.mount_at = mount_at + self.generate_amount_of_files = amount_files + self.max_file_size = max_file_size + self.mode = mode + self.save_file_system_at = save_file_system_at + + def create_file_system(self): + self.make_block_device() + self.format_file_system() + if self.generate_amount_of_files and self.max_file_size: + if self.file_system_type != "zfs": + self.mount_at = os.path.join(self.mount_at, self.file_system_name) + FileSystemCreator.mount_at = self.mount_at + logging.info("Mounting...") + self.mount_file_system() + self.populate_file_system() + self.unmount_file_system() + if self.file_system_type == "zfs" and not self.generate_amount_of_files: + self.unmount_zfs() + + def format_file_system(self): + if self.file_system_type in ["ext2", "ext3", "ext4"]: + self.make_ext() + elif self.file_system_type == "zfs": + self.make_zfs() + logging.debug("{} was created successfully".format(self.file_system_name)) + + def make_block_device(self): + cmd_find_loopdev = "losetup -f" + self.block_device = subprocess.check_output(cmd_find_loopdev.split(), encoding="UTF-8").strip() + cmd = "losetup {} {}".format(self.block_device, self.path_to_file_system) + subprocess.check_output(cmd.split(), encoding="UTF-8").strip() + logging.debug("block device {} created".format(self.block_device)) + + def destroy_block_device(self): + cmd_destroy_loopdev = "losetup -d {}".format(self.block_device) + subprocess.call(cmd_destroy_loopdev.split(), stdout=subprocess.DEVNULL) + + def make_ext(self): + cmd = "/sbin/mkfs.{} -v {}".format(self.file_system_type, self.path_to_file_system) + subprocess.call(cmd.split(), stdout=subprocess.DEVNULL) + + def make_zfs(self): + self.file_system_name = "pool_" + self.file_system_name + subprocess.call("zpool create {} {}".format(self.file_system_name, self.block_device).split()) + subprocess.call("zfs set mountpoint=/mnt/{} {}".format(self.file_system_name, self.file_system_name).split()) + subprocess.call("zfs set atime=off {}".format(self.file_system_name).split()) + self.mount_at = os.path.join("/mnt", self.file_system_name) + FileSystemCreator.mount_at = self.mount_at + + def mount_file_system(self): + pathlib.Path(self.mount_at).mkdir(parents=True, exist_ok=True) + cmd_mount = "/bin/mount -t {} {} {}".format(self.file_system_type, self.block_device, self.mount_at) + logging.debug(cmd_mount) + try: + subprocess.call(cmd_mount.split(), stdout=subprocess.DEVNULL) + except subprocess.CalledProcessError: + logging.error("Failed to mount {} during populating phase".format(self.file_system_name)) + self.destroy_block_device() + sys.exit(1) + except RuntimeError as e: + logging.error(e) + self.destroy_block_device() + sys.exit(1) + + def unmount_file_system(self): + if self.file_system_type in ["ext2", "ext3", "ext4"]: + self.unmount_ext_or_ufs() + if self.file_system_type == "zfs": + self.unmount_zfs() + + def unmount_ext_or_ufs(self): + cmd_umnt = "/bin/umount {}".format(self.block_device) + try: + subprocess.call(cmd_umnt.split(), stdout=subprocess.DEVNULL) + except RuntimeError as e: + logging.error(e) + finally: + self.destroy_block_device() + + def unmount_zfs(self): + cmd_export_pool = "zpool export {}".format(self.file_system_name) + try: + subprocess.call(cmd_export_pool.split(), stdout=subprocess.DEVNULL) + self.destroy_block_device() + except RuntimeError as e: + logging.warning(e) + sys.exit(1) + + +####################################################################################################################### +####################################################################################################################### +####################################################################################################################### + + +class FreeBSD(FileSystemCreator): + def __init__( + self, fs_type, fs_size, fs_name, path_to_file_system, mount_at, amount_files, max_file_size, mode, save_file_system_at, + ): + super(FreeBSD, self).__init__() + self.file_system_type = fs_type + self.file_system_size = fs_size + self.file_system_name = fs_name + self.path_to_file_system = path_to_file_system + self.block_device = None + self.mount_at = mount_at + self.generate_amount_of_files = amount_files + self.max_file_size = max_file_size + self.mode = mode + self.save_file_system_at = save_file_system_at + + def create_file_system(self): + self.make_block_device() + self.format_file_system() + if self.generate_amount_of_files and self.max_file_size: + if self.file_system_type != "zfs": + self.mount_at = os.path.join(self.mount_at, self.file_system_name) + FileSystemCreator.mount_at = self.mount_at + logging.info("Mounting...") + self.mount_file_system() + self.populate_file_system() + self.unmount_file_system() + if self.file_system_type == "zfs" and not self.generate_amount_of_files: + self.unmount_zfs() + + def format_file_system(self): + if self.file_system_type in ["ext2", "ext3", "ext4"]: + self.make_ext() + elif "ufs" in self.file_system_type: + self.make_ufs() + elif self.file_system_type == "zfs": + self.make_zfs() + logging.debug("{} was created successfully".format(self.file_system_name)) + + def make_block_device(self): + cmd = "/sbin/mdconfig -a -t vnode -f {}".format(self.path_to_file_system) + _blk_dev = subprocess.check_output(cmd.split(), encoding="UTF-8").strip() + self.block_device = os.path.join("/dev", _blk_dev) + logging.debug("block device {} created".format(self.block_device)) + return self.block_device + + def destroy_block_device(self): + cmd_del_blk_dev = "/sbin/mdconfig -d -u {}".format(self.block_device) + subprocess.call(cmd_del_blk_dev.split(), stdout=subprocess.DEVNULL) + + def make_ufs(self): + if self.file_system_type == "ufs1": + cmd = "/sbin/newfs -O 1 {}".format(self.block_device) + else: + cmd = "/sbin/newfs {}".format(self.block_device) + subprocess.call(cmd.split(), close_fds=True, stdout=subprocess.DEVNULL) + + def make_ext(self): + cmd = "/usr/local/sbin/mkfs.{} -v {}".format(self.file_system_type, self.path_to_file_system) + subprocess.call(cmd.split(), stdout=subprocess.DEVNULL) + + def make_zfs(self): + self.file_system_name = "pool_" + self.file_system_name + subprocess.call("zpool create {} {}".format(self.file_system_name, self.block_device).split()) + subprocess.call("zfs set mountpoint=/mnt/{} {}".format(self.file_system_name, self.file_system_name).split()) + subprocess.call("zfs set atime=off {}".format(self.file_system_name).split()) + self.mount_at = os.path.join("/mnt", self.file_system_name) + FileSystemCreator.mount_at = self.mount_at + + def mount_file_system(self): + mnt_param = "" + if self.file_system_type in ["ext2", "ext3", "ext4"]: + mnt_param = "ext2fs" + elif "ufs" in self.file_system_type: + mnt_param = "ufs" + pathlib.Path(self.mount_at).mkdir(parents=True, exist_ok=True) + cmd_mount = "/sbin/mount -t {} {} {}".format(mnt_param, self.block_device, self.mount_at) + logging.debug(cmd_mount) + try: + subprocess.call(cmd_mount.split(), stdout=subprocess.DEVNULL) + except subprocess.CalledProcessError: + logging.error("Failed to mount {} during populating phase".format(self.file_system_name)) + self.destroy_block_device() + sys.exit(1) + except RuntimeError as e: + logging.error(e) + self.destroy_block_device() + sys.exit(1) + + def unmount_file_system(self): + if self.file_system_type in ["ext2", "ext3", "ext4", "ufs1", "ufs2"]: + self.unmount_ext_or_ufs() + if self.file_system_type == "zfs": + self.unmount_zfs() + + def unmount_ext_or_ufs(self): + cmd_umnt = "/sbin/umount {}".format(self.block_device) + try: + subprocess.call(cmd_umnt.split(), stdout=subprocess.DEVNULL) + except RuntimeError as e: + logging.error(e) + finally: + self.destroy_block_device() + + def unmount_zfs(self): + cmd_export_pool = "zpool export {}".format(self.file_system_name) + try: + subprocess.call(cmd_export_pool.split(), stdout=subprocess.DEVNULL) + self.destroy_block_device() + except RuntimeError as e: + logging.warning(e) + sys.exit(1) + + +####################################################################################################################### +####################################################################################################################### +####################################################################################################################### + + +class NetBSD(FileSystemCreator): + def __init__( + self, fs_type, fs_size, fs_name, path_to_file_system, mount_at, amount_files, max_file_size, mode, save_file_system_at, + ): + super(NetBSD, self).__init__() + self.file_system_type = fs_type + self.file_system_size = fs_size + self.file_system_name = fs_name + self.path_to_file_system = path_to_file_system + self.block_device = None + self.mount_at = mount_at + self.generate_amount_of_files = amount_files + self.max_file_size = max_file_size + self.mode = mode + self.save_file_system_at = save_file_system_at + + def create_file_system(self): + self.make_block_device() + self.format_file_system() + if self.generate_amount_of_files and self.max_file_size: + self.mount_at = os.path.join(self.mount_at, self.file_system_name) + FileSystemCreator.mount_at = self.mount_at + logging.info("Mounting...") + self.mount_file_system() + self.populate_file_system() + self.unmount_file_system() + + def format_file_system(self): + if self.file_system_type == "ext2": + self.make_ext() + elif "ufs" in self.file_system_type or "4.3bsd" in self.file_system_type: + self.make_ufs() + logging.debug("{} was created successfully".format(self.file_system_name)) + + def make_block_device(self): + cmd = "/usr/sbin/vndconfig vnd0 {}".format(self.path_to_file_system) + _ = subprocess.check_output(cmd.split(), encoding="utf-8").strip() + self.block_device = "/dev/vnd0" + cmd_disklabel = "/sbin/disklabel {}".format(self.block_device) + subprocess.call(cmd_disklabel.split(), stdout=subprocess.DEVNULL) + self.block_device = "/dev/rvnd0" + logging.debug("block device {} created".format(self.block_device)) + + def destroy_block_device(self): + cmd_del_blk_dev = "/usr/sbin/vndconfig -u {}".format(self.block_device.split("/")[-1]) + subprocess.call(cmd_del_blk_dev.split(), stdout=subprocess.DEVNULL) + + def make_ufs(self): + if self.file_system_type == "4.3bsd": + cmd = "/sbin/newfs -O 0 {}".format(self.block_device) + elif self.file_system_type == "ufs1": + cmd = "/sbin/newfs -O 1 {}".format(self.block_device) + else: + cmd = "/sbin/newfs -O 2 {}".format(self.block_device) + subprocess.call(cmd.split(), close_fds=True, stdout=subprocess.DEVNULL) + + def make_ext(self): + cmd = "/sbin/newfs_ext2fs {}".format(self.block_device) + subprocess.call(cmd.split(), stdout=subprocess.DEVNULL) + + def mount_file_system(self): + mnt_param = "" + self.block_device = self.block_device.translate({ord(c): None for c in "r"}) + if self.file_system_type == "ext2": + mnt_param = "ext2fs" + elif "ufs" in self.file_system_type or "4.3bsd" in self.file_system_type: + mnt_param = "ufs" + pathlib.Path(self.mount_at).mkdir(parents=True, exist_ok=True) + cmd_mount = "/sbin/mount -t {} {} {}".format(mnt_param, self.block_device, self.mount_at) + logging.debug(cmd_mount) + try: + subprocess.call(cmd_mount.split(), stdout=subprocess.DEVNULL) + except subprocess.CalledProcessError: + logging.error("Failed to mount {} during populating phase".format(self.file_system_name)) + self.destroy_block_device() + sys.exit(1) + except RuntimeError as e: + logging.error(e) + self.destroy_block_device() + sys.exit(1) + + def unmount_file_system(self): + self.unmount_ext_or_ufs() + + def unmount_ext_or_ufs(self): + cmd_umnt = "/sbin/umount {}".format(self.block_device) + try: + subprocess.call(cmd_umnt.split(), stdout=subprocess.DEVNULL) + except RuntimeError as e: + logging.error(e) + finally: + self.destroy_block_device() + + +####################################################################################################################### +####################################################################################################################### +####################################################################################################################### + + +class OpenBSD(FileSystemCreator): + def __init__( + self, fs_type, fs_size, fs_name, path_to_file_system, mount_at, amount_files, max_file_size, mode, save_file_system_at, + ): + super(OpenBSD, self).__init__() + self.file_system_type = fs_type + self.file_system_size = fs_size + self.file_system_name = fs_name + self.path_to_file_system = path_to_file_system + self.block_device = None + self.mount_at = mount_at + self.generate_amount_of_files = amount_files + self.max_file_size = max_file_size + self.mode = mode + self.save_file_system_at = save_file_system_at + + def create_file_system(self): + self.make_block_device() + self.format_file_system() + if self.generate_amount_of_files and self.max_file_size: + self.mount_at = os.path.join(self.mount_at, self.file_system_name) + FileSystemCreator.mount_at = self.mount_at + logging.info("Mounting...") + self.mount_file_system() + self.populate_file_system() + self.unmount_file_system() + + def format_file_system(self): + if self.file_system_type == "ext2": + self.make_ext() + elif "ufs" in self.file_system_type or "4.3bsd" in self.file_system_type: + self.make_ufs() + logging.debug("{} was created successfully".format(self.file_system_name)) + + def make_block_device(self): + cmd = "/sbin/vnconfig vnd0 {}".format(self.path_to_file_system) + _ = subprocess.check_output(cmd.split(), encoding="utf-8").strip() + self.block_device = "/dev/vnd0" + cmd_disklabel = "/sbin/disklabel -A {}".format(self.block_device.split("/")[-1]) + subprocess.call(cmd_disklabel.split(), stdout=subprocess.DEVNULL) + self.block_device = ( + subprocess.check_output(cmd_disklabel.split(), stderr=subprocess.STDOUT, encoding="utf-8").split()[1][:-1].strip() + ) + logging.debug("block device {} created".format(self.block_device)) + + def destroy_block_device(self): + cmd_del_blk_dev = "/sbin/vnconfig -u {}".format(self.block_device.split("/")[-1]) + subprocess.call(cmd_del_blk_dev.split(), stdout=subprocess.DEVNULL) + + def make_ufs(self): + if self.file_system_type == "4.3bsd": + cmd = "/sbin/newfs -O 0 {}".format(self.block_device) + elif self.file_system_type == "ufs1": + cmd = "/sbin/newfs -O 1 {}".format(self.block_device) + else: + cmd = "/sbin/newfs -O 2 {}".format(self.block_device) + subprocess.call(cmd.split(), close_fds=True, stdout=subprocess.DEVNULL) + + def make_ext(self): + cmd = "/sbin/newfs_ext2fs -I {}".format(self.block_device) + subprocess.call(cmd.split(), stdout=subprocess.DEVNULL) + + def mount_file_system(self): + mnt_param = "" + self.block_device = self.block_device.translate({ord(c): None for c in "r"}) + if self.file_system_type == "ext2": + mnt_param = "ext2fs" + elif "ufs" in self.file_system_type or "4.3bsd" in self.file_system_type: + mnt_param = "ffs" + pathlib.Path(self.mount_at).mkdir(parents=True, exist_ok=True) + cmd_mount = "/sbin/mount -t {} {} {}".format(mnt_param, self.block_device, self.mount_at) + logging.debug(cmd_mount) + try: + subprocess.call(cmd_mount.split(), stdout=subprocess.DEVNULL) + except subprocess.CalledProcessError: + logging.error("Failed to mount {} during populating phase".format(self.file_system_name)) + self.destroy_block_device() + sys.exit(1) + except RuntimeError as e: + logging.error(e) + self.destroy_block_device() + sys.exit(1) + + def unmount_file_system(self): + self.unmount_ext_or_ufs() + + def unmount_ext_or_ufs(self): + cmd_umnt = "/sbin/umount {}".format(self.block_device) + try: + subprocess.call(cmd_umnt.split(), stdout=subprocess.DEVNULL) + except RuntimeError as e: + logging.error(e) + finally: + self.destroy_block_device() + + +####################################################################################################################### +####################################################################################################################### +####################################################################################################################### + + +class FreeBSDShaper(FreeBSD): + def __init__( + self, log_data, amount_files, fs_name, fs_size, fs_type, max_file_size, path_to_file_system, mode, + ): + self.data = log_data + self.shaper_random_generator = random.Random() + self.generate_amount_of_files = amount_files + self.file_system_type = fs_type + self.file_system_size = fs_size << 20 + self.path_to_file_system = path_to_file_system + self.max_file_size = max_file_size << 10 + self.file_system_name = "REPLICA_{}MB_OF_".format(int(self.file_system_size) >> 20) + fs_name + self.mount_at = os.path.join("/mnt/", self.file_system_name) + self.mode = mode + super(FreeBSDShaper, self).__init__( + amount_files=self.generate_amount_of_files, + fs_name=self.file_system_name, + fs_size=self.file_system_size, + fs_type=self.file_system_type, + max_file_size=self.max_file_size, + mode=1, + mount_at=self.mount_at, + path_to_file_system=self.path_to_file_system, + ) + + def init_file_system_creation(self): + self.create_raw_disk_image() + if get_os_platform() == "freebsd": + self.create_file_system() + + def create_file_system(self): + self.create_raw_disk_image() + self.make_block_device() + self.format_file_system() + self.populate_file_system() + self.unmount_file_system() + + def populate_file_system(self): + if self.file_system_type != "zfs": + self.mount_file_system() + self._setup_serialize() + self._init_files() + for file_ctr in range(self.generate_amount_of_files): + self.seed = self.data["files"]["seed_{}".format(file_ctr)]["seed_value"] + self.file_system_logger["files"]["seed_{}".format(file_ctr)] = {} + self.file_system_logger["files"]["seed_{}".format(file_ctr)]["seed_value"] = self.seed + self.shaper_random_generator.seed(self.seed) + coin_toss = self.shaper_random_generator.randint(1, 7) + all_dirs = get_all_directories_in_path(self.mount_at) + self._populate_on_coin_toss(all_dirs, coin_toss, file_ctr) + if ( + self.file_system_logger["files"]["seed_{}".format(file_ctr)]["file_name"] + != self.data["files"]["seed_{}".format(file_ctr)]["file_name"] + ): + self._populate_fs_error_logging(file_ctr) + print("[+] Finished populating the replica fs successfully!") + json.dumps(self.file_system_logger) + + def _populate_fs_error_logging(self, f): + logging.error("Name mismatching for {}".format(self.data["files"]["seed_{}".format(f)])) + logging.error("Expected: {}".format(self.data["files"]["seed_{}".format(f)]["file_name"])) + logging.error("Got: {}".format(self.file_system_logger["files"]["seed_{}".format(f)]["file_name"])) + sys.exit(1) + + def _populate_on_coin_toss(self, all_dirs, coin_toss, file_ctr): + if coin_toss in range(1, 4): + self.file_system_logger["files"]["seed_{}".format(file_ctr)]["file_type"] = "FILE" + self.create_new_file_in_current_file_system_structure(self.create_new_path_at_random_location(all_dirs), file_ctr) + if coin_toss in range(4, 6): + self.file_system_logger["files"]["seed_{}".format(file_ctr)]["file_type"] = "DIR" + self.create_new_directory_in_current_file_system_structure( + self.create_new_path_at_random_location(all_dirs), file_ctr + ) + if coin_toss == 6: + self.file_system_logger["files"]["seed_{}".format(file_ctr)]["file_type"] = "SYM_LINK" + all_files = get_all_files_in_path(self.mount_at) + self.create_new_symlink_in_current_file_system_structure(all_files, all_dirs, file_ctr) + if coin_toss == 7: + self.file_system_logger["files"]["seed_{}".format(file_ctr)]["file_type"] = "HARD_LINK" + only_files_no_dirs = get_only_files_in_path(self.mount_at) + self.create_new_hardlink_in_current_file_system_structure(only_files_no_dirs, all_dirs, file_ctr) + + def _init_files(self): + for i, v in list(enumerate(["FILE", "SYM_LINK", "DIR"])): + self.seed = self.data["files"]["init_files"]["init_{}".format(i)]["seed"] + self.shaper_random_generator.seed(self.seed) + name_length = self.shaper_random_generator.randint(1, 255) + _name = self.get_random_string_of_size(size=name_length) + _path = os.path.join(self.mount_at, _name) + if "FILE" in v: + _touch_fn = _name + pathlib.Path(_path).touch() + elif "SYM_LINK" in v: + lnk_path = os.path.join(self.mount_at, _name) + os.symlink(os.path.join(self.mount_at, _touch_fn), lnk_path) + else: + pathlib.Path(_path).mkdir(parents=True, exist_ok=True) + self._set_serialize_init_files_data(_name, _path, i, v) + if _name != self.data["files"]["init_files"]["init_{}".format(i)]["name"]: + self._init_fs_error_logging(i) + + def _init_fs_error_logging(self, i): + logging.error("[!] Name mismatching for {}".format(self.data["files"]["init_files"]["init_{}".format(i)]["file_type"])) + logging.error("Expected: {}".format(self.data["files"]["init_files"]["init_{}".format(i)]["name"])) + logging.error("Got: {}".format(self.file_system_logger["files"]["init_files"]["init_{}".format(i)]["name"])) + sys.exit(1) + + +def main(): + if os.geteuid() != 0: + print("[!] Script needs to be run as root!") + sys.exit(1) + logging.basicConfig(level="ERROR") + file_system_maker = FileSystemCreator() + arguments = file_system_maker.parse_arguments() + file_system_maker.setup( + file_system_name=arguments.name, + file_system_type=str(arguments.filesystem).lower(), + file_system_size=arguments.size, + generate_amount_of_files=arguments.populate, + max_file_size=arguments.populate_size, + mode=arguments.mode, + save_file_system_at=arguments.output_dir, + ) + file_system_maker.init_file_system_creation() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/config/__init__.py b/src/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/config/fuzzing_config.py b/src/config/fuzzing_config.py new file mode 100644 index 0000000..9d89e0f --- /dev/null +++ b/src/config/fuzzing_config.py @@ -0,0 +1,21 @@ +# [fuzzing task specs] +# List of dictionaries specifying each fuzzing instance +fuzzer = [ + { + "name": "fuzz1", # Name for internal bookkeeping + "fs_creator_vm": "genBox", # Name as specified in libvirt for the VM handling the file system generation + "fuzzing_vm": "fuzzBox", # Name as specified in libvirt for the VM handling the file system generation + "mutation_engine": "radamsa, 0", # Mutation Engine that is to be used, and size of mutation + "target_fs": "ufs2", # Target file system + "target_size": 15, # Max file system size in Megabyte + "populate_with_files": 10, # Amount of file that will be generated + "max_file_size": 1024, # Maximum file size in bytes for each generated file + "enable_dyn_scaling": False, # Dynamic scaling will increase the filesystem size periodically + }, +] + +# [credentials] +# Credentials for the root user for the VMs +# It is expected that these are the same across all instances +user = "root" +pw = "root" diff --git a/src/makeFS2.py b/src/makeFS2.py new file mode 100644 index 0000000..e6ce2f1 --- /dev/null +++ b/src/makeFS2.py @@ -0,0 +1,895 @@ +import argparse +import json +import logging +import os +import pathlib +import platform +import random +import string +import subprocess +import sys +import uuid +from shutil import rmtree +from typing import List + +CHARSET_EASY = string.ascii_letters + string.digits # excluding special characters due to parsing difficulties + +SUPPORTED_FILE_SYSTEMS = { + "freebsd": ["ufs1", "ufs2", "zfs", "ext2", "ext3", "ext4"], + "netbsd": ["4.3bsd", "ufs1", "ufs2", "ext2"], + "openbsd": ["4.3bsd", "ufs1", "ufs2", "ext2"], + "linux": ["uf1", "ufs2", "ext2", "ext3", "ext4", "zfs"], + "darwin": ["apfs"], +} + + +def _mk_dir(_path: str): + pathlib.Path(_path).mkdir(parents=True, exist_ok=True) + + +def _get_all_dirs(_path: str): + return [x[0] for x in os.walk(_path)] + + +def _get_all_files(_path: str): + files = [] + for (_dir, _, file_names) in os.walk(_path): + files += [os.path.join(_dir, file) for file in file_names] + for (_dir, dir_names, _) in os.walk(_path): + files += [os.path.join(_dir, d) for d in dir_names] + return files + + +def _get_all_data_files(_path: str): + files = [] + for (_dir, _, file_names) in os.walk(_path): + files += [os.path.join(_dir, file) for file in file_names] + return files + + +def _chk_availability(cmd: str): + return not subprocess.call(["which", f"{cmd}"], stdout=subprocess.DEVNULL) + + +class GenericFilesystemCreator: + def __init__(self): + self.fs_name = None + self.fs_type = None + self.fs_size = None + self.n_files = None + self.max_fsize = None + self.mount_pt = "/mnt" + self.save_pt = "/tmp/" + self.path = None + self.seed = None + self.logger = {} + self.mode = None + self.rng = random.Random() # Class bound number generator + self.host = platform.system().lower() + self.data = None + + def __setup__(self, **kwargs): + if "fs_name" in kwargs: + self.fs_name = kwargs["fs_name"] + if "fs_type" in kwargs: + self.fs_type = kwargs["fs_type"] + if "fs_size" in kwargs: + self.fs_size = kwargs["fs_size"] + if "n_files" in kwargs: + self.n_files = kwargs["n_files"] + if "max_fsize" in kwargs: + self.max_fsize = kwargs["max_fsize"] + if "mount_pt" in kwargs: + self.mount_pt = kwargs["mount_pt"] + if "save_pt" in kwargs: + self.save_pt = kwargs["save_pt"] + if "mode" in kwargs: + self.mode = kwargs["mode"] + if "data" in kwargs: + self.data = kwargs["data"] + + def mk_file_system(self): + self._parse_opts() + if not any(x == self.fs_type for x in SUPPORTED_FILE_SYSTEMS[self.host]): + logging.error(f"Requested file system not supported on current host os: {self.host}") + sys.exit(1) + self._init_mk_fs() + host = self._set_target() + self._create_fs(host) + + def _set_target(self): + target = None + if self.host == "freebsd": + target = FreeBSD + elif self.host == "netbsd": + target = NetBSD + elif self.host == "openbsd": + target = OpenBSD + elif self.host == "linux": + target = Ubuntu + elif self.host == "darwin": + target = Darwin + return target( + fs=self.fs_type, + size=self.fs_size, + name=self.fs_name, + location=self.path, + mount_pt=self.mount_pt, + n_files=self.n_files, + max_fsize=self.max_fsize, + mode=self.mode, + save_pt=self.save_pt, + ) + + def _create_fs(self, target): + target.mk_fs() + if self.n_files and self.max_fsize: + self._logger_setup() + self._mount(target) + self._init_fs_dummy_data() + self._populate_fs() + target.unmount_fs() + rmtree(self.mount_pt) + else: + print(f"Created empty {self.fs_type} disk: {self.path} {self.fs_name}") + if target.fs_type == "zfs": + target.unmount_fs() + + def _mount(self, target): + if self.fs_type != "zfs": + target.mount_pt = os.path.join(self.mount_pt, self.fs_name) + self.mount_pt = target.mount_pt + logging.info("Mounting...") + target.mount_fs() + + @staticmethod + def generic_mount(flag, dev, location): + try: + subprocess.call( + f"/sbin/mount -t {flag} {dev} {location}".split(), stdout=subprocess.DEVNULL, + ) + return 1 + except subprocess.CalledProcessError: + return 0 + + @staticmethod + def _generic_mk_zfs(name, dev): + if not _chk_availability("zpool"): + logging.error("Could not find zfs utils.") + logging.error("Please install the appropriate tooling: e.g.: zfsutils-linux on Debian.") + sys.exit(1) + try: + subprocess.call(f"zpool create {name} {dev}".split()) + subprocess.call(f"zfs set mountpoint=/mnt/{name} {name}".split()) + subprocess.call(f"zfs set atime=off {name}".split()) + return os.path.join("/mnt", name) + except subprocess.CalledProcessError: + logging.error("Failed in genericMakeZFS routine!") + sys.exit(1) + + def _init_mk_fs(self): + if not self.fs_name: + self._set_fs_name() + self._mk_raw_disk() + + def _mk_raw_disk(self): + self.path = os.path.join(self.save_pt, self.fs_name) + pathlib.Path(self.path).write_bytes(b"0" * self.fs_size) + + def _set_fs_name(self): + self.fs_name = "fs_" + str(uuid.uuid4()) + + def _populate_fs(self): + for f_ctr in range(self.n_files): + if self.data: + self.seed = self.data["files"][f"seed_{f_ctr}"]["seed_value"] + self.rng.seed(self.seed) + else: + self._set_seed() + self._set_logger_seed(f_ctr) + coin_toss = self.rng.randint(0, 7) + all_dirs = _get_all_dirs(self.mount_pt) + self._create_files(all_dirs, coin_toss, f_ctr) + self._hierarchy_sanity_check(f_ctr) + print(json.dumps(self.logger, separators=(",", ":"), indent=4)) + + def _hierarchy_sanity_check(self, f_ctr): + if self.data and self.logger["files"][f"seed_{f_ctr}"]["file_name"] != self.data["files"][f"seed_{f_ctr}"]["file_name"]: + self._shpr_hierarchy_verification(f_ctr) + + def _shpr_hierarchy_verification(self, fctr): + print("[!] Error reproducing same data hierarchy!!\n\n") + print(f"During seed {fctr}") + _expected = self.data["files"][f"seed_{fctr}"]["file_name"] + _actual = self.logger["files"][f"seed_{fctr}"]["file_name"] + print(f"Expected: {_expected}") + print(f"Got: {_actual}") + + def _create_files(self, all_dirs, coin_toss, fctr): + if coin_toss in range(0, 4): + self._create_data_file(self._get_new_rndm_file_path(all_dirs), fctr) + if coin_toss in range(4, 6): + self._create_dir(self._get_new_rndm_file_path(all_dirs), fctr) + if coin_toss == 6: + all_files = _get_all_files(self.mount_pt) + self._create_new_link(all_files, all_dirs, fctr, "SYM_LINK") + if coin_toss == 7: + all_data_files = _get_all_data_files(self.mount_pt) + self._create_new_link(all_data_files, all_dirs, fctr, "HARD_LINK") + + def _logger_setup(self): + self.logger["fs_name"] = self.fs_name + self.logger["fs_type"] = self.fs_type + self.logger["save_at"] = self.save_pt + self.logger["fs_size (MB)"] = str(int(self.fs_size) >> 20) + self.logger["amount_files"] = self.n_files + self.logger["max_file_size (MB)"] = str(int(self.max_fsize) >> 20) + self.logger["files"] = {} + self.logger["files"]["init_files"] = {} + + def _get_rndm_str(self, size: int, chars=CHARSET_EASY): + self.rng.seed(self.seed) + generated_string = "".join(self.rng.choice(chars) for x in range(size)) + return generated_string + + def _get_rndm_path_from_lst(self, dirs: List, ignore_system_dirs=False): + self.rng.seed(self.seed) + rndm_idx = self.rng.randint(0, len(dirs) - 1) + if ignore_system_dirs: + if dirs[rndm_idx] not in [ + os.path.join(self.mount_pt, "lost+found"), + os.path.join(self.mount_pt, ".snap"), + ]: + return dirs[rndm_idx] + else: + logging.debug("lost+found or .snap reached, recalling method...") + return self._get_rndm_path_from_lst(dirs) + else: + return dirs[rndm_idx] + + def _get_new_rndm_file_path(self, dirs: List): + self.rng.seed(self.seed) + return os.path.join(self._get_rndm_path_from_lst(dirs), self._get_rndm_fname()) + + def _get_rndm_fname(self): + n_len = self.rng.randint(1, 255) + return self._get_rndm_str(size=n_len) + + def _create_new_link(self, files: List, dirs: List, ctr: int, ftype: str): + try: + src = self.rng.choice(files) + dst = self._get_new_rndm_file_path(dirs) + if ftype == "SYM_LINK": + self._create_symlink(src, dst) + if ftype == "HARD_LINK": + self._create_hardlink(src, dst) + self._set_logger_generic(ctr, dst) + self._set_logger_specific(ctr, ftype=ftype, src=str(src)) + except OSError: + pass + + @staticmethod + def _create_hardlink(src: str, dst: str): + os.link(src, dst) + + @staticmethod + def _create_symlink(src: str, dst: str): + os.symlink(src, dst) + + def _create_data_file(self, location: str, ctr: int): + try: + fsize = self.rng.randrange(0.25 * self.max_fsize, self.max_fsize, 50) + self._set_logger_generic(ctr, location) + self._set_logger_specific(ctr, ftype="FILE", fsize=fsize) + pathlib.Path(location).write_bytes(os.urandom(fsize)) + except OSError: + pass + + def _create_dir(self, dpath: str, ctr: int): + if not os.path.exists(dpath): + try: + _mk_dir(dpath) + self._set_logger_specific(ctr, ftype="DIR") + self._set_logger_generic(ctr, dpath) + except (OSError, BlockingIOError): + self._create_dir(dpath[-3], ctr) + + def _set_logger_generic(self, ctr: int, _path: str): + self.logger["files"][f"seed_{ctr}"]["file_name"] = str(pathlib.Path(_path).name) + self.logger["files"][f"seed_{ctr}"]["file_path"] = str(pathlib.Path(_path).parent) + self.logger["files"][f"seed_{ctr}"]["full_path"] = str(_path) + + def _set_logger_specific(self, ctr: int, ftype=None, src=None, fsize=None): + if ftype: + self.logger["files"][f"seed_{ctr}"]["file_type"] = ftype + if src: + self.logger["files"][f"seed_{ctr}"]["source"] = src + if fsize: + self.logger["files"][f"seed_{ctr}"]["file_size"] = fsize + + def _set_seed(self): + if self.mode: + self.seed = self.rng.getrandbits(random.randint(1, 1024)) + else: + self.seed = None + self.rng.seed(self.seed) + + def _set_logger_seed(self, f_ctr: int): + self.logger["files"][f"seed_{f_ctr}"] = {} + self.logger["files"][f"seed_{f_ctr}"]["seed_value"] = self.seed + + def _init_fs_dummy_data(self): + for i, v in list(enumerate(["FILE", "SYM_LINK", "DIR"])): + if self.data: + self.seed = self.data["files"]["init_files"][f"init_{i}"]["seed"] + self.rng.seed(self.seed) + else: + self._set_seed() + _name = self._get_rndm_fname() + _path = os.path.join(self.mount_pt, _name) + if "FILE" in v: + _touch_fn = _name + pathlib.Path(_path).touch() + elif "SYM_LINK" in v: + lnk_path = os.path.join(self.mount_pt, _name) + os.symlink(os.path.join(self.mount_pt, _touch_fn), lnk_path) + else: + pathlib.Path(_path).mkdir(parents=True, exist_ok=True) + self._set_logger_dummy_data(_name, _path, i, v) + if self.data: + if _name != self.data["files"]["init_files"][f"init_{i}"]["name"]: + self._shpr_dummy_sanity_check(i) + + def _shpr_dummy_sanity_check(self, ctr: int): + _data = self.data["files"]["init_files"][f"init_{ctr}"]["file_type"] + print(f"[!] Name mismatching for {_data}") + _expected = self.data["files"]["init_files"][f"init_{ctr}"]["name"] + _actual = self.logger["files"]["init_files"][f"init_{ctr}"]["name"] + print(f" Expected: {_expected}") + print(f" Got: {_actual}") + sys.exit(1) + + def _set_logger_dummy_data(self, name: str, _path: str, i: int, ftype: str): + self.logger["files"]["init_files"][f"init_{i}"] = {} + self.logger["files"]["init_files"][f"init_{i}"]["seed"] = self.seed + self.logger["files"]["init_files"][f"init_{i}"]["file_type"] = ftype + self.logger["files"]["init_files"][f"init_{i}"]["name"] = name + self.logger["files"]["init_files"][f"init_{i}"]["path"] = self.mount_pt + self.logger["files"]["init_files"][f"init_{i}"]["full_path"] = _path + if ftype == "SYM_LINK": + self.logger["files"]["init_files"][f"init_{i}"]["source"] = self.logger["files"]["init_files"]["init_0"]["full_path"] + + def _parse_opts(self): + log_data = None + parser = argparse.ArgumentParser() + parser.add_argument( + "-fs", "--filesystem", type=str, help="ext2, ext3, ext4, ufs1, ufs2, zfs, apfs", + ) + parser.add_argument( + "-s", + "--size", + type=int, + default=10, + help="Specify the size in MB of the newly created file system, (default: %(default)s)", + ) + parser.add_argument( + "-n", "--name", type=str, help="custom name you want to give the file system", + ) + parser.add_argument( + "-o", + "--output_dir", + type=str, + default="/tmp/", + help="Path to store the newly created file system, (default: %(default)s)", + ) + parser.add_argument( + "-p", "--populate", type=int, help="Number of files/directories that will be created on the fresh file system", + ) + parser.add_argument( + "-ps", "--populate_size", type=int, help="Max file size limit in KB for -p option", + ) + parser.add_argument( + "-mnt", + "--mount", + type=str, + default="/mnt/", + help="path to mount the filesystem for populating it, (default %(default)s)", + ) + parser.add_argument( + "-m", + "--mode", + type=int, + default=1, + help="1 for determinism, or 0 for random (does not use seeds and does no logging), " "(default: %(default)s)", + ) + parser.add_argument( + "-shp", + "--shaper", + action="append", + nargs=2, + help="Requires a valid json log file from the file system creation process and " + "the desired new file system size to reshape the create a new file system " + "with the same layout but of the new size!", + ) + args = parser.parse_args() + if args.shaper: + log_data = json.loads(pathlib.Path(args.shaper[0][0]).read_text()) + args.name = f"SHP_{args.shaper[0][1]}__" + log_data["fs_name"] + args.filesystem = str(log_data["fs_type"]) + args.size = int(args.shaper[0][1]) + args.populate = int(log_data["amount_files"]) + args.populate_size = int(log_data["max_file_size (MB)"]) << 10 + args.mode = 1 + args.output_dir = str(log_data["save_at"]) + if not args.size or not args.filesystem: + parser.print_help() + sys.exit(1) + if args.size < 64 and args.filesystem == "zfs": + parser.error("ZFS needs at least 64MB of disk size") + sys.exit(1) + elif args.size < 2 and args.filesystem == "ext3": + parser.error("EXT3 needs at least 2MB of disk size") + sys.exit(1) + if (args.populate and not args.populate_size) or (args.populate_size and not args.populate): + parser.error("-p and -ps depend on each other. Set both or neither of them!") + sys.exit(1) + elif args.populate_size and args.populate: + args.populate_size = args.populate_size << 10 # shift bytes into Megabytes + args.size = args.size << 20 + if args.populate and args.populate_size and (args.populate * args.populate_size > args.size): + parser.error("New file system does not hold enough free space to write all requested files!") + sys.exit(1) + if args.output_dir: + _mk_dir(args.output_dir) + else: + _mk_dir(self.save_pt) + if args.mount: + _mk_dir(args.mount) + else: + _mk_dir(self.mount_pt) + if args.mode: + self.mode = 1 + else: + self.mode = 0 + + self.__setup__( + fs_name=args.name, + fs_type=str(args.filesystem).lower(), + fs_size=args.size, + n_files=args.populate, + max_fsize=args.populate_size, + mode=args.mode, + save_pt=args.output_dir, + data=log_data, + ) + + +####################################################################################################################### +# DARTWIN SPECIFIC FILE SYSTEM CREATION STEPS # +####################################################################################################################### + + +class Darwin(GenericFilesystemCreator): + def __init__(self, fs, size, name, location, mount_pt, n_files, max_fsize, mode, save_pt): + super(Darwin, self).__init__() + self.fs_type = fs + self.fs_size = size + self.fs_name = name + self.path = location + self.dev = None + self.mount_pt = mount_pt + self.n_files = n_files + self.max_fsize = max_fsize + self.mode = mode + self.save_pt = save_pt + + def _attach_disk(self): + hdiutil_out = subprocess.check_output( + f"/usr/bin/hdiutil attach -imagekey diskimage-class=CRawDiskImage -nomount {self.path}".split(), encoding="utf-8", + ).strip() + self.dev = hdiutil_out.split()[-2].strip() # needs better sanity checks + logging.debug(f"block device {self.dev} created") + return self.dev + + def _detach_disk(self): + subprocess.call(f"/usr/bin/hdiutil detach {self.dev}".split(), stdout=subprocess.DEVNULL) + + def mk_fs(self): + if self.fs_type == "apfs": + self._mk_apfs() + self._attach_disk() + logging.debug(f"{self.fs_name} was created successfully") + + def _mk_apfs(self): + subprocess.call(f"/sbin/newfs_{self.fs_type} -v {self.fs_name} {self.path}".split()) + + def mount_fs(self): + _mk_dir(self.mount_pt) + try: + subprocess.call( + f"/sbin/mount_{self.fs_type} {self.dev} {self.mount_pt}".split(), stdout=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError: + logging.error(f"Failed to mount {self.fs_name} during populating phase") + sys.exit(1) + except RuntimeError as e: + logging.error(e) + sys.exit(1) + finally: + self._detach_disk() + + def unmount_fs(self): + if self.fs_type == "apfs": + self._unmount_apfs() + + def _unmount_apfs(self): + try: + subprocess.call(f"/sbin/umount {self.dev}".split(), stdout=subprocess.DEVNULL) + except RuntimeError as e: + logging.error(e) + finally: + self._detach_disk() + + +####################################################################################################################### +# UBUNTU SPECIFIC FILE SYSTEM CREATION STEPS # +####################################################################################################################### + + +class Ubuntu(GenericFilesystemCreator): + def __init__(self, fs, size, name, location, mount_pt, n_files, max_fsize, mode, save_pt): + super(Ubuntu, self).__init__() + self.fs_type = fs + self.fs_size = size + self.fs_name = name + self.path = location + self.dev = None + self.mount_pt = mount_pt + self.n_files = n_files + self.max_fsize = max_fsize + self.mode = mode + self.save_pt = save_pt + + def _mk_blk_dev(self): + self.dev = subprocess.check_output("losetup -f".split(), encoding="utf-8").strip() + subprocess.check_output(f"losetup {self.dev} {self.path}".split(), encoding="utf-8").strip() + logging.debug(f"block device {self.dev} created") + return self.dev + + def _unmk_blk_dev(self): + subprocess.call(f"losetup -d {self.dev}".split(), stdout=subprocess.DEVNULL) + + def mk_fs(self): + self._mk_blk_dev() + if self.fs_type in ["ufs1", "ufs2"]: + self._mk_ufs() + if self.fs_type in ["ext2", "ext3", "ext4"]: + self._mk_ext() + if self.fs_type == "zfs": + self._mk_zfs() + logging.debug(f"{self.fs_name} was created successfully") + + def _mk_ufs(self): + if not _chk_availability("mkfs.ufs"): + logging.error("Could not find mkfs.ufs") + logging.error( + "Please install legacy package from:" + "\thttps://mirrors.mediatemple.net/debian-archive/debian/pool/main/u/ufsutils/ufsutils_8.2-3_amd64.deb" + ) + sys.exit(1) + if self.fs_type == "ufs1": + flag = 1 + else: + flag = 2 + # -b and -f flags ensure the same default result compared to FreeBSD + cmd = f"/sbin/mkfs.ufs -O {flag} -b 32768 -f 4096 {self.dev}" + subprocess.call(cmd.split(), close_fds=True, stdout=subprocess.DEVNULL) + print( + f"[*] The Ubuntu kernel has by default no write permissions for UFS.\n\tEmpty file system '{self.fs_name}' created." + ) + sys.exit(0) + + def _mk_ext(self): + subprocess.call( + f"/sbin/mkfs.{self.fs_type} -v {self.path}".split(), stdout=subprocess.DEVNULL, + ) + + def _mk_zfs(self): + self.fs_name = "pool_" + self.fs_name + GenericFilesystemCreator.mountAt = GenericFilesystemCreator._generic_mk_zfs(self.fs_name, self.dev) + + def mount_fs(self): + _mk_dir(self.mount_pt) + try: + subprocess.call( + f"/bin/mount -t {self.fs_type} {self.dev} {self.mount_pt}".split(), stdout=subprocess.DEVNULL, + ) + except subprocess.CalledProcessError: + logging.error(f"Failed to mount {self.fs_name} during populating phase") + self._unmk_blk_dev() + sys.exit(1) + except RuntimeError as e: + logging.error(e) + self._unmk_blk_dev() + sys.exit(1) + + def unmount_fs(self): + if self.fs_type in ["ext2", "ext3", "ext4"]: + self._unmount_ext() + if self.fs_type == "zfs": + self._unmount_zfs() + + def _unmount_ext(self): + try: + subprocess.call(f"/bin/umount {self.dev}".split(), stdout=subprocess.DEVNULL) + except RuntimeError as e: + logging.error(e) + finally: + self._unmk_blk_dev() + + def _unmount_zfs(self): + cmd_export_pool = "zpool export {}".format(self.fs_name) + try: + subprocess.call(cmd_export_pool.split(), stdout=subprocess.DEVNULL) + self._unmk_blk_dev() + except RuntimeError as e: + logging.warning(e) + sys.exit(1) + + +####################################################################################################################### +# FreeBSD SPECIFIC FILE SYSTEM CREATION STEPS # +####################################################################################################################### + + +class FreeBSD(GenericFilesystemCreator): + def __init__( + self, fs, size, name, location, mount_pt, n_files, max_fsize, mode, save_pt, log_data=None, + ): + super(FreeBSD, self).__init__() + self.fs_type = fs + self.fs_size = size + self.fs_name = name + self.path = location + self.dev = None + self.mount_pt = mount_pt + self.n_files = n_files + self.max_fsize = max_fsize + self.mode = mode + self.save_pt = save_pt + self.data = log_data + + def _mk_blk_dev(self): + dev = subprocess.check_output(f"/sbin/mdconfig -a -t vnode -f {self.path}".split(), encoding="utf-8").strip() + self.dev = os.path.join("/dev", dev) + logging.debug(f"block device {self.dev} created") + return self.dev + + def _unmk_blk_dev(self): + subprocess.call(f"/sbin/mdconfig -d -u {self.dev}".split(), stdout=subprocess.DEVNULL) + + def mk_fs(self): + self._mk_blk_dev() + if self.fs_type in ["ext2", "ext3", "ext4"]: + self._mk_ext() + if self.fs_type == "zfs": + self._mk_zfs() + if self.fs_type in ["4.3bsd", "ufs1", "ufs2"]: + self._mk_ufs() + logging.debug(f"{self.fs_name} was created successfully") + + def _mk_ufs(self): + if self.fs_type == "ufs1": + cmd = f"/sbin/newfs -O 1 {self.dev}" + else: + cmd = f"/sbin/newfs {self.dev}" + subprocess.call(cmd.split(), close_fds=True, stdout=subprocess.DEVNULL) + + def _mk_ext(self): + subprocess.call( + f"/usr/local/sbin/mkfs.{self.fs_type} -v {self.path}".split(), stdout=subprocess.DEVNULL, + ) + + def _mk_zfs(self): + self.fs_name = "pool_" + self.fs_name + GenericFilesystemCreator.mountAt = GenericFilesystemCreator._generic_mk_zfs(self.fs_name, self.dev) + + def mount_fs(self): + _mk_dir(self.mount_pt) + flag = "" + if self.fs_type in ["ext2", "ext3", "ext4"]: + flag = "ext2fs" + elif "ufs" in self.fs_type: + flag = "ufs" + if not GenericFilesystemCreator.generic_mount(flag, self.dev, self.mount_pt): + self._unmk_blk_dev() + logging.error(f"Failed to mount {self.fs_name} during populating phase") + sys.exit(1) + + def unmount_fs(self): + if self.fs_type in ["ext2", "ext3", "ext4", "ufs1", "ufs2"]: + self._unmount_ext_ufs() + if self.fs_type == "zfs": + self._unmount_zfs() + + def _unmount_ext_ufs(self): + try: + subprocess.call(f"/sbin/umount {self.dev}".split(), stdout=subprocess.DEVNULL) + except RuntimeError as e: + logging.error(e) + finally: + self._unmk_blk_dev() + + def _unmount_zfs(self): + cmd_export_pool = "zpool export {}".format(self.fs_name) + try: + subprocess.call(cmd_export_pool.split(), stdout=subprocess.DEVNULL) + self._unmk_blk_dev() + except RuntimeError as e: + logging.warning(e) + sys.exit(1) + + +####################################################################################################################### +# OpenBSD SPECIFIC FILE SYSTEM CREATION STEPS # +####################################################################################################################### + + +class OpenBSD: + def __init__(self, fs, size, name, location, mount_pt, n_files, max_fsize, mode, save_pt): + super(OpenBSD, self).__init__() + self.fs_type = fs + self.fs_size = size + self.fs_name = name + self.path = location + self.dev = None + self.mount_pt = mount_pt + self.n_files = n_files + self.max_fsize = max_fsize + self.mode = mode + self.save_pt = save_pt + + def _mk_blk_dev(self): + subprocess.check_output(f"/sbin/vnconfig vnd0 {self.path}".split(), stderr=subprocess.STDOUT, encoding="utf-8",).strip() + self.dev = ( + subprocess.check_output("/sbin/disklabel -A vnd0", stderr=subprocess.STDOUT, encoding="utf-8").split()[1][:-1].strip() + ) + logging.debug(f"block device {self.dev} created") + return self.dev + + def _unmk_blk_dev(self): + subprocess.call( + f'/sbin/vnconfig -u {self.dev.split("/")[-1]}'.split(), stdout=subprocess.DEVNULL, + ) + + def mk_fs(self): + self._mk_blk_dev() + if self.fs_type == "ext2": + self._mk_ext() + if self.fs_type in ["4.3bsd", "ufs1", "ufs2"]: + self._mk_ufs() + logging.debug(f"{self.fs_name} was created successfully") + + def _mk_ufs(self): + if self.fs_type == "4.3bsd": + cmd = f"/sbin/newfs -O 0 {self.dev}" + elif self.fs_type == "ufs1": + cmd = f"/sbin/newfs -O 1 {self.dev}" + else: + cmd = f"/sbin/newfs -O 2 {self.dev}" + subprocess.call(cmd.split(), stdout=subprocess.DEVNULL) + + def _mk_ext(self): + subprocess.call(f"/sbin/newfs_ext2fs -I {self.dev}".split(), stdout=subprocess.DEVNULL) + + def mount_fs(self): + _mk_dir(self.mount_pt) + flag = "" + if self.fs_type == "ext2": + flag = "ext2fs" + if self.fs_type in ["ufs", "4.3bsd"]: + flag = "ffs" + if not GenericFilesystemCreator.generic_mount(flag, self.dev, self.mount_pt): + self._unmk_blk_dev() + logging.error(f"Failed to mount {self.fs_name} during populating phase") + sys.exit(1) + + def unmount_fs(self): + self._unmount_ext_ufs() + + def _unmount_ext_ufs(self): + try: + subprocess.call(f"/bin/umount {self.dev}".split(), stdout=subprocess.DEVNULL) + except RuntimeError as e: + logging.error(e) + finally: + self._unmk_blk_dev() + + +####################################################################################################################### +# NetBSD SPECIFIC FILE SYSTEM CREATION STEPS # +####################################################################################################################### + + +class NetBSD: + def __init__(self, fs, size, name, location, mount_pt, n_files, max_fsize, mode, save_pt): + super(NetBSD, self).__init__() + self.fs_type = fs + self.fs_size = size + self.fs_name = name + self.path = location + self.dev = None + self.mount_pt = mount_pt + self.n_files = n_files + self.max_fsize = max_fsize + self.mode = mode + self.save_pt = save_pt + + def _mk_blk_dev(self): + _ = subprocess.check_output(f"/usr/sbin/vndconfig vnd0 {self.path}".split(), encoding="utf-8").strip() + self.dev = "/dev/vnd0" + subprocess.call(f"/sbin/disklabel {self.dev}".split(), stdout=subprocess.DEVNULL) + self.dev = "/dev/rvnd0" + logging.debug(f"block device {self.dev} created") + return self.dev + + def _unmk_blk_dev(self): + subprocess.call( + f'/usr/sbin/vndconfig -u {self.dev.split("/")[-1]}'.split(), stdout=subprocess.DEVNULL, + ) + + def mk_fs(self): + self._mk_blk_dev() + if self.fs_type in ["ext2", "ext3", "ext4"]: + self._mk_ext() + if self.fs_type in ["4.3bsd", "ufs1", "ufs2"]: + self._mk_ufs() + logging.debug(f"{self.fs_name} was created successfully") + + def _mk_ufs(self): + if self.fs_type == "4.3bsd": + cmd = f"/sbin/newfs -O 0 {self.dev}" + elif self.fs_type == "ufs1": + cmd = f"/sbin/newfs -O 1 {self.dev}" + else: + cmd = f"/sbin/newfs -O 2 {self.dev}" + subprocess.call(cmd.split(), stdout=subprocess.DEVNULL) + + def _mk_ext(self): + subprocess.call(f"/sbin/newfs_ext2fs {self.dev}".split(), stdout=subprocess.DEVNULL) + + def mount_fs(self): + _mk_dir(self.mount_pt) + flag = "" + self.dev = self.dev.translate({ord(c): None for c in "r"}) + if self.fs_type == "ext2": + flag = "ext2fs" + if self.fs_type in ["ufs", "4.3bsd"]: + flag = "ufs" + if not GenericFilesystemCreator.generic_mount(flag, self.dev, self.mount_pt): + self._unmk_blk_dev() + logging.error(f"Failed to mount {self.fs_name} during populating phase") + sys.exit(1) + + def unmount_fs(self): + self._unmount_ext_ufs() + + def _unmount_ext_ufs(self): + try: + subprocess.call(f"/bin/umount {self.dev}".split(), stdout=subprocess.DEVNULL) + except RuntimeError as e: + logging.error(e) + finally: + self._unmk_blk_dev() + + +def main(): + if os.geteuid() != 0: + print("[!] Script needs to be run as root!") + sys.exit(1) + logging.basicConfig(level="ERROR") + return GenericFilesystemCreator().mk_file_system() + + +if __name__ == "__main__": + main() diff --git a/src/run.py b/src/run.py new file mode 100644 index 0000000..7c0f968 --- /dev/null +++ b/src/run.py @@ -0,0 +1,54 @@ +import json +import logging +import subprocess +import sys + +from config import fuzzing_config + + +def build_tmux_session(): + subprocess.Popen( + "tmux new -d -s fsfuzzer", shell=True, start_new_session=True, stdout=subprocess.DEVNULL, + ) + + +def build_new_tmux_window(): + subprocess.call("tmux new-window -t fsfuzzer", shell=True) + + +def to_dict(input_ordered_dict): + return json.loads(json.dumps(input_ordered_dict)) + + +def kill_tmux(): + subprocess.call("tmux kill-session -t fsfuzzer", shell=True, stdout=subprocess.PIPE) + + +def main(): + build_tmux_session() + + for i in range(len(fuzzing_config.fuzzer)): + build_new_tmux_window() + cmd = "python3 Fuzzer/Fuzzer.py {} {} {} '{}' {} {} {} {} {}".format( + fuzzing_config.fuzzer[i]["name"], + fuzzing_config.fuzzer[i]["fs_creator_vm"], + fuzzing_config.fuzzer[i]["fuzzing_vm"], + fuzzing_config.fuzzer[i]["mutation_engine"], + fuzzing_config.fuzzer[i]["target_fs"], + fuzzing_config.fuzzer[i]["target_size"], + fuzzing_config.fuzzer[i]["populate_with_files"], + fuzzing_config.fuzzer[i]["max_file_size"], + fuzzing_config.fuzzer[i]["enable_dyn_scaling"], + ) + print(cmd) + fuzz_task = subprocess.Popen('tmux send-keys -t fsfuzzer "{}" C-m'.format(cmd), shell=True, stdout=subprocess.PIPE) + if fuzz_task.poll() is not None: + logging.debug("Failed to spawn at least one subprocess. Aborting!") + kill_tmux() + sys.exit(1) + print("\x1b[6;30;42m" + 'Attach to tmux session via "{}"!'.format("tmux attach-session -t fsfuzzer") + "\x1b[0m") + print("\x1b[6;30;42m" + "Ctrl+C to kill tmux session!" + "\x1b[0m") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/scripts/fuzzer.sh b/src/scripts/fuzzer.sh new file mode 100644 index 0000000..7a4768d --- /dev/null +++ b/src/scripts/fuzzer.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env sh +# Some stuff to setup for the FreeBSD fuzzer instances.. + +pkg install -y vim python3 e2fsprogs p7zip gdb wget curl +kldload ext2fs +sed -i -e 's/#PermitRootLogin no/PermitRootLogin yes/g' /etc/ssh/sshd_config +sed -i -e 's/#PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config + +/etc/rc.d/sshd restart + +echo "kern.panic_reboot_wait_time=-1" >> /etc/sysctl.conf # Doesnt reboot on crash +# echo "kern.panic_reboot_wait_time=0" >> /etc/sysctl.conf # immediate reboot on crash +echo "vm.redzone.panic=1" >> /etc/sysctl.conf \ No newline at end of file diff --git a/src/scripts/kern.diff b/src/scripts/kern.diff new file mode 100644 index 0000000..3098f35 --- /dev/null +++ b/src/scripts/kern.diff @@ -0,0 +1,204 @@ +1,19d0 +< # +< # GENERIC -- Generic kernel configuration file for FreeBSD/amd64 +< # +< # For more information on this file, please read the config(5) manual page, +< # and/or the handbook section on Kernel Configuration Files: +< # +< # https://www.FreeBSD.org/doc/en_US.ISO8859-1/books/handbook/kernelconfig-config.html +< # +< # The handbook is also available locally in /usr/share/doc/handbook +< # if you've installed the doc distribution, otherwise always see the +< # FreeBSD World Wide Web server (https://www.FreeBSD.org/) for the +< # latest information. +< # +< # An exhaustive list of options and more detailed explanations of the +< # device lines is also present in the ../../conf/NOTES and NOTES files. +< # If you are in doubt as to the purpose or necessity of a line, check first +< # in NOTES. +< # +< # $FreeBSD: releng/12.1/sys/amd64/conf/GENERIC 346179 2019-04-13 08:52:52Z dim $ +90a72,81 +> # ADDED_START +> options DDB +> options KDB_UNATTENDED +> # options DDB_NUMSYM +> options INVARIANTS +> options INVARIANT_SUPPORT +> options DIAGNOSTIC +> options DEBUG_REDZONE +> #options PANIC_REBOOT_WAIT_TIME=0 +> # ADDED_END +91a83 +> +112,114d103 +< # Floppy drives +< device fdc +< +121,136d109 +< # SCSI Controllers +< device ahc # AHA2940 and onboard AIC7xxx devices +< device ahd # AHA39320/29320 and onboard AIC79xx devices +< device esp # AMD Am53C974 (Tekram DC-390(T)) +< device hptiop # Highpoint RocketRaid 3xxx series +< device isp # Qlogic family +< #device ispfw # Firmware for QLogic HBAs- normally a module +< device mpt # LSI-Logic MPT-Fusion +< device mps # LSI-Logic MPT-Fusion 2 +< device mpr # LSI-Logic MPT-Fusion 3 +< #device ncr # NCR/Symbios Logic +< device sym # NCR/Symbios Logic (newer chipsets + those of `ncr') +< device trm # Tekram DC395U/UW/F DC315U adapters +< device isci # Intel C600 SAS controller +< device ocs_fc # Emulex FC adapters +< +147,175d119 +< # RAID controllers interfaced to the SCSI subsystem +< device amr # AMI MegaRAID +< device arcmsr # Areca SATA II RAID +< device ciss # Compaq Smart RAID 5* +< device dpt # DPT Smartcache III, IV - See NOTES for options +< device hptmv # Highpoint RocketRAID 182x +< device hptnr # Highpoint DC7280, R750 +< device hptrr # Highpoint RocketRAID 17xx, 22xx, 23xx, 25xx +< device hpt27xx # Highpoint RocketRAID 27xx +< device iir # Intel Integrated RAID +< device ips # IBM (Adaptec) ServeRAID +< device mly # Mylex AcceleRAID/eXtremeRAID +< device twa # 3ware 9000 series PATA/SATA RAID +< device smartpqi # Microsemi smartpqi driver +< device tws # LSI 3ware 9750 SATA+SAS 6Gb/s RAID controller +< +< # RAID controllers +< device aac # Adaptec FSA RAID +< device aacp # SCSI passthrough for aac (requires CAM) +< device aacraid # Adaptec by PMC RAID +< device ida # Compaq Smart RAID +< device mfi # LSI MegaRAID SAS +< device mlx # Mylex DAC960 family +< device mrsas # LSI/Avago MegaRAID SAS/SATA, 6Gb/s and 12Gb/s +< device pmspcv # PMC-Sierra SAS/SATA Controller driver +< #XXX pointer/int warnings +< #device pst # Promise Supertrak SX6000 +< device twe # 3ware ATA RAID +< +180,186d123 +< # atkbdc0 controls both the keyboard and the PS/2 mouse +< device atkbdc # AT keyboard controller +< device atkbd # AT keyboard +< device psm # PS/2 mouse +< +< device kbdmux # keyboard multiplexer +< +203,211d139 +< # PCCARD (PCMCIA) support +< # PCMCIA and cardbus bridge support +< device cbb # cardbus (yenta) bridge +< device pccard # PC Card (16-bit) bus +< device cardbus # CardBus (32-bit) bus +< +< # Serial (COM) ports +< device uart # Generic UART driver +< +215d142 +< device lpt # Printer +217d143 +< #device vpo # Requires scbus and da +219d144 +< device puc # Multi I/O cards and multi-channel UARTs +224,228d148 +< device ix # Intel PRO/10GbE PCIE PF Ethernet +< device ixv # Intel PRO/10GbE PCIE VF Ethernet +< device ixl # Intel 700 Series Physical Function +< device iavf # Intel Adaptive Virtual Function +< device vmx # VMware VMXNET3 Ethernet +230,302d149 +< # PCI Ethernet NICs. +< device bxe # Broadcom NetXtreme II BCM5771X/BCM578XX 10GbE +< device de # DEC/Intel DC21x4x (``Tulip'') +< device le # AMD Am7900 LANCE and Am79C9xx PCnet +< device ti # Alteon Networks Tigon I/II gigabit Ethernet +< device txp # 3Com 3cR990 (``Typhoon'') +< device vx # 3Com 3c590, 3c595 (``Vortex'') +< +< # PCI Ethernet NICs that use the common MII bus controller code. +< # NOTE: Be sure to keep the 'device miibus' line in order to use these NICs! +< device miibus # MII bus support +< device ae # Attansic/Atheros L2 FastEthernet +< device age # Attansic/Atheros L1 Gigabit Ethernet +< device alc # Atheros AR8131/AR8132 Ethernet +< device ale # Atheros AR8121/AR8113/AR8114 Ethernet +< device bce # Broadcom BCM5706/BCM5708 Gigabit Ethernet +< device bfe # Broadcom BCM440x 10/100 Ethernet +< device bge # Broadcom BCM570xx Gigabit Ethernet +< device cas # Sun Cassini/Cassini+ and NS DP83065 Saturn +< device dc # DEC/Intel 21143 and various workalikes +< device et # Agere ET1310 10/100/Gigabit Ethernet +< device fxp # Intel EtherExpress PRO/100B (82557, 82558) +< device gem # Sun GEM/Sun ERI/Apple GMAC +< device hme # Sun HME (Happy Meal Ethernet) +< device jme # JMicron JMC250 Gigabit/JMC260 Fast Ethernet +< device lge # Level 1 LXT1001 gigabit Ethernet +< device msk # Marvell/SysKonnect Yukon II Gigabit Ethernet +< device nfe # nVidia nForce MCP on-board Ethernet +< device nge # NatSemi DP83820 gigabit Ethernet +< device pcn # AMD Am79C97x PCI 10/100 (precedence over 'le') +< device re # RealTek 8139C+/8169/8169S/8110S +< device rl # RealTek 8129/8139 +< device sf # Adaptec AIC-6915 (``Starfire'') +< device sge # Silicon Integrated Systems SiS190/191 +< device sis # Silicon Integrated Systems SiS 900/SiS 7016 +< device sk # SysKonnect SK-984x & SK-982x gigabit Ethernet +< device ste # Sundance ST201 (D-Link DFE-550TX) +< device stge # Sundance/Tamarack TC9021 gigabit Ethernet +< device tl # Texas Instruments ThunderLAN +< device tx # SMC EtherPower II (83c170 ``EPIC'') +< device vge # VIA VT612x gigabit Ethernet +< device vr # VIA Rhine, Rhine II +< device wb # Winbond W89C840F +< device xl # 3Com 3c90x (``Boomerang'', ``Cyclone'') +< +< # Wireless NIC cards +< device wlan # 802.11 support +< options IEEE80211_DEBUG # enable debug msgs +< options IEEE80211_AMPDU_AGE # age frames in AMPDU reorder q's +< options IEEE80211_SUPPORT_MESH # enable 802.11s draft support +< device wlan_wep # 802.11 WEP support +< device wlan_ccmp # 802.11 CCMP support +< device wlan_tkip # 802.11 TKIP support +< device wlan_amrr # AMRR transmit rate control algorithm +< device an # Aironet 4500/4800 802.11 wireless NICs. +< device ath # Atheros NICs +< device ath_pci # Atheros pci/cardbus glue +< device ath_hal # pci/cardbus chip support +< options AH_SUPPORT_AR5416 # enable AR5416 tx/rx descriptors +< options AH_AR5416_INTERRUPT_MITIGATION # AR5416 interrupt mitigation +< options ATH_ENABLE_11N # Enable 802.11n support for AR5416 and later +< device ath_rate_sample # SampleRate tx rate control for ath +< #device bwi # Broadcom BCM430x/BCM431x wireless NICs. +< #device bwn # Broadcom BCM43xx wireless NICs. +< device ipw # Intel 2100 wireless NICs. +< device iwi # Intel 2200BG/2225BG/2915ABG wireless NICs. +< device iwn # Intel 4965/1000/5000/6000 wireless NICs. +< device malo # Marvell Libertas wireless NICs. +< device mwl # Marvell 88W8363 802.11n wireless NICs. +< device ral # Ralink Technology RT2500 wireless NICs. +< device wi # WaveLAN/Intersil/Symbol 802.11 wireless NICs. +< device wpi # Intel 3945ABG wireless NICs. +< +330,344d176 +< +< # Sound support +< device sound # Generic sound driver (required) +< device snd_cmi # CMedia CMI8338/CMI8738 +< device snd_csa # Crystal Semiconductor CS461x/428x +< device snd_emu10kx # Creative SoundBlaster Live! and Audigy +< device snd_es137x # Ensoniq AudioPCI ES137x +< device snd_hda # Intel High Definition Audio +< device snd_ich # Intel, NVidia and other ICH AC'97 Audio +< device snd_via8233 # VIA VT8233x Audio +< +< # MMC/SD +< device mmc # MMC/SD bus +< device mmcsd # MMC/SD memory card +< device sdhci # Generic PCI SD Host Controller diff --git a/src/scripts/loader.conf b/src/scripts/loader.conf new file mode 100644 index 0000000..f01e728 --- /dev/null +++ b/src/scripts/loader.conf @@ -0,0 +1 @@ +autoboot_delay='1' diff --git a/src/scripts/make_kern.sh b/src/scripts/make_kern.sh new file mode 100755 index 0000000..6a6f193 --- /dev/null +++ b/src/scripts/make_kern.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env sh +cd /usr/src +make -j4 buildkernel KERNCONF=CUSTOM +make installkernel KERNCONF=CUSTOM diff --git a/src/scripts/prepBSD_FSmakerVM.sh b/src/scripts/prepBSD_FSmakerVM.sh new file mode 100755 index 0000000..96c38d7 --- /dev/null +++ b/src/scripts/prepBSD_FSmakerVM.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env sh + +pkg install -y vim python3 e2fsprogs p7zip +kldload ext2fs +# Needs to be run as root on the FreeBSD VMs since we require root privs via ssh +sed -i -e 's/#PermitRootLogin no/PermitRootLogin yes/g' /etc/ssh/sshd_config +sed -i -e 's/#PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config +sed -i -e 's/#PermitEmptyPasswords no/PermitEmptyPasswords yes/g' /etc/ssh/sshd_config +/etc/rc.d/sshd restart + +echo "tmpfs_load='YES'" >> /boot/loader.conf +echo "tmpfs /tmp tmpfs rw,mode=1777 0 0" >> /etc/fstab +# NOTE: To use /tmp as a tmpfs in memory partition for performance speed ups we have to either: +# 1. echo "tmpfs_load='YES'" >> /boot/loader.conf, or +# 2. add options TMPFS to the Kernel file and recompile it +# Afterwards we can add tmpfs /tmp tmpfs rw,mode=1777 0 0 in /etc/fstab +# Source: https://www.chruetertee.ch/blog/archive/2007/08/28/ram-disk-mit-tmpfs-auf-freebsd-erstellen.html +# Source2: Alternatively mdmfs: https://www.freebsd.org/doc/handbook/disks-virtual.html diff --git a/src/scripts/prepBSD_fuzzVM.sh b/src/scripts/prepBSD_fuzzVM.sh new file mode 100644 index 0000000..37afe5d --- /dev/null +++ b/src/scripts/prepBSD_fuzzVM.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env sh + +pkg install -y vim python3 e2fsprogs p7zip gdb +kldload ext2fs +# Needs to be run as root on the FreeBSD VMs since we require root privs via ssh +sed -i -e 's/#PermitRootLogin no/PermitRootLogin yes/g' /etc/ssh/sshd_config +sed -i -e 's/#PasswordAuthentication no/PasswordAuthentication yes/g' /etc/ssh/sshd_config +sed -i -e 's/#PermitEmptyPasswords no/PermitEmptyPasswords yes/g' /etc/ssh/sshd_config + +/etc/rc.d/sshd restart + +echo "kern.panic_reboot_wait_time=-1" >> /etc/sysctl.conf +echo "vm.redzone.panic=1" >> /etc/sysctl.conf + +# https://gist.github.com/bijanebrahimi/f2eb0c620d81aa6234e121a0ddd88cc2 +echo "options DDB" >> /usr/src/sys/amd64/conf/GENERIC +echo "options KDB" >> /usr/src/sys/amd64/conf/GENERIC +echo "options KDB_TRACE" >> /usr/src/sys/amd64/conf/GENERIC +# echo "options KDB_UNATTENDED" >> /usr/src/sys/amd64/conf/GENERIC +echo "options INVARIANTS" >> /usr/src/sys/amd64/conf/GENERIC +echo "options INVARIANT_SUPPORT" >> /usr/src/sys/amd64/conf/GENERIC +echo "options DIAGNOSTIC" >> /usr/src/sys/amd64/conf/GENERIC +echo "options DEBUG_REDZONE" >> /usr/src/sys/amd64/conf/GENERIC +echo "options PANIC_REBOOT_WAIT_TIME=0" >> /usr/src/sys/amd64/conf/GENERIC +cd /usr/src/ || exit +# -DKERNFAST +make -j4 buildkernel KERNCONF=GENERIC +make installkernel KERNCONF=GENERIC +# kgdb /boot/kernel/kernel /var/crash/vmcore. diff --git a/src/scripts/prepLinux_VM.sh b/src/scripts/prepLinux_VM.sh new file mode 100644 index 0000000..ade1227 --- /dev/null +++ b/src/scripts/prepLinux_VM.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +sudo apt update +sudo apt upgrade -y +sudo apt-get install -yqq linux-crashdump python3-dev python3-pip zfsutils-linux file p7zip-full openssh-server vim +# systemd specific solution for tmpfs mounting! +sudo ln -s /usr/share/systemd/tmp.mount /etc/systemd/system/tmp.mount +sudo systemctl enable tmp.mount +sudo systemctl start tmp.mount +sed -i -e 's/#Port 22/Port 22/g' /etc/ssh/sshd_config +sudo ufw allow 22 +sudo systemctl restart ssh + +echo "panic=0" >> /etc/sysctl.conf +echo "kernel.core_pattern=/var/crash/core.%t.%p" >> /etc/sysctl.conf +echo "kernel.panic=10" >> /etc/sysctl.conf +echo "kernel.unknown_nmi_panic=1" >> /etc/sysctl.conf \ No newline at end of file diff --git a/src/scripts/rc.conf b/src/scripts/rc.conf new file mode 100644 index 0000000..1856642 --- /dev/null +++ b/src/scripts/rc.conf @@ -0,0 +1,5 @@ +hostname="" +keymap="us.kbd" +ifconfig_em0="DHCP" +sshd_enable="YES" +dumpdev="AUTO" diff --git a/src/scripts/sysctl.conf b/src/scripts/sysctl.conf new file mode 100644 index 0000000..324ff68 --- /dev/null +++ b/src/scripts/sysctl.conf @@ -0,0 +1,2 @@ +kern.panic_reboot_wait_time=0 +vm.redzone.panic=1 diff --git a/src/utility/__init__.py b/src/utility/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/utility/_unused/__init__.py b/src/utility/_unused/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/utility/_unused/find_latest_core.py b/src/utility/_unused/find_latest_core.py new file mode 100644 index 0000000..fb27b03 --- /dev/null +++ b/src/utility/_unused/find_latest_core.py @@ -0,0 +1,20 @@ +import os +import sys +import glob + + +def find_latest_core(): + # find /var/crash -name "core*" -print0 | xargs -0 ls -t | head -n1 + list_of_files = glob.glob("/var/crash/*") + list_of_files = [file for file in list_of_files if file.startswith("/var/crash/core")] + latest_file = max(list_of_files, key=os.path.getctime) + return latest_file + + +def main(): + f = find_latest_core() + print(f) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/utility/_unused/sha256sum.py b/src/utility/_unused/sha256sum.py new file mode 100644 index 0000000..c41fc14 --- /dev/null +++ b/src/utility/_unused/sha256sum.py @@ -0,0 +1,34 @@ +import sys +import hashlib + + +def pure_trace(core_dump): + with open(core_dump, "r") as f: + data = f.read() + panic_name = data.split("panic:")[1].split(":")[0].strip().split("\n")[0].strip() + pure_stack_backtrace = data.split("KDB: stack backtrace:")[1].split("Uptime:")[0].strip() + rem_noise = "" + for line in pure_stack_backtrace.split("\n"): + rem_noise += line.split("at")[1].lstrip() + "\n" + # print(rem_noise) + return panic_name, rem_noise + + +def write_sum(return_val): + with open("/var/crash/sha256sum.txt", "w") as f: + f.write("; ".join(return_val)) + + +def main(): + """ + Creates a sha256 hexdigest of the pure stack backtrace functions+offsets without any noise + """ + panic, backtrace = pure_trace(sys.argv[1]) + return_val = list() + return_val.append(panic) + return_val.append((hashlib.sha256(backtrace.encode()).hexdigest())) + write_sum(return_val) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/utility/extract_core_features.py b/src/utility/extract_core_features.py new file mode 100755 index 0000000..a8e1b0a --- /dev/null +++ b/src/utility/extract_core_features.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +import hashlib +import re +import sys + + +def get_panic_name(data): + return ( + str(data) + .split("panic:")[1] + .split(":")[0] + .split("(")[0] + .split("bp")[0] + .split("fip")[0] + .split("\\")[0] + .split(", addr:")[0] + .strip() + .replace(" ", "_") + .split("_/")[0] + ) + + +def get_core_details(data): + full_strace = ( + str(data) + .split("KDB: stack backtrace:")[1] + .split("--- syscall")[0] + .split("Uptime")[0] + .replace("\\", "\n") + .replace("\nn", "\n") + .strip() + ) + clean_strace = "" + for line in full_strace.split("\n"): + if re.match(r"---\strap\s", line): + continue + if re.match(r"#[0-9]{1,3}\s0x[0-9a-f]{0,16}\sat\s", line): + clean_strace += line.split(" at ")[1] + "\n" + else: + clean_strace += line.split("/frame")[0] + "\n" + return clean_strace + + +def get_sha256_sum(sanitized_stack_trace): + return hashlib.sha256(sanitized_stack_trace.encode()).hexdigest() + + +def main(): + with open(sys.argv[1], "rb") as f: + data = f.read() + + clean_stack_trace = get_core_details(data) + print(get_sha256_sum(clean_stack_trace)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/utility/file_traversal.py b/src/utility/file_traversal.py new file mode 100644 index 0000000..4bfee52 --- /dev/null +++ b/src/utility/file_traversal.py @@ -0,0 +1,65 @@ +import os +import pathlib +import sys + + +def get_all_dirs_in_a_path(dir_path): + only_dirs = "" + for f in os.scandir(dir_path): + if f.is_dir(): + only_dirs += ", " + str(f.path) + only_dirs += ", " + str(dir_path) + return only_dirs.lstrip(",").lstrip() + + +def get_all_files_in_a_path(dir_path): + all_files = "" + for path, subdirs, files in os.walk(dir_path): + for name in files: + all_files += ", " + str(pathlib.PurePath(path, name)) + for subs in subdirs: + all_files += ", " + str(pathlib.PurePath(path, subs)) + return all_files.lstrip(",").lstrip() + + +def get_only_files_in_a_path(dir_path): + only_files = "" + for path, subdirs, files in os.walk(dir_path): + for name in files: + only_files += ", " + str(pathlib.PurePath(path, name)) + return only_files.lstrip(",").lstrip() + + +def get_all_links_in_a_path(dir_path): + all_links = "" + for path, subdirs, files in os.walk(dir_path): + for name in files: + full_path = pathlib.Path(path, name) + if pathlib.Path.is_symlink(full_path): + all_links += ", " + str(full_path) + return all_links.lstrip(",").lstrip() + + +def main(): + results = "" + try: + if sys.argv[2] == "all": + results = get_all_dirs_in_a_path(sys.argv[1]) + results += " {}".format(get_all_files_in_a_path(sys.argv[1])) + results += " {}".format(get_only_files_in_a_path(sys.argv[1])) + results += " {}".format(get_all_links_in_a_path(sys.argv[1])) + elif sys.argv[2] == "dir": + results = get_all_dirs_in_a_path(sys.argv[1]) + elif sys.argv[2] == "files": + results += get_all_files_in_a_path(sys.argv[1]) + elif sys.argv[2] == "file": + results += get_only_files_in_a_path(sys.argv[1]) + elif sys.argv[2] == "link": + results += get_all_links_in_a_path(sys.argv[1]) + print(results) + except FileNotFoundError: + pass + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/utility/get_users_and_groups.py b/src/utility/get_users_and_groups.py new file mode 100644 index 0000000..3c0affb --- /dev/null +++ b/src/utility/get_users_and_groups.py @@ -0,0 +1,32 @@ +import grp +import pwd +import sys + + +def get_groups(): + groups = "" + for group in grp.getgrall(): + groups += ", " + group[0] + return groups.lstrip(",").lstrip() + + +def get_users(): + users = "" + for user in pwd.getpwall(): + users += ", " + user[0] + return users.lstrip(",").lstrip() + + +def main(): + """ + Fetches the Users and Groups located on a system + """ + grps = get_groups() + usrs = get_users() + result = usrs + " " + grps + print(result) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/utility/test_core.txt b/src/utility/test_core.txt new file mode 100644 index 0000000..9cef498 --- /dev/null +++ b/src/utility/test_core.txt @@ -0,0 +1,1740 @@ + dumped core - see /var/crash/vmcore.0 + +Sat Mar 16 04:01:57 CET 2019 + +FreeBSD 12.0-RELEASE FreeBSD 12.0-RELEASE GENERIC amd64 + +panic: page fault + +GNU gdb 6.1.1 [FreeBSD] +Copyright 2004 Free Software Foundation, Inc. +GDB is free software, covered by the GNU General Public License, and you are +welcome to change it and/or distribute copies of it under certain conditions. +Type "show copying" to see the conditions. +There is absolutely no warranty for GDB. Type "show warranty" for details. +This GDB was configured as "amd64-marcel-freebsd"... + +Unread portion of the kernel message buffer: + + +Fatal trap 12: page fault while in kernel mode +cpuid = 1; apic id = 01 +fault virtual address = 0xfffff7fe9a87894c +fault code = supervisor write data, page not present +instruction pointer = 0x20:0xffffffff8282e56c +stack pointer = 0x28:0xfffffe001b81d5d0 +frame pointer = 0x28:0xfffffe001b81d630 +code segment = base 0x0, limit 0xfffff, type 0x1b + = DPL 0, pres 1, long 1, def32 0, gran 1 +processor eflags = interrupt enabled, resume, IOPL = 0 +current process = 1946 (cp) +trap number = 12 +panic: page fault +cpuid = 1 +time = 1552705261 +KDB: stack backtrace: +db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe001b81d280 +vpanic() at vpanic+0x1a3/frame 0xfffffe001b81d2e0 +panic() at panic+0x43/frame 0xfffffe001b81d340 +trap_fatal() at trap_fatal+0x35f/frame 0xfffffe001b81d390 +trap_pfault() at trap_pfault+0x49/frame 0xfffffe001b81d3f0 +trap() at trap+0x2ba/frame 0xfffffe001b81d500 +calltrap() at calltrap+0x8/frame 0xfffffe001b81d500 +--- trap 0xc, rip = 0xffffffff8282e56c, rsp = 0xfffffe001b81d5d0, rbp = 0xfffffe001b81d630 --- +ext2_vget() at ext2_vget+0x2ec/frame 0xfffffe001b81d630 +ext2_valloc() at ext2_valloc+0x501/frame 0xfffffe001b81d6c0 +ext2_mkdir() at ext2_mkdir+0x80/frame 0xfffffe001b81d740 +VOP_MKDIR_APV() at VOP_MKDIR_APV+0x9e/frame 0xfffffe001b81d770 +kern_mkdirat() at kern_mkdirat+0x1be/frame 0xfffffe001b81d980 +amd64_syscall() at amd64_syscall+0x28d/frame 0xfffffe001b81dab0 +fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe001b81dab0 +--- syscall (136, FreeBSD ELF64, sys_mkdir), rip = 0x8003fd55a, rsp = 0x7fffffffe6b8, rbp = 0x7fffffffe7f0 --- +Uptime: 2m5s +Dumping 157 out of 2009 MB:..11%..21%..31%..41%..51%..62%..72%..82%..92% + +Reading symbols from /boot/kernel/intpm.ko...Reading symbols from /usr/lib/debug//boot/kernel/intpm.ko.debug...done. +done. +Loaded symbols for /boot/kernel/intpm.ko +Reading symbols from /boot/kernel/smbus.ko...Reading symbols from /usr/lib/debug//boot/kernel/smbus.ko.debug...done. +done. +Loaded symbols for /boot/kernel/smbus.ko +Reading symbols from /boot/kernel/ext2fs.ko...Reading symbols from /usr/lib/debug//boot/kernel/ext2fs.ko.debug...done. +done. +Loaded symbols for /boot/kernel/ext2fs.ko +#0 doadump (textdump=1) at pcpu.h:230 +230 pcpu.h: No such file or directory. + in pcpu.h +(kgdb) #0 doadump (textdump=1) at pcpu.h:230 +#1 0xffffffff80b79b60 in kern_reboot (howto=260) + at /usr/src/sys/kern/kern_shutdown.c:446 +#2 0xffffffff80b79fb3 in vpanic (fmt=, + ap=) at /usr/src/sys/kern/kern_shutdown.c:872 +#3 0xffffffff80b79d13 in panic (fmt=) + at /usr/src/sys/kern/kern_shutdown.c:799 +#4 0xffffffff8104c18f in trap_fatal (frame=0xfffffe001b81d510, + eva=18446735271619168588) at /usr/src/sys/amd64/amd64/trap.c:929 +#5 0xffffffff8104c1e9 in trap_pfault (frame=0xfffffe001b81d510, usermode=0) + at pcpu.h:230 +#6 0xffffffff8104b82a in trap (frame=0xfffffe001b81d510) + at /usr/src/sys/amd64/amd64/trap.c:441 +#7 0xffffffff810265c5 in calltrap () + at /usr/src/sys/amd64/amd64/exception.S:232 +#8 0xffffffff8282e56c in ext2_vget (mp=0xfffff8002f9e9000, ino=1933, + flags=, vpp=0xfffffe001b81d6f0) + at /usr/src/sys/fs/ext2fs/ext2_vfsops.c:1062 +#9 0xffffffff8281efc1 in ext2_valloc (pvp=0xfffff8002f57d1e0, + mode=, cred=0xfffff8002f232100, + vpp=0xfffffe001b81d6f0) at /usr/src/sys/fs/ext2fs/ext2_alloc.c:410 +#10 0xffffffff82830f70 in ext2_mkdir (ap=0xfffffe001b81d910) + at /usr/src/sys/fs/ext2fs/ext2_vnops.c:1315 +#11 0xffffffff811d3d6e in VOP_MKDIR_APV (vop=0xffffffff82833838, + a=0xfffffe001b81d910) at vnode_if.c:1610 +#12 0xffffffff80c5518e in kern_mkdirat (td=0xfffff8002f9e5580, fd=-100, + path=0x206030
, segflg=UIO_USERSPACE, + mode=16877) at vnode_if.h:665 +#13 0xffffffff8104cbcd in amd64_syscall (td=0xfffff8002f9e5580, traced=0) + at subr_syscall.c:135 +#14 0xffffffff81026ead in fast_syscall_common () + at /usr/src/sys/amd64/amd64/exception.S:504 +#15 0x00000008003fd55a in ?? () +Previous frame inner to this frame (corrupt stack?) +Current language: auto; currently minimal +(kgdb) + +------------------------------------------------------------------------ +ps -axlww + +UID PID PPID CPU PRI NI VSZ RSS MWCHAN STAT TT TIME COMMAND + 0 0 0 0 -16 0 0 0 swapin DLs - 0:00.02 [kernel] + 0 1 0 0 20 0 9952 1000 wait DLs - 0:00.01 [init] + 0 2 0 0 -16 0 0 0 crypto_w DL - 0:00.00 [crypto] + 0 3 0 0 -16 0 0 0 crypto_r DL - 0:00.00 [crypto returns 0] + 0 4 0 0 -16 0 0 0 crypto_r DL - 0:00.00 [crypto returns 1] + 0 5 0 0 -16 0 0 0 - RL - 0:00.18 [cam] + 0 6 0 0 -16 0 0 0 - DL - 0:00.00 [soaiod1] + 0 7 0 0 -16 0 0 0 - DL - 0:00.00 [soaiod2] + 0 8 0 0 -16 0 0 0 - DL - 0:00.00 [soaiod3] + 0 9 0 0 -16 0 0 0 - DL - 0:00.00 [soaiod4] + 0 10 0 0 -16 0 0 0 audit_wo DL - 0:00.00 [audit] + 0 11 0 0 155 0 0 0 - RL - 0:18.34 [idle] + 0 12 0 0 -52 0 0 0 - WL - 0:00.38 [intr] + 0 13 0 0 -8 0 0 0 - DL - 0:00.04 [geom] + 0 14 0 0 -16 0 0 0 seqstate DL - 0:00.00 [sequencer 00] + 0 15 0 0 -68 0 0 0 - DL - 0:00.00 [usb] + 0 16 0 0 -16 0 0 0 waiting_ DL - 0:00.00 [sctp_iterator] + 0 17 0 0 -16 0 0 0 - DL - 0:00.00 [rand_harvestq] + 0 18 0 0 -16 0 0 0 psleep DL - 0:00.01 [pagedaemon] + 0 19 0 0 -16 0 0 0 psleep DL - 0:00.00 [vmdaemon] + 0 20 0 0 -16 0 0 0 qsleep DL - 0:00.01 [bufdaemon] + 0 21 0 0 -16 0 0 0 vlruwt DL - 0:00.00 [vnlru] + 0 22 0 0 16 0 0 0 syncer DL - 0:00.00 [syncer] + 0 355 1 0 52 0 11352 2588 select Ds - 0:00.00 [dhclient] + 0 358 1 0 52 0 11572 2720 select Ds - 0:00.00 [dhclient] + 65 405 1 0 52 0 11740 2840 select DCs - 0:00.00 [dhclient] + 0 406 1 0 20 0 10480 1432 select Ds - 0:00.00 [devd] + 0 549 1 0 20 0 11372 2636 select Ds - 0:00.01 [syslogd] + 0 731 1 0 20 0 19580 7896 select Ds - 0:00.00 [sshd] + 0 734 1 0 20 0 16916 6928 select Ds - 0:00.00 [sendmail] + 25 737 1 0 52 0 16616 6684 pause Ds - 0:00.00 [sendmail] + 0 741 1 0 20 0 11384 2604 nanslp Ds - 0:00.00 [cron] + 0 791 1 0 52 0 10848 2248 ttyin Ds+ - 0:00.00 [getty] + 0 792 1 0 52 0 10848 2248 ttyin Ds+ - 0:00.00 [getty] + 0 793 1 0 52 0 10848 2248 ttyin Ds+ - 0:00.00 [getty] + 0 794 1 0 52 0 10848 2248 ttyin Ds+ - 0:00.00 [getty] + 0 795 1 0 52 0 10848 2248 ttyin Ds+ - 0:00.00 [getty] + 0 796 1 0 52 0 10848 2248 ttyin Ds+ - 0:00.00 [getty] + 0 797 1 0 52 0 10848 2248 ttyin Ds+ - 0:00.00 [getty] + 0 798 1 0 52 0 10848 2248 ttyin Ds+ - 0:00.00 [getty] + 0 801 731 0 52 0 25184 13804 select Ds - 0:00.00 [sshd] + 0 1832 0 0 -8 0 0 0 mdwait DL - 0:00.00 [md0] + 0 1944 801 0 52 0 12200 3264 pause Ds+ - 0:00.01 [csh] + 0 1946 1944 0 72 0 11844 2156 - R+ - 0:00.00 [cp] + +------------------------------------------------------------------------ +vmstat -s + + 289456 cpu context switches + 65626 device interrupts + 2366 software interrupts + 755792 traps + 1315463 system calls + 28 kernel threads created + 1696 fork() calls + 220 vfork() calls + 2 rfork() calls + 0 swap pager pageins + 0 swap pager pages paged in + 0 swap pager pageouts + 0 swap pager pages paged out + 1095 vnode pager pageins + 8278 vnode pager pages paged in + 0 vnode pager pageouts + 0 vnode pager pages paged out + 0 page daemon wakeups + 0 pages examined by the page daemon + 0 clean page reclamation shortfalls + 0 pages reactivated by the page daemon + 133698 copy-on-write faults + 27 copy-on-write optimized faults + 605321 zero fill pages zeroed + 0 zero fill pages prezeroed + 15 intransit blocking page faults + 799926 total VM faults taken + 950 page faults requiring I/O + 0 pages affected by kernel thread creation + 104654 pages affected by fork() + 7911 pages affected by vfork() + 114 pages affected by rfork() + 882958 pages freed + 0 pages freed by daemon + 0 pages freed by exiting processes + 0 pages active + 0 pages inactive + 0 pages in the laundry queue + 0 pages wired down + 0 pages free + 0 bytes per page + 0 total name lookups + cache hits (0% pos + 0% neg) system 0% per-directory + deletions 0%, falsehits 0%, toolong 0% + +------------------------------------------------------------------------ +vmstat -m + + Type InUse MemUse HighUse Requests Size(s) + CAM DEV 5 20K - 11 4096 + CAM CCB 0 0K - 8207 4096 + CAM path 7 4K - 46 512 + CAM periph 6 3K - 69 512,1024 + feeder 7 4K - 7 512 +CAM I/O Scheduler 1 1K - 1 512 + CAM queue 8 12K - 28 512,2048 + UART 3 5K - 3 512,2048 + scsi_cd 0 0K - 6 512 + USB 24 44K - 24 512,1024,8192 + USBdev 20 10K - 20 512 +CAM dev queue 3 2K - 3 512 + ata_pci 1 1K - 1 512 + vtbuf 24 2064K - 46 8192 + vt 11 11K - 11 1024 + DEVFS3 111 56K - 1529 512 + DEVFS1 91 91K - 455 1024 + DEVFS_RULE 56 54K - 56 512,1024 + DEVFS 14 7K - 15 512 + DEVFSP 2 1K - 2 512 +NFSD V4client 1 1K - 1 512 + NFSD lckfile 1 1K - 1 512 + NFSD session 1 2K - 1 2048 + pfs_nodes 20 10K - 20 512 + GEOM 69 39K - 1194 512,1024,2048,4096,8192,16384,32768 + raid_data 0 0K - 162 512 + isadev 7 4K - 7 512 + ddb_capture 1 64K - 1 65536 + cdev 4 2K - 4 512 + filedesc 1 8K - 1 8192 + sigio 0 0K - 1 512 + filecaps 1 1K - 65 512 + kdtrace 147 125K - 3953 512,1024 + kenv 90 61K - 94 512,16384 + kqueue 44 22K - 1947 512 + proc-args 37 19K - 1947 512 + hhook 13 7K - 13 512 + ithread 69 35K - 69 512 + prison 4 2K - 4 512 + KTRACE 100 50K - 100 512 + linker 219 282K - 239 512,1024,2048,4096,8192,16384,65536 + lockf 24 12K - 3214 512 + loginclass 3 2K - 11 512 + devbuf 1826 1670K - 2151 512,1024,2048,4096,8192,16384,65536 + temp 17 40K - 4904 512,1024,2048,4096,8192,16384 + module 504 252K - 505 512 + mtx_pool 2 32K - 2 16384 + osd 3 2K - 9 512 + pmchooks 1 1K - 1 512 + pmc 1 1K - 1 512 + pgrp 20 10K - 403 512 + session 20 10K - 403 512 + proc 2 32K - 2 16384 + subproc 117 425K - 2020 1024,8192 + cred 41 21K - 7338 512 + acpiintr 1 1K - 1 512 + plimit 17 17K - 5705 1024 + uidinfo 5 10K - 7 512,8192 + dumper 1 2K - 1 2048 + sysctl 0 0K - 1 512 + sysctloid 2675 1338K - 2753 512 + sysctltmp 0 0K - 4785 512,1024,4096 + acpica 815 424K - 48871 512,1024,2048,4096,16384 + tidhash 1 32K - 1 32768 + callout 3 1692K - 3 + umtx 272 136K - 272 512 + p1003.1b 1 1K - 1 512 + SWAP 1 76K - 1 + bus 1051 526K - 5935 512,4096,8192 + bus-sc 38 300K - 2039 512,1024,2048,4096,8192,16384,32768,65536 + acpitask 1 64K - 1 65536 + devstat 6 50K - 6 512,16384 + epoch 4 2K - 4 512 + eventhandler 123 62K - 123 512 + gtaskqueue 22 73K - 22 512,16384 + kobj 342 2736K - 639 8192 + Per-cpu 1 1K - 1 512 + kbdmux 6 28K - 6 512,1024,2048,4096,16384 + rman 100 50K - 395 512 + sbuf 0 0K - 1170 512,1024,2048,4096,8192,16384,32768 + toponodes 10 5K - 10 512 + taskqueue 30 15K - 30 512 + terminal 11 6K - 11 512 + Unitno 23 12K - 2413 512 + vmem 3 32K - 4 4096,8192,16384 + ioctlops 0 0K - 87 512,1024,2048,4096 + select 27 14K - 27 512 + iov 0 0K - 171214 512,1024 + msg 4 68K - 4 4096,16384,32768 + sem 4 120K - 4 4096,8192 + shm 1 32K - 1 32768 + tty 14 28K - 364 2048 + pts 1 1K - 351 512 + mbuf_tag 0 0K - 16 512 + shmfd 1 16K - 1 16384 + soname 4 2K - 12286 512 + pcb 22 620K - 75 512,4096,8192,16384 + acl 0 0K - 5 16384 + vfscache 4 1081K - 4 1024,16384,32768 + cl_savebuf 0 0K - 83 512,2048 + vfs_hash 1 520K - 1 + vnodes 1 1K - 1 1024 + mount 29 16K - 198 512,1024 + statfs 0 0K - 5546 8192 + md_disk 1 8K - 6 8192 + vnodemarker 0 0K - 33 1024 +chacha20random 1 1K - 1 1024 + BPF 10 68K - 10 512,1024,16384 + ifnet 3 9K - 3 512,4096 + ifaddr 36 45K - 36 512,1024,8192,16384 + ether_multi 13 7K - 18 512 + clone 7 4K - 7 512 + ipsec 1 1K - 1 1024 + lltable 10 8K - 10 512,1024 + iflib 14 120K - 18 512,2048,4096,8192,16384,32768 + routetbl 25 14K - 29 512,1024 + vnet 1 1K - 1 512 + vnet_data 1 248K - 1 +vnet_data_free 1 1K - 1 512 + igmp 2 1K - 2 512 + in_multi 2 1K - 3 512 +encap_export_host 8 4K - 8 512 + sctp_a_it 0 0K - 2 512 + sctp_vrf 1 1K - 1 512 + sctp_ifa 4 2K - 4 512 + sctp_ifn 2 1K - 2 512 + sctp_iter 0 0K - 2 512 + tfo_ccache 1 136K - 1 + hostcache 1 64K - 1 65536 + LRO 2 48K - 2 16384,32768 + tcpfunc 1 1K - 1 512 + syncache 1 76K - 1 + in6_multi 9 5K - 9 512 + mld 2 1K - 2 512 + ip6ndp 3 2K - 3 512 + inpcbpolicy 7 4K - 158 512 + secasvar 1 4K - 1 4096 + sahead 1 4K - 1 4096 + ipsecpolicy 2 5K - 2 512,4096 + ipsec-saq 2 8K - 2 4096 + crypto 2 3K - 2 512,2048 + rpc 2 16K - 2 8192 +audit_evclass 230 115K - 285 512 + pagedep 8 140K - 36 512 + inodedep 13 532K - 109 1024 + bmsafemap 4 18K - 72 512,16384 + newblk 485 1274K - 7207 512 + indirdep 1 1K - 10 512 + freefrag 0 0K - 36 512 + freeblks 4 2K - 39 512 + freefile 1 1K - 45 512 + diradd 8 4K - 56 512 + mkdir 3 2K - 32 512 + dirrem 0 0K - 48 512 + newdirblk 2 1K - 16 512 + freework 22 11K - 107 512 + freedep 1 1K - 52 512 + jaddref 0 0K - 88 512 + jremref 0 0K - 66 512 + jnewblk 0 0K - 7206 512 + jfreefrag 0 0K - 36 512 + jseg 25 13K - 37 512 + jsegdep 498 249K - 7396 512 + sbdep 0 0K - 7 512 + savedino 0 0K - 44 1024 + jblocks 2 1K - 2 512 + softdep 1 1K - 1 1024 + ufs_dirhash 63 42K - 63 512,1024 + ufs_quota 1 520K - 1 + ufs_mount 3 33K - 5 1024,16384 + vm_pgdata 1 1K - 1 512 + UMAHash 4 24K - 9 2048,4096,8192,16384 + memdesc 1 8K - 1 8192 + pci_link 10 5K - 10 512 + atkbddev 2 1K - 2 512 + acpisem 23 12K - 23 512 + acpidev 22 11K - 22 512 + CAM SIM 3 2K - 3 512 + apmdev 1 1K - 1 512 + madt_table 0 0K - 2 512,8192 + intr 4 408K - 4 65536 + io_apic 1 4K - 1 4096 + local_apic 1 4K - 1 4096 + MCA 10 5K - 10 512 + cpus 2 1K - 2 512 + nexusdev 5 3K - 5 512 + CAM XPT 23 12K - 89 512,1024,2048,4096 + entropy 1 1K - 196 512,16384 + ext2_node 24 24K - 110 1024 + ext2_mount 9 13K - 54 512,2048,4096 + +------------------------------------------------------------------------ +vmstat -z + +ITEM SIZE LIMIT USED FREE REQ FAIL SLEEP + +UMA Kegs: 224, 0, 123, 13, 123, 0, 0 +UMA Zones: 648, 0, 125, 0, 125, 0, 0 +UMA Slabs: 112, 0, 2137, 33, 2308, 0, 0 +UMA Hash: 256, 0, 7, 8, 11, 0, 0 +4 Bucket: 32, 0, 84, 784, 10781, 0, 0 +6 Bucket: 48, 0, 46, 784, 3618, 0, 0 +8 Bucket: 64, 0, 13, 545, 992, 21, 0 +12 Bucket: 96, 0, 12, 398, 800, 0, 0 +16 Bucket: 128, 0, 31, 620, 2441, 1, 0 +32 Bucket: 256, 0, 354, 141, 12693, 4, 0 +64 Bucket: 512, 0, 70, 70, 884,18166, 0 +128 Bucket: 1024, 0, 33, 35, 433, 1, 0 +256 Bucket: 2048, 0, 43, 23, 1231, 881, 0 +vmem: 1792, 0, 3, 1, 3, 0, 0 +vmem btag: 56, 0, 6188, 415, 10296, 47, 0 +VM OBJECT: 256, 0, 1207, 203, 44937, 0, 0 +RADIX NODE: 144, 0, 4063, 3902, 134544, 0, 0 +MAP: 232, 0, 3, 65, 3, 0, 0 +KMAP ENTRY: 120, 0, 8, 256, 8, 0, 0 +MAP ENTRY: 120, 0, 842, 544, 193215, 0, 0 +VMSPACE: 2512, 0, 21, 12, 1920, 0, 0 +fakepg: 104, 0, 0, 0, 0, 0, 0 +64 pcpu: 8, 0, 2541, 275, 2541, 0, 0 +mt_stats_zone: 64, 0, 426, 86, 426, 0, 0 +mt_zone: 24, 0, 426, 238, 426, 0, 0 +16: 16, 0, 0, 0, 0, 0, 0 +32: 32, 0, 0, 0, 0, 0, 0 +64: 64, 0, 0, 0, 0, 0, 0 +128: 128, 0, 0, 0, 0, 0, 0 +256: 256, 0, 0, 0, 0, 0, 0 +512: 512, 0, 9650, 8767, 293299, 0, 0 +1024: 1024, 0, 847, 33, 15772, 0, 0 +2048: 2048, 0, 48, 12, 2278, 0, 0 +4096: 4096, 0, 39, 56, 8997, 0, 0 +8192: 8192, 0, 424, 9, 9011, 0, 0 +16384: 16384, 0, 53, 3, 760, 0, 0 +32768: 32768, 0, 6, 3, 199, 0, 0 +65536: 65536, 0, 11, 0, 219, 0, 0 +SLEEPQUEUE: 88, 0, 137, 142, 137, 0, 0 +Files: 80, 0, 71, 223, 46666, 0, 0 +filedesc0: 1104, 0, 44, 28, 1947, 0, 0 +TURNSTILE: 136, 0, 137, 63, 137, 0, 0 +rl_entry: 40, 0, 47, 547, 47, 0, 0 +umtx pi: 96, 0, 0, 0, 0, 0, 0 +umtx_shm: 88, 0, 0, 0, 0, 0, 0 +MAC labels: 40, 0, 0, 0, 0, 0, 0 +PROC: 1320, 0, 43, 29, 1946, 0, 0 +THREAD: 1400, 0, 127, 9, 127, 0, 0 +cpuset: 104, 0, 7, 272, 7, 0, 0 +domainset: 40, 0, 0, 0, 0, 0, 0 +audit_record: 1280, 0, 0, 0, 0, 0, 0 +mbuf_packet: 256, 801780, 6, 507, 1446, 0, 0 +mbuf: 256, 801780, 1068, 1216, 515316, 0, 0 +mbuf_cluster: 2048, 125276, 1530, 512, 70726, 0, 0 +mbuf_jumbo_page: 4096, 62638, 0, 114, 39306, 0, 0 +mbuf_jumbo_9k: 9216, 55677, 0, 0, 0, 0, 0 +mbuf_jumbo_16k: 16384, 41756, 0, 0, 0, 0, 0 +epoch_record pcpu: 256, 0, 4, 60, 4, 0, 0 +g_bio: 400, 0, 0, 297, 25617, 0, 0 +DMAR_MAP_ENTRY: 120, 0, 0, 0, 0, 0, 0 +ttyinq: 160, 0, 135, 177, 5520, 0, 0 +FPU_save_area: 832, 0, 0, 0, 0, 0, 0 +ttyoutq: 256, 0, 72, 183, 2944, 0, 0 +nvme_request: 128, 0, 0, 0, 0, 0, 0 +cryptop: 128, 0, 0, 0, 0, 0, 0 +cryptodesc: 120, 0, 0, 0, 0, 0, 0 +crypto_session: 24, 0, 0, 0, 0, 0, 0 +vtnet_tx_hdr: 24, 0, 0, 0, 0, 0, 0 +VNODE: 480, 0, 783, 33, 1624, 0, 0 +VNODEPOLL: 120, 0, 0, 0, 0, 0, 0 +BUF TRIE: 144, 0, 681, 12846, 12720, 0, 0 +NAMEI: 1024, 0, 1, 39, 118388, 0, 0 +rentr: 24, 0, 0, 0, 0, 0, 0 +S VFS Cache: 108, 0, 779, 236, 3180, 0, 0 +STS VFS Cache: 148, 0, 0, 0, 0, 0, 0 +L VFS Cache: 328, 0, 11, 49, 43, 0, 0 +LTS VFS Cache: 368, 0, 0, 0, 0, 0, 0 +DIRHASH: 1024, 0, 116, 16, 116, 0, 0 +NCLNODE: 592, 0, 0, 0, 0, 0, 0 +AIO: 208, 0, 0, 0, 0, 0, 0 +AIOP: 32, 0, 0, 0, 0, 0, 0 +AIOCB: 752, 0, 0, 0, 0, 0, 0 +AIOLIO: 280, 0, 0, 0, 0, 0, 0 +pipe: 760, 0, 3, 37, 698, 0, 0 +procdesc: 136, 0, 1, 86, 2, 0, 0 +Mountpoints: 2744, 0, 3, 3, 8, 0, 0 +ksiginfo: 112, 0, 53, 997, 423, 0, 0 +itimer: 352, 0, 0, 0, 0, 0, 0 +KNOTE: 160, 0, 0, 0, 0, 0, 0 +socket: 872, 64304, 18, 34, 1618, 0, 0 +unpcb: 256, 64305, 10, 125, 1449, 0, 0 +ipq: 56, 3976, 0, 0, 0, 0, 0 +udp_inpcb: 488, 64304, 2, 62, 131, 0, 0 +udpcb: 32, 64356, 2, 618, 131, 0, 0 +tcp_inpcb: 488, 64304, 4, 60, 26, 0, 0 +tcpcb: 976, 64304, 4, 36, 26, 0, 0 +tcptw: 88, 12870, 0, 0, 0, 0, 0 +syncache: 168, 15364, 0, 69, 21, 0, 0 +hostcache: 96, 15375, 0, 0, 0, 0, 0 +sackhole: 32, 0, 0, 0, 0, 0, 0 +tfo: 4, 0, 0, 0, 0, 0, 0 +tfo_ccache_entries: 80, 0, 0, 0, 0, 0, 0 +tcpreass: 48, 7885, 0, 0, 0, 0, 0 +tcp_log: 408, 1000008, 0, 0, 0, 0, 0 +tcp_log_bucket: 144, 0, 0, 0, 0, 0, 0 +tcp_log_node: 120, 0, 0, 0, 0, 0, 0 +sctp_ep: 1280, 64305, 0, 0, 0, 0, 0 +sctp_asoc: 2408, 40000, 0, 0, 0, 0, 0 +sctp_laddr: 48, 80012, 0, 332, 3, 0, 0 +sctp_raddr: 736, 80000, 0, 0, 0, 0, 0 +sctp_chunk: 152, 400010, 0, 0, 0, 0, 0 +sctp_readq: 152, 400010, 0, 0, 0, 0, 0 +sctp_stream_msg_out: 112, 400015, 0, 0, 0, 0, 0 +sctp_asconf: 40, 400059, 0, 0, 0, 0, 0 +sctp_asconf_ack: 48, 400060, 0, 0, 0, 0, 0 +udplite_inpcb: 488, 64304, 0, 0, 0, 0, 0 +ripcb: 488, 64304, 1, 31, 1, 0, 0 +IPsec SA lft_c: 16, 0, 0, 0, 0, 0, 0 +rtentry: 208, 0, 11, 122, 14, 0, 0 +selfd: 64, 0, 46, 760, 81334, 0, 0 +swpctrie: 144, 250560, 0, 0, 0, 0, 0 +swblk: 136, 250560, 0, 0, 0, 0, 0 +FFS inode: 160, 0, 725, 91, 770, 0, 0 +FFS1 dinode: 128, 0, 0, 0, 0, 0, 0 +FFS2 dinode: 256, 0, 725, 70, 770, 0, 0 + + +------------------------------------------------------------------------ +vmstat -i + +interrupt total rate +irq1: atkbd0 2 0 +irq11: em0:irq0++ 57446 3781 +irq14: ata0 8178 538 +cpu0:timer 8336 549 +cpu1:timer 6490 427 +Total 80452 5295 + +------------------------------------------------------------------------ +pstat -T + + 71/64303 files +0M/0M swap space + +------------------------------------------------------------------------ +pstat -s + +Device 512-blocks Used Avail Capacity + +------------------------------------------------------------------------ +iostat + + tty md0 ada0 cd0 cpu + tin tout KB/t tps MB/s KB/t tps MB/s KB/t tps MB/s us ni sy in id + 0 3091 0.00 2 0.00 75.75 65 4.77 0.00 0 0.00 6 0 9 2 83 + +------------------------------------------------------------------------ +ipcs -a + +Message Queues: +T ID KEY MODE OWNER GROUP CREATOR CGROUP CBYTES QNUM QBYTES LSPID LRPID STIME RTIME CTIME + +Shared Memory: +T ID KEY MODE OWNER GROUP CREATOR CGROUP NATTCH SEGSZ CPID LPID ATIME DTIME CTIME + +Semaphores: +T ID KEY MODE OWNER GROUP CREATOR CGROUP NSEMS OTIME CTIME + + +------------------------------------------------------------------------ +ipcs -T + +msginfo: + msgmax: 16384 (max characters in a message) + msgmni: 40 (# of message queues) + msgmnb: 2048 (max characters in a message queue) + msgtql: 40 (max # of messages in system) + msgssz: 8 (size of a message segment) + msgseg: 2048 (# of message segments in system) + +shminfo: + shmmax: 536870912 (max shared memory segment size) + shmmin: 1 (min shared memory segment size) + shmmni: 192 (max number of shared memory identifiers) + shmseg: 128 (max shared memory segments per process) + shmall: 131072 (max amount of shared memory in pages) + +seminfo: + semmni: 50 (# of semaphore identifiers) + semmns: 340 (# of semaphores in system) + semmnu: 150 (# of undo structures in system) + semmsl: 340 (max # of semaphores per id) + semopm: 100 (max # of operations per semop call) + semume: 50 (max # of undo entries per process) + semusz: 632 (size in bytes of undo structure) + semvmx: 32767 (semaphore maximum value) + semaem: 16384 (adjust on exit max value) + + +------------------------------------------------------------------------ +nfsstat + +Rpc Counts: + Getattr Setattr Lookup Readlink Read Write Create Remove + 0 0 0 0 0 0 0 0 + Rename Link Symlink Mkdir Rmdir Readdir RdirPlus Access + 0 0 0 0 0 0 0 0 + Mknod Fsstat Fsinfo PathConf Commit + 0 0 0 0 0 +Rpc Info: + TimedOut Invalid X Replies Retries Requests + 0 0 0 0 0 +Cache Info: + Attr Hits Attr Misses Lkup Hits Lkup Misses BioR Hits BioR Misses BioW Hits BioW Misses + 0 0 0 0 0 0 0 0 + BioRL Hits BioRL Misses BioD Hits BioD Misses DirE Hits DirE Misses Accs Hits Accs Misses + 0 0 0 0 0 0 0 0 +Server Info: + Getattr Setattr Lookup Readlink Read Write Create Remove + 0 0 0 0 0 0 0 0 + Rename Link Symlink Mkdir Rmdir Readdir RdirPlus Access + 0 0 0 0 0 0 0 0 + Mknod Fsstat Fsinfo PathConf Commit + 0 0 0 0 0 +Server Re-Failed: + 140737488346800 +Server Faults: +140737488346800 +Server Write + WriteOps WriteRPC Opsaved + 0 0 0 +Server Cache + Inprog Idem Non-Idem Misses + 0 0 0 0 + +------------------------------------------------------------------------ +netstat -s + +tcp: + 163845 packets sent + 128355 data packets (162803686 bytes) + 0 data packets (0 bytes) retransmitted + 0 data packets unnecessarily retransmitted + 0 resends initiated by MTU discovery + 33332 ack-only packets (413 delayed) + 0 URG only packets + 0 window probe packets + 2158 window update packets + 0 control packets + 126037 packets received + 60156 acks (for 162802947 bytes) + 1 duplicate ack + 0 acks for unsent data + 69695 packets (95380584 bytes) received in-sequence + 0 completely duplicate packets (0 bytes) + 0 old duplicate packets + 0 packets with some dup. data (0 bytes duped) + 0 out-of-order packets (0 bytes) + 0 packets (0 bytes) of data after window + 0 window probes + 1 window update packet + 0 packets received after close + 0 discarded for bad checksums + 0 discarded for bad header offset fields + 0 discarded because packet too short + 0 discarded due to memory problems + 0 connection requests + 21 connection accepts + 0 bad connection attempts + 0 listen queue overflows + 0 ignored RSTs in the windows + 21 connections established (including accepts) + 0 times used RTT from hostcache + 0 times used RTT variance from hostcache + 0 times used slow-start threshold from hostcache + 22 connections closed (including 20 drops) + 0 connections updated cached RTT on close + 0 connections updated cached RTT variance on close + 0 connections updated cached ssthresh on close + 0 embryonic connections dropped + 60156 segments updated rtt (of 11890 attempts) + 0 retransmit timeouts + 0 connections dropped by rexmit timeout + 0 persist timeouts + 0 connections dropped by persist timeout + 0 Connections (fin_wait_2) dropped because of timeout + 0 keepalive timeouts + 0 keepalive probes sent + 0 connections dropped by keepalive + 43640 correct ACK header predictions + 65380 correct data packet header predictions + 21 syncache entries added + 0 retransmitted + 0 dupsyn + 0 dropped + 21 completed + 0 bucket overflow + 0 cache overflow + 0 reset + 0 stale + 0 aborted + 0 badack + 0 unreach + 0 zone failures + 21 cookies sent + 21 cookies received + 0 hostcache entries added + 0 bucket overflow + 0 SACK recovery episodes + 0 segment rexmits in SACK recovery episodes + 0 byte rexmits in SACK recovery episodes + 0 SACK options (SACK blocks) received + 0 SACK options (SACK blocks) sent + 0 SACK scoreboard overflow + 0 packets with ECN CE bit set + 0 packets with ECN ECT(0) bit set + 0 packets with ECN ECT(1) bit set + 0 successful ECN handshakes + 0 times ECN reduced the congestion window + 0 packets with matching signature received + 0 packets with bad signature received + 0 times failed to make signature due to no SA + 0 times unexpected signature received + 0 times no signature provided by segment + 0 Path MTU discovery black hole detection activations + 0 Path MTU discovery black hole detection min MSS activations + 0 Path MTU discovery black hole detection failures +TCP connection count by state: + 0 connections in CLOSED state + 3 connections in LISTEN state + 0 connections in SYN_SENT state + 0 connections in SYN_RCVD state + 1 connection in ESTABLISHED state + 0 connections in CLOSE_WAIT state + 0 connections in FIN_WAIT_1 state + 0 connections in CLOSING state + 0 connections in LAST_ACK state + 0 connections in FIN_WAIT_2 state + 0 connections in TIME_WAIT state +udp: + 44 datagrams received + 0 with incomplete header + 0 with bad data length field + 0 with bad checksum + 0 with no checksum + 0 dropped due to no socket + 1 broadcast/multicast datagram undelivered + 0 dropped due to full socket buffers + 0 not for hashed pcb + 43 delivered + 43 datagrams output + 0 times multicast source filter matched +ip: + 126087 total packets received + 0 bad header checksums + 0 with size smaller than minimum + 0 with data size < data length + 0 with ip length > max ip packet size + 0 with header length < data size + 0 with data length < header length + 0 with bad options + 0 with incorrect version number + 0 fragments received + 0 fragments dropped (dup or out of space) + 0 fragments dropped after timeout + 0 packets reassembled ok + 126081 packets for this host + 0 packets for unknown/unsupported protocol + 0 packets forwarded (0 packets fast forwarded) + 6 packets not forwardable + 0 packets received for unknown multicast group + 0 redirects sent + 163890 packets sent from this host + 0 packets sent with fabricated ip header + 0 output packets dropped due to no bufs, etc. + 0 output packets discarded due to no route + 0 output datagrams fragmented + 0 fragments created + 0 datagrams that can't be fragmented + 0 tunneling packets that can't find gif + 0 datagrams with bad address in header +icmp: + 0 calls to icmp_error + 0 errors not generated in response to an icmp message + 0 messages with bad code fields + 0 messages less than the minimum length + 0 messages with bad checksum + 0 messages with bad length + 0 multicast echo requests ignored + 0 multicast timestamp requests ignored + 0 message responses generated + 0 invalid return addresses + 0 no return routes +ipsec: + 0 inbound packets violated process security policy + 0 inbound packets failed due to insufficient memory + 0 invalid inbound packets + 0 outbound packets violated process security policy + 0 outbound packets with no SA available + 0 outbound packets failed due to insufficient memory + 0 outbound packets with no route available + 0 invalid outbound packets + 0 outbound packets with bundled SAs + 0 spd cache hits + 0 spd cache misses + 0 clusters copied during clone + 0 mbufs inserted during makespace +ah: + 0 packets shorter than header shows + 0 packets dropped; protocol family not supported + 0 packets dropped; no TDB + 0 packets dropped; bad KCR + 0 packets dropped; queue full + 0 packets dropped; no transform + 0 replay counter wraps + 0 packets dropped; bad authentication detected + 0 packets dropped; bad authentication length + 0 possible replay packets detected + 0 packets in + 0 packets out + 0 packets dropped; invalid TDB + 0 bytes in + 0 bytes out + 0 packets dropped; larger than IP_MAXPACKET + 0 packets blocked due to policy + 0 crypto processing failures + 0 tunnel sanity check failures +esp: + 0 packets shorter than header shows + 0 packets dropped; protocol family not supported + 0 packets dropped; no TDB + 0 packets dropped; bad KCR + 0 packets dropped; queue full + 0 packets dropped; no transform + 0 packets dropped; bad ilen + 0 replay counter wraps + 0 packets dropped; bad encryption detected + 0 packets dropped; bad authentication detected + 0 possible replay packets detected + 0 packets in + 0 packets out + 0 packets dropped; invalid TDB + 0 bytes in + 0 bytes out + 0 packets dropped; larger than IP_MAXPACKET + 0 packets blocked due to policy + 0 crypto processing failures + 0 tunnel sanity check failures +ipcomp: + 0 packets shorter than header shows + 0 packets dropped; protocol family not supported + 0 packets dropped; no TDB + 0 packets dropped; bad KCR + 0 packets dropped; queue full + 0 packets dropped; no transform + 0 replay counter wraps + 0 packets in + 0 packets out + 0 packets dropped; invalid TDB + 0 bytes in + 0 bytes out + 0 packets dropped; larger than IP_MAXPACKET + 0 packets blocked due to policy + 0 crypto processing failures + 0 packets sent uncompressed; size < compr. algo. threshold + 0 packets sent uncompressed; compression was useless +arp: + 2 ARP requests sent + 1 ARP reply sent + 1 ARP request received + 1 ARP reply received + 3 ARP packets received + 0 total packets dropped due to no ARP entry + 0 ARP entrys timed out + 0 Duplicate IPs seen +ip6: + 0 total packets received + 0 with size smaller than minimum + 0 with data size < data length + 0 with bad options + 0 with incorrect version number + 0 fragments received + 0 fragments dropped (dup or out of space) + 0 fragments dropped after timeout + 0 fragments that exceeded limit + 0 packets reassembled ok + 0 packets for this host + 0 packets forwarded + 0 packets not forwardable + 0 redirects sent + 0 packets sent from this host + 0 packets sent with fabricated ip header + 0 output packets dropped due to no bufs, etc. + 0 output packets discarded due to no route + 0 output datagrams fragmented + 0 fragments created + 0 datagrams that can't be fragmented + 0 packets that violated scope rules + 0 multicast packets which we don't join + Mbuf statistics: + 0 one mbuf + 0 one ext mbuf + 0 two or more ext mbuf + 0 packets whose headers are not contiguous + 0 tunneling packets that can't find gif + 0 packets discarded because of too many headers + 0 failures of source address selection +icmp6: + 0 calls to icmp6_error + 0 errors not generated in response to an icmp6 message + 0 errors not generated because of rate limitation + 0 messages with bad code fields + 0 messages < minimum length + 0 bad checksums + 0 messages with bad length + Histogram of error messages to be generated: + 0 no route + 0 administratively prohibited + 0 beyond scope + 0 address unreachable + 0 port unreachable + 0 packet too big + 0 time exceed transit + 0 time exceed reassembly + 0 erroneous header field + 0 unrecognized next header + 0 unrecognized option + 0 redirect + 0 unknown + 0 message responses generated + 0 messages with too many ND options + 0 messages with bad ND options + 0 bad neighbor solicitation messages + 0 bad neighbor advertisement messages + 0 bad router solicitation messages + 0 bad router advertisement messages + 0 bad redirect messages + 0 path MTU changes +ipsec6: + 0 inbound packets violated process security policy + 0 inbound packets failed due to insufficient memory + 0 invalid inbound packets + 0 outbound packets violated process security policy + 0 outbound packets with no SA available + 0 outbound packets failed due to insufficient memory + 0 outbound packets with no route available + 0 invalid outbound packets + 0 outbound packets with bundled SAs + 0 spd cache hits + 0 spd cache misses + 0 clusters copied during clone + 0 mbufs inserted during makespace +rip6: + 0 messages received + 0 checksum calculations on inbound + 0 messages with bad checksum + 0 messages dropped due to no socket + 0 multicast messages dropped due to no socket + 0 messages dropped due to full socket buffers + 0 delivered + 0 datagrams output +pfkey: + 0 requests sent from userland + 0 bytes sent from userland + 0 messages with invalid length field + 0 messages with invalid version field + 0 messages with invalid message type field + 0 messages too short + 0 messages with memory allocation failure + 0 messages with duplicate extension + 0 messages with invalid extension type + 0 messages with invalid sa type + 0 messages with invalid address extension + 0 requests sent to userland + 0 bytes sent to userland + 0 messages toward single socket + 0 messages toward all sockets + 0 messages toward registered sockets + 0 messages with memory allocation failure + +------------------------------------------------------------------------ +netstat -m + +1074/1723/2797 mbufs in use (current/cache/total) +1023/1019/2042/125276 mbuf clusters in use (current/cache/total/max) +6/507 mbuf+clusters out of packet secondary zone in use (current/cache) +0/114/114/62638 4k (page size) jumbo clusters in use (current/cache/total/max) +0/0/0/55677 9k jumbo clusters in use (current/cache/total/max) +0/0/0/41756 16k jumbo clusters in use (current/cache/total/max) +2314K/2924K/5239K bytes allocated to network (current/cache/total) +0/0/0 requests for mbufs denied (mbufs/clusters/mbuf+clusters) +0/0/0 requests for mbufs delayed (mbufs/clusters/mbuf+clusters) +0/0/0 requests for jumbo clusters delayed (4k/9k/16k) +0/0/0 requests for jumbo clusters denied (4k/9k/16k) +0 sendfile syscalls +0 sendfile syscalls completed without I/O request +0 requests for I/O initiated by sendfile +0 pages read by sendfile as part of a request +0 pages were valid at time of a sendfile request +0 pages were valid and substituted to bogus page +0 pages were requested for read ahead by applications +0 pages were read ahead by sendfile +0 times sendfile encountered an already busy page +0 requests for sfbufs denied +0 requests for sfbufs delayed + +------------------------------------------------------------------------ +netstat -anA + +Active Internet connections (including servers) +Tcpcb Proto Recv-Q Send-Q Local Address Foreign Address (state) +fffff80003b591e8 udp4 0 0 *.514 *.* +fffff80003b58d58 udp6 0 0 *.514 *.* + +------------------------------------------------------------------------ +netstat -aL + +Current listen queue sizes (qlen/incqlen/maxqlen) +Proto Listen Local Address + +------------------------------------------------------------------------ +fstat + +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 5 at 0x4400000000 +fstat: can't read file 11 at 0x4400000000 +fstat: can't read file 17 at 0x4200000000 +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 4 at 0x780000ffff +fstat: can't read file 5 at 0x800000000 +fstat: can't read file 7 at 0x20007ffffffffff +fstat: can't read file 8 at 0x4000000001fffff +fstat: can't read file 10 at 0x780000ffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 4 at 0x780000ffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 4 at 0x780000ffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 4 at 0x780000ffff +fstat: can't read file 5 at 0x600000000 +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 4 at 0x780000ffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 4 at 0x780000ffff +fstat: can't read file 5 at 0x400000000 +fstat: can't read file 7 at 0x20007ffffffffff +fstat: can't read file 8 at 0x4000000001fffff +fstat: can't read file 10 at 0x780000ffff +fstat: can't read file 11 at 0x400000000 +fstat: can't read file 13 at 0x20007ffffffffff +fstat: can't read file 14 at 0x4000000001fffff +fstat: can't read file 16 at 0x780000ffff +fstat: can't read file 17 at 0x400000000 +fstat: can't read file 19 at 0x20007ffffffffff +fstat: can't read file 20 at 0x4000000001fffff +fstat: can't read file 22 at 0x780000ffff +fstat: can't read file 23 at 0x2600000001 +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 4 at 0x780000ffff +fstat: can't read file 5 at 0x800000000 +fstat: can't read file 7 at 0x20007ffffffffff +fstat: can't read file 8 at 0x4000000001fffff +fstat: can't read file 1 at 0x200000000000000 +fstat: can't read file 2 at 0x400000000000000 +fstat: can't read file 5 at 0x800000000 +fstat: can't read file 7 at 0x200000000000002 +fstat: can't read file 8 at 0x400000000000000 +fstat: can't read file 11 at 0x400000000 +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 4 at 0x780000ffff +fstat: can't read file 5 at 0x800000000 +fstat: can't read file 7 at 0x20007ffffffffff +fstat: can't read file 8 at 0x4000000001fffff +fstat: can't read file 10 at 0x780000ffff +fstat: can't read file 1 at 0x20007ffffffffff +fstat: can't read file 2 at 0x4000000001fffff +fstat: can't read file 4 at 0x780000ffff +fstat: can't read file 5 at 0x800000000 +USER CMD PID FD MOUNT INUM MODE SZ|DV R/W +root cp 1946 root / 2 drwxr-xr-x 1024 r +root cp 1946 wd / 1685376 drwxr-xr-x 512 r +root cp 1946 text / 240801 -r-xr-xr-x 31824 r +root cp 1946 ctty /dev 102 crw--w---- pts/0 rw +root cp 1946 0 /dev 102 crw--w---- pts/0 rw +root csh 1944 root / 2 drwxr-xr-x 1024 r +root csh 1944 wd / 1685376 drwxr-xr-x 512 r +root csh 1944 text / 240777 -r-xr-xr-x 429104 r +root csh 1944 ctty /dev 102 crw--w---- pts/0 rw +root md0 1832 root / 2 drwxr-xr-x 1024 r +root md0 1832 wd / 2 drwxr-xr-x 1024 r +root sshd 801 root / 2 drwxr-xr-x 1024 r +root sshd 801 wd / 2 drwxr-xr-x 1024 r +root sshd 801 text / 2006533 -r-xr-xr-x 319304 r +root sshd 801 0 /dev 40 crw-rw-rw- null rw +root sshd 801 6 /dev 40 crw-rw-rw- null rw +root getty 798 root / 2 drwxr-xr-x 1024 r +root getty 798 wd / 2 drwxr-xr-x 1024 r +root getty 798 text / 2006894 -r-xr-xr-x 36032 r +root getty 798 ctty /dev 83 crw------- ttyv7 rw +root getty 798 0 /dev 83 crw------- ttyv7 rw +root getty 797 root / 2 drwxr-xr-x 1024 r +root getty 797 wd / 2 drwxr-xr-x 1024 r +root getty 797 text / 2006894 -r-xr-xr-x 36032 r +root getty 797 ctty /dev 82 crw------- ttyv6 rw +root getty 797 0 /dev 82 crw------- ttyv6 rw +root getty 796 root / 2 drwxr-xr-x 1024 r +root getty 796 wd / 2 drwxr-xr-x 1024 r +root getty 796 text / 2006894 -r-xr-xr-x 36032 r +root getty 796 ctty /dev 81 crw------- ttyv5 rw +root getty 796 0 /dev 81 crw------- ttyv5 rw +root getty 795 root / 2 drwxr-xr-x 1024 r +root getty 795 wd / 2 drwxr-xr-x 1024 r +root getty 795 text / 2006894 -r-xr-xr-x 36032 r +root getty 795 ctty /dev 80 crw------- ttyv4 rw +root getty 795 0 /dev 80 crw------- ttyv4 rw +root getty 794 root / 2 drwxr-xr-x 1024 r +root getty 794 wd / 2 drwxr-xr-x 1024 r +root getty 794 text / 2006894 -r-xr-xr-x 36032 r +root getty 794 ctty /dev 79 crw------- ttyv3 rw +root getty 794 0 /dev 79 crw------- ttyv3 rw +root getty 793 root / 2 drwxr-xr-x 1024 r +root getty 793 wd / 2 drwxr-xr-x 1024 r +root getty 793 text / 2006894 -r-xr-xr-x 36032 r +root getty 793 ctty /dev 78 crw------- ttyv2 rw +root getty 793 0 /dev 78 crw------- ttyv2 rw +root getty 792 root / 2 drwxr-xr-x 1024 r +root getty 792 wd / 2 drwxr-xr-x 1024 r +root getty 792 text / 2006894 -r-xr-xr-x 36032 r +root getty 792 ctty /dev 77 crw------- ttyv1 rw +root getty 792 0 /dev 77 crw------- ttyv1 rw +root getty 791 root / 2 drwxr-xr-x 1024 r +root getty 791 wd / 2 drwxr-xr-x 1024 r +root getty 791 text / 2006894 -r-xr-xr-x 36032 r +root getty 791 ctty /dev 76 crw------- ttyv0 rw +root getty 791 0 /dev 76 crw------- ttyv0 rw +root cron 741 root / 2 drwxr-xr-x 1024 r +root cron 741 wd / 1605136 drwxr-x--- 512 r +root cron 741 text / 2006447 -r-xr-xr-x 52272 r +root cron 741 0 /dev 40 crw-rw-rw- null rw +smmsp sendmail 737 root / 2 drwxr-xr-x 1024 r +smmsp sendmail 737 wd / 1605148 drwxrwx--- 512 r +smmsp sendmail 737 text / 2007120 -r-xr-sr-x 740512 r +smmsp sendmail 737 0 /dev 40 crw-rw-rw- null r +root sendmail 734 root / 2 drwxr-xr-x 1024 r +root sendmail 734 wd / 1605147 drwxr-xr-x 512 r +root sendmail 734 text / 2007120 -r-xr-sr-x 740512 r +root sendmail 734 0 /dev 40 crw-rw-rw- null r +root sshd 731 root / 2 drwxr-xr-x 1024 r +root sshd 731 wd / 2 drwxr-xr-x 1024 r +root sshd 731 text / 2006533 -r-xr-xr-x 319304 r +root sshd 731 0 /dev 40 crw-rw-rw- null rw +root syslogd 549 root / 2 drwxr-xr-x 1024 r +root syslogd 549 wd / 2 drwxr-xr-x 1024 r +root syslogd 549 text / 2006612 -r-xr-xr-x 60616 r +root syslogd 549 0 /dev 40 crw-rw-rw- null rw +root syslogd 549 6 /dev 40 crw-rw-rw- null rw +root syslogd 549 12 /dev 40 crw-rw-rw- null rw +root syslogd 549 18* pipe fffff80003f058e8 <-> fffff80003f05a50 0 rw +root devd 406 root / 2 drwxr-xr-x 1024 r +root devd 406 wd / 2 drwxr-xr-x 1024 r +root devd 406 text / 160589 -r-xr-xr-x 993232 r +root devd 406 0 /dev 40 crw-rw-rw- null rw +root devd 406 6 /dev 40 crw-rw-rw- null rw +_dhcp dhclient 405 root / 2 drwxr-xr-x 1024 r +_dhcp dhclient 405 wd / 2 drwxr-xr-x 1024 r +_dhcp dhclient 405 text / 160605 -r-xr-xr-x 106856 r +_dhcp dhclient 405 0 /dev 40 crw-rw-rw- null rw +_dhcp dhclient 405 6 /dev 40 crw-rw-rw- null rw +root dhclient 358 root / 2 drwxr-xr-x 1024 r +root dhclient 358 wd / 2 drwxr-xr-x 1024 r +root dhclient 358 text / 160605 -r-xr-xr-x 106856 r +root dhclient 358 0 /dev 40 crw-rw-rw- null rw +root dhclient 358 6 /dev 40 crw-rw-rw- null rw +root dhclient 355 root / 2 drwxr-xr-x 1024 r +root dhclient 355 wd / 2 drwxr-xr-x 1024 r +root dhclient 355 text / 160605 -r-xr-xr-x 106856 r +root dhclient 355 0 /dev 40 crw-rw-rw- null rw +root dhclient 355 6 /dev 40 crw-rw-rw- null rw +root init 1 root / 2 drwxr-xr-x 1024 r +root init 1 wd / 2 drwxr-xr-x 1024 r +root init 1 text / 160555 -r-xr-xr-x 1240504 r +root kernel 0 root / 2 drwxr-xr-x 1024 r +root kernel 0 wd / 2 drwxr-xr-x 1024 r + +------------------------------------------------------------------------ +dmesg + +---<>--- +Copyright (c) 1992-2018 The FreeBSD Project. +Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 + The Regents of the University of California. All rights reserved. +FreeBSD is a registered trademark of The FreeBSD Foundation. +FreeBSD 12.0-RELEASE GENERIC amd64 +FreeBSD clang version 6.0.1 (tags/RELEASE_601/final 335540) (based on LLVM 6.0.1) +WARNING: DIAGNOSTIC option enabled, expect reduced performance. +Entering uma_startup with 6 boot pages configured +startup_alloc from "UMA Kegs", 5 boot pages left +startup_alloc from "UMA Zones", 4 boot pages left +startup_alloc from "UMA Zones", 3 boot pages left +startup_alloc from "UMA Hash", 2 boot pages left +startup_alloc from "UMA Zones", 1 boot pages left +Entering uma_startup1 with 0 boot pages left +Entering uma_startup2 with 0 boot pages left +VT(vga): text 80x25 +CPU: Intel Xeon E3-12xx v2 (Ivy Bridge, IBRS) (3400.06-MHz K8-class CPU) + Origin="GenuineIntel" Id=0x306a9 Family=0x6 Model=0x3a Stepping=9 + Features=0x783fbff + Features2=0xffb82203 + AMD Features=0x28100800 + AMD Features2=0x1 + Structured Extended Features=0x281 + Structured Extended Features3=0x4000000 + XSAVE Features=0x1 +Hypervisor: Origin = "KVMKVMKVM" +real memory = 2147483648 (2048 MB) +avail memory = 2042863616 (1948 MB) +Event timer "LAPIC" quality 600 +ACPI APIC Table: +FreeBSD/SMP: Multiprocessor System Detected: 2 CPUs +FreeBSD/SMP: 2 package(s) x 1 core(s) +random: unblocking device. +ioapic0 irqs 0-23 on motherboard +Launching APs: 1 +random: entropy device external interface +[ath_hal] loaded +module_register_init: MOD_LOAD (vesa, 0xffffffff810cfc20, 0) error 19 +kbd1 at kbdmux0 +random: registering fast source Intel Secure Key RNG +random: fast provider: "Intel Secure Key RNG" +netmap: loaded module +nexus0 +vtvga0: on motherboard +cryptosoft0: on motherboard +acpi0: on motherboard +acpi0: Power Button (fixed) +cpu0: on acpi0 +atrtc0: port 0x70-0x71,0x72-0x77 irq 8 on acpi0 +atrtc0: registered as a time-of-day clock, resolution 1.000000s +Event timer "RTC" frequency 32768 Hz quality 0 +Timecounter "ACPI-fast" frequency 3579545 Hz quality 900 +acpi_timer0: <24-bit timer at 3.579545MHz> port 0x608-0x60b on acpi0 +pcib0: port 0xcf8-0xcff on acpi0 +pci0: on pcib0 +isab0: at device 1.0 on pci0 +isa0: on isab0 +atapci0: port 0x1f0-0x1f7,0x3f6,0x170-0x177,0x376,0xc0c0-0xc0cf at device 1.1 on pci0 +ata0: at channel 0 on atapci0 +ata1: at channel 1 on atapci0 +pci0: at device 1.3 (no driver attached) +vgapci0: mem 0xfc000000-0xfdffffff,0xfebb0000-0xfebb0fff at device 2.0 on pci0 +vgapci0: Boot video device +em0: port 0xc000-0xc03f mem 0xfeb80000-0xfeb9ffff irq 11 at device 3.0 on pci0 +em0: attach_pre capping queues at 1 +em0: using 1024 tx descriptors and 1024 rx descriptors +em0: allocated for 1 tx_queues +em0: allocated for 1 rx_queues +em0: Ethernet address: 52:54:00:10:b0:81 +em0: netmap queues/slots: TX 1/1024, RX 1/1024 +uhci0: port 0xc040-0xc05f irq 11 at device 4.0 on pci0 +usbus0 on uhci0 +usbus0: 12Mbps Full Speed USB v1.0 +uhci1: port 0xc060-0xc07f irq 10 at device 4.1 on pci0 +usbus1 on uhci1 +usbus1: 12Mbps Full Speed USB v1.0 +uhci2: port 0xc080-0xc09f irq 10 at device 4.2 on pci0 +usbus2 on uhci2 +usbus2: 12Mbps Full Speed USB v1.0 +ehci0: mem 0xfebb1000-0xfebb1fff irq 11 at device 4.7 on pci0 +usbus3: EHCI version 1.0 +usbus3 on ehci0 +usbus3: 480Mbps High Speed USB v2.0 +virtio_pci0: port 0xc0a0-0xc0bf mem 0xfe000000-0xfe003fff irq 10 at device 5.0 on pci0 +vtballoon0: on virtio_pci0 +acpi_syscontainer0: on acpi0 +acpi_syscontainer1: port 0xaf00-0xaf0b on acpi0 +acpi_syscontainer2: port 0xafe0-0xafe3 on acpi0 +acpi_syscontainer3: port 0xae00-0xae13 on acpi0 +atkbdc0: port 0x60,0x64 irq 1 on acpi0 +atkbd0: irq 1 on atkbdc0 +kbd0 at atkbd0 +atkbd0: [GIANT-LOCKED] +psm0: irq 12 on atkbdc0 +psm0: [GIANT-LOCKED] +psm0: model IntelliMouse Explorer, device ID 4 +fdc0: port 0x3f2-0x3f5,0x3f7 irq 6 drq 2 on acpi0 +fdc0: does not respond +device_attach: fdc0 attach returned 6 +uart0: <16550 or compatible> port 0x3f8-0x3ff irq 4 flags 0x10 on acpi0 +orm0: at iomem 0xc0000-0xc97ff,0xec800-0xeffff pnpid ORM0000 on isa0 +vga0: at port 0x3c0-0x3df iomem 0xa0000-0xbffff pnpid PNP0900 on isa0 +attimer0: at port 0x40 on isa0 +Timecounter "i8254" frequency 1193182 Hz quality 0 +Event timer "i8254" frequency 1193182 Hz quality 100 +attimer0: non-PNP ISA device will be removed from GENERIC in FreeBSD 12. +fdc0: No FDOUT register! +NULL mp in getnewvnode(9), tag crossmp +Timecounters tick every 10.000 msec +ugen2.1: at usbus2 +ugen3.1: at usbus3 +uhub0: on usbus2 +uhub1: on usbus3 +ugen1.1: at usbus1 +uhub2: on usbus1 +ugen0.1: at usbus0 +uhub3: on usbus0 +ada0 at ata0 bus 0 scbus0 target 0 lun 0 +ada0: ATA-7 device +ada0: Serial Number QM00001 +ada0: 16.700MB/s transfers (WDMA2, PIO 8192bytes) +ada0: 25600MB (52428800 512 byte sectors) +cd0 at ata0 bus 0 scbus0 target 1 lun 0 +cd0: Removable CD-ROM SCSI device +cd0: Serial Number QM00002 +cd0: 16.700MB/s transfers (WDMA2, ATAPI 12bytes, PIO 65534bytes) +cd0: Attempt to query device size failed: NOT READY, Medium not present +WARNING: DIAGNOSTIC option enabled, expect reduced performance. +Trying to mount root from ufs:/dev/ada0p2 [rw]... +Expensive timeout(9) function: 0xffffffff809ebc20(0xffffffff81af2cb0) 0.005693740 s +WARNING: / was not properly dismounted +WARNING: /: mount pending error: blocks 128 files 1 +Setting hostuuid: df521590-c04c-4022-b9a9-7d38f800ec2e. +Setting hostid: 0x5f2988f0. +Starting file system checks: +** SU+J Recovering /dev/ada0p2 +** Reading 33554432 byte journal from inode 4. +** Building recovery table. +** Resolving unreferenced inode list. +** Processing journal entries. +uhub0: 2 ports with 2 removable, self powered +uhub2: 2 ports with 2 removable, self powered +uhub3: 2 ports with 2 removable, self powered +** 3471 journal records in 126464 bytes for 87.83% utilization +** Freed 6 inodes (6 dirs) 1446 blocks, and 6 frags. + +***** FILE SYSTEM MARKED CLEAN ***** +Mounting local filesystems:. +ELF ldconfig path: /lib /usr/lib /usr/lib/compat /usr/local/lib /usr/local/lib/e2fsprogs /usr/local/lib/perl5/5.26/mach/CORE +32-bit compatibility ldconfig path: /usr/lib32 +Setting up harvesting: PURE_RDRAND,[UMA],[FS_ATIME],SWI,INTERRUPT,NET_NG,NET_ETHER,NET_TUN,MOUSE,KEYBOARD,ATTACH,CACHED +Feeding entropy: . +lo0: link state changed to UP +uhub1: 6 ports with 6 removable, self powered +em0: link state changed to UP +Starting Network: lo0 em0. +lo0: flags=8049 metric 0 mtu 16384 + options=680003 + inet6 ::1 prefixlen 128 + inet6 fe80::1%lo0 prefixlen 64 scopeid 0x2 + inet 127.0.0.1 netmask 0xff000000 + groups: lo + nd6 options=21 +em0: flags=8843 metric 0 mtu 1500 + options=81209b + ether 52:54:00:10:b0:81 + media: Ethernet autoselect (1000baseT ) + status: active + nd6 options=29 +Starting devd. +Autoloading module: intpm.ko +intsmb0: irq 9 at device 1.3 on pci0 +intsmb0: intr IRQ 9 enabled revision 0 +smbus0: on intsmb0 +Starting dhclient. +DHCPREQUEST on em0 to 255.255.255.255 port 67 +DHCPACK from 192.168.122.1 +if_delmulti_locked: detaching ifnet instance 0xfffff8000351a800 +bound to 192.168.122.107 -- renewal in 1800 seconds. +add host 127.0.0.1: gateway lo0 fib 0: route already in table +add host ::1: gateway lo0 fib 0: route already in table +add net fe80::: gateway ::1 +add net ff02::: gateway ::1 +add net ::ffff:0.0.0.0: gateway ::1 +add net ::0.0.0.0: gateway ::1 +Creating and/or trimming log files. +Starting syslogd. +savecore 559 - - reboot after panic: ext2_dirbad: /mnt/radamsa_fuzz3_ext2_15MB: bad dir ino 11 at offset 1024: mangled entry +savecore 559 - - writing core to /var/crash/vmcore.0 +Writing crash summary to /var/crash/core.txt.0. +Clearing /tmp (X related). +Updating motd:. +Mounting late filesystems:. +Configuring vt: keymap blanktime. +Performing sanity check on sshd configuration. +Starting sshd. +Starting sendmail_submit. +Starting sendmail_msp_queue. +Starting cron. +Starting background file system checks in 60 seconds. + +Sat Mar 16 03:59:12 CET 2019 +Expensive timeout(9) function: 0xffffffff80da0a00(0xfffff8002f669b70) 0.015339379 s + + +Fatal trap 12: page fault while in kernel mode +cpuid = 1; apic id = 01 +fault virtual address = 0xfffff7fe9a87894c +fault code = supervisor write data, page not present +instruction pointer = 0x20:0xffffffff8282e56c +stack pointer = 0x28:0xfffffe001b81d5d0 +frame pointer = 0x28:0xfffffe001b81d630 +code segment = base 0x0, limit 0xfffff, type 0x1b + = DPL 0, pres 1, long 1, def32 0, gran 1 +processor eflags = interrupt enabled, resume, IOPL = 0 +current process = 1946 (cp) +trap number = 12 +panic: page fault +cpuid = 1 +time = 1552705261 +KDB: stack backtrace: +db_trace_self_wrapper() at db_trace_self_wrapper+0x2b/frame 0xfffffe001b81d280 +vpanic() at vpanic+0x1a3/frame 0xfffffe001b81d2e0 +panic() at panic+0x43/frame 0xfffffe001b81d340 +trap_fatal() at trap_fatal+0x35f/frame 0xfffffe001b81d390 +trap_pfault() at trap_pfault+0x49/frame 0xfffffe001b81d3f0 +trap() at trap+0x2ba/frame 0xfffffe001b81d500 +calltrap() at calltrap+0x8/frame 0xfffffe001b81d500 +--- trap 0xc, rip = 0xffffffff8282e56c, rsp = 0xfffffe001b81d5d0, rbp = 0xfffffe001b81d630 --- +ext2_vget() at ext2_vget+0x2ec/frame 0xfffffe001b81d630 +ext2_valloc() at ext2_valloc+0x501/frame 0xfffffe001b81d6c0 +ext2_mkdir() at ext2_mkdir+0x80/frame 0xfffffe001b81d740 +VOP_MKDIR_APV() at VOP_MKDIR_APV+0x9e/frame 0xfffffe001b81d770 +kern_mkdirat() at kern_mkdirat+0x1be/frame 0xfffffe001b81d980 +amd64_syscall() at amd64_syscall+0x28d/frame 0xfffffe001b81dab0 +fast_syscall_common() at fast_syscall_common+0x101/frame 0xfffffe001b81dab0 +--- syscall (136, FreeBSD ELF64, sys_mkdir), rip = 0x8003fd55a, rsp = 0x7fffffffe6b8, rbp = 0x7fffffffe7f0 --- +Uptime: 2m5s +Dumping 157 out of 2009 MB:..11%..21%..31%..41%..51%..62%..72%..82%..92% + +------------------------------------------------------------------------ +kernel config + +options CONFIG_AUTOGENERATED +ident GENERIC +machine amd64 +cpu HAMMER +makeoptions WITH_CTF=1 +makeoptions DEBUG=-g +options PANIC_REBOOT_WAIT_TIME=0 +options DEBUG_REDZONE +options DIAGNOSTIC +options INVARIANT_SUPPORT +options INVARIANTS +options KDB_UNATTENDED +options DDB +options XENHVM +options USB_DEBUG +options ATH_ENABLE_11N +options AH_AR5416_INTERRUPT_MITIGATION +options AH_SUPPORT_AR5416 +options IEEE80211_SUPPORT_MESH +options IEEE80211_AMPDU_AGE +options IEEE80211_DEBUG +options SC_PIXEL_MODE +options VESA +options PCI_IOV +options PCI_HP +options ACPI_DMAR +options EARLY_AP_STARTUP +options SMP +options NETDUMP +options ZSTDIO +options GZIO +options EKCD +options KDB_TRACE +options KDB +options RCTL +options RACCT_DEFAULT_TO_DISABLED +options RACCT +options INCLUDE_CONFIG_FILE +options DDB_CTF +options KDTRACE_HOOKS +options KDTRACE_FRAME +options MAC +options CAPABILITIES +options CAPABILITY_MODE +options AUDIT +options HWPMC_HOOKS +options KBD_INSTALL_CDEV +options PRINTF_BUFR_SIZE=128 +options _KPOSIX_PRIORITY_SCHEDULING +options SYSVSEM +options SYSVMSG +options SYSVSHM +options STACK +options KTRACE +options SCSI_DELAY=5000 +options COMPAT_FREEBSD11 +options COMPAT_FREEBSD10 +options COMPAT_FREEBSD9 +options COMPAT_FREEBSD7 +options COMPAT_FREEBSD6 +options COMPAT_FREEBSD5 +options COMPAT_FREEBSD4 +options COMPAT_FREEBSD32 +options EFIRT +options GEOM_LABEL +options GEOM_RAID +options PSEUDOFS +options PROCFS +options CD9660 +options MSDOSFS +options NFS_ROOT +options NFSLOCKD +options NFSD +options NFSCL +options MD_ROOT +options QUOTA +options UFS_GJOURNAL +options UFS_DIRHASH +options UFS_ACL +options SOFTUPDATES +options FFS +options SCTP +options TCP_RFC7413 +options TCP_HHOOK +options TCP_BLACKBOX +options TCP_OFFLOAD +options IPSEC_SUPPORT +options IPSEC +options INET6 +options INET +options VIMAGE +options PREEMPTION +options NUMA +options SCHED_ULE +options NEW_PCIB +options GEOM_PART_GPT +options GEOM_PART_MBR +options GEOM_PART_EBR_COMPAT +options GEOM_PART_EBR +options GEOM_PART_BSD +device isa +device mem +device io +device uart_ns8250 +device cpufreq +device acpi +device pci +device fdc +device ahci +device ata +device mvs +device siis +device ahc +device ahd +device esp +device hptiop +device isp +device mpt +device mps +device mpr +device sym +device trm +device isci +device ocs_fc +device scbus +device ch +device da +device sa +device cd +device pass +device ses +device amr +device arcmsr +device ciss +device dpt +device hptmv +device hptnr +device hptrr +device hpt27xx +device iir +device ips +device mly +device twa +device smartpqi +device tws +device aac +device aacp +device aacraid +device ida +device mfi +device mlx +device mrsas +device pmspcv +device twe +device nvme +device nvd +device atkbdc +device atkbd +device psm +device kbdmux +device vga +device splash +device sc +device vt +device vt_vga +device vt_efifb +device agp +device cbb +device pccard +device cardbus +device uart +device ppc +device ppbus +device lpt +device ppi +device puc +device bxe +device de +device em +device ix +device ixv +device ixl +device iavf +device le +device ti +device txp +device vx +device miibus +device ae +device age +device alc +device ale +device bce +device bfe +device bge +device cas +device dc +device et +device fxp +device gem +device hme +device jme +device lge +device msk +device nfe +device nge +device pcn +device re +device rl +device sf +device sge +device sis +device sk +device ste +device stge +device tl +device tx +device vge +device vr +device wb +device xl +device wlan +device wlan_wep +device wlan_ccmp +device wlan_tkip +device wlan_amrr +device an +device ath +device ath_pci +device ath_hal +device ath_rate_sample +device ipw +device iwi +device iwn +device malo +device mwl +device ral +device wi +device wpi +device crypto +device loop +device random +device padlock_rng +device rdrand_rng +device ether +device vlan +device tun +device md +device gif +device firmware +device bpf +device uhci +device ohci +device ehci +device xhci +device usb +device ukbd +device umass +device sound +device snd_cmi +device snd_csa +device snd_emu10kx +device snd_es137x +device snd_hda +device snd_ich +device snd_via8233 +device mmc +device mmcsd +device sdhci +device virtio +device virtio_pci +device vtnet +device virtio_blk +device virtio_scsi +device virtio_balloon +device hyperv +device xenpci +device vmx +device netmap + +------------------------------------------------------------------------ +ddb capture buffer + + diff --git a/src/utility/urls.py b/src/utility/urls.py new file mode 100644 index 0000000..7c6fddd --- /dev/null +++ b/src/utility/urls.py @@ -0,0 +1,6 @@ +# FreeBSD latest + +FreeBSD11_2_QCOW_XZ = ( + "https://download.freebsd.org/ftp/releases/VM-IMAGES/11.2-RELEASE/amd64/Latest/FreeBSD-11.2-RELEASE-amd64.qcow2.xz" +) +FreeBSD11_2_ISO = "https://download.freebsd.org/ftp/releases/amd64/amd64/ISO-IMAGES/11.2/FreeBSD-11.2-RELEASE-amd64-dvd1.iso" diff --git a/src/utility/vmcore_remover.py b/src/utility/vmcore_remover.py new file mode 100644 index 0000000..4043704 --- /dev/null +++ b/src/utility/vmcore_remover.py @@ -0,0 +1,19 @@ +import os +import sys +import pathlib + + +def main(): + # python3 cleaner.py crash_dumps/ + subfolders = sorted([f.path for f in os.scandir(sys.argv[1]) if f.is_dir()]) + for entry in subfolders: + try: + file_list_in_entry = [f for f in os.listdir(entry) if os.path.isfile(os.path.join(entry, f))] + vmcore = list(filter(lambda element: "vmcore" in element, file_list_in_entry))[0] + pathlib.Path(os.path.join(entry, vmcore)).unlink() + except: + continue + + +if __name__ == "__main__": + sys.exit(main())