From 2820c20cc3436794c961a648af3b6daff1d4aee2 Mon Sep 17 00:00:00 2001 From: msever Date: Mon, 23 Dec 2024 15:50:05 +0100 Subject: [PATCH 1/9] Update user agent to match RLSDK versions --- user_agent.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/user_agent.json b/user_agent.json index 554c15c..f19f545 100644 --- a/user_agent.json +++ b/user_agent.json @@ -1,3 +1,3 @@ { - "user_agent": "ReversingLabs SDK Cookbook v1.4.0" + "user_agent": "ReversingLabs SDK Cookbook v2.8.0" } From 8075ef4831873a4c5731698f5508edb462ed5442 Mon Sep 17 00:00:00 2001 From: msever Date: Mon, 23 Dec 2024 20:54:09 +0100 Subject: [PATCH 2/9] Add download_yara_retro_matches_a1000.ipynb --- .../download_yara_retro_matches_a1000.ipynb | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 Scenarios and Workflows/download_yara_retro_matches_a1000.ipynb diff --git a/Scenarios and Workflows/download_yara_retro_matches_a1000.ipynb b/Scenarios and Workflows/download_yara_retro_matches_a1000.ipynb new file mode 100644 index 0000000..b9851fa --- /dev/null +++ b/Scenarios and Workflows/download_yara_retro_matches_a1000.ipynb @@ -0,0 +1,204 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Download YARA Retro matches - A1000\n", + "This notebook offers the option of downloading multiple files matched by YARA Retroactive hunt. \n", + "To create YARA rulesets and start YARA Retroactive hunt, use other Python methods from the ReversingLabs SDK.\n" + ], + "metadata": { + "collapsed": false + }, + "id": "1b2c7214bc2ae860" + }, + { + "cell_type": "markdown", + "source": [ + "### Used A1000 functions\n", + "- **get_yara_ruleset_matches_v2**\n", + "- **download_sample**\n", + "\n", + "### Credentials\n", + "Credentials are loaded from a local file instead of being written here in plain text.\n", + "To learn how to creat the credentials file, see the **Storing and using the credentials** section in the [README file](./README.md)" + ], + "metadata": { + "collapsed": false + }, + "id": "568bb5260ad71495" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from ReversingLabs.SDK.a1000 import A1000\n", + "\n", + "\n", + "CREDENTIALS = json.load(open('credentials.json'))\n", + "HOST = CREDENTIALS.get(\"a1000\").get(\"a1000_url\")\n", + "TOKEN = CREDENTIALS.get(\"a1000\").get(\"token\")\n", + "\n", + "cwd = os.getcwd()\n", + "upper_level = os.path.dirname(cwd)\n", + "USER_AGENT = json.load(open(os.path.join(upper_level, \"user_agent.json\")))[\"user_agent\"]\n", + "\n", + "a1000 = A1000(\n", + " host=HOST,\n", + " token=TOKEN,\n", + " user_agent=USER_AGENT,\n", + " verify=False\n", + ")\n" + ], + "metadata": { + "collapsed": false + }, + "id": "3a6fa19993a33b2c" + }, + { + "cell_type": "markdown", + "source": [ + "### 1. Batch-download function\n", + "This function will be used for downloading each batch of matched samples. Execute it so you can use it further on in the notebook." + ], + "metadata": { + "collapsed": false + }, + "id": "355732a0fa3c05b8" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "\n", + "def download_batch(sample_list, download_path):\n", + " for sample in sample_list:\n", + " response = a1000.download_sample(sample_hash=sample.get(\"sha1\"))\n", + " \n", + " with open(os.path.join(download_path, sample.get(\"sha1\")), \"wb\") as file_handle:\n", + " file_handle.write(response.content)\n", + " " + ], + "metadata": { + "collapsed": false + }, + "id": "8be925b62c2cd3e8" + }, + { + "cell_type": "markdown", + "source": [ + "### 2. Main function\n", + "To retrieve batches of samples matched by a YARA Retro ruleset and download them as files, use the `download_retro_hunt_matches` function. \n", + "This function accepts the following parameters:\n", + "- `ruleset_name`: the name of the ruleset whose matches you want to download as files\n", + "- `download_path`: needs to be a full path to the existing folder\n", + "- `max_results`: the maximum number of matches you want retrieved and downloaded\n", + "- `records_per_page`: matches are retrieved in pages. this number defines how big each page will be\n", + "\n", + "Execute the function so you can use it multiple times.\n" + ], + "metadata": { + "collapsed": false + }, + "id": "489a65d42264e55e" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "def download_retro_hunt_matches(ruleset_name, download_path, records_per_page=500, max_results=None): \n", + " page_number = 1\n", + " more_pages = True\n", + " result_count = 0\n", + " \n", + " while more_pages: \n", + " response = a1000.get_yara_ruleset_matches_v2(\n", + " ruleset_name=ruleset_name,\n", + " page_size=records_per_page,\n", + " page=page_number\n", + " )\n", + " \n", + " resp_json = response.json()\n", + " results = resp_json.get(\"results\", [])\n", + " more_pages = resp_json.get(\"next\")\n", + " \n", + " result_count += len(results)\n", + " page_number += 1\n", + " \n", + " if max_results:\n", + " \n", + " if result_count >= max_results:\n", + " excess = result_count - max_results \n", + " cutoff = len(results) - excess\n", + " \n", + " results = results[:cutoff]\n", + " download_batch(sample_list=results, download_path=download_path)\n", + " \n", + " break\n", + " \n", + " download_batch(sample_list=results, download_path=download_path)\n", + " \n", + " if not more_pages:\n", + " break\n", + " " + ], + "metadata": { + "collapsed": false + }, + "id": "761b526762c97952" + }, + { + "cell_type": "markdown", + "source": [ + "### 3. Run" + ], + "metadata": { + "collapsed": false + }, + "id": "66f29338218bb874" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "download_retro_hunt_matches(\n", + " ruleset_name=\"ChangeMe\",\n", + " download_path=\"/change/me\",\n", + " records_per_page=20,\n", + " max_results=100\n", + ")" + ], + "metadata": { + "collapsed": false + }, + "id": "ca75a6f951a8d3c4" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From e15522c9b5853c894fcd58ec34c61e9e114eed80 Mon Sep 17 00:00:00 2001 From: msever Date: Mon, 23 Dec 2024 23:59:37 +0100 Subject: [PATCH 3/9] Correct a typo --- Scenarios and Workflows/directory_scanning.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scenarios and Workflows/directory_scanning.ipynb b/Scenarios and Workflows/directory_scanning.ipynb index 1221cbe..72e39b8 100644 --- a/Scenarios and Workflows/directory_scanning.ipynb +++ b/Scenarios and Workflows/directory_scanning.ipynb @@ -8,7 +8,7 @@ }, "source": [ "# Directory Scanning\n", - "This notebook contains and example of how to use the ReversingLabs SDK to **collect files from a local directory and send them for analysis on TitaniumCloud and A1000**." + "This notebook contains examples of how to use the ReversingLabs SDK to **collect files from a local directory and send them for analysis on TitaniumCloud and A1000**." ] }, { From d9ff404c950a1b17a42b644dcd2d231edc803073 Mon Sep 17 00:00:00 2001 From: msever Date: Mon, 23 Dec 2024 23:59:58 +0100 Subject: [PATCH 4/9] Add download_yara_retro_matches_titaniumcloud.ipynb --- ...oad_yara_retro_matches_titaniumcloud.ipynb | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 Scenarios and Workflows/download_yara_retro_matches_titaniumcloud.ipynb diff --git a/Scenarios and Workflows/download_yara_retro_matches_titaniumcloud.ipynb b/Scenarios and Workflows/download_yara_retro_matches_titaniumcloud.ipynb new file mode 100644 index 0000000..0befa60 --- /dev/null +++ b/Scenarios and Workflows/download_yara_retro_matches_titaniumcloud.ipynb @@ -0,0 +1,181 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Download YARA Retro matches - TitaniumCloud\n", + "This notebook offers the option of downloading multiple files matched by YARA Retroactive hunt. \n", + "To create YARA rulesets and start YARA Retroactive hunt, use other Python methods from the ReversingLabs SDK." + ], + "metadata": { + "collapsed": false + }, + "id": "5cbf3effb9c2d685" + }, + { + "cell_type": "markdown", + "source": [ + "\n", + "### Used TitaniumCloud classes\n", + "- **YARARetroHunting** (*TCA-0319*)\n", + "- **FileDownload** (*TCA-0201*)" + ], + "metadata": { + "collapsed": false + }, + "id": "138c1a1fe3ad7f3a" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from ReversingLabs.SDK.ticloud import YARARetroHunting, FileDownload\n", + "\n", + "\n", + "CREDENTIALS = json.load(open(\"credentials.json\"))\n", + "USERNAME = CREDENTIALS.get(\"ticloud\").get(\"username\")\n", + "PASSWORD = CREDENTIALS.get(\"ticloud\").get(\"password\")\n", + "\n", + "cwd = os.getcwd()\n", + "upper_level = os.path.dirname(cwd)\n", + "USER_AGENT = json.load(open(os.path.join(upper_level, \"user_agent.json\")))[\"user_agent\"]\n", + "\n", + "yara_retro = YARARetroHunting(\n", + " host=\"https://data.reversinglabs.com\",\n", + " username=USERNAME,\n", + " password=PASSWORD,\n", + " user_agent=USER_AGENT\n", + ")\n", + "\n", + "file_download = FileDownload(\n", + " host=\"https://data.reversinglabs.com\",\n", + " username=USERNAME,\n", + " password=PASSWORD,\n", + " user_agent=USER_AGENT\n", + ")\n", + " \n", + " " + ], + "metadata": { + "collapsed": false + }, + "id": "e85f200c89683c8a" + }, + { + "cell_type": "markdown", + "source": [ + "### 1. Batch-download function\n", + "This function will be used for downloading each batch of matched samples. Execute it so you can use it further on in the notebook." + ], + "metadata": { + "collapsed": false + }, + "id": "7dfb3a1f629015c9" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "def download_batch(sample_list, download_path):\n", + " for sample in sample_list:\n", + " response = file_download.download_sample(hash_input=sample.get(\"sha1\"))\n", + " \n", + " with open(os.path.join(download_path, sample.get(\"sha1\")), \"wb\") as file_handle:\n", + " file_handle.write(response.content)\n" + ], + "metadata": { + "collapsed": false + }, + "id": "9c908568a1731d7d" + }, + { + "cell_type": "markdown", + "source": [ + "### 2. Main function\n", + "To retrieve batches of samples matched by a YARA Retro ruleset and download them as files, use the `download_retro_hunt_matches` function. \n", + "This function accepts the following parameters:\n", + "- `time_format`: defines the format of the used time value. possible options are 'utc' and 'timestamp'\n", + "- `time_value`: point back in time from which the retro hunt will start producing matches\n", + "- `download_path`: needs to be a full path to the existing folder\n", + "\n", + "Execute the function so you can use it multiple times." + ], + "metadata": { + "collapsed": false + }, + "id": "bc3934d897e55c41" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "def download_retro_hunt_matches(time_format, time_value, download_path):\n", + " response = yara_retro.yara_retro_matches_feed(\n", + " time_format=time_format,\n", + " time_value=time_value\n", + " )\n", + " \n", + " resp_json = response.json()\n", + " entries = resp_json.get(\"rl\", {}).get(\"feed\", {}).get(\"entries\", [])\n", + " \n", + " download_batch(sample_list=entries, download_path=download_path)" + ], + "metadata": { + "collapsed": false + }, + "id": "8397a0c5fec20c40" + }, + { + "cell_type": "markdown", + "source": [ + "### 3. Run" + ], + "metadata": { + "collapsed": false + }, + "id": "1786817698e6cd41" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "download_retro_hunt_matches(\n", + " time_format=\"timestamp\",\n", + " time_value=\"1734994735\",\n", + " download_path=\"/change/me\"\n", + ")" + ], + "metadata": { + "collapsed": false + }, + "id": "ca468b52d8d260e8" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 4b3214675ca1100f541fe3d2ee44751a7450a6b6 Mon Sep 17 00:00:00 2001 From: msever Date: Tue, 24 Dec 2024 10:31:34 +0100 Subject: [PATCH 5/9] Add download_advanced_search_matches_a1000.ipynb --- ...wnload_advanced_search_matches_a1000.ipynb | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 Scenarios and Workflows/download_advanced_search_matches_a1000.ipynb diff --git a/Scenarios and Workflows/download_advanced_search_matches_a1000.ipynb b/Scenarios and Workflows/download_advanced_search_matches_a1000.ipynb new file mode 100644 index 0000000..29cbbb1 --- /dev/null +++ b/Scenarios and Workflows/download_advanced_search_matches_a1000.ipynb @@ -0,0 +1,213 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Download Advanced Search matches - A1000\n", + "This notebook offers the option of downloading multiple files matched by the Advanced Search API." + ], + "metadata": { + "collapsed": false + }, + "id": "69ccdccfdf3cf4dc" + }, + { + "cell_type": "markdown", + "source": [ + "### Used A1000 functions\n", + "- **advanced_search_v3**\n", + "- **download_sample**\n", + "\n", + "### Credentials\n", + "Credentials are loaded from a local file instead of being written here in plain text.\n", + "To learn how to creat the credentials file, see the **Storing and using the credentials** section in the [README file](./README.md)" + ], + "metadata": { + "collapsed": false + }, + "id": "942b493d1c741fe6" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from ReversingLabs.SDK.a1000 import A1000\n", + "\n", + "\n", + "CREDENTIALS = json.load(open('credentials.json'))\n", + "HOST = CREDENTIALS.get(\"a1000\").get(\"a1000_url\")\n", + "TOKEN = CREDENTIALS.get(\"a1000\").get(\"token\")\n", + "\n", + "cwd = os.getcwd()\n", + "upper_level = os.path.dirname(cwd)\n", + "USER_AGENT = json.load(open(os.path.join(upper_level, \"user_agent.json\")))[\"user_agent\"]\n", + "\n", + "a1000 = A1000(\n", + " host=HOST,\n", + " token=TOKEN,\n", + " user_agent=USER_AGENT,\n", + " verify=False\n", + ")" + ], + "metadata": { + "collapsed": false + }, + "id": "2ceb51c62e6da64e" + }, + { + "cell_type": "markdown", + "source": [ + "### 1. Batch-download function\n", + "This function will be used for downloading each batch of matched samples. Execute it so you can use it further on in the notebook." + ], + "metadata": { + "collapsed": false + }, + "id": "870cb3e0eba4d554" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "def download_batch(sample_list, download_path):\n", + " for sample in sample_list:\n", + " response = a1000.download_sample(sample_hash=sample.get(\"sha1\"))\n", + " \n", + " with open(os.path.join(download_path, sample.get(\"sha1\")), \"wb\") as file_handle:\n", + " file_handle.write(response.content)" + ], + "metadata": { + "collapsed": false + }, + "id": "1645904dfa5a2cc" + }, + { + "cell_type": "markdown", + "source": [ + "### 2. Main function\n", + "To retrieve batches of samples matched by the Advanced Search API and download them as files, use the `download_advanced_search_matches` function. \n", + "This function accepts the following parameters:\n", + "- `query_string`: advanced search query\n", + "- `download_path`: needs to be a full path to the existing folder\n", + "- `ticloud`: use cloud results in the search query\n", + "- `start_search_date`: start date for the search (optional)\n", + "- `end_search_date`: end date for the search (optional)\n", + "- `sorting_criteria`: field used to sort the results\n", + "- `sorting_order`: 'desc' or 'asc'\n", + "- `records_per_page`: matches are retrieved in pages. this number defines how big each page will be\n", + "- `max_results`: the maximum number of matches you want retrieved and downloaded\n", + "\n", + "Execute the function so you can use it multiple times." + ], + "metadata": { + "collapsed": false + }, + "id": "afdb4f18d433057a" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "def download_advanced_search_matches(query_string, download_path, ticloud=False, start_search_date=None, \n", + " end_search_date=None, sorting_criteria=None, sorting_order=\"desc\", \n", + " records_per_page=100, max_results=500):\n", + " page_number = 1\n", + " more_pages = True\n", + " result_count = 0\n", + " \n", + " while more_pages:\n", + " response = a1000.advanced_search_v3(\n", + " query_string=query_string,\n", + " ticloud=ticloud,\n", + " start_search_date=start_search_date,\n", + " end_search_date=end_search_date,\n", + " page_number=page_number,\n", + " records_per_page=records_per_page,\n", + " sorting_criteria=sorting_criteria,\n", + " sorting_order=sorting_order\n", + " )\n", + " \n", + " resp_json = response.json()\n", + " results = resp_json.get(\"rl\", {}).get(\"web_search_api\", {}).get(\"entries\", [])\n", + " more_pages = resp_json.get(\"rl\", {}).get(\"web_search_api\", {}).get(\"more_pages\")\n", + " \n", + " result_count += len(results)\n", + " page_number += 1\n", + " \n", + " if max_results:\n", + " \n", + " if result_count >= max_results:\n", + " excesss = result_count - max_results\n", + " cutoff = len(results) - excesss\n", + " \n", + " results = results[:cutoff]\n", + " download_batch(sample_list=results, download_path=download_path)\n", + " \n", + " break\n", + " \n", + " download_batch(sample_list=results, download_path=download_path)\n", + " \n", + " if not more_pages:\n", + " break" + ], + "metadata": { + "collapsed": false + }, + "id": "a8dec3a5697a2e95" + }, + { + "cell_type": "markdown", + "source": [ + "### 3. Run" + ], + "metadata": { + "collapsed": false + }, + "id": "df873aa93ad4c0e3" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "download_advanced_search_matches(\n", + " query_string=\"change_me\",\n", + " download_path=\"/change/me\",\n", + " ticloud=False,\n", + " records_per_page=20,\n", + " max_results=100\n", + ")" + ], + "metadata": { + "collapsed": false + }, + "id": "88430e2876d2fd5" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 3c187b555cc6020d74b550e9bd8ecf356203e1a3 Mon Sep 17 00:00:00 2001 From: msever Date: Tue, 24 Dec 2024 10:32:25 +0100 Subject: [PATCH 6/9] Correct notebooks --- .../download_yara_retro_matches_a1000.ipynb | 1 - .../download_yara_retro_matches_titaniumcloud.ipynb | 7 +++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Scenarios and Workflows/download_yara_retro_matches_a1000.ipynb b/Scenarios and Workflows/download_yara_retro_matches_a1000.ipynb index b9851fa..bf05137 100644 --- a/Scenarios and Workflows/download_yara_retro_matches_a1000.ipynb +++ b/Scenarios and Workflows/download_yara_retro_matches_a1000.ipynb @@ -74,7 +74,6 @@ "execution_count": null, "outputs": [], "source": [ - "\n", "def download_batch(sample_list, download_path):\n", " for sample in sample_list:\n", " response = a1000.download_sample(sample_hash=sample.get(\"sha1\"))\n", diff --git a/Scenarios and Workflows/download_yara_retro_matches_titaniumcloud.ipynb b/Scenarios and Workflows/download_yara_retro_matches_titaniumcloud.ipynb index 0befa60..4b3b67a 100644 --- a/Scenarios and Workflows/download_yara_retro_matches_titaniumcloud.ipynb +++ b/Scenarios and Workflows/download_yara_retro_matches_titaniumcloud.ipynb @@ -15,10 +15,13 @@ { "cell_type": "markdown", "source": [ - "\n", "### Used TitaniumCloud classes\n", "- **YARARetroHunting** (*TCA-0319*)\n", - "- **FileDownload** (*TCA-0201*)" + "- **FileDownload** (*TCA-0201*)\n", + "\n", + "### Credentials\n", + "Credentials are loaded from a local file instead of being written here in plain text.\n", + "To learn how to creat the credentials file, see the **Storing and using the credentials** section in the [README file](./README.md)" ], "metadata": { "collapsed": false From 84db648a7931cba359f31e8e10784542f560d85c Mon Sep 17 00:00:00 2001 From: msever Date: Tue, 24 Dec 2024 10:32:39 +0100 Subject: [PATCH 7/9] Rename notebook --- ...ating.ipynb => retro_hunt_with_timegating_titaniumcloud.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Scenarios and Workflows/{retro_hunt_with_timegating.ipynb => retro_hunt_with_timegating_titaniumcloud.ipynb} (100%) diff --git a/Scenarios and Workflows/retro_hunt_with_timegating.ipynb b/Scenarios and Workflows/retro_hunt_with_timegating_titaniumcloud.ipynb similarity index 100% rename from Scenarios and Workflows/retro_hunt_with_timegating.ipynb rename to Scenarios and Workflows/retro_hunt_with_timegating_titaniumcloud.ipynb From 4ee3f8105f24beffb908611f077311d3fe88e14a Mon Sep 17 00:00:00 2001 From: msever Date: Tue, 24 Dec 2024 10:49:55 +0100 Subject: [PATCH 8/9] Add download_advanced_search_matches_titaniumcloud.ipynb --- ...dvanced_search_matches_titaniumcloud.ipynb | 211 ++++++++++++++++++ 1 file changed, 211 insertions(+) create mode 100644 Scenarios and Workflows/download_advanced_search_matches_titaniumcloud.ipynb diff --git a/Scenarios and Workflows/download_advanced_search_matches_titaniumcloud.ipynb b/Scenarios and Workflows/download_advanced_search_matches_titaniumcloud.ipynb new file mode 100644 index 0000000..796279a --- /dev/null +++ b/Scenarios and Workflows/download_advanced_search_matches_titaniumcloud.ipynb @@ -0,0 +1,211 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Download Advanced Search matches - TitaniumCloud\n", + "This notebook offers the option of downloading multiple files matched by the Advanced Search API." + ], + "metadata": { + "collapsed": false + }, + "id": "af34fda1aa5e36" + }, + { + "cell_type": "markdown", + "source": [ + "### Used TitaniumCloud classes\n", + "- **AdvancedSearch** (*TCA-0320*)\n", + "- **FileDownload** (*TCA-0201*)\n", + "\n", + "### Credentials\n", + "Credentials are loaded from a local file instead of being written here in plain text.\n", + "To learn how to creat the credentials file, see the **Storing and using the credentials** section in the [README file](./README.md)" + ], + "metadata": { + "collapsed": false + }, + "id": "532408c567e8eef9" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from ReversingLabs.SDK.ticloud import AdvancedSearch, FileDownload\n", + "\n", + "CREDENTIALS = json.load(open(\"credentials.json\"))\n", + "USERNAME = CREDENTIALS.get(\"ticloud\").get(\"username\")\n", + "PASSWORD = CREDENTIALS.get(\"ticloud\").get(\"password\")\n", + "\n", + "cwd = os.getcwd()\n", + "upper_level = os.path.dirname(cwd)\n", + "USER_AGENT = json.load(open(os.path.join(upper_level, \"user_agent.json\")))[\"user_agent\"]\n", + "\n", + "advanced_search = AdvancedSearch(\n", + " host=\"https://data.reversinglabs.com\",\n", + " username=USERNAME,\n", + " password=PASSWORD,\n", + " user_agent=USER_AGENT\n", + ")\n", + "\n", + "file_download = FileDownload(\n", + " host=\"https://data.reversinglabs.com\",\n", + " username=USERNAME,\n", + " password=PASSWORD,\n", + " user_agent=USER_AGENT\n", + ")" + ], + "metadata": { + "collapsed": false + }, + "id": "1a39e4718f394d1b" + }, + { + "cell_type": "markdown", + "source": [ + "### 1. Batch-download function\n", + "This function will be used for downloading each batch of matched samples. Execute it so you can use it further on in the notebook." + ], + "metadata": { + "collapsed": false + }, + "id": "49f4750049c38316" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "def download_batch(sample_list, download_path):\n", + " for sample in sample_list:\n", + " response = file_download.download_sample(sample_hash=sample.get(\"sha1\"))\n", + " \n", + " with open(os.path.join(download_path, sample.get(\"sha1\")), \"wb\") as file_handle:\n", + " file_handle.write(response.content)" + ], + "metadata": { + "collapsed": false + }, + "id": "9eaf31ec7e340997" + }, + { + "cell_type": "markdown", + "source": [ + "### 2. Main function\n", + "To retrieve batches of samples matched by the Advanced Search API and download them as files, use the `download_advanced_search_matches` function. \n", + "This function accepts the following parameters:\n", + "- `query_string`: advanced search query\n", + "- `download_path`: needs to be a full path to the existing folder\n", + "- `sorting_criteria`: field used to sort the results\n", + "- `sorting_order`: 'desc' or 'asc'\n", + "- `records_per_page`: matches are retrieved in pages. this number defines how big each page will be\n", + "- `max_results`: the maximum number of matches you want retrieved and downloaded\n", + "\n", + "Execute the function so you can use it multiple times." + ], + "metadata": { + "collapsed": false + }, + "id": "a455c9d37c5d273c" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "def download_advanced_search_matches(query_string, download_path, sorting_criteria=None, sorting_order=\"desc\", \n", + " records_per_page=100, max_results=500):\n", + " page_number = 1\n", + " more_pages = True\n", + " result_count = 0\n", + " \n", + " while more_pages:\n", + " response = advanced_search.search(\n", + " query_string=query_string,\n", + " page_number=page_number,\n", + " records_per_page=records_per_page,\n", + " sorting_criteria=sorting_criteria,\n", + " sorting_order=sorting_order\n", + " )\n", + " \n", + " resp_json = response.json()\n", + " results = resp_json.get(\"rl\", {}).get(\"web_search_api\", {}).get(\"entries\", [])\n", + " more_pages = resp_json.get(\"rl\", {}).get(\"web_search_api\", {}).get(\"more_pages\")\n", + " \n", + " result_count += len(results)\n", + " page_number += 1\n", + " \n", + " if max_results:\n", + " \n", + " if result_count >= max_results:\n", + " excesss = result_count - max_results\n", + " cutoff = len(results) - excesss\n", + " \n", + " results = results[:cutoff]\n", + " download_batch(sample_list=results, download_path=download_path)\n", + " \n", + " break\n", + " \n", + " download_batch(sample_list=results, download_path=download_path)\n", + " \n", + " if not more_pages:\n", + " break" + ], + "metadata": { + "collapsed": false + }, + "id": "69cc23a9710ca4c5" + }, + { + "cell_type": "markdown", + "source": [ + "### 3. Run" + ], + "metadata": { + "collapsed": false + }, + "id": "89076d7c3de14473" + }, + { + "cell_type": "code", + "execution_count": null, + "outputs": [], + "source": [ + "download_advanced_search_matches(\n", + " query_string=\"change_me\",\n", + " download_path=\"/change/me\",\n", + " records_per_page=20,\n", + " max_results=100\n", + ")" + ], + "metadata": { + "collapsed": false + }, + "id": "fbc5a6fa36ad7871" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 795868a16a2209419b67be4b8c9028c2e56485ca Mon Sep 17 00:00:00 2001 From: msever Date: Tue, 24 Dec 2024 22:23:23 +0100 Subject: [PATCH 9/9] Add the download_advanced_search_matches_a1000.ipynb, download_advanced_search_matches_titaniumcloud.ipynb, download_yara_retro_matches_a1000.ipynb and download_yara_retro_matches_titaniumcloud.ipynb notebooks Add the cyber_defense_alliance.py command line tool --- CHANGELOG.md | 15 +- Command line tools and scripts/README.md | 7 + .../cyber_defense_alliance.py | 157 ++++++++++++++++++ 3 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 Command line tools and scripts/README.md create mode 100644 Command line tools and scripts/cyber_defense_alliance.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c481ad..ca81e1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,4 +50,17 @@ v1.4.0 (2024-10-18) - Added File Inspection Engine support. - **File Inspection Engine** notebooks: - - Added the `scan_and_report_actions.ipynb` notebook. \ No newline at end of file + - Added the `scan_and_report_actions.ipynb` notebook. + + +v2.8.0 (2024-12-24) +------------------- + +#### Improvements +- Added File Inspection Engine support. + +- **Scenarios and Workflows** notebooks: + - Added the `download_advanced_search_matches_a1000.ipynb`, `download_advanced_search_matches_titaniumcloud.ipynb`, `download_yara_retro_matches_a1000.ipynb` and `download_yara_retro_matches_titaniumcloud.ipynb` notebooks. + +- **Command line tools and scripts**: + - Added the `cyber_defense_alliance.py` command line tool. \ No newline at end of file diff --git a/Command line tools and scripts/README.md b/Command line tools and scripts/README.md new file mode 100644 index 0000000..b2ea919 --- /dev/null +++ b/Command line tools and scripts/README.md @@ -0,0 +1,7 @@ +# Scenarios and Workflows + +This directory contains various scripts that can either be used as command line tools or standalone fragments of code. + +## Running +To ron a command line tool, position yourself inside of this folder and run: +`python name_of_the_script.py` \ No newline at end of file diff --git a/Command line tools and scripts/cyber_defense_alliance.py b/Command line tools and scripts/cyber_defense_alliance.py new file mode 100644 index 0000000..68cc1c1 --- /dev/null +++ b/Command line tools and scripts/cyber_defense_alliance.py @@ -0,0 +1,157 @@ +import json +import argparse +import configparser +import sys + +from ReversingLabs.SDK.ticloud import AdvancedSearch, FileAnalysis + + +def parse_arguments(): + parser = argparse.ArgumentParser() + parser.add_argument("--url", type=str, help="ReversingLabs host (e.g., https://data.reversinglabs.com)") + parser.add_argument("-u", "--username", type=str, help="ReversingLabs account username") + parser.add_argument("-p", "--password", type=str, help="ReversingLabs account password") + parser.add_argument("--since", type=str, help="Start date for first seen range (e.g., 2019-01-03T04:10:00Z)") + parser.add_argument("--until", type=str, help="End date for first seen range (e.g., 2019-01-03T04:10:00Z)") + parser.add_argument("--family", type=str, help="Threat family name") + parser.add_argument("--output", type=str, help="Output file name") + parser.add_argument("--config", type=str, help="Alternate config file path (defaults to config.ini)") + return parser.parse_args() + + +def load_config(args): + config = configparser.ConfigParser() + config.read(args.config if args.config else 'config.ini') + return config + + +def get_config_value(args, config, key): + if getattr(args, key) is not None: + return getattr(args, key) + elif key in config["DEFAULT"]: + return config["DEFAULT"][key] + elif key == "url": + return "https://data.reversinglabs.com" + else: + print(f"Missing argument {key} - Try running '{sys.argv[0]} -h' for help") + sys.exit(0) + + +def perform_advanced_search(advanced_search, first_seen_from, first_seen_to, family_name, file_analysis): + more_pages = True + next_page = 1 + output_results = [] + + while more_pages: + response = advanced_search.search( + query_string=f"firstseen:[{first_seen_from} TO {first_seen_to}] threatname:*{family_name}", + page_number=next_page, + records_per_page=100 + ) + + response_json = response.json() + + total_count = response_json.get("rl", {}).get("web_search_api", {}).get("total_count") + if not total_count: + print("No files found within range") + sys.exit(0) + + search_entries = response_json.get("rl", {}).get("web_search_api", {}).get("entries", []) + next_page = response_json.get("rl", {}).get("web_search_api", {}).get("next_page") + more_pages = response_json.get("rl", {}).get("web_search_api", {}).get("more_pages") + + output_data = {} + + for entry in search_entries: + sha1 = entry["sha1"] + output_data[sha1] = { + "first_seen": entry["firstseen"], + "sha1": sha1, + "md5": entry["md5"], + "threat_name": entry["threatname"], + "file_type": entry["sampletype"] + } + + output_results.extend(fetch_sample_details(search_entries, file_analysis, sha1, output_data)) + + return output_results + + +def fetch_sample_details(search_entries, file_analysis, sha1, output_data): + hashes = [entry["sha256"] for entry in search_entries] + if not hashes: + print("Empty hashes list") + return [] + + response = file_analysis.get_analysis_results( + hash_input=hashes + ) + + entries = response.json().get("rl", {}).get("entries", []) + for entry in entries: + if "dynamic_analysis" in entry: + output_data[sha1]["dynamic_analysis"] = parse_dynamic_analysis(dynamic_analysis=entry["dynamic_analysis"]) + + return list(output_data.values()) + +def parse_dynamic_analysis(dynamic_analysis): + analysis_results = [] + for da in dynamic_analysis.get("entries", []): + result = {} + + if "dynamic_analysis_report" in da: + result["network"] = da["dynamic_analysis_report"].get("network", {}) + result["mutexes"] = da["dynamic_analysis_report"].get("summary", {}).get("mutexes", []) + + if "dynamic_analysis_report_joe_sandbox" in da: + result["network"] = da["dynamic_analysis_report_joe_sandbox"].get("network", {}) + result["mutexes"] = da["dynamic_analysis_report_joe_sandbox"].get("summary", {}).get("mutexes", []) + + analysis_results.append(result) + + return analysis_results + + +def save_results(output_results, output_file): + with open(output_file, "w") as f: + json.dump(output_results, f, indent=4) + + +def main(): + args = parse_arguments() + config = load_config(args) + + first_seen_from = get_config_value(args, config, "since") + first_seen_to = get_config_value(args, config, "until") + family_name = get_config_value(args, config, "family") + url = get_config_value(args, config, "url") + username = get_config_value(args, config, "username") + password = get_config_value(args, config, "password") + output_file = args.output or config["DEFAULT"].get("output", "output.json") + + advanced_search = AdvancedSearch( + host=url, + username=username, + password=password + ) + + file_analysis = FileAnalysis( + host=url, + username=username, + password=password + ) + + output_results = perform_advanced_search( + advanced_search=advanced_search, + first_seen_from=first_seen_from, + first_seen_to=first_seen_to, + family_name=family_name, + file_analysis=file_analysis + ) + + print(f"Total interesting samples: {len(output_results)}") + save_results(output_results, output_file) + + +if __name__ == "__main__": + main()