From 43e8d7fe92a062eee7f85636df5cde712e90d63d Mon Sep 17 00:00:00 2001 From: Aaron Peterson Date: Thu, 5 Oct 2023 10:37:21 -0700 Subject: [PATCH 1/8] Update report filtering --- .../api/cli/turbinia_client/core/commands.py | 13 +++++-- .../cli/turbinia_client/helpers/formatter.py | 39 ++++++++++++------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/turbinia/api/cli/turbinia_client/core/commands.py b/turbinia/api/cli/turbinia_client/core/commands.py index 35350ecc4..454cbc09e 100644 --- a/turbinia/api/cli/turbinia_client/core/commands.py +++ b/turbinia/api/cli/turbinia_client/core/commands.py @@ -123,13 +123,20 @@ def get_jobs(ctx: click.Context) -> None: @click.pass_context @click.argument('request_id') @click.option( - '--show_all', '-a', help='Shows all field regardless of priority.', + '--priority_filter', '-p', help='This sets what report sections are ' + 'shown in full detail in report output. Any tasks that have set a ' + 'report_priority value equal to or lower than this setting will be ' + 'shown in full detail, and tasks with a higher value will only have ' + 'a summary shown. To see all tasks report output in full detail, ' + 'set --priority_filter=100', is_flag=True, required=False) +@click.option( + '--show_all', '-a', help='Shows all fields including saved output paths.', is_flag=True, required=False) @click.option( '--json_dump', '-j', help='Generates JSON output.', is_flag=True, required=False) def get_request( - ctx: click.Context, request_id: str, show_all: bool, + ctx: click.Context, request_id: str, priority_filter: int, show_all: bool, json_dump: bool) -> None: """Gets Turbinia request status.""" client: api_client.ApiClient = ctx.obj.api_client @@ -145,7 +152,7 @@ def get_request( formatter.echo_json(api_response) else: report = formatter.RequestMarkdownReport(api_response).generate_markdown( - show_all=show_all) + priority_filter=priority_filter, show_all=show_all) click.echo(report) except exceptions.ApiException as exception: log.error( diff --git a/turbinia/api/cli/turbinia_client/helpers/formatter.py b/turbinia/api/cli/turbinia_client/helpers/formatter.py index ef7e8d37b..df35c853c 100644 --- a/turbinia/api/cli/turbinia_client/helpers/formatter.py +++ b/turbinia/api/cli/turbinia_client/helpers/formatter.py @@ -271,7 +271,8 @@ def __init__(self, request_data: dict = None): super().__init__() self._request_data: dict = request_data - def generate_markdown(self, show_all=False, compact=False) -> str: + def generate_markdown( + self, priority_filter=None, show_all=False, compact=False) -> str: """Generate a markdown report.""" report: list[str] = [] task: dict = self._request_data @@ -291,16 +292,21 @@ def generate_markdown(self, show_all=False, compact=False) -> str: name = f'{task.get("name")} ({"LOW PRIORITY"})' try: - report.append(self.heading2(name)) - line = f"{self.bold('Evidence:'):s} {task.get('evidence_name')!s}" - report.append(self.bullet(line)) - line = f"{self.bold('Status:'):s} {task.get('status')!s}" - report.append(self.bullet(line)) - if show_all or priority <= MEDIUM_PRIORITY: + # Only show Task details if the Task has more priority than the + # priority_filter + if priority > priority_filter: + report.append(f'{self.heading2(name)}: {task.get('status')!s}') + else: + report.append(self.heading2(name)) + line = f"{self.bold('Evidence:'):s} {task.get('evidence_name')!s}" + report.append(self.bullet(line)) + line = f"{self.bold('Status:'):s} {task.get('status')!s}" + report.append(self.bullet(line)) + report.append(self.bullet(f"Task Id: {task.get('id')!s}")) report.append( self.bullet(f"Executed on worker {task.get('worker_name')!s}")) - if show_all or priority <= HIGH_PRIORITY: + if task.get('report_data'): if not compact: report.append('') @@ -308,7 +314,8 @@ def generate_markdown(self, show_all=False, compact=False) -> str: report.extend(task.get('report_data').splitlines()) if not compact: report.append('') - if show_all or priority <= CRITICAL_PRIORITY: + + if show_all and priority <= priority_filter: if not compact: report.append('') report.append(self.heading3('Saved Task Files:')) @@ -318,6 +325,8 @@ def generate_markdown(self, show_all=False, compact=False) -> str: report.append(self.bullet(self.code(path))) if not compact: report.append('') + else: + report.append('No saved files') report.append('') except TypeError as exception: log.warning(f'Error formatting the Markdown report: {exception!s}') @@ -433,7 +442,7 @@ def __init__(self, workers_status: dict, days: int): def generate_markdown(self) -> str: """Generates a Markdown version of tasks per worker. - + Returns: markdown (str): Markdown version of tasks per worker. """ @@ -486,7 +495,7 @@ def __init__(self, statistics: dict): def stat_to_row(self, task: str, stat_dict: dict): """Generates a row of the statistics table. - + Args: task (str): Name of the current task. stat_dict (dict): Dictionary with information about current row. @@ -502,8 +511,8 @@ def generate_data_frame(self) -> pandas.DataFrame: Args: markdown (bool): Bool defining if the tasks should be in markdown format. - - Returns: + + Returns: data_frame (DataFrame): Statistics table in pandas DataFrame format. """ for stat_group, stat_dict in self._statistics.items(): @@ -520,7 +529,7 @@ def generate_data_frame(self) -> pandas.DataFrame: def generate_markdown(self) -> str: """Generates a Markdown version of task statistics. - + Returns: markdown(str): Markdown version of task statistics. """ @@ -532,7 +541,7 @@ def generate_markdown(self) -> str: def generate_csv(self) -> str: """Generates a CSV version of task statistics. - + Returns: csv(str): CSV version of task statistics. """ From 51af080f1d170c2cebb1216becda0a41e0dc7417 Mon Sep 17 00:00:00 2001 From: Aaron Peterson Date: Thu, 5 Oct 2023 10:55:18 -0700 Subject: [PATCH 2/8] fix quotes --- turbinia/api/cli/turbinia_client/helpers/formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbinia/api/cli/turbinia_client/helpers/formatter.py b/turbinia/api/cli/turbinia_client/helpers/formatter.py index df35c853c..9d783630b 100644 --- a/turbinia/api/cli/turbinia_client/helpers/formatter.py +++ b/turbinia/api/cli/turbinia_client/helpers/formatter.py @@ -295,7 +295,7 @@ def generate_markdown( # Only show Task details if the Task has more priority than the # priority_filter if priority > priority_filter: - report.append(f'{self.heading2(name)}: {task.get('status')!s}') + report.append(f'{self.heading2(name)}: {task.get("status")!s}') else: report.append(self.heading2(name)) line = f"{self.bold('Evidence:'):s} {task.get('evidence_name')!s}" From 301754f12cfaf9f1d74063159101665e3880b97e Mon Sep 17 00:00:00 2001 From: Aaron Peterson Date: Sat, 7 Oct 2023 13:57:48 -0700 Subject: [PATCH 3/8] plumb through priority_filter --- turbinia/api/cli/turbinia_client/helpers/formatter.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/turbinia/api/cli/turbinia_client/helpers/formatter.py b/turbinia/api/cli/turbinia_client/helpers/formatter.py index 9d783630b..87da51ea6 100644 --- a/turbinia/api/cli/turbinia_client/helpers/formatter.py +++ b/turbinia/api/cli/turbinia_client/helpers/formatter.py @@ -281,6 +281,7 @@ def generate_markdown( priority = task.get('report_priority') if task.get( 'report_priority') else MEDIUM_PRIORITY + priority_filter = priority_filter if priority_filter else HIGH_PRIORITY if priority <= CRITICAL_PRIORITY: name = f'{task.get("name")} ({"CRITICAL PRIORITY"})' @@ -364,7 +365,7 @@ def add_components(self, components: list[MarkdownReportComponent]) -> None: self.components.append(component) component.parent = self - def generate_markdown(self, show_all=False) -> str: + def generate_markdown(self, priority_filter=None, show_all=False) -> str: """Generates a Markdown version of Requests results.""" report: list[str] = [] request_dict: dict = self._request_data @@ -400,7 +401,9 @@ def generate_markdown(self, show_all=False) -> str: log.warning(f'Error formatting the Markdown report: {exception!s}') for task in self.components: - report.append(task.generate_markdown(show_all=show_all, compact=True)) + report.append( + task.generate_markdown( + priority_filter=priority_filter, show_all=show_all, compact=True)) self.report = '\n'.join(report) return self.report From 2aa3810998b0567c7439d500be1b55e3ba890a54 Mon Sep 17 00:00:00 2001 From: Aaron Peterson Date: Sat, 7 Oct 2023 14:21:48 -0700 Subject: [PATCH 4/8] Set default to 80 --- turbinia/api/cli/turbinia_client/core/commands.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/turbinia/api/cli/turbinia_client/core/commands.py b/turbinia/api/cli/turbinia_client/core/commands.py index 454cbc09e..66f50ef84 100644 --- a/turbinia/api/cli/turbinia_client/core/commands.py +++ b/turbinia/api/cli/turbinia_client/core/commands.py @@ -127,8 +127,10 @@ def get_jobs(ctx: click.Context) -> None: 'shown in full detail in report output. Any tasks that have set a ' 'report_priority value equal to or lower than this setting will be ' 'shown in full detail, and tasks with a higher value will only have ' - 'a summary shown. To see all tasks report output in full detail, ' - 'set --priority_filter=100', is_flag=True, required=False) + 'a summary shown. The default is 20 which corresponds to "HIGH_PRIORITY"' + 'To see all tasks report output in full detail, set --priority_filter=100 ' + 'or to see CRITICAL only set --priority_filter=10', is_flag=True, type=int, + default=80, required=False) @click.option( '--show_all', '-a', help='Shows all fields including saved output paths.', is_flag=True, required=False) From 4af7a13ada981ff8ff5c0420233a6712a3086c51 Mon Sep 17 00:00:00 2001 From: Aaron Peterson Date: Sat, 7 Oct 2023 14:32:07 -0700 Subject: [PATCH 5/8] fix arg def --- turbinia/api/cli/turbinia_client/core/commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/turbinia/api/cli/turbinia_client/core/commands.py b/turbinia/api/cli/turbinia_client/core/commands.py index 66f50ef84..dcbac399b 100644 --- a/turbinia/api/cli/turbinia_client/core/commands.py +++ b/turbinia/api/cli/turbinia_client/core/commands.py @@ -129,8 +129,8 @@ def get_jobs(ctx: click.Context) -> None: 'shown in full detail, and tasks with a higher value will only have ' 'a summary shown. The default is 20 which corresponds to "HIGH_PRIORITY"' 'To see all tasks report output in full detail, set --priority_filter=100 ' - 'or to see CRITICAL only set --priority_filter=10', is_flag=True, type=int, - default=80, required=False) + 'or to see CRITICAL only set --priority_filter=10', show_default=True, + default=80, type=int, required=False) @click.option( '--show_all', '-a', help='Shows all fields including saved output paths.', is_flag=True, required=False) From 3c9e080b645476c73568d0fde56c14b9e974c257 Mon Sep 17 00:00:00 2001 From: Aaron Peterson Date: Sun, 8 Oct 2023 13:30:53 -0700 Subject: [PATCH 6/8] fix default --- turbinia/api/cli/turbinia_client/core/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbinia/api/cli/turbinia_client/core/commands.py b/turbinia/api/cli/turbinia_client/core/commands.py index dcbac399b..44ff60b74 100644 --- a/turbinia/api/cli/turbinia_client/core/commands.py +++ b/turbinia/api/cli/turbinia_client/core/commands.py @@ -130,7 +130,7 @@ def get_jobs(ctx: click.Context) -> None: 'a summary shown. The default is 20 which corresponds to "HIGH_PRIORITY"' 'To see all tasks report output in full detail, set --priority_filter=100 ' 'or to see CRITICAL only set --priority_filter=10', show_default=True, - default=80, type=int, required=False) + default=20, type=int, required=False) @click.option( '--show_all', '-a', help='Shows all fields including saved output paths.', is_flag=True, required=False) From 20c12c5d56d4545e0b95e608a18ab47aac3af6f9 Mon Sep 17 00:00:00 2001 From: Aaron Peterson Date: Sun, 8 Oct 2023 15:39:18 -0700 Subject: [PATCH 7/8] Sort/Uniq -c tasks --- .../cli/turbinia_client/helpers/formatter.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/turbinia/api/cli/turbinia_client/helpers/formatter.py b/turbinia/api/cli/turbinia_client/helpers/formatter.py index 87da51ea6..8cadbfc80 100644 --- a/turbinia/api/cli/turbinia_client/helpers/formatter.py +++ b/turbinia/api/cli/turbinia_client/helpers/formatter.py @@ -20,6 +20,7 @@ from abc import ABC, abstractmethod from click import echo as click_echo +from collections import defaultdict import logging import json import pandas @@ -345,10 +346,28 @@ def __init__(self, request_data: dict): self._request_data: dict = request_data sorted_tasks = sorted( - request_data.get('tasks'), key=lambda x: x['report_priority']) + request_data.get('tasks'), key=lambda x: + (x['report_priority'], x['name'])) tasks = [TaskMarkdownReport(task) for task in sorted_tasks] - self.add_components(tasks) + task_counter = defaultdict(int) + unique_tasks = [] + filtered_tasks = [] + + # Get unique tasks and task counts + for task in tasks: + task_counter[task] += 1 + if task not in unique_tasks: + unique_tasks.append(task) + + # Generate task list with counts + for task in unique_tasks: + if task_counter[task] > 1: + filtered_tasks.append(f'{task_counter[task]} x {task}') + else: + filtered_tasks.append(task) + + self.add_components(filtered_tasks) def add(self, component: MarkdownReportComponent) -> None: if component: From a26e1c11e067d923564e1c032e3ab3a30dcc55f7 Mon Sep 17 00:00:00 2001 From: Aaron Peterson Date: Wed, 25 Oct 2023 15:28:37 -0700 Subject: [PATCH 8/8] Update task count multiplier string --- turbinia/api/cli/turbinia_client/helpers/formatter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbinia/api/cli/turbinia_client/helpers/formatter.py b/turbinia/api/cli/turbinia_client/helpers/formatter.py index 8cadbfc80..bf90a3995 100644 --- a/turbinia/api/cli/turbinia_client/helpers/formatter.py +++ b/turbinia/api/cli/turbinia_client/helpers/formatter.py @@ -363,7 +363,7 @@ def __init__(self, request_data: dict): # Generate task list with counts for task in unique_tasks: if task_counter[task] > 1: - filtered_tasks.append(f'{task_counter[task]} x {task}') + filtered_tasks.append(f'{task} ({task_counter[task]}x)') else: filtered_tasks.append(task)