From b6122e94cc978c5d2e860bf2c570c515090a7d43 Mon Sep 17 00:00:00 2001 From: kondoumizuki Date: Mon, 23 Dec 2019 20:04:09 +0900 Subject: [PATCH 1/9] =?UTF-8?q?=E4=BC=91=E6=86=A9=E3=81=8C=E5=83=8D?= =?UTF-8?q?=E3=81=84=E3=81=A6=E3=81=84=E3=81=AA=E3=81=84=E4=BA=BA=E7=99=BA?= =?UTF-8?q?=E8=A6=8B=E3=83=84=E3=83=BC=E3=83=AB=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- annofabcli/experimental/found_break_error.py | 185 ++++++++++++++++++ .../experimental/subcommand_experimental.py | 6 +- 2 files changed, 188 insertions(+), 3 deletions(-) create mode 100644 annofabcli/experimental/found_break_error.py diff --git a/annofabcli/experimental/found_break_error.py b/annofabcli/experimental/found_break_error.py new file mode 100644 index 00000000..195215bb --- /dev/null +++ b/annofabcli/experimental/found_break_error.py @@ -0,0 +1,185 @@ +# flake8: noqa +# type: ignore +# pylint: skip-file +import argparse +import datetime +import json +import logging +from typing import Any, Dict, List, Optional, Tuple # pylint: disable=unused-import + +import annofabapi +import dateutil.parser + +import annofabcli +import annofabcli.common.cli +from annofabcli import AnnofabApiFacade +from annofabcli.common.cli import AbstractCommandLineInterface, ArgumentParser, build_annofabapi_resource_and_login +from annofabcli.common.utils import read_lines_except_blank_line + +logger = logging.getLogger(__name__) + +TASK_STATUS = { + "not_started": "未着手", + "working": "作業中", # 誰かが実際にエディタ上で作業している状態。 + "on_hold": "保留,", # 作業ルールの確認などで作業できない状態。 + "break": "休憩中", + "complete": "完了", # 次のフェーズへ進む + "rejected": "差戻し", # 修正のため、annotationフェーズへ戻る。 + "cancelled": "提出取消し" # 修正のため、前フェーズへ戻る。 +} + + +class FoundBreakError(AbstractCommandLineInterface): + + def __init__(self, service: annofabapi.Resource, facade: AnnofabApiFacade, args: argparse.Namespace): + super().__init__(service, facade, args) + self.project_id = args.project_id + + def _get_username(self, account_id: Optional[str]) -> Optional[str]: + """ + プロジェクトメンバのusernameを取得する。プロジェクトメンバでなければ、account_idを返す。 + account_idがNoneならばNoneを返す。 + """ + if account_id is None: + return None + + member = self.facade.get_organization_member_from_account_id(self.project_id, account_id) + if member is not None: + return member["username"] + else: + return account_id + + def _get_all_tasks(self, project_id: str, task_query: str = None) -> List[Dict[str, Any]]: + """ + task一覧を取得する + """ + tasks = self.service.wrapper.get_all_tasks(project_id, query_params=task_query) + return tasks + + def _project_task_history_events(self, project_id: str, output: str, import_file: str = None): + """ + タスク履歴イベント全件ファイルを取得する。 + import_fileがTrue:outputで指定されたパスから読み込む + import_fileがFalse:outputで指定されたパスへ一旦保存し、読み込む + """ + if not import_file: + self.service.wrapper.download_project_task_history_events_url(project_id, output) + try: + history_events = read_lines_except_blank_line(output) + project_task_history_events = json.loads(history_events[0]) + except: + logger.warning(f"ファイル '{output}' は読み込めませんでした。") + else: + try: + history_events = read_lines_except_blank_line(import_file) + project_task_history_events = json.loads(history_events[0]) + except: + logger.warning(f"ファイル '{import_file}' は読み込めませんでした。") + + return project_task_history_events + + def found_err_task(self, tasks: List[Dict[str, Any]]) -> List[str]: + """ + タスクリストから作業時間合計がしきい値以上のタスクだけを返す + """ + return [task["task_id"] for task in tasks if task["work_time_span"] > (self.args.task_time_threshold * 60000)] + + def get_err_history_events(self, task_list: List[str], task_history_events: List[Dict[str, Any]]) -> Dict[ + str, List[Dict[str, Any]]]: + """ + しきい値以上のタスクリストのtask_idが含まれるhistory_eventsを返す + """ + err_history_events_dict = {} + for task_history_event in task_history_events: + if task_history_event["task_id"] in task_list: + if task_history_event["task_id"] in err_history_events_dict: + err_history_events_dict[task_history_event["task_id"]].append(task_history_event) + else: + err_history_events_dict[task_history_event["task_id"]] = [task_history_event] + return err_history_events_dict + + def get_err_events(self, err_history_events: Dict[str, List[Dict[str, Any]]]): + """ + しきい値以上の作業時間になっている開始と終了のhistory_eventsのペアを返す + """ + err_events_list = [] + for k, v in err_history_events.items(): + v.sort(key=lambda x: x["created_datetime"]) + for i, history_events in enumerate(v): + if history_events["status"] == "working": + if v[i + 1]["status"] in ["on_hold", "break", "complete"]: + working_time = dateutil.parser.parse(v[i + 1]["created_datetime"]) - dateutil.parser.parse( + history_events["created_datetime"]) + if working_time > datetime.timedelta(minutes=self.args.task_history_time_threshold): + err_events_list.append((history_events, v[i + 1])) + return err_events_list + + def output_err_events(self, err_events_list: List[Tuple[Dict[str, Any], Dict[str, Any]]], output: str = None): + """ + 開始と終了のhistory_eventsのペアから出力する + :param err_events_list: + :param output: + :return: + """ + + def _timedelta_to_HM(td: datetime.timedelta): + sec = td.total_seconds() + return str(sec // 3600) + "時間" + str(sec % 3600 // 60) + "分" + + output_lines: List[str] = [] + output_lines.append(f"task_id,フェーズ,担当者,開始日時,完了日時,実作業時間") + for start_data, end_data in err_events_list: + task_id = start_data["task_id"] + phase = str(start_data["phase"]) + username = self._get_username(start_data["account_id"]) + start_time = dateutil.parser.parse(start_data["created_datetime"]).strftime('%Y/%m/%d %H:%M:%S') + end_time = dateutil.parser.parse(end_data["created_datetime"]).strftime('%Y/%m/%d %H:%M:%S') + working_time = _timedelta_to_HM(dateutil.parser.parse(end_data["created_datetime"]) - dateutil.parser.parse( + start_data["created_datetime"])) + output_lines.append(",".join([task_id, phase, username, start_time, end_time, working_time])) + annofabcli.utils.output_string("\n".join(output_lines), output) + + def main(self): + args = self.args + tasks = self._get_all_tasks(project_id=args.project_id) + task_history_events = self._project_task_history_events(project_id=args.project_id, + output=args.task_history_events_path, + import_file=args.import_file) + err_task_list = self.found_err_task(tasks) + err_history_events = self.get_err_history_events(task_list=err_task_list, + task_history_events=task_history_events) + err_events = self.get_err_events(err_history_events=err_history_events) + self.output_err_events(err_events_list=err_events, output=self.output) + + +def main(args): + service = build_annofabapi_resource_and_login() + facade = AnnofabApiFacade(service) + FoundBreakError(service, facade, args).main() + + +def parse_args(parser: argparse.ArgumentParser): + argument_parser = ArgumentParser(parser) + + parser.add_argument('--task_history_events_path', type=str, default="task_history_events.txt", + help="タスク履歴イベント全件ファイルを保存するパス。指定しない場合カレントの'task_history_events.txt'に保存する") + parser.add_argument('--task_time_threshold', type=int, default=600, + help="1タスク何分以上を検知対象とするか。指定しない場合は600分(10時間)") + parser.add_argument('--task_history_time_threshold', type=int, default=300, + help="1履歴何分以上を検知対象とするか。指定しない場合は300分(5時間)") + parser.add_argument('--import_file', type=str, default=None, + help="importするタスク履歴イベント全件ファイル") + + argument_parser.add_output() + argument_parser.add_project_id() + + parser.set_defaults(subcommand_func=main) + + +def add_parser(subparsers: argparse._SubParsersAction): + subcommand_name = "found_break_error" + subcommand_help = "不当に長い作業時間の作業履歴を出力します" + description = ("自動休憩が作動せず不当に長い作業時間になっている履歴を出力します。") + + parser = annofabcli.common.cli.add_parser(subparsers, subcommand_name, subcommand_help, description) + parse_args(parser) diff --git a/annofabcli/experimental/subcommand_experimental.py b/annofabcli/experimental/subcommand_experimental.py index 9f8a8a5c..540133fc 100644 --- a/annofabcli/experimental/subcommand_experimental.py +++ b/annofabcli/experimental/subcommand_experimental.py @@ -2,15 +2,15 @@ import annofabcli import annofabcli.common.cli -import annofabcli.experimental.list_labor_worktime +from annofabcli.experimental import list_labor_worktime, found_break_error def parse_args(parser: argparse.ArgumentParser): - subparsers = parser.add_subparsers(dest='subcommand_name') # サブコマンドの定義 - annofabcli.experimental.list_labor_worktime.add_parser(subparsers) # type: ignore + list_labor_worktime.add_parser(subparsers) # type: ignore + found_break_error.add_parser(subparsers) # type: ignore def add_parser(subparsers: argparse._SubParsersAction): From 10a881e7c02303aba76d2cc99eeb3bfa40d383f1 Mon Sep 17 00:00:00 2001 From: kondoumizuki Date: Tue, 24 Dec 2019 13:44:33 +0900 Subject: [PATCH 2/9] =?UTF-8?q?=E5=90=8D=E7=A7=B0=E3=82=92found=E2=87=92fi?= =?UTF-8?q?nd=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{found_break_error.py => find_break_error.py} | 4 ++-- annofabcli/experimental/subcommand_experimental.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename annofabcli/experimental/{found_break_error.py => find_break_error.py} (98%) diff --git a/annofabcli/experimental/found_break_error.py b/annofabcli/experimental/find_break_error.py similarity index 98% rename from annofabcli/experimental/found_break_error.py rename to annofabcli/experimental/find_break_error.py index 195215bb..47c1d093 100644 --- a/annofabcli/experimental/found_break_error.py +++ b/annofabcli/experimental/find_break_error.py @@ -29,7 +29,7 @@ } -class FoundBreakError(AbstractCommandLineInterface): +class FindBreakError(AbstractCommandLineInterface): def __init__(self, service: annofabapi.Resource, facade: AnnofabApiFacade, args: argparse.Namespace): super().__init__(service, facade, args) @@ -155,7 +155,7 @@ def main(self): def main(args): service = build_annofabapi_resource_and_login() facade = AnnofabApiFacade(service) - FoundBreakError(service, facade, args).main() + FindBreakError(service, facade, args).main() def parse_args(parser: argparse.ArgumentParser): diff --git a/annofabcli/experimental/subcommand_experimental.py b/annofabcli/experimental/subcommand_experimental.py index 540133fc..8e5c0fc7 100644 --- a/annofabcli/experimental/subcommand_experimental.py +++ b/annofabcli/experimental/subcommand_experimental.py @@ -2,7 +2,7 @@ import annofabcli import annofabcli.common.cli -from annofabcli.experimental import list_labor_worktime, found_break_error +from annofabcli.experimental import list_labor_worktime, find_break_error def parse_args(parser: argparse.ArgumentParser): @@ -10,7 +10,7 @@ def parse_args(parser: argparse.ArgumentParser): # サブコマンドの定義 list_labor_worktime.add_parser(subparsers) # type: ignore - found_break_error.add_parser(subparsers) # type: ignore + find_break_error.add_parser(subparsers) # type: ignore def add_parser(subparsers: argparse._SubParsersAction): From 473e25440b094d5df1af310cfcff4965f18d1f68 Mon Sep 17 00:00:00 2001 From: kondoumizuki Date: Wed, 25 Dec 2019 18:03:05 +0900 Subject: [PATCH 3/9] =?UTF-8?q?args=E3=81=AE=E6=8C=87=E5=AE=9A=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E5=A4=89=E6=9B=B4=E3=83=BBhistory=5Fevents=5Furl?= =?UTF-8?q?=E3=81=8B=E3=82=89=E7=9B=B4=E6=8E=A5=E8=AA=AD=E3=81=BF=E8=BE=BC?= =?UTF-8?q?=E3=82=80=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- annofabcli/experimental/find_break_error.py | 41 +++++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/annofabcli/experimental/find_break_error.py b/annofabcli/experimental/find_break_error.py index 47c1d093..cd3b241a 100644 --- a/annofabcli/experimental/find_break_error.py +++ b/annofabcli/experimental/find_break_error.py @@ -5,11 +5,12 @@ import datetime import json import logging +import tempfile from typing import Any, Dict, List, Optional, Tuple # pylint: disable=unused-import import annofabapi import dateutil.parser - +import requests import annofabcli import annofabcli.common.cli from annofabcli import AnnofabApiFacade @@ -27,6 +28,16 @@ "rejected": "差戻し", # 修正のため、annotationフェーズへ戻る。 "cancelled": "提出取消し" # 修正のため、前フェーズへ戻る。 } +def download_content(url: str)->Any: + """ + HTTP GETで取得した内容を保存せずそのまま返す + + Args: + url: ダウンロード対象のURL + """ + response = requests.get(url) + annofabapi.utils.raise_for_status(response) + return response.content class FindBreakError(AbstractCommandLineInterface): @@ -56,25 +67,26 @@ def _get_all_tasks(self, project_id: str, task_query: str = None) -> List[Dict[s tasks = self.service.wrapper.get_all_tasks(project_id, query_params=task_query) return tasks - def _project_task_history_events(self, project_id: str, output: str, import_file: str = None): + def _project_task_history_events(self, project_id: str, import_file_path: Optional[str] = None): """ タスク履歴イベント全件ファイルを取得する。 - import_fileがTrue:outputで指定されたパスから読み込む - import_fileがFalse:outputで指定されたパスへ一旦保存し、読み込む + import_fileがNone:history_events_urlを取得してパスから読み込む + import_fileがNoneではない:import_fileで指定されたパスへ一旦保存し、読み込む """ - if not import_file: - self.service.wrapper.download_project_task_history_events_url(project_id, output) + if import_file_path is None: + content, _ = self.service.wrapper.api.get_project_task_history_events_url(project_id=project_id) + url = content["url"] try: - history_events = read_lines_except_blank_line(output) - project_task_history_events = json.loads(history_events[0]) + history_events = download_content(url) + project_task_history_events = json.loads(history_events) except: - logger.warning(f"ファイル '{output}' は読み込めませんでした。") + logger.warning(f"history_events_urlを読み込めませんでした。") else: try: - history_events = read_lines_except_blank_line(import_file) + history_events = read_lines_except_blank_line(import_file_path) project_task_history_events = json.loads(history_events[0]) except: - logger.warning(f"ファイル '{import_file}' は読み込めませんでした。") + logger.warning(f"ファイル '{import_file_path}' は読み込めませんでした。") return project_task_history_events @@ -143,8 +155,7 @@ def main(self): args = self.args tasks = self._get_all_tasks(project_id=args.project_id) task_history_events = self._project_task_history_events(project_id=args.project_id, - output=args.task_history_events_path, - import_file=args.import_file) + import_file_path=args.import_file_path) err_task_list = self.found_err_task(tasks) err_history_events = self.get_err_history_events(task_list=err_task_list, task_history_events=task_history_events) @@ -161,13 +172,11 @@ def main(args): def parse_args(parser: argparse.ArgumentParser): argument_parser = ArgumentParser(parser) - parser.add_argument('--task_history_events_path', type=str, default="task_history_events.txt", - help="タスク履歴イベント全件ファイルを保存するパス。指定しない場合カレントの'task_history_events.txt'に保存する") parser.add_argument('--task_time_threshold', type=int, default=600, help="1タスク何分以上を検知対象とするか。指定しない場合は600分(10時間)") parser.add_argument('--task_history_time_threshold', type=int, default=300, help="1履歴何分以上を検知対象とするか。指定しない場合は300分(5時間)") - parser.add_argument('--import_file', type=str, default=None, + parser.add_argument('--import_file_path', type=str, default=None, help="importするタスク履歴イベント全件ファイル") argument_parser.add_output() From dc57fbfc06d552281d0a2e09febf8578ef27809c Mon Sep 17 00:00:00 2001 From: kondoumizuki Date: Wed, 25 Dec 2019 18:58:48 +0900 Subject: [PATCH 4/9] =?UTF-8?q?lint=E9=80=9A=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- annofabcli/experimental/find_break_error.py | 73 ++++++++++----------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/annofabcli/experimental/find_break_error.py b/annofabcli/experimental/find_break_error.py index cd3b241a..707e756d 100644 --- a/annofabcli/experimental/find_break_error.py +++ b/annofabcli/experimental/find_break_error.py @@ -1,16 +1,13 @@ -# flake8: noqa -# type: ignore -# pylint: skip-file import argparse import datetime import json import logging -import tempfile from typing import Any, Dict, List, Optional, Tuple # pylint: disable=unused-import import annofabapi import dateutil.parser import requests + import annofabcli import annofabcli.common.cli from annofabcli import AnnofabApiFacade @@ -19,16 +16,8 @@ logger = logging.getLogger(__name__) -TASK_STATUS = { - "not_started": "未着手", - "working": "作業中", # 誰かが実際にエディタ上で作業している状態。 - "on_hold": "保留,", # 作業ルールの確認などで作業できない状態。 - "break": "休憩中", - "complete": "完了", # 次のフェーズへ進む - "rejected": "差戻し", # 修正のため、annotationフェーズへ戻る。 - "cancelled": "提出取消し" # 修正のため、前フェーズへ戻る。 -} -def download_content(url: str)->Any: + +def download_content(url: str) -> Any: """ HTTP GETで取得した内容を保存せずそのまま返す @@ -40,6 +29,21 @@ def download_content(url: str)->Any: return response.content +def get_err_history_events(task_list: List[str], task_history_events: List[Dict[str, Any]]) \ + -> Dict[str, List[Dict[str, Any]]]: + """ + しきい値以上のタスクリストのtask_idが含まれるhistory_eventsを返す + """ + err_history_events_dict: Dict[str, List[Dict[str, Any]]] = {} + for task_history_event in task_history_events: + if task_history_event["task_id"] in task_list: + if task_history_event["task_id"] in err_history_events_dict: + err_history_events_dict[task_history_event["task_id"]].append(task_history_event) + else: + err_history_events_dict[task_history_event["task_id"]] = [task_history_event] + return err_history_events_dict + + class FindBreakError(AbstractCommandLineInterface): def __init__(self, service: annofabapi.Resource, facade: AnnofabApiFacade, args: argparse.Namespace): @@ -67,26 +71,30 @@ def _get_all_tasks(self, project_id: str, task_query: str = None) -> List[Dict[s tasks = self.service.wrapper.get_all_tasks(project_id, query_params=task_query) return tasks - def _project_task_history_events(self, project_id: str, import_file_path: Optional[str] = None): + def _project_task_history_events(self, project_id: str, import_file_path: Optional[str] = None) \ + -> List[Dict[str, Any]]: """ タスク履歴イベント全件ファイルを取得する。 - import_fileがNone:history_events_urlを取得してパスから読み込む - import_fileがNoneではない:import_fileで指定されたパスへ一旦保存し、読み込む + import_fileがNone:history_events_urlパスから直接読み込む + import_fileがNoneではない:import_fileで指定されたファイルから読み込む """ + project_task_history_events: List[Dict[str, Any]] = [] if import_file_path is None: - content, _ = self.service.wrapper.api.get_project_task_history_events_url(project_id=project_id) - url = content["url"] try: + content, _ = self.service.wrapper.api.get_project_task_history_events_url(project_id=project_id) + url = content["url"] history_events = download_content(url) project_task_history_events = json.loads(history_events) - except: + except NameError: logger.warning(f"history_events_urlを読み込めませんでした。") + except requests.exceptions.HTTPError: + logger.warning(f"history_eventsをurlから読み込めませんでした。") else: try: history_events = read_lines_except_blank_line(import_file_path) project_task_history_events = json.loads(history_events[0]) - except: - logger.warning(f"ファイル '{import_file_path}' は読み込めませんでした。") + except FileNotFoundError: + logger.warning(f"ファイル '{import_file_path}' は存在しません。") return project_task_history_events @@ -96,26 +104,12 @@ def found_err_task(self, tasks: List[Dict[str, Any]]) -> List[str]: """ return [task["task_id"] for task in tasks if task["work_time_span"] > (self.args.task_time_threshold * 60000)] - def get_err_history_events(self, task_list: List[str], task_history_events: List[Dict[str, Any]]) -> Dict[ - str, List[Dict[str, Any]]]: - """ - しきい値以上のタスクリストのtask_idが含まれるhistory_eventsを返す - """ - err_history_events_dict = {} - for task_history_event in task_history_events: - if task_history_event["task_id"] in task_list: - if task_history_event["task_id"] in err_history_events_dict: - err_history_events_dict[task_history_event["task_id"]].append(task_history_event) - else: - err_history_events_dict[task_history_event["task_id"]] = [task_history_event] - return err_history_events_dict - def get_err_events(self, err_history_events: Dict[str, List[Dict[str, Any]]]): """ しきい値以上の作業時間になっている開始と終了のhistory_eventsのペアを返す """ err_events_list = [] - for k, v in err_history_events.items(): + for v in err_history_events.values(): v.sort(key=lambda x: x["created_datetime"]) for i, history_events in enumerate(v): if history_events["status"] == "working": @@ -148,7 +142,8 @@ def _timedelta_to_HM(td: datetime.timedelta): end_time = dateutil.parser.parse(end_data["created_datetime"]).strftime('%Y/%m/%d %H:%M:%S') working_time = _timedelta_to_HM(dateutil.parser.parse(end_data["created_datetime"]) - dateutil.parser.parse( start_data["created_datetime"])) - output_lines.append(",".join([task_id, phase, username, start_time, end_time, working_time])) + output_lines.append( + ",".join([task_id, phase, "" if username is None else username, start_time, end_time, working_time])) annofabcli.utils.output_string("\n".join(output_lines), output) def main(self): @@ -177,7 +172,7 @@ def parse_args(parser: argparse.ArgumentParser): parser.add_argument('--task_history_time_threshold', type=int, default=300, help="1履歴何分以上を検知対象とするか。指定しない場合は300分(5時間)") parser.add_argument('--import_file_path', type=str, default=None, - help="importするタスク履歴イベント全件ファイル") + help="importするタスク履歴イベント全件ファイル,指定しない場合はタスク履歴イベント全件を新規取得する") argument_parser.add_output() argument_parser.add_project_id() From 0eb9f877b711d5497725b3a41b411750c1d02bd4 Mon Sep 17 00:00:00 2001 From: kondoumizuki Date: Thu, 26 Dec 2019 17:10:45 +0900 Subject: [PATCH 5/9] =?UTF-8?q?=E5=87=BA=E5=8A=9B=E3=82=92pandas=E3=82=92?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- annofabcli/experimental/find_break_error.py | 23 +++++++++++---------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/annofabcli/experimental/find_break_error.py b/annofabcli/experimental/find_break_error.py index 707e756d..845d16c2 100644 --- a/annofabcli/experimental/find_break_error.py +++ b/annofabcli/experimental/find_break_error.py @@ -6,6 +6,7 @@ import annofabapi import dateutil.parser +import pandas as pd import requests import annofabcli @@ -64,7 +65,7 @@ def _get_username(self, account_id: Optional[str]) -> Optional[str]: else: return account_id - def _get_all_tasks(self, project_id: str, task_query: str = None) -> List[Dict[str, Any]]: + def _get_all_tasks(self, project_id: str, task_query: Optional[Dict[str, Any]] = None) -> List[Dict[str, Any]]: """ task一覧を取得する """ @@ -130,21 +131,21 @@ def output_err_events(self, err_events_list: List[Tuple[Dict[str, Any], Dict[str def _timedelta_to_HM(td: datetime.timedelta): sec = td.total_seconds() - return str(sec // 3600) + "時間" + str(sec % 3600 // 60) + "分" + return str(round(sec // 3600)) + "時間" + str(round(sec % 3600 // 60)) + "分" - output_lines: List[str] = [] - output_lines.append(f"task_id,フェーズ,担当者,開始日時,完了日時,実作業時間") + dict_list = [] for start_data, end_data in err_events_list: task_id = start_data["task_id"] phase = str(start_data["phase"]) username = self._get_username(start_data["account_id"]) - start_time = dateutil.parser.parse(start_data["created_datetime"]).strftime('%Y/%m/%d %H:%M:%S') - end_time = dateutil.parser.parse(end_data["created_datetime"]).strftime('%Y/%m/%d %H:%M:%S') + start_time = start_data["created_datetime"] + end_time = end_data["created_datetime"] working_time = _timedelta_to_HM(dateutil.parser.parse(end_data["created_datetime"]) - dateutil.parser.parse( start_data["created_datetime"])) - output_lines.append( - ",".join([task_id, phase, "" if username is None else username, start_time, end_time, working_time])) - annofabcli.utils.output_string("\n".join(output_lines), output) + dict_list.append({"task_id": task_id, "phase": phase, "username": "" if username is None else username, + "start_time": start_time, "end_time": end_time, "working_time": working_time}) + + annofabcli.utils.print_csv(pd.DataFrame(dict_list), output=output) def main(self): args = self.args @@ -152,8 +153,8 @@ def main(self): task_history_events = self._project_task_history_events(project_id=args.project_id, import_file_path=args.import_file_path) err_task_list = self.found_err_task(tasks) - err_history_events = self.get_err_history_events(task_list=err_task_list, - task_history_events=task_history_events) + err_history_events = get_err_history_events(task_list=err_task_list, + task_history_events=task_history_events) err_events = self.get_err_events(err_history_events=err_history_events) self.output_err_events(err_events_list=err_events, output=self.output) From bf5cb8aa2a3362e7c98517c893dcda9091fbbd4c Mon Sep 17 00:00:00 2001 From: kondoumizuki Date: Thu, 26 Dec 2019 17:13:20 +0900 Subject: [PATCH 6/9] =?UTF-8?q?to=5Fcsv=5Fkwargs=20=3D=20None=20=E3=81=AE?= =?UTF-8?q?=E5=A0=B4=E5=90=88=E3=81=AB**=E3=81=A7=E5=B1=95=E9=96=8B?= =?UTF-8?q?=E5=87=BA=E6=9D=A5=E3=81=9A=E3=81=AB=E6=AD=BB=E3=81=AC=E3=81=A8?= =?UTF-8?q?=E3=81=93=E3=82=8D=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- annofabcli/common/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/annofabcli/common/utils.py b/annofabcli/common/utils.py index 38c27fb2..4f2b519c 100644 --- a/annofabcli/common/utils.py +++ b/annofabcli/common/utils.py @@ -131,7 +131,11 @@ def print_csv(df: pandas.DataFrame, output: Optional[str] = None, to_csv_kwargs: Path(output).parent.mkdir(parents=True, exist_ok=True) path_or_buf = sys.stdout if output is None else output - df.to_csv(path_or_buf, **to_csv_kwargs) + + if to_csv_kwargs is None: + df.to_csv(path_or_buf) + else: + df.to_csv(path_or_buf, **to_csv_kwargs) def print_id_list(id_list: List[Any], output: Optional[str]): From 44199fa9199cfd66c2d4c3ddef77de60bb3710b5 Mon Sep 17 00:00:00 2001 From: kondoumizuki Date: Thu, 26 Dec 2019 18:42:57 +0900 Subject: [PATCH 7/9] =?UTF-8?q?pandas=E3=81=AE=E5=87=BA=E5=8A=9B=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E3=82=92=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- annofabcli/experimental/find_break_error.py | 40 ++++++++++++++------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/annofabcli/experimental/find_break_error.py b/annofabcli/experimental/find_break_error.py index 845d16c2..7a59853a 100644 --- a/annofabcli/experimental/find_break_error.py +++ b/annofabcli/experimental/find_break_error.py @@ -133,19 +133,35 @@ def _timedelta_to_HM(td: datetime.timedelta): sec = td.total_seconds() return str(round(sec // 3600)) + "時間" + str(round(sec % 3600 // 60)) + "分" - dict_list = [] - for start_data, end_data in err_events_list: - task_id = start_data["task_id"] - phase = str(start_data["phase"]) + data_list = [] + for i, data in enumerate(err_events_list): + start_data, end_data = data username = self._get_username(start_data["account_id"]) - start_time = start_data["created_datetime"] - end_time = end_data["created_datetime"] - working_time = _timedelta_to_HM(dateutil.parser.parse(end_data["created_datetime"]) - dateutil.parser.parse( - start_data["created_datetime"])) - dict_list.append({"task_id": task_id, "phase": phase, "username": "" if username is None else username, - "start_time": start_time, "end_time": end_time, "working_time": working_time}) - - annofabcli.utils.print_csv(pd.DataFrame(dict_list), output=output) + start_data["user_name"] = "" if username is None else username + end_data["user_name"] = "" if username is None else username + start_time = dateutil.parser.parse(start_data["created_datetime"]) + end_time = dateutil.parser.parse(end_data["created_datetime"]) + start_data["datetime"] = start_time.isoformat() + end_data["datetime"] = end_time.isoformat() + start_data["working_time"] = "" + end_data["working_time"] = \ + _timedelta_to_HM(end_time - start_time) + + df = pd.DataFrame([start_data, end_data]) + df["no"] = i + 1 + + del df["project_id"] + del df["phase_stage"] + del df["account_id"] + del df["created_datetime"] + data_list.append(df) + + pd_data = pd.concat(data_list) + pd_data.set_index(["no", "task_id"]) + + annofabcli.utils.print_csv( + pd_data[["no", "task_id", "user_name", "phase", "status", "datetime", "task_history_id", "working_time"]], + output=output) def main(self): args = self.args From 91b49acccdb4b230f81523261f0b13ad3420fc98 Mon Sep 17 00:00:00 2001 From: kondoumizuki Date: Thu, 26 Dec 2019 18:49:51 +0900 Subject: [PATCH 8/9] =?UTF-8?q?index=E3=82=92=E9=9D=9E=E8=A1=A8=E7=A4=BA?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- annofabcli/experimental/find_break_error.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/annofabcli/experimental/find_break_error.py b/annofabcli/experimental/find_break_error.py index 7a59853a..ff80ba65 100644 --- a/annofabcli/experimental/find_break_error.py +++ b/annofabcli/experimental/find_break_error.py @@ -161,7 +161,8 @@ def _timedelta_to_HM(td: datetime.timedelta): annofabcli.utils.print_csv( pd_data[["no", "task_id", "user_name", "phase", "status", "datetime", "task_history_id", "working_time"]], - output=output) + output=output, + to_csv_kwargs={"index": False}) def main(self): args = self.args From 11a53aaae0d5415c7868aa8ee33bacb43eec39d5 Mon Sep 17 00:00:00 2001 From: kondoumizuki Date: Fri, 27 Dec 2019 14:54:46 +0900 Subject: [PATCH 9/9] =?UTF-8?q?try=20catch=20=E3=82=92=E3=81=AA=E3=81=8F?= =?UTF-8?q?=E3=81=97=E3=81=A6=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=AE?= =?UTF-8?q?=E5=AD=98=E5=9C=A8=E7=A2=BA=E8=AA=8D=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- annofabcli/experimental/find_break_error.py | 39 +++++++++++++-------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/annofabcli/experimental/find_break_error.py b/annofabcli/experimental/find_break_error.py index ff80ba65..176176be 100644 --- a/annofabcli/experimental/find_break_error.py +++ b/annofabcli/experimental/find_break_error.py @@ -2,6 +2,8 @@ import datetime import json import logging +import sys +from pathlib import Path from typing import Any, Dict, List, Optional, Tuple # pylint: disable=unused-import import annofabapi @@ -81,21 +83,14 @@ def _project_task_history_events(self, project_id: str, import_file_path: Option """ project_task_history_events: List[Dict[str, Any]] = [] if import_file_path is None: - try: - content, _ = self.service.wrapper.api.get_project_task_history_events_url(project_id=project_id) - url = content["url"] - history_events = download_content(url) - project_task_history_events = json.loads(history_events) - except NameError: - logger.warning(f"history_events_urlを読み込めませんでした。") - except requests.exceptions.HTTPError: - logger.warning(f"history_eventsをurlから読み込めませんでした。") + content, _ = self.service.wrapper.api.get_project_task_history_events_url(project_id=project_id) + url = content["url"] + history_events = download_content(url) + project_task_history_events = json.loads(history_events) + else: - try: - history_events = read_lines_except_blank_line(import_file_path) - project_task_history_events = json.loads(history_events[0]) - except FileNotFoundError: - logger.warning(f"ファイル '{import_file_path}' は存在しません。") + history_events = read_lines_except_blank_line(import_file_path) + project_task_history_events = json.loads(history_events[0]) return project_task_history_events @@ -164,8 +159,22 @@ def _timedelta_to_HM(td: datetime.timedelta): output=output, to_csv_kwargs={"index": False}) + @staticmethod + def validate(args: argparse.Namespace) -> bool: + COMMON_MESSAGE = "annofabcli experimental find_break_error: error:" + if args.import_file_path is not None: + if not Path(args.import_file_path).is_file(): + print(f"{COMMON_MESSAGE} argument --import_file_path: ファイルパスが存在しません。 '{args.import_file_path}'", + file=sys.stderr) + return False + + return True + def main(self): args = self.args + if not self.validate(args): + return + tasks = self._get_all_tasks(project_id=args.project_id) task_history_events = self._project_task_history_events(project_id=args.project_id, import_file_path=args.import_file_path) @@ -189,7 +198,7 @@ def parse_args(parser: argparse.ArgumentParser): help="1タスク何分以上を検知対象とするか。指定しない場合は600分(10時間)") parser.add_argument('--task_history_time_threshold', type=int, default=300, help="1履歴何分以上を検知対象とするか。指定しない場合は300分(5時間)") - parser.add_argument('--import_file_path', type=str, default=None, + parser.add_argument('--import_file_path', type=str, help="importするタスク履歴イベント全件ファイル,指定しない場合はタスク履歴イベント全件を新規取得する") argument_parser.add_output()