From 3b0ada03445ffcdb8d312e0494bab25cb44b8544 Mon Sep 17 00:00:00 2001 From: Animesh404 Date: Mon, 29 Jul 2024 22:20:22 +0530 Subject: [PATCH] use pygetwindow for window and pywinauto for a11y and display a11y on dashboard --- openadapt/app/dashboard/api/recordings.py | 8 +-- .../components/ActionEvent/ActionEvent.tsx | 29 ++++++--- .../components/ActionEvent/style.css | 20 ++++++ openadapt/app/dashboard/types/action-event.ts | 2 +- openadapt/record.py | 1 - openadapt/window/_windows.py | 61 +++++++++++-------- 6 files changed, 82 insertions(+), 39 deletions(-) create mode 100644 openadapt/app/dashboard/components/ActionEvent/style.css diff --git a/openadapt/app/dashboard/api/recordings.py b/openadapt/app/dashboard/api/recordings.py index b252a020e..dec392d02 100644 --- a/openadapt/app/dashboard/api/recordings.py +++ b/openadapt/app/dashboard/api/recordings.py @@ -11,6 +11,7 @@ from openadapt.models import Recording from openadapt.plotting import display_event from openadapt.utils import image2utf8, row2dict +from openadapt.visualize import dict2html class RecordingsAPI: @@ -145,10 +146,7 @@ def convert_to_str(event_dict: dict) -> dict: for action_event in action_events: event_dict = row2dict(action_event) a11y_dict = row2dict(action_event.window_event.a11y_event) - a11y_event_dict = a11y_dict["data"] - a11y_data_dict = a11y_event_dict["data"] - a11y_texts = extract_a11y_texts_value(a11y_data_dict, "texts", "Static") - + event_dict["a11y_data"] = dict2html(a11y_dict) try: image = display_event(action_event) width, height = image.size @@ -170,8 +168,6 @@ def convert_to_str(event_dict: dict) -> dict: ): words.append(words_with_timestamps[word_index]["word"]) word_index += 1 - - event_dict["a11y_data"] = a11y_texts event_dict["words"] = words convert_to_str(event_dict) await websocket.send_json({"type": "action_event", "value": event_dict}) diff --git a/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx b/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx index 38b42f3a1..c368b0956 100644 --- a/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx +++ b/openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx @@ -5,6 +5,7 @@ import { ActionEvent as ActionEventType } from '@/types/action-event' import { Accordion, Box, Grid, Image, Table } from '@mantine/core' import { useHover } from '@mantine/hooks'; import { RemoveActionEvent } from './RemoveActionEvent'; +import './style.css'; type Props = { event: ActionEventType; @@ -41,7 +42,7 @@ export const ActionEvent = ({ let content = ( - +
{typeof event.id === 'number' && ( @@ -99,12 +100,6 @@ export const ActionEvent = ({ {event.text} )} - {event.a11y_data && ( - - a11y_data - {event.a11y_data.join(", ")} - - )} {event.canonical_text && ( canonical text @@ -142,6 +137,26 @@ export const ActionEvent = ({
+ {event.a11y_data && ( + + + + +
+ Accessibility Data +
+ + + + +
+ +
+ + + + + )} diff --git a/openadapt/app/dashboard/components/ActionEvent/style.css b/openadapt/app/dashboard/components/ActionEvent/style.css new file mode 100644 index 000000000..f04e4b186 --- /dev/null +++ b/openadapt/app/dashboard/components/ActionEvent/style.css @@ -0,0 +1,20 @@ +.table-nested table { + border-collapse: collapse; + border: 1px solid #d1d5db; +} + +.table-nested th, +.table-nested td { + border: 1px solid #d1d5db; + padding: 8px; + text-align: left; +} + +.table-nested th { + background-color: #f9fafb; + font-weight: bold; +} + +.table-nested td { + background-color: #ffffff; +} \ No newline at end of file diff --git a/openadapt/app/dashboard/types/action-event.ts b/openadapt/app/dashboard/types/action-event.ts index 663a82197..5e48b4b37 100644 --- a/openadapt/app/dashboard/types/action-event.ts +++ b/openadapt/app/dashboard/types/action-event.ts @@ -27,7 +27,7 @@ export type ActionEvent = { dimensions?: { width: number, height: number }; children?: ActionEvent[]; words?: string[]; - a11y_data: string[]; + a11y_data: string; isComputed?: boolean; isOriginal?: boolean; } diff --git a/openadapt/record.py b/openadapt/record.py index fdb3ecea8..0535b47e4 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -256,7 +256,6 @@ def process_events( num_window_events.value += 1 prev_saved_window_timestamp = prev_window_event.timestamp elif event.type == "a11y": - logger.debug(f"Processing A11yEvent: {event}") try: window_event = window_events_waiting_for_a11y.get_nowait() except queue.Empty as exc: diff --git a/openadapt/window/_windows.py b/openadapt/window/_windows.py index 7067833c8..d12e6d7a1 100644 --- a/openadapt/window/_windows.py +++ b/openadapt/window/_windows.py @@ -4,6 +4,7 @@ from loguru import logger import pywinauto +import pygetwindow as gw def get_active_window_state(read_a11y_data: bool) -> dict: @@ -19,35 +20,47 @@ def get_active_window_state(read_a11y_data: bool) -> dict: - "height": Height of the active window. - "meta": Meta information of the active window. - "data": None (to be filled with window data). - - "window_id": ID of the active window. + - "handle": ID of the active window. """ # catch specific exceptions, when except happens do log.warning - try: - active_window, handle = get_active_window() - except RuntimeError as e: - logger.warning(e) - return {} - meta = get_active_window_meta(active_window) - rectangle_dict = dictify_rect(meta["rectangle"]) if read_a11y_data: + try: + active_window, handle = get_active_window() + except RuntimeError as e: + logger.warning(e) + return {} + meta = get_active_window_meta(active_window) + rectangle_dict = dictify_rect(meta["rectangle"]) data = get_element_properties(active_window) + state = { + "title": meta["texts"][0], + "left": meta["rectangle"].left, + "top": meta["rectangle"].top, + "width": meta["rectangle"].width(), + "height": meta["rectangle"].height(), + "meta": {**meta, "rectangle": rectangle_dict}, + "data": data, + "handle": handle, + } + try: + pickle.dumps(state) + except Exception as exc: + logger.warning(f"{exc=}") + state.pop("data") else: - data = {} - state = { - "title": meta["texts"][0], - "left": meta["rectangle"].left, - "top": meta["rectangle"].top, - "width": meta["rectangle"].width(), - "height": meta["rectangle"].height(), - "meta": {**meta, "rectangle": rectangle_dict}, - "data": data, - "handle": handle, - } - try: - pickle.dumps(state) - except Exception as exc: - logger.warning(f"{exc=}") - state.pop("data") + try: + active_window = gw.getActiveWindow() + except RuntimeError as e: + logger.warning(e) + return {} + state = { + "title": active_window.title if active_window.title else "None", + "left": active_window.left, + "top": active_window.top, + "width": active_window.width, + "height": active_window.height, + "handle": active_window._hWnd, + } return state