use pygetwindow for window and pywinauto for a11y and display a11y on…

… dashboard

openadapt/app/dashboard/api/recordings.py

@@ -11,6 +11,7 @@
 from openadapt.models import Recording
 from openadapt.plotting import display_event
 from openadapt.utils import image2utf8, row2dict
+from openadapt.visualize import dict2html
 
 
 class RecordingsAPI:
@@ -145,10 +146,7 @@ def convert_to_str(event_dict: dict) -> dict:
             for action_event in action_events:
                 event_dict = row2dict(action_event)
                 a11y_dict = row2dict(action_event.window_event.a11y_event)
-                a11y_event_dict = a11y_dict["data"]
-                a11y_data_dict = a11y_event_dict["data"]
-                a11y_texts = extract_a11y_texts_value(a11y_data_dict, "texts", "Static")
-
+                event_dict["a11y_data"] = dict2html(a11y_dict)
                 try:
                     image = display_event(action_event)
                     width, height = image.size
@@ -170,8 +168,6 @@ def convert_to_str(event_dict: dict) -> dict:
                 ):
                     words.append(words_with_timestamps[word_index]["word"])
                     word_index += 1
-
-                event_dict["a11y_data"] = a11y_texts
                 event_dict["words"] = words
                 convert_to_str(event_dict)
                 await websocket.send_json({"type": "action_event", "value": event_dict})

openadapt/app/dashboard/components/ActionEvent/ActionEvent.tsx

@@ -5,6 +5,7 @@ import { ActionEvent as ActionEventType } from '@/types/action-event'
 import { Accordion, Box, Grid, Image, Table } from '@mantine/core'
 import { useHover } from '@mantine/hooks';
 import { RemoveActionEvent } from './RemoveActionEvent';
+import './style.css'; 
 
 type Props = {
     event: ActionEventType;
@@ -41,7 +42,7 @@ export const ActionEvent = ({
     let content = (
         <Grid align='center'>
             <Grid.Col span={8}>
-                <Table w={400} withTableBorder withColumnBorders my={20} className='border-2 border-gray-300 border-solid'>
+                <Table withTableBorder withColumnBorders my={20} className='border-2 border-gray-300 border-solid'>
                     <Table.Tbody>
                         {typeof event.id === 'number' && (
                             <TableRowWithBorder>
@@ -99,12 +100,6 @@ export const ActionEvent = ({
                                 <TableCellWithBorder>{event.text}</TableCellWithBorder>
                             </TableRowWithBorder>
                         )}
-                        {event.a11y_data && (
-                            <TableRowWithBorder>
-                                <TableCellWithBorder>a11y_data</TableCellWithBorder>
-                                <TableCellWithBorder>{event.a11y_data.join(", ")}</TableCellWithBorder>
-                            </TableRowWithBorder>
-                        )}
                         {event.canonical_text && (
                             <TableRowWithBorder>
                                 <TableCellWithBorder>canonical text</TableCellWithBorder>
@@ -142,6 +137,26 @@ export const ActionEvent = ({
                     </Table.Tbody>
                 </Table>
             </Grid.Col>
+            {event.a11y_data && (
+                <Grid.Col span={12}>
+                        <Accordion className="w-full">
+                            <Accordion.Item value="a11y_data">
+                                <Accordion.Control>
+                                    <div className="text-blue-500 w-full cursor-pointer">
+                                        Accessibility Data
+                                    </div>
+                                </Accordion.Control>
+                                <Accordion.Panel>
+                                    <Table  withTableBorder withColumnBorders my={20} className="border-2 border-gray-300 border-solid w-full">
+                                        <Table.Tbody>
+                                            <div className="table-nested border-2 border-gray-300 border-solid w-full" dangerouslySetInnerHTML={{ __html: event.a11y_data }} />
+                                        </Table.Tbody>
+                                    </Table>
+                                </Accordion.Panel>
+                            </Accordion.Item>
+                        </Accordion>   
+                </Grid.Col>
+            )}
             <Grid.Col span={4}>
                 <RemoveActionEvent event={event} />
             </Grid.Col>

openadapt/app/dashboard/components/ActionEvent/style.css

@@ -0,0 +1,20 @@
+.table-nested table {
+  border-collapse: collapse;
+  border: 1px solid #d1d5db;
+}
+
+.table-nested th,
+.table-nested td {
+  border: 1px solid #d1d5db;
+  padding: 8px; 
+  text-align: left;
+}
+
+.table-nested th {
+  background-color: #f9fafb;
+  font-weight: bold;
+}
+
+.table-nested td {
+  background-color: #ffffff;
+}

openadapt/app/dashboard/types/action-event.ts

@@ -27,7 +27,7 @@ export type ActionEvent = {
     dimensions?: { width: number, height: number };
     children?: ActionEvent[];
     words?: string[];
-    a11y_data: string[];
+    a11y_data: string;
     isComputed?: boolean;
     isOriginal?: boolean;
 }

openadapt/record.py

@@ -256,7 +256,6 @@ def process_events(
                 num_window_events.value += 1
                 prev_saved_window_timestamp = prev_window_event.timestamp
         elif event.type == "a11y":
-            logger.debug(f"Processing A11yEvent: {event}")
             try:
                 window_event = window_events_waiting_for_a11y.get_nowait()
             except queue.Empty as exc:

openadapt/window/_windows.py

@@ -4,6 +4,7 @@
 
 from loguru import logger
 import pywinauto
+import pygetwindow as gw
 
 
 def get_active_window_state(read_a11y_data: bool) -> dict:
@@ -19,35 +20,47 @@ def get_active_window_state(read_a11y_data: bool) -> dict:
                 - "height": Height of the active window.
                 - "meta": Meta information of the active window.
                 - "data": None (to be filled with window data).
-                - "window_id": ID of the active window.
+                - "handle": ID of the active window.
     """
     # catch specific exceptions, when except happens do log.warning
-    try:
-        active_window, handle = get_active_window()
-    except RuntimeError as e:
-        logger.warning(e)
-        return {}
-    meta = get_active_window_meta(active_window)
-    rectangle_dict = dictify_rect(meta["rectangle"])
     if read_a11y_data:
+        try:
+            active_window, handle = get_active_window()
+        except RuntimeError as e:
+            logger.warning(e)
+            return {}
+        meta = get_active_window_meta(active_window)
+        rectangle_dict = dictify_rect(meta["rectangle"])
         data = get_element_properties(active_window)
+        state = {
+            "title": meta["texts"][0],
+            "left": meta["rectangle"].left,
+            "top": meta["rectangle"].top,
+            "width": meta["rectangle"].width(),
+            "height": meta["rectangle"].height(),
+            "meta": {**meta, "rectangle": rectangle_dict},
+            "data": data,
+            "handle": handle,
+        }
+        try:
+            pickle.dumps(state)
+        except Exception as exc:
+            logger.warning(f"{exc=}")
+            state.pop("data")
     else:
-        data = {}
-    state = {
-        "title": meta["texts"][0],
-        "left": meta["rectangle"].left,
-        "top": meta["rectangle"].top,
-        "width": meta["rectangle"].width(),
-        "height": meta["rectangle"].height(),
-        "meta": {**meta, "rectangle": rectangle_dict},
-        "data": data,
-        "handle": handle,
-    }
-    try:
-        pickle.dumps(state)
-    except Exception as exc:
-        logger.warning(f"{exc=}")
-        state.pop("data")
+        try:
+            active_window = gw.getActiveWindow()
+        except RuntimeError as e:
+            logger.warning(e)
+            return {}
+        state = {
+            "title": active_window.title if active_window.title else "None",
+            "left": active_window.left,
+            "top": active_window.top,
+            "width": active_window.width,
+            "height": active_window.height,
+            "handle": active_window._hWnd,
+        }
     return state