Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Extend SegmentReplayStrategy #806

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Extend SegmentReplayStrategy #806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
066bcaa
include_segmentations; include_segmentation_centroids
abrichr Jun 26, 2024
d8b8fee
remove include_segmentation_centroids
abrichr Jun 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions openadapt/strategies/segment.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
INCLUDE_WINDOW_DATA = False
FILTER_MASKS = True
INCLUDE_CURRENT_SCREENSHOT = False
INCLUDE_SEGMENTATIONS = True


class SegmentReplayStrategy(strategies.base.BaseReplayStrategy):
Expand Down Expand Up @@ -80,6 +81,7 @@ def get_next_action_event(
include_active_window_data: bool = INCLUDE_WINDOW_DATA,
include_replay_instructions: bool = INCLUDE_REPLAY_INSTRUCTIONS,
include_current_screenshot: bool = INCLUDE_CURRENT_SCREENSHOT,
include_segmentations: bool = INCLUDE_SEGMENTATIONS,
) -> models.ActionEvent | None:
"""Get the next ActionEvent for replay.

Expand All @@ -102,6 +104,8 @@ def get_next_action_event(
in the prompt.
include_current_screenshot (bool): Whether to include the current screenshot
in the prompt.
include_segmentations (bool): Whether to include window segmentations
in the prompt.

Returns:
models.ActionEvent or None: The next ActionEvent for replay or None
Expand Down Expand Up @@ -148,7 +152,8 @@ def get_next_action_event(
logger.info(f"{active_window=}")

if (
generated_action_event.name in common.MOUSE_EVENTS
include_segmentations
and generated_action_event.name in common.MOUSE_EVENTS
and generated_action_event.active_segment_description
):
generated_action_event.screenshot = active_screenshot
Expand All @@ -168,17 +173,18 @@ def get_next_action_event(
exceptions.append(exc)
# TODO XXX this does not update the prompts, even though it should
logger.exception(exc)
import ipdb

ipdb.set_trace()
import ipdb; ipdb.set_trace()
logger.warning(f"{exc=} {len(exceptions)=}")
else:
break
target_centroid = active_window_segmentation.centroids[target_segment_idx]

# <image space position> = scale_ratio * <window/action space position>
# TODO: move this
width_ratio, height_ratio = utils.get_scale_ratios(generated_action_event)
target_mouse_x = target_centroid[0] / width_ratio + active_window.left
target_mouse_y = target_centroid[1] / height_ratio + active_window.top

generated_action_event.mouse_x = target_mouse_x
generated_action_event.mouse_y = target_mouse_y
else:
Expand Down
3 changes: 2 additions & 1 deletion openadapt/strategies/vanilla.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,10 +198,11 @@ def generate_action_event(
replay_instructions=replay_instructions,
)
prompt_adapter = adapters.get_default_prompt_adapter()
images = [current_image]
content = prompt_adapter.prompt(
prompt,
system_prompt,
[current_image],
images,
)
action_dict = utils.parse_code_snippet(content)
logger.info(f"{action_dict=}")
Expand Down
Loading