feat: added io toolkit to control mouse and keyboard

pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
     "ai-exchange>=0.9.2",
     "click>=8.1.7",
     "prompt-toolkit>=3.0.47",
+    "pyautogui>=0.9.53"
 ]
 author = [{ name = "Block", email = "ai-oss-tools@block.xyz" }]
 packages = [{ include = "goose", from = "src" }]
@@ -26,6 +27,7 @@ developer = "goose.toolkit.developer:Developer"
 github = "goose.toolkit.github:Github"
 jira = "goose.toolkit.jira:Jira"
 screen = "goose.toolkit.screen:Screen"
+io = "goose.toolkit.io:IO"
 repo_context = "goose.toolkit.repo_context.repo_context:RepoContext"
 
 [project.entry-points."goose.profile"]

src/goose/toolkit/io.py

@@ -0,0 +1,52 @@
+import pyautogui
+
+from goose.toolkit.base import Toolkit, tool
+
+
+class IO(Toolkit):
+    """Provides tools to control mouse and keyboard inputs."""
+
+    @tool
+    def move_mouse(self, x: int, y: int) -> str:
+        """
+        Move the mouse cursor to the specified (x, y) coordinates.
+
+        Args:
+            x (int): The x-coordinate to move the mouse to.
+            y (int): The y-coordinate to move the mouse to.
+
+        Return:
+            (str) a message indicating the mouse has been moved.
+        """
+        pyautogui.moveTo(x, y)
+        return f"Mouse moved to ({x}, {y})"
+
+    @tool
+    def click_mouse(self) -> str:
+        """
+        Perform a mouse click at the current cursor position.
+
+        Return:
+            (str) a message indicating the mouse has been clicked.
+        """
+        pyautogui.click()
+        return "Mouse clicked"
+
+    @tool
+    def type_text(self, text: str) -> str:
+        """
+        Type the given text using the keyboard.
+
+        Args:
+            text (str): The text to type.
+
+        Return:
+            (str) a message indicating the text has been typed.
+        """
+        pyautogui.write(text)
+        return f"Typed text: {text}"
+
+    # Provide any system instructions for the model
+    # This can be generated dynamically, and is run at startup time
+    def system(self) -> str:
+        return """**You can control the mouse and keyboard using the tools provided by the IO toolkit.**"""

tests/toolkit/test_io.py

@@ -0,0 +1,30 @@
+import pytest
+from unittest.mock import patch, MagicMock
+from goose.toolkit.io import IO
+
+
+@pytest.fixture
+def io_toolkit():
+    notifier = MagicMock()
+    return IO(notifier)
+
+
+@patch("goose.toolkit.io.pyautogui.moveTo")
+def test_move_mouse(mock_move_to, io_toolkit):
+    result = io_toolkit.move_mouse(100, 200)
+    mock_move_to.assert_called_once_with(100, 200)
+    assert result == "Mouse moved to (100, 200)"
+
+
+@patch("goose.toolkit.io.pyautogui.click")
+def test_click_mouse(mock_click, io_toolkit):
+    result = io_toolkit.click_mouse()
+    mock_click.assert_called_once()
+    assert result == "Mouse clicked"
+
+
+@patch("goose.toolkit.io.pyautogui.write")
+def test_type_text(mock_write, io_toolkit):
+    result = io_toolkit.type_text("Hello, World!")
+    mock_write.assert_called_once_with("Hello, World!")
+    assert result == "Typed text: Hello, World!"