Fallback to Talon actions when focus is not on the text editor

changelog/2024-03-fallBackToTalonActions.md

@@ -0,0 +1,6 @@
+---
+tags: [enhancement]
+pullRequest: 2235
+---
+
+- Fall back to text-based Talon actions when editor is not focused. This allows you to say things like "take token", "bring air", etc, when in the terminal, search bar, etc.

cursorless-talon/src/command.py

@@ -3,10 +3,13 @@
 
 from talon import Module, actions, speech_system
 
+from .fallback import perform_fallback
+from .versions import COMMAND_VERSION
+
 
 @dataclasses.dataclass
 class CursorlessCommand:
-    version = 6
+    version = COMMAND_VERSION
     spokenForm: str
     usePrePhraseSnapshot: bool
     action: dict
@@ -30,10 +33,12 @@ def on_phrase(d):
 class Actions:
     def private_cursorless_command_and_wait(action: dict):
         """Execute cursorless command and wait for it to finish"""
-        actions.user.private_cursorless_run_rpc_command_and_wait(
+        response = actions.user.private_cursorless_run_rpc_command_get(
             CURSORLESS_COMMAND_ID,
             construct_cursorless_command(action),
         )
+        if "fallback" in response:
+            perform_fallback(response["fallback"])
 
     def private_cursorless_command_no_wait(action: dict):
         """Execute cursorless command without waiting"""
@@ -44,10 +49,15 @@ def private_cursorless_command_no_wait(action: dict):
 
     def private_cursorless_command_get(action: dict):
         """Execute cursorless command and return result"""
-        return actions.user.private_cursorless_run_rpc_command_get(
+        response = actions.user.private_cursorless_run_rpc_command_get(
             CURSORLESS_COMMAND_ID,
             construct_cursorless_command(action),
         )
+        if "fallback" in response:
+            return perform_fallback(response["fallback"])
+        if "returnValue" in response:
+            return response["returnValue"]
+        return None
 
 
 def construct_cursorless_command(action: dict) -> dict:

cursorless-talon/src/fallback.py

@@ -0,0 +1,107 @@
+from typing import Callable
+
+from talon import actions
+
+from .versions import COMMAND_VERSION
+
+# This ensures that we remember to update fallback if the response payload changes
+assert COMMAND_VERSION == 7
+
+action_callbacks = {
+    "getText": lambda: [actions.edit.selected_text()],
+    "setSelection": actions.skip,
+    "setSelectionBefore": actions.edit.left,
+    "setSelectionAfter": actions.edit.right,
+    "copyToClipboard": actions.edit.copy,
+    "cutToClipboard": actions.edit.cut,
+    "pasteFromClipboard": actions.edit.paste,
+    "clearAndSetSelection": actions.edit.delete,
+    "remove": actions.edit.delete,
+    "editNewLineBefore": actions.edit.line_insert_up,
+    "editNewLineAfter": actions.edit.line_insert_down,
+}
+
+modifier_callbacks = {
+    "extendThroughStartOf.line": actions.user.select_line_start,
+    "extendThroughEndOf.line": actions.user.select_line_end,
+    "containingScope.document": actions.edit.select_all,
+    "containingScope.paragraph": actions.edit.select_paragraph,
+    "containingScope.line": actions.edit.select_line,
+    "containingScope.token": actions.edit.select_word,
+}
+
+
+def call_as_function(callee: str):
+    wrap_with_paired_delimiter(f"{callee}(", ")")
+
+
+def wrap_with_paired_delimiter(left: str, right: str):
+    selected = actions.edit.selected_text()
+    actions.insert(f"{left}{selected}{right}")
+    for _ in right:
+        actions.edit.left()
+
+
+def containing_token_if_empty():
+    if actions.edit.selected_text() == "":
+        actions.edit.select_word()
+
+
+def perform_fallback(fallback: dict):
+    try:
+        modifier_callbacks = get_modifier_callbacks(fallback)
+        action_callback = get_action_callback(fallback)
+        for callback in reversed(modifier_callbacks):
+            callback()
+        return action_callback()
+    except ValueError as ex:
+        actions.app.notify(str(ex))
+
+
+def get_action_callback(fallback: dict) -> Callable:
+    action = fallback["action"]
+
+    if action in action_callbacks:
+        return action_callbacks[action]
+
+    match action:
+        case "insert":
+            return lambda: actions.insert(fallback["text"])
+        case "callAsFunction":
+            return lambda: call_as_function(fallback["callee"])
+        case "wrapWithPairedDelimiter":
+            return lambda: wrap_with_paired_delimiter(
+                fallback["left"], fallback["right"]
+            )
+
+    raise ValueError(f"Unknown Cursorless fallback action: {action}")
+
+
+def get_modifier_callbacks(fallback: dict) -> list[Callable]:
+    return [get_modifier_callback(modifier) for modifier in fallback["modifiers"]]
+
+
+def get_modifier_callback(modifier: dict) -> Callable:
+    modifier_type = modifier["type"]
+
+    match modifier_type:
+        case "containingTokenIfEmpty":
+            return containing_token_if_empty
+        case "containingScope":
+            scope_type_type = modifier["scopeType"]["type"]
+            return get_simple_modifier_callback(f"{modifier_type}.{scope_type_type}")
+        case "extendThroughStartOf":
+            if "modifiers" not in modifier:
+                return get_simple_modifier_callback(f"{modifier_type}.line")
+        case "extendThroughEndOf":
+            if "modifiers" not in modifier:
+                return get_simple_modifier_callback(f"{modifier_type}.line")
+
+    raise ValueError(f"Unknown Cursorless fallback modifier: {modifier_type}")
+
+
+def get_simple_modifier_callback(key: str) -> Callable:
+    try:
+        return modifier_callbacks[key]
+    except KeyError:
+        raise ValueError(f"Unknown Cursorless fallback modifier: {key}")

cursorless-talon/src/versions.py

@@ -0,0 +1 @@
+COMMAND_VERSION = 7

packages/common/src/FakeCommandServerApi.ts

@@ -0,0 +1,25 @@
+import {
+  CommandServerApi,
+  FocusedElementType,
+  InboundSignal,
+} from "./types/CommandServerApi";
+
+export class FakeCommandServerApi implements CommandServerApi {
+  private focusedElementType: FocusedElementType | undefined;
+  signals: { prePhrase: InboundSignal };
+
+  constructor() {
+    this.signals = { prePhrase: { getVersion: async () => null } };
+    this.focusedElementType = "textEditor";
+  }
+
+  getFocusedElementType(): FocusedElementType | undefined {
+    return this.focusedElementType;
+  }
+
+  setFocusedElementType(
+    focusedElementType: FocusedElementType | undefined,
+  ): void {
+    this.focusedElementType = focusedElementType;
+  }
+}

packages/common/src/index.ts

@@ -85,16 +85,18 @@ export * from "./types/command/legacy/ActionCommandV5";
 export * from "./types/command/legacy/CommandV5.types";
 export * from "./types/command/legacy/PartialTargetDescriptorV5.types";
 export * from "./types/command/CommandV6.types";
+export * from "./types/command/CommandV7.types";
 export * from "./types/command/legacy/PartialTargetDescriptorV3.types";
 export * from "./types/command/legacy/PartialTargetDescriptorV4.types";
 export * from "./types/CommandServerApi";
 export * from "./util/itertools";
 export * from "./extensionDependencies";
-export * from "./getFakeCommandServerApi";
+export * from "./FakeCommandServerApi";
 export * from "./types/TestCaseFixture";
 export * from "./util/getEnvironmentVariableStrict";
 export * from "./util/CompositeKeyDefaultMap";
 export * from "./util/toPlainObject";
+export * from "./util/clientSupportsFallback";
 export * from "./scopeSupportFacets/scopeSupportFacets.types";
 export * from "./scopeSupportFacets/scopeSupportFacetInfos";
 export * from "./scopeSupportFacets/textualScopeSupportFacetInfos";

packages/common/src/testUtil/serializeTestFixture.ts

@@ -7,6 +7,7 @@ function reorderFields(
 ): EnforceUndefined<TestCaseFixtureLegacy> {
   return {
     languageId: fixture.languageId,
+    focusedElementType: fixture.focusedElementType,
     postEditorOpenSleepTimeMs: fixture.postEditorOpenSleepTimeMs,
     postCommandSleepTimeMs: fixture.postCommandSleepTimeMs,
     command: fixture.command,
@@ -15,6 +16,7 @@ function reorderFields(
     initialState: fixture.initialState,
     finalState: fixture.finalState,
     returnValue: fixture.returnValue,
+    fallback: fixture.fallback,
     thrownError: fixture.thrownError,
     ide: fixture.ide,
   };

packages/common/src/types/CommandServerApi.ts

@@ -2,11 +2,15 @@
  * API object for interacting with the command server
  */
 export interface CommandServerApi {
+  getFocusedElementType: () => FocusedElementType | undefined;
+
   signals: {
     prePhrase: InboundSignal;
   };
 }
 
+export type FocusedElementType = "textEditor" | "terminal";
+
 export interface InboundSignal {
   getVersion(): Promise<string | null>;
 }

packages/common/src/types/TestCaseFixture.ts

@@ -1,6 +1,6 @@
-import { Command, CommandLatest } from "..";
-import { TestCaseSnapshot } from "../testUtil/TestCaseSnapshot";
-import { PlainSpyIDERecordedValues } from "../testUtil/spyToPlainObject";
+import type { Command, CommandLatest, Fallback, FocusedElementType } from "..";
+import type { TestCaseSnapshot } from "../testUtil/TestCaseSnapshot";
+import type { PlainSpyIDERecordedValues } from "../testUtil/spyToPlainObject";
 
 export type ThrownError = {
   name: string;
@@ -12,6 +12,11 @@ interface TestCaseFixtureBase {
   postCommandSleepTimeMs?: number;
   spokenFormError?: string;
 
+  /**
+   * The type of element that is focused before the command is executed. If undefined default to text editor.
+   */
+  focusedElementType?: FocusedElementType | "other";
+
   /**
    * A list of marks to check in the case of navigation map test otherwise undefined
    */
@@ -30,6 +35,11 @@ interface TestCaseFixtureBase {
    * error test case.
    */
   returnValue?: unknown;
+
+  /**
+   * The fallback of the command. Will be undefined if the command was executed by the extension.
+   */
+  fallback?: Fallback;
 }
 
 export interface TestCaseFixture extends TestCaseFixtureBase {

packages/common/src/types/command/CommandV7.types.ts

@@ -0,0 +1,8 @@
+import type { CommandV6 } from "./CommandV6.types";
+
+export interface CommandV7 extends Omit<CommandV6, "version"> {
+  /**
+   * The version number of the command API
+   */
+  version: 7;
+}

packages/common/src/types/command/command.types.ts

@@ -1,4 +1,7 @@
-import { CommandV6 } from "./CommandV6.types";
+import type { ActionDescriptor } from "./ActionDescriptor";
+import type { CommandV6 } from "./CommandV6.types";
+import type { CommandV7 } from "./CommandV7.types";
+import type { Modifier } from "./PartialTargetDescriptor.types";
 import type { CommandV0, CommandV1 } from "./legacy/CommandV0V1.types";
 import type { CommandV2 } from "./legacy/CommandV2.types";
 import type { CommandV3 } from "./legacy/CommandV3.types";
@@ -7,7 +10,7 @@ import type { CommandV5 } from "./legacy/CommandV5.types";
 
 export type CommandComplete = Required<Omit<CommandLatest, "spokenForm">> &
   Pick<CommandLatest, "spokenForm">;
-export const LATEST_VERSION = 6 as const;
+export const LATEST_VERSION = 7 as const;
 
 export type CommandLatest = Command & {
   version: typeof LATEST_VERSION;
@@ -20,4 +23,20 @@ export type Command =
   | CommandV3
   | CommandV4
   | CommandV5
-  | CommandV6;
+  | CommandV6
+  | CommandV7;
+
+export type CommandResponse = { returnValue: unknown } | { fallback: Fallback };
+
+export type FallbackModifier = Modifier | { type: "containingTokenIfEmpty" };
+
+export type Fallback =
+  | { action: ActionDescriptor["name"]; modifiers: FallbackModifier[] }
+  | { action: "insert"; modifiers: FallbackModifier[]; text: string }
+  | { action: "callAsFunction"; modifiers: FallbackModifier[]; callee: string }
+  | {
+      action: "wrapWithPairedDelimiter" | "rewrapWithPairedDelimiter";
+      modifiers: FallbackModifier[];
+      left: string;
+      right: string;
+    };

packages/common/src/util/clientSupportsFallback.ts

@@ -0,0 +1,5 @@
+import type { Command } from "../types/command/command.types";
+
+export function clientSupportsFallback(command: Command): boolean {
+  return command.version >= 7;
+}

packages/cursorless-engine/src/CommandRunner.ts

@@ -1,5 +1,5 @@
-import { CommandComplete } from "@cursorless/common";
+import type { CommandComplete, CommandResponse } from "@cursorless/common";
 
 export interface CommandRunner {
-  run(command: CommandComplete): Promise<unknown>;
+  run(command: CommandComplete): Promise<CommandResponse>;
 }

packages/cursorless-engine/src/api/CursorlessEngineApi.ts

@@ -1,9 +1,14 @@
-import { Command, HatTokenMap, IDE } from "@cursorless/common";
-import { Snippets } from "../core/Snippets";
-import { StoredTargetMap } from "../core/StoredTargets";
-import { ScopeProvider } from "@cursorless/common";
-import { CommandRunner } from "../CommandRunner";
-import { ReadOnlyHatMap } from "@cursorless/common";
+import type {
+  Command,
+  CommandResponse,
+  HatTokenMap,
+  IDE,
+  ReadOnlyHatMap,
+  ScopeProvider,
+} from "@cursorless/common";
+import type { CommandRunner } from "../CommandRunner";
+import type { Snippets } from "../core/Snippets";
+import type { StoredTargetMap } from "../core/StoredTargets";
 
 export interface CursorlessEngine {
   commandApi: CommandApi;
@@ -34,13 +39,13 @@ export interface CommandApi {
    * Runs a command.  This is the core of the Cursorless engine.
    * @param command The command to run
    */
-  runCommand(command: Command): Promise<unknown>;
+  runCommand(command: Command): Promise<CommandResponse | unknown>;
 
   /**
    * Designed to run commands that come directly from the user.  Ensures that
    * the command args are of the correct shape.
    */
-  runCommandSafe(...args: unknown[]): Promise<unknown>;
+  runCommandSafe(...args: unknown[]): Promise<CommandResponse | unknown>;
 }
 
 export interface CommandRunnerDecorator {