Web socket interface supports 'load-subtitles'

README.md

@@ -260,7 +260,7 @@ asbplayer can be setup to support one-click mining workflows by integrating with
 1. Install [Go](https://go.dev/doc/install).
 2. Clone this repository and start the AnkiConnect proxy server:
     ```
-    cd scripts/anki-connect-proxy
+    cd scripts/web-socket-server
     go run main.go
     ```
 3. Enable asbplayer's WebSocket client from the [settings](https://killergerbah.github.io/asbplayer?view=settings#misc-settings).
@@ -270,42 +270,75 @@ asbplayer can be setup to support one-click mining workflows by integrating with
 
 The proxy is very lightweight, so it's fine to leave it running in the background. On Windows, [RBTray](https://github.com/benbuck/rbtray) can be used to minimise it to the taskbar.
 
-See the proxy's [example configuration file](https://github.com/killergerbah/asbplayer/blob/main/scripts/anki-connect-proxy/.env.example) for how to further configure it.
+See the proxy's [example configuration file](https://github.com/killergerbah/asbplayer/blob/main/scripts/web-socket-server/.env.example) for how to further configure it.
 
 #### WebSocket interface
 
-The asbplayer website can be controlled remotely through a WebSocket connection, which enables [one-click mining flows](#one-click-mining-flow) with the right setup. Currently asbplayer responds to one type of payload:
-
-```javascript
-{
-    "command": "mine-subtitle",
-    // Message ID to correlate with asbplayer's response
-    "messageId": "10281760-d787-4356-8572-f698d8ff3884",
-    "body": {
-        // 0 = "None", 1 = "Show anki dialog", 2 = "Update last card", 3 = "Export card"
-        "postMineAction": 1,
-        // Key-value pairs corresponding to an Anki note type
-        "fields": {
-            "key1": "value1",
-            "key2": "value2"
+The asbplayer website can be controlled remotely through a WebSocket connection, which enables [one-click mining flows](#one-click-mining-flow) with the right setup. Currently asbplayer responds to two types of payloads:
+
+-   `mine-subtitle` request:
+
+    ```javascript
+    {
+        "command": "mine-subtitle",
+        // Message ID to correlate with asbplayer's response
+        "messageId": "10281760-d787-4356-8572-f698d8ff3884",
+        "body": {
+            // 0 = "None", 1 = "Show anki dialog", 2 = "Update last card", 3 = "Export card"
+            "postMineAction": 1,
+            // Key-value pairs corresponding to an Anki note type
+            "fields": {
+                "key1": "value1",
+                "key2": "value2"
+            }
         }
     }
-}
-```
+    ```
 
-Response:
+    `mine-subtitle` response:
 
-```javascript
-{
-    "command": "response",
-    // Same message ID received in request
-    "messageId": "10281760-d787-4356-8572-f698d8ff3884",
-    "body": {
-        // Whether the command was successfully published to the website
-        "published": true
+    ```javascript
+    {
+        "command": "response",
+        // Same message ID received in request
+        "messageId": "10281760-d787-4356-8572-f698d8ff3884",
+        "body": {
+            // Whether the command was successfully published to an asbplayer client
+            "published": true
+        }
     }
-}
-```
+    ```
+
+-   `load-subtitles` request:
+
+    ```javascript
+    {
+        "command": "load-subtitles",
+        // Message ID to correlate with asbplayer's response
+        "messageId": "3565510c-342f-4cec-ad2e-dee81af88d75",
+        "body": {
+            "files": [{
+                // Name of the file, including its extension
+                "name": "some-file.srt",
+                // Base64-encoded file contents
+                "base64": "Zm9vYmFyY..."
+            }]
+        }
+    }
+    ```
+
+    `load-subtitles` response:
+
+    ```javascript
+    {
+        "command": "response",
+        // Same message ID received in request
+        "messageId": "3565510c-342f-4cec-ad2e-dee81af88d75",
+        "body": {}
+    }
+    ```
+
+    The [Web Socket server](https://github.com/killergerbah/asbplayer/blob/main/scripts/web-socket-server) implements this protocol and can load subtitles into connected asbplayer clients through its HTTP endpoint `POST asbplayer/load-subtitles` using the `body` of the payload documented above. See the [CLI script](https://github.com/killergerbah/asbplayer/blob/main/scripts/web-socket-server/cli/load-subtitles) for an example of how this is done.
 
 ## Usage on Android
 

common/app/components/App.tsx

@@ -55,6 +55,8 @@ import { useAnki } from '../hooks/use-anki';
 import { usePlaybackPreferences } from '../hooks/use-playback-preferences';
 import { MiningContext } from '../services/mining-context';
 import { timeDurationDisplay } from '../services/util';
+import { useAppWebSocketClient } from '../hooks/use-app-web-socket-client';
+import { LoadSubtitlesCommand } from '../../web-socket-client';
 
 const latestExtensionVersion = '1.5.0';
 const extensionUrl =
@@ -216,6 +218,7 @@ function App({ origin, logoUrl, settings, extension, fetcher, onSettingsChanged,
             pgsWorkerFactory,
         });
     }, [settings.subtitleRegexFilter, settings.subtitleRegexFilterTextReplacement]);
+    const webSocketClient = useAppWebSocketClient({ settings });
     const [subtitles, setSubtitles] = useState<DisplaySubtitleModel[]>([]);
     const playbackPreferences = usePlaybackPreferences(settings, extension);
     const theme = useMemo<Theme>(() => createTheme(settings.themeType), [settings.themeType]);
@@ -818,6 +821,20 @@ function App({ origin, logoUrl, settings, extension, fetcher, onSettingsChanged,
         [handleError, handleFiles, t]
     );
 
+    useEffect(() => {
+        if (!webSocketClient) {
+            return;
+        }
+
+        webSocketClient.onLoadSubtitles = async (command: LoadSubtitlesCommand) => {
+            const { files } = command.body;
+            const filePromises = (files ?? []).map(
+                async (f) => new File([await (await fetch('data:text/plain;base64,' + f.base64)).blob()], f.name)
+            );
+            handleFiles({ files: await Promise.all(filePromises) });
+        };
+    }, [webSocketClient, handleFiles]);
+
     useEffect(() => {
         if (inVideoPlayer) {
             extension.videoPlayer = true;
@@ -1347,6 +1364,7 @@ function App({ origin, logoUrl, settings, extension, fetcher, onSettingsChanged,
                                 disableKeyEvents={disableKeyEvents}
                                 miningContext={miningContext}
                                 keyBinder={keyBinder}
+                                webSocketClient={webSocketClient}
                             />
                         </Content>
                     </Paper>

common/app/components/Player.tsx

@@ -32,6 +32,7 @@ import { useWindowSize } from '../hooks/use-window-size';
 import { useAppBarHeight } from '../hooks/use-app-bar-height';
 import { createBlobUrl } from '../../blob-url';
 import { MiningContext } from '../services/mining-context';
+import { WebSocketClient } from '../../web-socket-client';
 
 const minVideoPlayerWidth = 300;
 
@@ -122,6 +123,7 @@ interface PlayerProps {
     rewindSubtitle?: SubtitleModel;
     hideControls?: boolean;
     forceCompressedMode?: boolean;
+    webSocketClient?: WebSocketClient;
 }
 
 const Player = React.memo(function Player({
@@ -160,6 +162,7 @@ const Player = React.memo(function Player({
     rewindSubtitle,
     hideControls,
     forceCompressedMode,
+    webSocketClient,
 }: PlayerProps) {
     const [playMode, setPlayMode] = useState<PlayMode>(PlayMode.normal);
     const [subtitlesSentThroughChannel, setSubtitlesSentThroughChannel] = useState<boolean>();
@@ -1057,6 +1060,7 @@ const Player = React.memo(function Player({
                         autoPauseContext={autoPauseContext}
                         settings={settings}
                         keyBinder={keyBinder}
+                        webSocketClient={webSocketClient}
                     />
                 </Grid>
             </Grid>

common/app/components/SubtitlePlayer.tsx

@@ -43,9 +43,10 @@ import Tooltip from '@material-ui/core/Tooltip';
 import Typography from '@material-ui/core/Typography';
 import Clock from '../services/clock';
 import { useAppBarHeight } from '../hooks/use-app-bar-height';
-import { MineSubtitleParams, useAppWebSocketClient } from '../hooks/use-app-web-socket-client';
+import { MineSubtitleParams } from '../hooks/use-app-web-socket-client';
 import { isMobile } from 'react-device-detect';
 import ChromeExtension, { ExtensionMessage } from '../services/chrome-extension';
+import { MineSubtitleCommand, WebSocketClient } from '../../web-socket-client';
 
 let lastKnownWidth: number | undefined;
 export const minSubtitlePlayerWidth = 200;
@@ -362,6 +363,7 @@ interface SubtitlePlayerProps {
     settings: AsbplayerSettings;
     keyBinder: KeyBinder;
     maxResizeWidth: number;
+    webSocketClient?: WebSocketClient;
 }
 
 export default function SubtitlePlayer({
@@ -394,6 +396,7 @@ export default function SubtitlePlayer({
     settings,
     keyBinder,
     maxResizeWidth,
+    webSocketClient,
 }: SubtitlePlayerProps) {
     const { t } = useTranslation();
     const clockRef = useRef<Clock>(clock);
@@ -828,11 +831,37 @@ export default function SubtitlePlayer({
         ]
     );
 
-    useAppWebSocketClient({
-        onMineSubtitle: copyFromWebSocketClient,
-        settings,
-        enabled: !extension.supportsWebSocketClient && subtitles !== undefined && subtitles.length > 0,
-    });
+    useEffect(() => {
+        if (!webSocketClient || extension.supportsWebSocketClient) {
+            // Do not handle mining commands here if the extension supports the web socket client.
+            // The extension will handle the commands for us.
+            return;
+        }
+
+        webSocketClient.onMineSubtitle = async ({
+            body: { fields: receivedFields, postMineAction: receivedPostMineAction },
+        }: MineSubtitleCommand) => {
+            const fields = receivedFields ?? {};
+            const word = fields[settings.wordField] || undefined;
+            const definition = fields[settings.definitionField] || undefined;
+            const text = fields[settings.sentenceField] || undefined;
+            const customFieldValues = Object.fromEntries(
+                Object.entries(settings.customAnkiFields)
+                    .map(([asbplayerFieldName, ankiFieldName]) => {
+                        const fieldValue = fields[ankiFieldName];
+
+                        if (fieldValue === undefined) {
+                            return undefined;
+                        }
+
+                        return [asbplayerFieldName, fieldValue];
+                    })
+                    .filter((entry) => entry !== undefined) as string[][]
+            );
+            const postMineAction = receivedPostMineAction ?? PostMineAction.showAnkiDialog;
+            return copyFromWebSocketClient({ postMineAction, text, word, definition, customFieldValues });
+        };
+    }, [webSocketClient, extension, settings, copyFromWebSocketClient]);
 
     useEffect(() => {
         if (extension.installed) {

common/app/hooks/use-app-web-socket-client.ts

@@ -1,70 +1,25 @@
-import { AnkiSettings, WebSocketClientSettings, ankiSettingsKeys } from '../../settings';
+import { WebSocketClientSettings } from '../../settings';
 import { CardTextFieldValues, PostMineAction } from '../../src/model';
-import { MineSubtitleCommand, WebSocketClient } from '../../web-socket-client';
+import { WebSocketClient } from '../../web-socket-client';
 import { useEffect, useState } from 'react';
-import { useDocumentHasFocus } from './use-document-has-focus';
 
 export interface MineSubtitleParams extends CardTextFieldValues {
     postMineAction: PostMineAction;
 }
 
-export const useAppWebSocketClient = ({
-    onMineSubtitle,
-    settings,
-    enabled,
-}: {
-    onMineSubtitle: (params: MineSubtitleParams) => boolean;
-    settings: WebSocketClientSettings & AnkiSettings;
-    enabled: boolean;
-}) => {
+export const useAppWebSocketClient = ({ settings }: { settings: WebSocketClientSettings }) => {
     const [client, setClient] = useState<WebSocketClient>();
-    const documentHasFocus = useDocumentHasFocus();
 
     useEffect(() => {
-        if (enabled && settings.webSocketClientEnabled && settings.webSocketServerUrl && documentHasFocus) {
+        if (settings.webSocketClientEnabled && settings.webSocketServerUrl) {
             const client = new WebSocketClient();
             client.bind(settings.webSocketServerUrl).catch(console.error);
             setClient(client);
             return () => client.unbind();
         }
 
         setClient(undefined);
-    }, [documentHasFocus, settings.webSocketServerUrl, settings.webSocketClientEnabled, enabled]);
+    }, [settings.webSocketServerUrl, settings.webSocketClientEnabled]);
 
-    useEffect(() => {
-        if (!client) {
-            return;
-        }
-
-        client.onMineSubtitle = async ({
-            body: { fields: receivedFields, postMineAction: receivedPostMineAction },
-        }: MineSubtitleCommand) => {
-            const fields = receivedFields ?? {};
-            const word = fields[settings.wordField] || undefined;
-            const definition = fields[settings.definitionField] || undefined;
-            const text = fields[settings.sentenceField] || undefined;
-            const customFieldValues = Object.fromEntries(
-                Object.entries(settings.customAnkiFields)
-                    .map(([asbplayerFieldName, ankiFieldName]) => {
-                        const fieldValue = fields[ankiFieldName];
-
-                        if (fieldValue === undefined) {
-                            return undefined;
-                        }
-
-                        return [asbplayerFieldName, fieldValue];
-                    })
-                    .filter((entry) => entry !== undefined) as string[][]
-            );
-            const postMineAction = receivedPostMineAction ?? PostMineAction.showAnkiDialog;
-            return onMineSubtitle({ postMineAction, text, word, definition, customFieldValues });
-        };
-    }, [
-        client,
-        settings.wordField,
-        settings.definitionField,
-        settings.sentenceField,
-        settings.customAnkiFields,
-        onMineSubtitle,
-    ]);
+    return client;
 };

common/src/message.ts

@@ -226,9 +226,15 @@ export interface ToggleRecordingMessage extends Message {
     readonly command: 'toggle-recording';
 }
 
+export interface SubtitleFile {
+    base64: string;
+    name: string;
+}
+
 export interface ToggleVideoSelectMessage extends Message {
     readonly command: 'toggle-video-select';
     readonly fromAsbplayerId?: string;
+    readonly subtitleFiles?: SubtitleFile[];
 }
 
 export interface ShowAnkiUiAfterRerecordMessage extends Message {