Streaming video overlay for better mobile support

killergerbah · Feb 22, 2024 · 02a2668 · 02a2668
1 parent bcfae2c
commit 02a2668
Show file tree

Hide file tree

Showing 37 changed files with 749 additions and 95 deletions.
diff --git a/common/app/components/Controls.tsx b/common/app/components/Controls.tsx
@@ -36,6 +36,7 @@ import MenuItem from '@material-ui/core/MenuItem';
 import useMediaQuery from '@material-ui/core/useMediaQuery';
 import MoreVertIcon from '@material-ui/icons/MoreVert';
 import { isMobile } from 'react-device-detect';
+import SubtitleOffsetInput from './SubtitleOffsetInput';
 const useControlStyles = makeStyles((theme) => ({
     container: {
         position: 'absolute',
@@ -680,7 +681,6 @@ export default function Controls({
     const lastNumberInputChangeTimestampRef = useRef<number>(Date.now());
     const lastShowRef = useRef<boolean>(true);
     const forceShowRef = useRef<boolean>(false);
-    const [offsetInputWidth, setOffsetInputWidth] = useState<number>(5);
     const [playbackRateInputWidth, setPlaybackRateInputWidth] = useState<number>(5);
     const offsetInputRef = useRef<HTMLInputElement>();
     const playbackRateInputRef = useRef<HTMLInputElement>();
@@ -766,22 +766,6 @@ export default function Controls({
 
     useEffect(() => onShow?.(show), [onShow, show]);
 
-    const updateOffset = useCallback((offset: number) => {
-        if (offsetInputRef.current) {
-            if (offset === 0) {
-                offsetInputRef.current.value = '';
-                setOffsetInputWidth(5);
-            } else {
-                const offsetSeconds = offset / 1000;
-                const value = offsetSeconds >= 0 ? '+' + offsetSeconds.toFixed(2) : String(offsetSeconds.toFixed(2));
-                offsetInputRef.current.value = value;
-                lastNumberInputChangeTimestampRef.current = Date.now();
-                setOffsetInputWidth(value.length);
-            }
-            offsetInputRef.current.blur();
-        }
-    }, []);
-
     const updatePlaybackRate = useCallback((playbackRate: number) => {
         if (playbackRateInputRef.current) {
             if (playbackRate === 1) {
@@ -797,27 +781,22 @@ export default function Controls({
         }
     }, []);
 
+    const handleOffsetChange = useCallback(
+        (offset: number) => {
+            lastNumberInputChangeTimestampRef.current = Date.now();
+            onOffsetChange(offset);
+        },
+        [onOffsetChange]
+    );
+
     useEffect(() => {
         if (disableKeyEvents) {
             return;
         }
 
         function handleKey(event: KeyboardEvent) {
             if (event.key === 'Enter') {
-                if (offsetInputRef.current === document.activeElement) {
-                    const newOffset = Number(offsetInputRef.current.value);
-
-                    if (newOffset === offset) {
-                        updateOffset(offset);
-                        return;
-                    }
-
-                    if (Number.isNaN(newOffset)) {
-                        return;
-                    }
-
-                    onOffsetChange(newOffset * 1000);
-                } else if (playbackRateInputRef.current === document.activeElement) {
+                if (playbackRateInputRef.current === document.activeElement) {
                     const newPlaybackRate = Number(playbackRateInputRef.current.value);
 
                     if (playbackRate === newPlaybackRate) {
@@ -839,32 +818,25 @@ export default function Controls({
         return () => {
             window.removeEventListener('keydown', handleKey);
         };
-    }, [
-        onOffsetChange,
-        onPlaybackRateChange,
-        updateOffset,
-        updatePlaybackRate,
-        offset,
-        playbackRate,
-        disableKeyEvents,
-    ]);
+    }, [onOffsetChange, onPlaybackRateChange, updatePlaybackRate, offset, playbackRate, disableKeyEvents]);
 
     const handleNumberInputClicked = useCallback((e: React.MouseEvent<HTMLInputElement>) => {
         const inputElement = e.target as HTMLInputElement;
         inputElement.setSelectionRange(0, inputElement.value?.length || 0);
     }, []);
 
     useEffect(() => {
-        const interval = setInterval(() => {
-            forceUpdate();
-        }, 100);
-
-        return () => clearInterval(interval);
-    }, [forceUpdate]);
+        clock.onEvent('settime', () => forceUpdate());
+    }, [clock, forceUpdate]);
 
     useEffect(() => {
-        updateOffset(offset);
-    }, [offset, updateOffset]);
+        if (!show || !playing) {
+            return;
+        }
+
+        const interval = setInterval(() => forceUpdate(), 100);
+        return () => clearInterval(interval);
+    }, [show, playing, forceUpdate]);
 
     useEffect(() => {
         updatePlaybackRate(playbackRate);
@@ -1116,19 +1088,12 @@ export default function Controls({
                             </Grid>
                             {offsetEnabled && !showVolumeBar && !isReallySmallScreen && (
                                 <Grid item>
-                                    <Tooltip title={t('controls.subtitleOffset')!}>
-                                        <Input
-                                            style={{
-                                                width: `${offsetInputWidth}ch`,
-                                            }}
-                                            inputRef={offsetInputRef}
-                                            disableUnderline={true}
-                                            className={classes.numberInput}
-                                            placeholder={'±' + Number(0).toFixed(2)}
-                                            onClick={handleNumberInputClicked}
-                                            onChange={(e) => setOffsetInputWidth(Math.max(5, e.target.value.length))}
-                                        />
-                                    </Tooltip>
+                                    <SubtitleOffsetInput
+                                        inputRef={offsetInputRef}
+                                        offset={offset}
+                                        onOffset={handleOffsetChange}
+                                        disableKeyEvents={disableKeyEvents}
+                                    />
                                 </Grid>
                             )}
                             {playbackRateEnabled && !showVolumeBar && !isReallySmallScreen && (

diff --git a/common/app/components/SettingsDialog.tsx b/common/app/components/SettingsDialog.tsx
@@ -65,6 +65,7 @@ export default function SettingsDialog({
                     anki={anki}
                     extensionInstalled={extension.installed}
                     extensionSupportsAppIntegration={extension.supportsAppIntegration}
+                    extensionSupportsOverlay={extension.supportsStreamingVideoOverlay}
                     insideApp
                     chromeKeyBinds={extension.extensionCommands}
                     onOpenChromeExtensionShortcuts={extension.openShortcuts}

diff --git a/common/app/components/SubtitleOffsetInput.tsx b/common/app/components/SubtitleOffsetInput.tsx
@@ -0,0 +1,107 @@
+import Input from '@material-ui/core/Input';
+import Tooltip from '@material-ui/core/Tooltip';
+import React, { MutableRefObject, useCallback, useEffect, useRef, useState } from 'react';
+import { makeStyles } from '@material-ui/core/styles';
+import { useTranslation } from 'react-i18next';
+
+const useStyles = makeStyles({
+    input: {
+        height: '100%',
+        display: 'flex',
+        flexDirection: 'column',
+        justifyContent: 'center',
+        fontSize: 20,
+        marginLeft: 10,
+        width: 100,
+        color: '#fff',
+        pointerEvents: 'auto',
+    },
+});
+
+interface Props {
+    inputRef: MutableRefObject<HTMLInputElement | undefined>;
+    offset: number;
+    onOffset: (offset: number) => void;
+    disableKeyEvents?: boolean;
+}
+
+export default function SubtitleOffsetInput({ inputRef, offset, onOffset, disableKeyEvents }: Props) {
+    const { t } = useTranslation();
+    const classes = useStyles();
+    const [offsetInputWidth, setOffsetInputWidth] = useState<number>(5);
+    const handleNumberInputClicked = useCallback((e: React.MouseEvent<HTMLInputElement>) => {
+        const inputElement = e.target as HTMLInputElement;
+        inputElement.setSelectionRange(0, inputElement.value?.length || 0);
+    }, []);
+
+    const updateOffset = useCallback(
+        (offset: number) => {
+            if (!inputRef.current) {
+                return;
+            }
+
+            if (offset === 0) {
+                inputRef.current.value = '';
+                setOffsetInputWidth(5);
+            } else {
+                const offsetSeconds = offset / 1000;
+                const value = offsetSeconds >= 0 ? '+' + offsetSeconds.toFixed(2) : String(offsetSeconds.toFixed(2));
+                inputRef.current.value = value;
+                setOffsetInputWidth(value.length);
+            }
+
+            inputRef.current.blur();
+        },
+        [inputRef]
+    );
+    useEffect(() => {
+        updateOffset(offset);
+    }, [offset, updateOffset]);
+
+    useEffect(() => {
+        if (disableKeyEvents) {
+            return;
+        }
+
+        function handleKey(event: KeyboardEvent) {
+            if (event.key === 'Enter') {
+                if (inputRef.current !== null && inputRef.current === document.activeElement) {
+                    const newOffset = Number(inputRef.current.value);
+
+                    if (newOffset === offset) {
+                        updateOffset(offset);
+                        return;
+                    }
+
+                    if (Number.isNaN(newOffset)) {
+                        return;
+                    }
+
+                    onOffset(newOffset * 1000);
+                }
+            }
+        }
+
+        window.addEventListener('keydown', handleKey);
+
+        return () => {
+            window.removeEventListener('keydown', handleKey);
+        };
+    }, [updateOffset, onOffset, disableKeyEvents, inputRef, offset]);
+
+    return (
+        <Tooltip title={t('controls.subtitleOffset')!}>
+            <Input
+                style={{
+                    width: `${offsetInputWidth}ch`,
+                }}
+                inputRef={inputRef}
+                disableUnderline={true}
+                className={classes.input}
+                placeholder={'±' + Number(0).toFixed(2)}
+                onClick={handleNumberInputClicked}
+                onChange={(e) => setOffsetInputWidth(Math.max(5, e.target.value.length))}
+            />
+        </Tooltip>
+    );
+}
diff --git a/common/app/components/SubtitlePlayer.tsx b/common/app/components/SubtitlePlayer.tsx
@@ -295,12 +295,6 @@ const SubtitleRow = React.memo(function SubtitleRow({
     );
 });
 
-interface ResizeHandleStylesProps {
-    isResizing: boolean;
-    appBarHidden: boolean;
-    appBarHeight: number;
-}
-
 interface ResizeHandleProps extends React.HTMLAttributes<HTMLDivElement> {
     isResizing: boolean;
 }

diff --git a/common/app/services/chrome-extension.ts b/common/app/services/chrome-extension.ts
@@ -107,6 +107,10 @@ export default class ChromeExtension {
         window.addEventListener('message', this.windowEventListener);
     }
 
+    get supportsStreamingVideoOverlay() {
+        return this.installed && gte(this.version, '1.1.0');
+    }
+
     get supportsAppIntegration() {
         return this.installed && gte(this.version, '1.0.0');
     }

diff --git a/common/app/services/clock.ts b/common/app/services/clock.ts
@@ -1,11 +1,11 @@
-export type ClockEvent = 'stop' | 'start';
+export type ClockEvent = 'stop' | 'start' | 'settime';
 
 export default class Clock {
     private _accumulated: number;
     private _started: boolean;
     private _startTime?: number;
     private _rate = 1;
-    private _callbacks: { [event in ClockEvent]: (() => void)[] } = { stop: [], start: [] };
+    private _callbacks: { [event in ClockEvent]: (() => void)[] } = { stop: [], start: [], settime: [] };
 
     constructor() {
         this._accumulated = 0;
@@ -65,6 +65,7 @@ export default class Clock {
         } else {
             this._accumulated = time;
         }
+        this._fireEvent('settime');
     }
 
     progress(max: number) {

diff --git a/common/components/SettingsForm.tsx b/common/components/SettingsForm.tsx
@@ -523,6 +523,7 @@ interface Props {
     anki: Anki;
     extensionInstalled: boolean;
     extensionSupportsAppIntegration: boolean;
+    extensionSupportsOverlay: boolean;
     insideApp?: boolean;
     settings: AsbplayerSettings;
     scrollToId?: string;
@@ -545,6 +546,7 @@ export default function SettingsForm({
     settings,
     extensionInstalled,
     extensionSupportsAppIntegration,
+    extensionSupportsOverlay,
     insideApp,
     scrollToId,
     chromeKeyBinds,
@@ -696,6 +698,7 @@ export default function SettingsForm({
         streamingCondensedPlaybackMinimumSkipIntervalMs,
         streamingScreenshotDelay,
         streamingSubtitleListPreference,
+        streamingEnableOverlay,
         webSocketClientEnabled,
         webSocketServerUrl,
     } = settings;
@@ -1556,6 +1559,21 @@ export default function SettingsForm({
                                 label={t('extension.settings.openSubtitleList')}
                                 labelPlacement="start"
                             />
+                            {extensionSupportsOverlay && (
+                                <LabelWithHoverEffect
+                                    className={classes.switchLabel}
+                                    control={
+                                        <Switch
+                                            checked={streamingEnableOverlay}
+                                            onChange={(e) =>
+                                                handleSettingChanged('streamingEnableOverlay', e.target.checked)
+                                            }
+                                        />
+                                    }
+                                    label={t('extension.settings.enableOverlay')}
+                                    labelPlacement="start"
+                                />
+                            )}
                             <LabelWithHoverEffect
                                 className={classes.switchLabel}
                                 control={

diff --git a/common/locales/de.json b/common/locales/de.json
@@ -165,7 +165,8 @@
             "subtitles": "Untertitel",
             "syncing": "Syncronisation",
             "takeScreenshot": "Screenshot erstellen",
-            "updateAvailable": "Aktualisierung verfügbar"
+            "updateAvailable": "Aktualisierung verfügbar",
+            "enableOverlay": "Enable controls overlay"
         },
         "videoDataSync": {
             "emptySubtitleTrack": "Empty",

diff --git a/common/locales/en.json b/common/locales/en.json
@@ -165,7 +165,8 @@
             "subtitles": "Subtitles",
             "syncing": "Syncing",
             "takeScreenshot": "Take screenshot",
-            "updateAvailable": "Update Available"
+            "updateAvailable": "Update Available",
+            "enableOverlay": "Enable controls overlay"
         },
         "videoDataSync": {
             "emptySubtitleTrack": "Empty",

diff --git a/common/locales/es.json b/common/locales/es.json
@@ -165,7 +165,8 @@
             "subtitles": "Subtítulos",
             "syncing": "Sincronización",
             "takeScreenshot": "Tomar captura de pantalla",
-            "updateAvailable": "Actualización Disponible"
+            "updateAvailable": "Actualización Disponible",
+            "enableOverlay": "Enable controls overlay"
         },
         "videoDataSync": {
             "emptySubtitleTrack": "En Blanco",

diff --git a/common/locales/ja.json b/common/locales/ja.json
@@ -165,7 +165,8 @@
             "subtitles": "字幕",
             "syncing": "同期",
             "takeScreenshot": "スクリーンショットを撮る",
-            "updateAvailable": "アップデートがあります"
+            "updateAvailable": "アップデートがあります",
+            "enableOverlay": "コントロールオーバーレイを表示"
         },
         "videoDataSync": {
             "emptySubtitleTrack": "空",