From fd435b23033927079150afdaf1c9539c8e1a9122 Mon Sep 17 00:00:00 2001 From: Maksim Sukharev Date: Wed, 27 Mar 2024 18:41:06 +0100 Subject: [PATCH] feat(media): rank and list device preferences Signed-off-by: Maksim Sukharev --- .../MediaSettings/MediaSettings.vue | 4 + src/composables/useDevices.js | 11 ++ src/services/mediaDevicePreferences.ts | 183 ++++++++++++++++++ src/store/conversationsStore.spec.js | 2 +- src/utils/webrtc/MediaDevicesManager.js | 65 +++++++ 5 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 src/services/mediaDevicePreferences.ts diff --git a/src/components/MediaSettings/MediaSettings.vue b/src/components/MediaSettings/MediaSettings.vue index 8c3619194cc7..3c46f0a86f90 100644 --- a/src/components/MediaSettings/MediaSettings.vue +++ b/src/components/MediaSettings/MediaSettings.vue @@ -285,6 +285,7 @@ export default { const { devices, updateDevices, + updatePreferences, currentVolume, currentThreshold, audioPreviewAvailable, @@ -305,6 +306,7 @@ export default { // useDevices devices, updateDevices, + updatePreferences, currentVolume, currentThreshold, audioPreviewAvailable, @@ -553,6 +555,8 @@ export default { if (this.videoDeviceStateChanged) { emit('local-video-control-button:toggle-video') } + + this.updatePreferences() this.closeModal() }, diff --git a/src/composables/useDevices.js b/src/composables/useDevices.js index d4b3d703717e..74605e141c28 100644 --- a/src/composables/useDevices.js +++ b/src/composables/useDevices.js @@ -45,6 +45,8 @@ export function useDevices(video, initializeOnMounted) { const videoTrackToStream = ref(null) const mediaDevicesManager = reactive(mediaDevicesManagerInstance) + window.OCA.Talk.mediaDevicesManager = mediaDevicesManagerInstance + // Public refs const currentVolume = ref(-100) const currentThreshold = ref(-100) @@ -169,6 +171,14 @@ export function useDevices(video, initializeOnMounted) { mediaDevicesManager._updateDevices() } + /** + * Update preference counters for devices (audio and video) + * @public + */ + function updatePreferences() { + mediaDevicesManager.updatePreferences() + } + /** * Stop tracking device events (audio and video) * @public @@ -391,6 +401,7 @@ export function useDevices(video, initializeOnMounted) { videoStreamError, // MediaSettings only initializeDevices, + updatePreferences, stopDevices, virtualBackground, } diff --git a/src/services/mediaDevicePreferences.ts b/src/services/mediaDevicePreferences.ts new file mode 100644 index 000000000000..6d4c5cc6cb49 --- /dev/null +++ b/src/services/mediaDevicePreferences.ts @@ -0,0 +1,183 @@ +/** + * @copyright Copyright (c) 2024 Maksim Sukharev + * + * @author Maksim Sukharev + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +/** + * List all registered devices in order of their preferences + * Show whether device is currently unplugged or selected, if information is available + * + * @param devices list of available devices + * @param audioInputId id of currently selected audio input + * @param videoInputId id of currently selected video input + * @param audioInputList list of registered audio devices in order of preference + * @param videoInputList list of registered video devices in order of preference + */ +function listMediaDevices(devices: MediaDeviceInfo[], audioInputId: string, videoInputId: string, audioInputList: MediaDeviceInfo[], videoInputList: MediaDeviceInfo[]) { + const availableDevices = devices.map(device => device.deviceId).filter(id => id !== 'default') + + const getDeviceString = (device: MediaDeviceInfo, index: number) => { + const isUnplugged = !availableDevices.includes(device.deviceId) ? ' (unplugged)' : '' + const isSelected = () => { + if (device.kind === 'audioinput') { + return device.deviceId === audioInputId ? ' (selected)' : '' + } else if (device.kind === 'videoinput') { + return device.deviceId === videoInputId ? ' (selected)' : '' + } + } + return ` ${index + 1}. ${device.label || device.deviceId}` + isUnplugged + isSelected() + } + + // eslint-disable-next-line no-console + console.log(`Media devices: + Audio input: +${audioInputList.map(getDeviceString).join('\n')} + + Video input: +${videoInputList.map(getDeviceString).join('\n')} +`) +} + +/** + * Modify devices list. + * + * @param device device + * @param devicesList list of registered devices in order of preference + * @param promote whether device should be promoted (to be used in updateMediaDevicesPreferences) + */ +function registerNewDevice(device: MediaDeviceInfo, devicesList: MediaDeviceInfo[], promote: boolean = false) { + const newDevicesList = devicesList.slice() + console.debug('Registering new device:', device) + + if (promote) { + newDevicesList.unshift(device) + } else { + newDevicesList.push(device) + } + + return newDevicesList +} + +/** + * Promote device in the preference list. + * Regardless if new unknown or registered device was selected, we promote it in the list: + * - if first item is plugged, then we prefer device out of all options and put it on the first place; + * - if first item is unplugged, then we don't consider it, and compare with the next in the list; + * - if second item is unplugged, compare with next in the list; + * - ... + * - if N-th item is plugged, then we prefer device to it and put it on the Nth place. + * + * Returns changed preference lists for audio / video devices (null, if it hasn't been changed) + * + * @param devices list of available devices + * @param inputList list of registered audio/video devices in order of preference + * @param inputId id of currently selected input + */ +function promoteDevice(devices: MediaDeviceInfo[], inputList: MediaDeviceInfo[], inputId: string | null) { + const newInputList = inputList.slice() + + // Get the index of the first plugged device + const availableDevices = devices.map(device => device.deviceId).filter(id => id !== 'default') + const firstPluggedIndex = newInputList.findIndex(device => availableDevices.includes(device.deviceId)) + const insertPosition = firstPluggedIndex === -1 ? newInputList.length : firstPluggedIndex + + // Get the index of the currently selected device + const currentDevicePosition = newInputList.findIndex(device => device.deviceId === inputId) + + if (currentDevicePosition === insertPosition) { + // preferences list is unchanged + return null + } + + let deviceToPromote = null + if (currentDevicePosition === -1 && inputId !== 'default' && inputId !== null) { + // If device was not registered in preferences list, get it from devices list + deviceToPromote = devices.find(device => device.deviceId === inputId) + } else if (currentDevicePosition > 0) { + // Otherwise extract it from preferences list + deviceToPromote = newInputList.splice(currentDevicePosition, 1)[0] + } + + if (deviceToPromote) { + // Put the device at the new position + newInputList.splice(insertPosition, 0, deviceToPromote) + return newInputList + } else { + return null + } +} + +/** + * Populate devices preferences. If device has not yet been registered in preference list, it will be added. + * + * Returns changed preference lists for audio / video devices (null, if it hasn't been changed) + * + * @param devices list of available devices + * @param audioInputList list of registered audio devices in order of preference + * @param videoInputList list of registered video devices in order of preference + */ +function populateMediaDevicesPreferences(devices: MediaDeviceInfo[], audioInputList: MediaDeviceInfo[], videoInputList: MediaDeviceInfo[]) { + let newAudioInputList = null + let newVideoInputList = null + + for (const device of devices) { + if (device.kind === 'audioinput') { + // Add to the list of known devices + if (device.deviceId !== 'default' && !audioInputList.some(input => input.deviceId === device.deviceId)) { + newAudioInputList = registerNewDevice(device, audioInputList) + } + } else if (device.kind === 'videoinput') { + // Add to the list of known devices + if (device.deviceId !== 'default' && !videoInputList.some(input => input.deviceId === device.deviceId)) { + newVideoInputList = registerNewDevice(device, videoInputList) + } + } + } + + return { + newAudioInputList, + newVideoInputList, + } +} + +/** + * Update devices preferences. Assuming that preferred devices were selected, should be called after applying the selection: + * so either with joining the call or changing device during the call + * + * Returns changed preference lists for audio / video devices (null, if it hasn't been changed) + * + * @param devices list of available devices + * @param audioInputId id of currently selected audio input + * @param videoInputId id of currently selected video input + * @param audioInputList list of registered audio devices in order of preference + * @param videoInputList list of registered video devices in order of preference + */ +function updateMediaDevicesPreferences(devices: MediaDeviceInfo[], audioInputId: string, videoInputId: string, audioInputList: MediaDeviceInfo[], videoInputList: MediaDeviceInfo[]) { + return { + newAudioInputList: promoteDevice(devices, audioInputList, audioInputId), + newVideoInputList: promoteDevice(devices, videoInputList, videoInputId), + } +} + +export { + listMediaDevices, + populateMediaDevicesPreferences, + updateMediaDevicesPreferences, +} diff --git a/src/store/conversationsStore.spec.js b/src/store/conversationsStore.spec.js index 04ec957d39b0..534252c393b6 100644 --- a/src/store/conversationsStore.spec.js +++ b/src/store/conversationsStore.spec.js @@ -65,7 +65,7 @@ jest.mock('../services/messagesService', () => ({ jest.mock('@nextcloud/event-bus') jest.mock('../services/BrowserStorage.js', () => ({ - getItem: jest.fn(), + getItem: jest.fn().mockReturnValue(null), setItem: jest.fn(), removeItem: jest.fn(), })) diff --git a/src/utils/webrtc/MediaDevicesManager.js b/src/utils/webrtc/MediaDevicesManager.js index 396843bb6f0f..21deb55fbc60 100644 --- a/src/utils/webrtc/MediaDevicesManager.js +++ b/src/utils/webrtc/MediaDevicesManager.js @@ -20,6 +20,11 @@ */ import BrowserStorage from '../../services/BrowserStorage.js' +import { + listMediaDevices, + populateMediaDevicesPreferences, + updateMediaDevicesPreferences, +} from '../../services/mediaDevicePreferences.ts' import EmitterMixin from '../EmitterMixin.js' /** @@ -91,6 +96,12 @@ export default function MediaDevicesManager() { this._fallbackAudioInputId = undefined this._fallbackVideoInputId = undefined + const audioInputPreferences = BrowserStorage.getItem('audioInputPreferences') + this._preferenceAudioInputList = audioInputPreferences !== null ? JSON.parse(audioInputPreferences) : [] + + const videoInputPreferences = BrowserStorage.getItem('videoInputPreferences') + this._preferenceVideoInputList = videoInputPreferences !== null ? JSON.parse(videoInputPreferences) : [] + this._tracks = [] this._updateDevicesBound = this._updateDevices.bind(this) @@ -220,6 +231,8 @@ MediaDevicesManager.prototype = { } this._pendingEnumerateDevicesPromise = null + + this._populatePreferences(devices) }).catch(function(error) { console.error('Could not update known media devices: ' + error.name + ': ' + error.message) @@ -227,6 +240,58 @@ MediaDevicesManager.prototype = { }) }, + _populatePreferences(devices) { + const { newAudioInputList, newVideoInputList } = populateMediaDevicesPreferences( + devices, + this._preferenceAudioInputList, + this._preferenceVideoInputList, + ) + + if (newAudioInputList) { + this._preferenceAudioInputList = newAudioInputList + BrowserStorage.setItem('audioInputPreferences', JSON.stringify(this._preferenceAudioInputList)) + } + if (newVideoInputList) { + this._preferenceVideoInputList = newVideoInputList + BrowserStorage.setItem('videoInputPreferences', JSON.stringify(this._preferenceVideoInputList)) + } + }, + + updatePreferences() { + const { newAudioInputList, newVideoInputList } = updateMediaDevicesPreferences( + this.attributes.devices, + this.attributes.audioInputId, + this.attributes.videoInputId, + this._preferenceAudioInputList, + this._preferenceVideoInputList, + ) + + if (newAudioInputList) { + this._preferenceAudioInputList = newAudioInputList + BrowserStorage.setItem('audioInputPreferences', JSON.stringify(newAudioInputList)) + } + if (newVideoInputList) { + this._preferenceVideoInputList = newVideoInputList + BrowserStorage.setItem('videoInputPreferences', JSON.stringify(newVideoInputList)) + } + }, + + /** + * List all registered devices in order of their preferences + * Show whether device is currently unplugged or selected, if information is available + */ + listDevices() { + navigator.mediaDevices.enumerateDevices().then(devices => { + listMediaDevices( + devices, + this.attributes.audioInputId, + this.attributes.videoInputId, + this._preferenceAudioInputList, + this._preferenceVideoInputList, + ) + }) + }, + _removeDevice(removedDevice) { const removedDeviceIndex = this.attributes.devices.findIndex(oldDevice => oldDevice.deviceId === removedDevice.deviceId && oldDevice.kind === removedDevice.kind) if (removedDeviceIndex >= 0) {