From 439377b75a28509d29efec5c7ce1b6c283c52508 Mon Sep 17 00:00:00 2001 From: "Grigorii K. Shartsev" Date: Wed, 3 Apr 2024 19:37:08 +0200 Subject: [PATCH 01/11] chore(deps): add @mdi/svg icons Signed-off-by: Grigorii K. Shartsev --- package-lock.json | 11 +++++++++++ package.json | 1 + 2 files changed, 12 insertions(+) diff --git a/package-lock.json b/package-lock.json index 9740af8e..f52d92c4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.27.0", "license": "AGPL-3.0", "dependencies": { + "@mdi/svg": "^7.4.47", "@nextcloud/axios": "^2.4.0", "@nextcloud/browser-storage": "^0.3.0", "@nextcloud/capabilities": "^1.1.0", @@ -4148,6 +4149,11 @@ "unist-util-is": "^3.0.0" } }, + "node_modules/@mdi/svg": { + "version": "7.4.47", + "resolved": "https://registry.npmjs.org/@mdi/svg/-/svg-7.4.47.tgz", + "integrity": "sha512-WQ2gDll12T9WD34fdRFgQVgO8bag3gavrAgJ0frN4phlwdJARpE6gO1YvLEMJR0KKgoc+/Ea/A0Pp11I00xBvw==" + }, "node_modules/@nextcloud/auth": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@nextcloud/auth/-/auth-2.2.1.tgz", @@ -23630,6 +23636,11 @@ } } }, + "@mdi/svg": { + "version": "7.4.47", + "resolved": "https://registry.npmjs.org/@mdi/svg/-/svg-7.4.47.tgz", + "integrity": "sha512-WQ2gDll12T9WD34fdRFgQVgO8bag3gavrAgJ0frN4phlwdJARpE6gO1YvLEMJR0KKgoc+/Ea/A0Pp11I00xBvw==" + }, "@nextcloud/auth": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/@nextcloud/auth/-/auth-2.2.1.tgz", diff --git a/package.json b/package.json index 062ca87b..6783bdde 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "release:package": "zx ./scripts/prepare-release-packages.mjs" }, "dependencies": { + "@mdi/svg": "^7.4.47", "@nextcloud/axios": "^2.4.0", "@nextcloud/browser-storage": "^0.3.0", "@nextcloud/capabilities": "^1.1.0", From 049ca586e32b45bac1a749f29193ff14a84878d5 Mon Sep 17 00:00:00 2001 From: "Grigorii K. Shartsev" Date: Fri, 5 Apr 2024 20:47:56 +0200 Subject: [PATCH 02/11] feat(screensharing): add app:getDesktopCaptureSources method Signed-off-by: Grigorii K. Shartsev --- src/main.js | 18 ++++++++++++++++-- src/preload.js | 6 ++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main.js b/src/main.js index bd0a1b33..10e3a792 100644 --- a/src/main.js +++ b/src/main.js @@ -20,7 +20,7 @@ */ const path = require('node:path') -const { app, dialog, BrowserWindow, ipcMain } = require('electron') +const { app, dialog, BrowserWindow, ipcMain, desktopCapturer, systemPreferences } = require('electron') const { setupMenu } = require('./app/app.menu.js') const { setupReleaseNotificationScheduler } = require('./app/githubReleaseNotification.service.js') const { enableWebRequestInterceptor, disableWebRequestInterceptor } = require('./app/webRequestInterceptor.js') @@ -28,7 +28,7 @@ const { createAuthenticationWindow } = require('./authentication/authentication. const { openLoginWebView } = require('./authentication/login.window.js') const { createHelpWindow } = require('./help/help.window.js') const { createUpgradeWindow } = require('./upgrade/upgrade.window.js') -const { getOs, isLinux } = require('./shared/os.utils.js') +const { getOs, isLinux, isMac } = require('./shared/os.utils.js') const { createTalkWindow } = require('./talk/talk.window.js') const { createWelcomeWindow } = require('./welcome/welcome.window.js') const { installVueDevtools } = require('./install-vue-devtools.js') @@ -82,6 +82,20 @@ ipcMain.on('app:relaunch', () => { app.relaunch() app.exit(0) }) +ipcMain.handle('app:getDesktopCapturerSources', async () => { + // macOS 10.15 Catalina or higher requires consent for screen access + if (isMac() && systemPreferences.getMediaAccessStatus('screen') !== 'granted') { + // TODO: show user-friendly error in this case + return [] + } + + const sources = await desktopCapturer.getSources({ types: ['window', 'screen'], fetchWindowIcons: true }) + return sources.map((source) => ({ + id: source.id, + name: source.name, + icon: source.appIcon?.toDataURL(), + })) +}) app.whenReady().then(async () => { try { diff --git a/src/preload.js b/src/preload.js index 9af7cf47..7c386ceb 100644 --- a/src/preload.js +++ b/src/preload.js @@ -83,6 +83,12 @@ const TALK_DESKTOP = { * @return {Promise} */ setBadgeCount: (count) => ipcRenderer.invoke('app:setBadgeCount', count), + /** + * Get available desktop capture sources: screens and windows + * + * @return {Promise<{ id: string, name: string, icon?: string }>} + */ + getDesktopCapturerSources: () => ipcRenderer.invoke('app:getDesktopCapturerSources'), /** * Relaunch the application */ From e6dca93bdca286acb295c330ce28e5a4f3e0724b Mon Sep 17 00:00:00 2001 From: "Grigorii K. Shartsev" Date: Sat, 6 Apr 2024 00:09:17 +0200 Subject: [PATCH 03/11] feat(screensharing): add DesktopMediaSource dialog Signed-off-by: Grigorii K. Shartsev --- .../components/DesktopMediaSourceDialog.vue | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 src/talk/renderer/components/DesktopMediaSourceDialog.vue diff --git a/src/talk/renderer/components/DesktopMediaSourceDialog.vue b/src/talk/renderer/components/DesktopMediaSourceDialog.vue new file mode 100644 index 00000000..83f77442 --- /dev/null +++ b/src/talk/renderer/components/DesktopMediaSourceDialog.vue @@ -0,0 +1,231 @@ + + + + + + + From 95a3ba9981f23f0a5de480428abe962463578ef4 Mon Sep 17 00:00:00 2001 From: "Grigorii K. Shartsev" Date: Thu, 4 Apr 2024 17:53:42 +0200 Subject: [PATCH 04/11] feat(screensharing): add public OCA.Talk.Desktop.getDesktopMediaSource Signed-off-by: Grigorii K. Shartsev --- .../renderer/AppGetDesktopMediaSource.vue | 56 +++++++++++++++++++ src/talk/renderer/getDesktopMediaSource.js | 41 ++++++++++++++ src/talk/renderer/talk.main.js | 5 ++ 3 files changed, 102 insertions(+) create mode 100644 src/talk/renderer/AppGetDesktopMediaSource.vue create mode 100644 src/talk/renderer/getDesktopMediaSource.js diff --git a/src/talk/renderer/AppGetDesktopMediaSource.vue b/src/talk/renderer/AppGetDesktopMediaSource.vue new file mode 100644 index 00000000..ea147f85 --- /dev/null +++ b/src/talk/renderer/AppGetDesktopMediaSource.vue @@ -0,0 +1,56 @@ + + + + + diff --git a/src/talk/renderer/getDesktopMediaSource.js b/src/talk/renderer/getDesktopMediaSource.js new file mode 100644 index 00000000..b4e0683c --- /dev/null +++ b/src/talk/renderer/getDesktopMediaSource.js @@ -0,0 +1,41 @@ +/* + * @copyright Copyright (c) 2024 Grigorii Shartsev + * + * @author Grigorii Shartsev + * + * @license AGPL-3.0-or-later + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import Vue from 'vue' + +import AppGetDesktopMediaSource from './AppGetDesktopMediaSource.vue' + +/** @type {import('vue').ComponentPublicInstance} */ +let appGetDesktopMediaSourceInstance + +/** + * Prompt user to select a desktop media source to share and return the selected sourceId or an empty string if canceled + * + * @return {Promise<{ sourceId: string }>} sourceId of the selected mediaSource or an empty string if canceled + */ +export async function getDesktopMediaSource() { + if (!appGetDesktopMediaSourceInstance) { + const container = document.body.appendChild(document.createElement('div')) + appGetDesktopMediaSourceInstance = new Vue(AppGetDesktopMediaSource).$mount(container) + } + + return appGetDesktopMediaSourceInstance.promptDesktopMediaSource() +} diff --git a/src/talk/renderer/talk.main.js b/src/talk/renderer/talk.main.js index a88384e8..167a59bf 100644 --- a/src/talk/renderer/talk.main.js +++ b/src/talk/renderer/talk.main.js @@ -25,6 +25,7 @@ import './assets/styles.css' import 'regenerator-runtime' // TODO: Why isn't it added on bundling import { init, initTalkHashIntegration } from './init.js' import { setupWebPage } from '../../shared/setupWebPage.js' +import { getDesktopMediaSource } from './getDesktopMediaSource.js' // Initially open the welcome page, if not specified if (!window.location.hash) { @@ -43,3 +44,7 @@ await import('@talk/src/main.js') initTalkHashIntegration(OCA.Talk.instance) await import('./notifications/notifications.store.js') + +window.OCA.Talk.Desktop = { + getDesktopMediaSource, +} From e79732ba1f446d44d7043b2b9de10e6c7c29822d Mon Sep 17 00:00:00 2001 From: "Grigorii K. Shartsev" Date: Fri, 5 Apr 2024 22:40:11 +0200 Subject: [PATCH 05/11] fix(screensharing): handle denied screen sharing access on Mac Signed-off-by: Grigorii K. Shartsev --- src/main.js | 9 ++++++--- src/preload.js | 2 +- .../renderer/components/DesktopMediaSourceDialog.vue | 5 +++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/main.js b/src/main.js index 10e3a792..dc74c89c 100644 --- a/src/main.js +++ b/src/main.js @@ -20,7 +20,7 @@ */ const path = require('node:path') -const { app, dialog, BrowserWindow, ipcMain, desktopCapturer, systemPreferences } = require('electron') +const { app, dialog, BrowserWindow, ipcMain, desktopCapturer, systemPreferences, shell } = require('electron') const { setupMenu } = require('./app/app.menu.js') const { setupReleaseNotificationScheduler } = require('./app/githubReleaseNotification.service.js') const { enableWebRequestInterceptor, disableWebRequestInterceptor } = require('./app/webRequestInterceptor.js') @@ -85,8 +85,11 @@ ipcMain.on('app:relaunch', () => { ipcMain.handle('app:getDesktopCapturerSources', async () => { // macOS 10.15 Catalina or higher requires consent for screen access if (isMac() && systemPreferences.getMediaAccessStatus('screen') !== 'granted') { - // TODO: show user-friendly error in this case - return [] + // Open System Preferences to allow screen recording + await shell.openExternal('x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture') + // We cannot detect that the user has granted access, so return no sources + // The user will have to try again after granting access + return null } const sources = await desktopCapturer.getSources({ types: ['window', 'screen'], fetchWindowIcons: true }) diff --git a/src/preload.js b/src/preload.js index 7c386ceb..451803a3 100644 --- a/src/preload.js +++ b/src/preload.js @@ -86,7 +86,7 @@ const TALK_DESKTOP = { /** * Get available desktop capture sources: screens and windows * - * @return {Promise<{ id: string, name: string, icon?: string }>} + * @return {Promise<{ id: string, name: string, icon?: string }[]|null>} */ getDesktopCapturerSources: () => ipcRenderer.invoke('app:getDesktopCapturerSources'), /** diff --git a/src/talk/renderer/components/DesktopMediaSourceDialog.vue b/src/talk/renderer/components/DesktopMediaSourceDialog.vue index 83f77442..3f376660 100644 --- a/src/talk/renderer/components/DesktopMediaSourceDialog.vue +++ b/src/talk/renderer/components/DesktopMediaSourceDialog.vue @@ -95,6 +95,11 @@ const getStreamForMediaSource = (mediaSourceId) => { const requestDesktopCapturerSources = async () => { sources.value = await window.TALK_DESKTOP.getDesktopCapturerSources() + // There is no source. Probably the user hasn't granted the permission. + if (!sources.value) { + emit('cancel') + } + const hasMultipleScreens = sources.value.filter((source) => source.id.startsWith('screen:')).length > 1 // There is no sourceId for the entire desktop in Electron - create a custom one From a18244f763281ef737c9995782268316346973c3 Mon Sep 17 00:00:00 2001 From: "Grigorii K. Shartsev" Date: Thu, 4 Apr 2024 18:07:21 +0200 Subject: [PATCH 06/11] chore(eslint): temporary disable import/default Signed-off-by: Grigorii K. Shartsev --- .eslintrc.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.eslintrc.js b/.eslintrc.js index 2eb57dea..0134474b 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -33,6 +33,10 @@ module.exports = { }, rules: { + // import/default is not compatible with SFC Setup .vue files + // It works fine on server because by default in @nextcloud/eslint-config .vue files are not inspected via eslint-plugin-import thus import/extensions doesn't include .vue + // See: https://github.com/import-js/eslint-plugin-import/blob/main/README.md#importextensions + 'import/default': 'off', /** * ESLint */ From 9b15300bbe1c69e0dd333a14b5736d6a4ce8c23e Mon Sep 17 00:00:00 2001 From: "Grigorii K. Shartsev" Date: Fri, 5 Apr 2024 08:31:23 +0200 Subject: [PATCH 07/11] chore(readme): remove screen sharing from unsupported features list Signed-off-by: Grigorii K. Shartsev --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index c917f268..f9e1bd5e 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,6 @@ ## 👾 Drawbacks - Currently not supported: - - Screen sharing ([#11](https://github.com/nextcloud/talk-desktop/issues/11)) - Setting User Status ([#26](https://github.com/nextcloud/talk-desktop/issues/26)) - Search ([#30](https://github.com/nextcloud/talk-desktop/issues/30)) - Untrusted certificate on Linux ([#23](https://github.com/nextcloud/talk-desktop/issues/23)) From ca40b846e34f740cc8346bde174515d639df5d15 Mon Sep 17 00:00:00 2001 From: "Grigorii K. Shartsev" Date: Tue, 9 Apr 2024 18:35:12 +0200 Subject: [PATCH 08/11] fix(screensharing): Wayland support Signed-off-by: Grigorii K. Shartsev --- src/main.js | 18 +++++-- src/shared/os.utils.js | 12 +++++ .../components/DesktopMediaSourceDialog.vue | 51 +++++++++++++++---- 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/src/main.js b/src/main.js index dc74c89c..cd63ae90 100644 --- a/src/main.js +++ b/src/main.js @@ -28,7 +28,7 @@ const { createAuthenticationWindow } = require('./authentication/authentication. const { openLoginWebView } = require('./authentication/login.window.js') const { createHelpWindow } = require('./help/help.window.js') const { createUpgradeWindow } = require('./upgrade/upgrade.window.js') -const { getOs, isLinux, isMac } = require('./shared/os.utils.js') +const { getOs, isLinux, isMac, isWayland } = require('./shared/os.utils.js') const { createTalkWindow } = require('./talk/talk.window.js') const { createWelcomeWindow } = require('./welcome/welcome.window.js') const { installVueDevtools } = require('./install-vue-devtools.js') @@ -92,11 +92,23 @@ ipcMain.handle('app:getDesktopCapturerSources', async () => { return null } - const sources = await desktopCapturer.getSources({ types: ['window', 'screen'], fetchWindowIcons: true }) + // We cannot show live previews on Wayland, so we show thumbnails + const thumbnailWidth = isWayland() ? 320 : 0 + + const sources = await desktopCapturer.getSources({ + types: ['screen', 'window'], + fetchWindowIcons: true, + thumbnailSize: { + width: thumbnailWidth, + height: thumbnailWidth * 9 / 16, + }, + }) + return sources.map((source) => ({ id: source.id, name: source.name, - icon: source.appIcon?.toDataURL(), + icon: source.appIcon && !source.appIcon.isEmpty() ? source.appIcon.toDataURL() : null, + thumbnail: source.thumbnail && !source.thumbnail.isEmpty() ? source.thumbnail.toDataURL() : null, })) }) diff --git a/src/shared/os.utils.js b/src/shared/os.utils.js index 2cf17019..7d50a16c 100644 --- a/src/shared/os.utils.js +++ b/src/shared/os.utils.js @@ -76,11 +76,21 @@ function isWindows() { return os.type() === 'Windows_NT' } +/** + * Is it Linux with Wayland window communication protocol? + * @return {boolean} + */ +function isWayland() { + // TODO: is it better than checking for XDG_SESSION_TYPE === 'wayland'? + return !!process.env.WAYLAND_DISPLAY +} + /** * @typedef OsVersion * @property {boolean} isLinux - Is Linux? * @property {boolean} isMac - Is Mac? * @property {boolean} isWindows - Is Windows? + * @property {boolean} isWayland - Is Linux with Wayland window communication protocol? * @property {string} version - Full string representation of OS version */ @@ -94,6 +104,7 @@ function getOs() { isLinux: isLinux(), isMac: isMac(), isWindows: isWindows(), + isWayland: isWayland(), version: getOsVersion(), } } @@ -104,5 +115,6 @@ module.exports = { isLinux, isMac, isWindows, + isWayland, getOs, } diff --git a/src/talk/renderer/components/DesktopMediaSourceDialog.vue b/src/talk/renderer/components/DesktopMediaSourceDialog.vue index 3f376660..5e55245d 100644 --- a/src/talk/renderer/components/DesktopMediaSourceDialog.vue +++ b/src/talk/renderer/components/DesktopMediaSourceDialog.vue @@ -24,6 +24,7 @@ import { computed, nextTick, onBeforeUnmount, onMounted, ref } from 'vue' import MdiCancel from '@mdi/svg/svg/cancel.svg?raw' import MdiMonitorShare from '@mdi/svg/svg/monitor-share.svg?raw' +import MdiApplicationOutline from 'vue-material-design-icons/ApplicationOutline.vue' import MdiMonitor from 'vue-material-design-icons/Monitor.vue' import MdiMonitorSpeaker from 'vue-material-design-icons/MonitorSpeaker.vue' @@ -35,6 +36,11 @@ import { translate as t } from '@nextcloud/l10n' const emit = defineEmits(['submit', 'cancel']) +// On Wayland getting each stream for the live preview requests user to select the source via system dialog again +// Instead - show static images. +// See: https://github.com/electron/electron/issues/27732 +const previewType = window.OS.isWayland ? 'thumbnail' : 'live' + const selectedSourceId = ref(null) const sources = ref(null) const videoElements = {} @@ -100,13 +106,23 @@ const requestDesktopCapturerSources = async () => { emit('cancel') } - const hasMultipleScreens = sources.value.filter((source) => source.id.startsWith('screen:')).length > 1 + // On Wayland we don't manually provide the source stream. It is covered by Wayland and custom id is not supported + if (!window.OS.isWayland) { + // There is no sourceId for the entire desktop with all the screens and audio in Electron. + // But it is possible to capture it. "entire-desktop:0:0" is a custom sourceId for this specific case. + const hasMultipleScreens = sources.value.filter((source) => source.id.startsWith('screen:')).length > 1 + sources.value.unshift({ + id: 'entire-desktop:0:0', + name: hasMultipleScreens ? t('talk_desktop', 'All screens with audio') : t('talk_desktop', 'Entire screen with audio'), + }) + } - // There is no sourceId for the entire desktop in Electron - create a custom one - sources.value.unshift({ - id: 'entire-desktop:0:0', - name: hasMultipleScreens ? t('talk_desktop', 'All screens with audio') : t('talk_desktop', 'Entire screen with audio'), - }) + // On Wayland there might be no name from the desktopCapturer + if (window.OS.isWayland) { + for (const source of sources.value) { + source.name ||= t('talk_desktop', 'Selected screen or window') + } + } // Preselect the first media source if any selectedSourceId.value = sources.value[0]?.id @@ -116,16 +132,23 @@ const setVideoSources = async () => Promise.allSettled(sources.value.map(async ( videoElements[source.id].srcObject = await getStreamForMediaSource(source.id) })) -onMounted(async () => { - await requestDesktopCapturerSources() +const showLivePreviews = async () => { // Wait for video elements to be mounted await nextTick() // Set streams for all ids await setVideoSources() +} + +onMounted(async () => { + await requestDesktopCapturerSources() + + if (previewType === 'live') { + await showLivePreviews() + } }) onBeforeUnmount(() => { - if (!sources.value) { + if (!sources.value || previewType !== 'live') { return } // Release all streams, otherwise they are still captured even if no video element is using them @@ -150,10 +173,17 @@ onBeforeUnmount(() => { class="capture-source__input" type="radio" :value="source.id"> -