[#65] Parse creator information on offcanvas

DOMParser is not available anymore in MV3 Conditionally use the old way or the offscreen page (which calls old fn)
cpiber · Aug 17, 2024 · 99603b5 · 99603b5
1 parent 37bf455
commit 99603b5
Show file tree

Hide file tree

Showing 8 changed files with 136 additions and 23 deletions.
diff --git a/src/manifest_v3.ts b/src/manifest_v3.ts
@@ -40,6 +40,7 @@ const manifest: browser._manifest.WebExtensionManifest = {
   ],
   permissions: [
     'storage',
+    'offscreen',
   ],
   host_permissions: [
     // Only firefox requires this, and the subdomain must be completely specified, otherwise the request fails

diff --git a/src/offscreen.html b/src/offscreen.html
@@ -0,0 +1,2 @@
+<!doctype html>
+<script src="scripts/offscreen.js"></script>
diff --git a/src/scripts/background/misc.ts b/src/scripts/background/misc.ts
@@ -1,19 +1,77 @@
+import { getBrowserInstance } from '../helpers/sharedExt';
+import { getInformation } from '../page/offscreen';
 import type { Creator } from './index';
 
-// TODO: In MV3, DOMParser needs to work with an offscreen document instead: https://developer.chrome.com/docs/extensions/develop/migrate/to-service-workers#move-dom-and-window
-export const loadCreators = async (): Promise<Creator[]> => {
-  const res = await fetch('https://talent.nebula.tv/creators/');
-  const body = await res.text();
-  const parser = new DOMParser();
-  const doc = parser.parseFromString(body, 'text/html');
-  return Array.from(doc.querySelectorAll('#creator-wall .youtube-creator')).map(c => ({
-    name: c.querySelector('img').alt,
-    nebula: c.querySelector<HTMLAnchorElement>('.link.nebula')?.href?.split('/')?.pop(),
-    nebulaAlt: new URL(c.querySelector<HTMLAnchorElement>('h3 a').href).pathname.split('/')[1],
-    channel: c.getAttribute('data-video'),
-    uploads: c.getAttribute('data-video') ? 'UU' + c.getAttribute('data-video').substring(2) : undefined,
-  }));
-};
+let creating: Promise<void> | undefined = undefined;
+async function setupOffscreenDocument(path: string) {
+  // Check all windows controlled by the service worker to see if one
+  // of them is the offscreen document with the given path
+  const offscreenUrl = getBrowserInstance().runtime.getURL(path);
+  // @ts-expect-error offscreen API not typed
+  const existingContexts = await getBrowserInstance().runtime.getContexts({
+    contextTypes: ['OFFSCREEN_DOCUMENT'],
+    documentUrls: [offscreenUrl]
+  });
+
+  if (existingContexts.length > 0) {
+    return;
+  }
+
+  // create offscreen document
+  if (creating) {
+    await creating;
+  } else {
+    // @ts-expect-error offscreen API not typed
+    creating = getBrowserInstance().offscreen.createDocument({
+      url: path,
+      reasons: ['DOM_PARSER'],
+      justification: 'needed for scraping the nebula creator page',
+    });
+    await creating;
+    creating = null;
+  }
+}
+async function closeOffscreenDocument() {
+  // @ts-expect-error offscreen API not typed
+  await getBrowserInstance().offscreen.closeDocument();
+}
+
+let dataPromiseResolve: (data: Creator[]) => void | undefined = undefined;
+getBrowserInstance().runtime.onMessage.addListener(async (message: { [key: string]: any; }) => {
+  if (message.target !== 'background') {
+    return false;
+  }
+  console.dev.log('Received message from offcanvas', message);
+  switch (message.type) {
+    case 'receiveCreatorInformation': {
+      console.dev.log('dataPromiseResolve is set?', !!dataPromiseResolve);
+      if (dataPromiseResolve) dataPromiseResolve(message.data);
+    } break;
+  }
+});
+
+export const loadCreators: () => Promise<Creator[]> = (() => {
+  if (__MV3__) {
+    console.info('MV3! Using offscreen API instead');
+    return async () => {
+      await setupOffscreenDocument('offscreen.html');
+      const data = await new Promise<Creator[]>(async resolve => {
+        dataPromiseResolve = resolve;
+
+        await getBrowserInstance().runtime.sendMessage({
+          type: 'getCreatorInformation',
+          target: 'offscreen',
+        });
+      });
+      dataPromiseResolve = undefined;
+      await closeOffscreenDocument();
+      console.dev.log('Creator data:', data);
+      return data;
+    };
+  }
+
+  return getInformation;
+})();
 
 // @ts-expect-error regex works, don't touch
 export const normalizeString = (str: string) => str.toLowerCase().normalize('NFD').replace(/\p{Pd}/g, '-')

diff --git a/src/scripts/background_script.ts b/src/scripts/background_script.ts
@@ -1,7 +1,7 @@
 import { Creator, loadCreators as _loadCreators, creatorHasNebulaVideo, creatorHasYTVideo, existsNebulaVideo, normalizeString } from './background';
 import { purgeCache, purgeCacheIfNecessary } from './background/ext';
 import type { CreatorSettings } from './content/nebula/creator-settings';
-import { BrowserMessage, getBase, getBrowserInstance, getFromStorage, nebulavideo, parseTimeString, parseTypeObject, setToStorage, toTimeString } from './helpers/sharedExt';
+import { BrowserMessage, getBase, getBrowserInstance, getFromStorage, isChrome, nebulavideo, parseTimeString, parseTypeObject, setToStorage, toTimeString } from './helpers/sharedExt';
 
 const videoFetchYt = 50;
 const videoFetchNebula = 50;
@@ -14,21 +14,38 @@ if (getBrowserInstance().action) {
   getBrowserInstance().browserAction.onClicked.addListener(() => openOptions());
 }
 
-getBrowserInstance().runtime.onMessage.addListener(async (message: string | { [key: string]: any; }) => {
+getBrowserInstance().runtime.onMessage.addListener((message: string | { [key: string]: any; }) => {
+  // Return early if this message isn't meant for the background script
+  if (typeof message !== 'string' && message.target !== undefined && message.target !== 'background') {
+    return;
+  }
+
   const keepAlive = setInterval(getBrowserInstance().runtime.getPlatformInfo, 25 * 1000);
+  let ret: Promise<any>;
   try {
     const msg = parseTypeObject(message);
     console.dev.log('Handling message', msg);
     switch (msg.type) {
       case BrowserMessage.INIT_PAGE:
-        return openChangelog();
+        ret = openChangelog();
+        break;
       case BrowserMessage.LOAD_CREATORS:
-        return console.debug(await loadCreators());
+        ret = loadCreators();
+        break;
       case BrowserMessage.GET_YTID:
-        return getYoutubeId(msg);
+        ret = getYoutubeId(msg);
+        break;
       case BrowserMessage.GET_VID:
-        return getNebulaVideo(msg);
+        ret = getNebulaVideo(msg);
+        break;
+    }
+    if (ret) {
+      ret.then(
+        res => console.dev.log('Result for', msg.type, ':', res),
+        err => console.dev.log('Error running', msg.type, ':', err),
+      );
     }
+    return ret;
   } catch {
   } finally {
     clearInterval(keepAlive);
@@ -168,7 +185,7 @@ const openOptions = (active = true, ...args: string[]) => {
   console.debug(await loadCreators());
 
   // debug code
-  if (!__DEV__) return;
+  if (!__DEV__ || isChrome()) return;
   (window as any).loadCreators = loadCreators;
   Object.defineProperty(window, 'loadCreatorsPromise', {
     get: () => promise, set(v) {

diff --git a/src/scripts/content/nebula/index.ts b/src/scripts/content/nebula/index.ts
@@ -240,6 +240,8 @@ const loadComments = async () => {
   try {
     // TODO: in MV3 this always returns false for some reason, even if the creator loading is fixed, so disable for now
     const vid: ytvideo = await getBrowserInstance().runtime.sendMessage({ type: BrowserMessage.GET_YTID, creator, title, nebula });
+    console.dev.log('got:', vid);
+    if (vid as any === false) throw new Error('Unknown error');
     e.querySelectorAll('.enhancer-yt, .enhancer-yt-err, .enhancer-yt-outside, .enhancer-yt-inside').forEach(e => e.remove());
     console.debug('Found video:', vid);
     const v = document.createElement('span');

diff --git a/src/scripts/helpers/sharedExt.ts b/src/scripts/helpers/sharedExt.ts
@@ -5,15 +5,16 @@ import './shared/prototype';
 export * from './shared';
 
 export const getBrowserInstance = () => browser; // poly-filled
-export const isChrome = () => (window as any).chrome !== undefined;
+export const isChrome = () => (globalThis as any).chrome !== undefined;
 
-const { sync, local } = browser.storage;
 export function getFromStorage<T extends { [key: string]: any; }>(key: T): Promise<T>;
 export function getFromStorage<T>(key?: string | string[] | null): Promise<T>;
 export function getFromStorage<T>(key?: string | string[] | { [key: string]: any; } | null) {
+  const { sync, local } = browser.storage;
   return (sync || local).get(key) as Promise<T>;
 }
 export function setToStorage(key: { [key: string]: any; }) {
+  const { sync, local } = browser.storage;
   return (sync || local).set(key);
 }
 

diff --git a/src/scripts/offscreen.ts b/src/scripts/offscreen.ts
@@ -0,0 +1,19 @@
+import browser from 'webextension-polyfill';
+import { getInformation } from './page/offscreen';
+
+browser.runtime.onMessage.addListener(async (message: { [key: string]: any; }) => {
+  if (message.target !== 'offscreen') {
+    return false;
+  }
+  console.dev.log('Received message for offcanvas', message);
+  switch (message.type) {
+    case 'getCreatorInformation': {
+      const data = await getInformation();
+      await browser.runtime.sendMessage({
+        type: 'receiveCreatorInformation',
+        target: 'background',
+        data
+      });
+    } break;
+  }
+});
diff --git a/src/scripts/page/offscreen.ts b/src/scripts/page/offscreen.ts
@@ -0,0 +1,13 @@
+export const getInformation = async () => {
+  const res = await fetch('https://talent.nebula.tv/creators/');
+  const body = await res.text();
+  const parser = new DOMParser();
+  const doc = parser.parseFromString(body, 'text/html');
+  return Array.from(doc.querySelectorAll('#creator-wall .youtube-creator')).map(c => ({
+    name: c.querySelector('img').alt,
+    nebula: c.querySelector<HTMLAnchorElement>('.link.nebula')?.href?.split('/')?.pop(),
+    nebulaAlt: new URL(c.querySelector<HTMLAnchorElement>('h3 a').href).pathname.split('/')[1],
+    channel: c.getAttribute('data-video'),
+    uploads: c.getAttribute('data-video') ? 'UU' + c.getAttribute('data-video').substring(2) : undefined,
+  }));
+};
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		<!doctype html>
		<script src="scripts/offscreen.js"></script>