340 ocr feature request (#434)

* Added vision feature

* Refactor background index.ts

* Fixed error handling

* Add dependencies for Firebase functions, core common, and core service

apps/acf-extension/src/background/index.ts

@@ -12,23 +12,20 @@ import { GoogleDriveBackground, RUNTIME_MESSAGE_GOOGLE_DRIVE } from '@dhruv-tech
 import { GoogleOauth2Background, RUNTIME_MESSAGE_GOOGLE_OAUTH } from '@dhruv-techapps/google-oauth';
 import { GoogleSheetsBackground, RUNTIME_MESSAGE_GOOGLE_SHEETS } from '@dhruv-techapps/google-sheets';
 import { registerNotifications } from '@dhruv-techapps/notifications';
+import { RUNTIME_MESSAGE_VISION, VisionBackground } from '@dhruv-techapps/vision';
 import XMLHttpRequest from 'xhr-shim';
 import { ACTION_POPUP } from '../common/constant';
-import { DISCORD_CLIENT_ID, EDGE_OAUTH_CLIENT_ID, FIREBASE_FUNCTIONS_URL, OPTIONS_PAGE_URL, UNINSTALL_URL, VARIANT } from '../common/environments';
+import { DISCORD_CLIENT_ID, EDGE_OAUTH_CLIENT_ID, FIREBASE_FUNCTIONS_URL, OPTIONS_PAGE_URL, VARIANT } from '../common/environments';
 import AcfBackup from './acf-backup';
 import registerContextMenus from './context-menu';
 import { auth } from './firebase';
 import { googleAnalytics } from './google-analytics';
 import './sync-config';
 import { TabsMessenger } from './tab';
-import { Update } from './update';
-self['XMLHttpRequest'] = XMLHttpRequest;
 
-let firebaseFirestoreBackground: FirebaseFirestoreBackground;
+self['XMLHttpRequest'] = XMLHttpRequest;
 
 try {
-  firebaseFirestoreBackground = new FirebaseFirestoreBackground(auth, EDGE_OAUTH_CLIENT_ID, OPTIONS_PAGE_URL);
-
   /**
    * Browser Action set to open option page / configuration page
    */
@@ -46,12 +43,6 @@ try {
     }
   });
 
-  auth.onAuthStateChanged((user) => {
-    if (user) {
-      Update.discord(firebaseFirestoreBackground);
-    }
-  });
-
   /**
    * Set Context Menu for right click
    */
@@ -62,13 +53,6 @@ try {
    */
   registerNotifications(OPTIONS_PAGE_URL);
 
-  /**
-   * Setup Uninstall action
-   */
-  if (UNINSTALL_URL) {
-    chrome.runtime.setUninstallURL(UNINSTALL_URL);
-  }
-
   /**
    * Setup on Message Listener
    */
@@ -85,6 +69,7 @@ try {
     [RUNTIME_MESSAGE_FIREBASE_OAUTH]: new FirebaseOauth2Background(auth, EDGE_OAUTH_CLIENT_ID),
     [RUNTIME_MESSAGE_FIREBASE_FIRESTORE]: new FirebaseFirestoreBackground(auth, EDGE_OAUTH_CLIENT_ID, OPTIONS_PAGE_URL),
     [RUNTIME_MESSAGE_FIREBASE_FUNCTIONS]: new FirebaseFunctionsBackground(auth, FIREBASE_FUNCTIONS_URL, EDGE_OAUTH_CLIENT_ID),
+    [RUNTIME_MESSAGE_VISION]: new VisionBackground(auth, FIREBASE_FUNCTIONS_URL, EDGE_OAUTH_CLIENT_ID),
   };
   Runtime.onMessageExternal(onMessageListener);
   Runtime.onMessage(onMessageListener);

apps/acf-extension/src/content_scripts/action.ts

@@ -31,7 +31,7 @@ const ActionProcessor = (() => {
     if (elements === undefined) {
       return ACTION_STATUS.SKIPPED;
     }
-    const value = action.value ? await ACFValue.getValue(action.value) : action.value;
+    const value = action.value ? await ACFValue.getValue(action.value, action.settings) : action.value;
     await ACFEvents.check(elementFinder, elements, value);
   };
 

apps/acf-extension/src/content_scripts/util/acf-value.ts

@@ -1,21 +1,42 @@
+import { ActionSettings } from '@dhruv-techapps/acf-common';
 import { Value } from '@dhruv-techapps/acf-util';
 import { GoogleSheetsValue } from '@dhruv-techapps/google-sheets';
 import { SandboxValue } from '@dhruv-techapps/sandbox';
+import { VisionService, VisionValue } from '@dhruv-techapps/vision';
+import Common from '../common';
 
 export const VALUE_MATCHER = {
   GOOGLE_SHEETS: /^GoogleSheets::/i,
   FUNC: /^Func::/i,
+  IMAGE: /^Image::/i,
 };
 
 export class ACFValue {
-  static async getValue(value: string) {
+  static async getValue(value: string, settings?: ActionSettings): Promise<string> {
     value = await Value.getValue(value);
     if (VALUE_MATCHER.GOOGLE_SHEETS.test(value)) {
       value = GoogleSheetsValue.getSheetValue(value);
     }
     if (VALUE_MATCHER.FUNC.test(value)) {
       value = await SandboxValue.getFuncValue(value);
     }
+    if (VALUE_MATCHER.IMAGE.test(value)) {
+      try {
+        const elementFinder = value.replace(/image::/i, '');
+        const elements = await Common.start(elementFinder, settings);
+        if (elements === undefined || typeof elements === 'number' || elements.length === 0) {
+          throw new Error('No element found with the given selector');
+        }
+        const data = VisionValue.getImageSrc(elements);
+        value = await VisionService.imagesAnnotate(data);
+        return value;
+      } catch (error) {
+        if (error instanceof Error) {
+          return error.message;
+        }
+      }
+      return value;
+    }
     return value;
   }
 }

apps/acf-options-page/src/components/data-list.components.tsx

@@ -165,6 +165,12 @@ export function DataList() {
         <option value='Element::scrollIntoView::{ "behavior": "smooth", "block": "end", "inline": "nearest" }'>
           Element::scrollIntoView::&#123; "behavior": "smooth", "block": "end", "inline": "nearest" &#123;
         </option>
+        <option value='Image::'>Image::</option>
+        <option value='Image::#'>Image::#</option>
+        <option value='Image::ClassName::'>Image::ClassName::</option>
+        <option value='Image::Name::'>Image::Name::</option>
+        <option value='Image::Selector::'>Image::Selector::</option>
+        <option value='Image:://input[@id="element-id"]'>Image:://input[@id=&quot;element-id&quot;]</option>
       </datalist>
       <datalist id='valueExtractor'>
         <option value='@id'>To get id attribute of element</option>

libs/shared/firebase-functions/src/lib/firebase-functions-background.ts

@@ -1,7 +1,7 @@
 import { Auth, FirebaseOauth2Background } from '@dhruv-techapps/firebase-oauth';
 import { GOOGLE_SCOPES } from '@dhruv-techapps/google-oauth';
 import { NotificationHandler } from '@dhruv-techapps/notifications';
-import { NOTIFICATIONS_ID, NOTIFICATIONS_TITLE } from './firebase-functions.constant';
+import { NOTIFICATIONS_ID } from './firebase-functions.constant';
 
 export class FirebaseFunctionsBackground extends FirebaseOauth2Background {
   cloudFunctionUrl: string;
@@ -11,11 +11,8 @@ export class FirebaseFunctionsBackground extends FirebaseOauth2Background {
     this.cloudFunctionUrl = cloudFunctionUrl;
   }
 
-  async visionImagesAnnotate<Res>(content: string): Promise<Res> {
+  async visionImagesAnnotate<Req, Res>(data: Req): Promise<Res> {
     const headers = await this._getFirebaseHeaders();
-    const data = {
-      requests: [{ image: { content }, features: [{ type: 'TEXT_DETECTION' }] }],
-    };
     const url = new URL(this.cloudFunctionUrl + '/visionImagesAnnotate');
     const response = await this.#fetch(url, headers, data);
     return response;
@@ -80,15 +77,21 @@ export class FirebaseFunctionsBackground extends FirebaseOauth2Background {
       const response = await fetch(url.href, init);
       const result = await response.json();
       if (result.error) {
-        NotificationHandler.notify(NOTIFICATIONS_ID, NOTIFICATIONS_TITLE, result.error, true);
-        throw new Error(result.error);
+        throw new CustomError(result.error, result.message);
       }
       return result;
     } catch (error) {
-      if (error instanceof Error) {
-        NotificationHandler.notify(NOTIFICATIONS_ID, NOTIFICATIONS_TITLE, error.message, true);
+      if (error instanceof CustomError || error instanceof Error) {
+        NotificationHandler.notify(NOTIFICATIONS_ID, error.name, error.message, true);
       }
       throw error;
     }
   }
 }
+
+class CustomError extends Error {
+  constructor(name: string, message: string) {
+    super(message);
+    this.name = name;
+  }
+}

libs/shared/google-sheets/src/lib/google-sheets-background.ts

@@ -9,24 +9,17 @@ const NOTIFICATIONS_ID = 'google-sheets';
 export class GoogleSheetsBackground extends FirebaseFunctionsBackground {
   scopes = [GOOGLE_SCOPES.SHEETS];
   async getSheets({ spreadsheetId, ranges }: GoogleSheetsRequest): Promise<GoogleSheetsResponse> {
-    try {
-      if (!spreadsheetId || !ranges) {
-        throw new Error('spreadsheetId or ranges is not defined');
-      }
+    if (!spreadsheetId || !ranges) {
+      throw new Error('spreadsheetId or ranges is not defined');
+    }
 
-      const response = await this.getValues<GoogleSheetsRequest, GoogleSheetsResponse>({ spreadsheetId, ranges });
-      return response.filter((result) => {
-        if (result.error) {
-          NotificationHandler.notify(NOTIFICATIONS_ID, NOTIFICATIONS_TITLE, result.error.message);
-          return false;
-        }
-        return true;
-      });
-    } catch (error) {
-      if (error instanceof Error) {
-        NotificationHandler.notify(NOTIFICATIONS_ID, NOTIFICATIONS_TITLE, error.message);
+    const response = await this.getValues<GoogleSheetsRequest, GoogleSheetsResponse>({ spreadsheetId, ranges });
+    return response.filter((result) => {
+      if (result.error) {
+        NotificationHandler.notify(NOTIFICATIONS_ID, NOTIFICATIONS_TITLE, result.error.message);
+        return false;
       }
-      throw error;
-    }
+      return true;
+    });
   }
 }

libs/shared/vision/.eslintrc.json

@@ -0,0 +1,25 @@
+{
+  "extends": ["../../../.eslintrc.json"],
+  "ignorePatterns": ["!**/*"],
+  "overrides": [
+    {
+      "files": ["*.ts", "*.tsx", "*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.ts", "*.tsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.js", "*.jsx"],
+      "rules": {}
+    },
+    {
+      "files": ["*.json"],
+      "parser": "jsonc-eslint-parser",
+      "rules": {
+        "@nx/dependency-checks": "error"
+      }
+    }
+  ]
+}

libs/shared/vision/jest.config.ts

@@ -0,0 +1,10 @@
+/* eslint-disable */
+export default {
+  displayName: 'vision',
+  preset: '../../../jest.preset.js',
+  transform: {
+    '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '<rootDir>/tsconfig.spec.json' }],
+  },
+  moduleFileExtensions: ['ts', 'js', 'html'],
+  coverageDirectory: '../../../coverage/libs/shared/vision',
+};

libs/shared/vision/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@dhruv-techapps/vision",
+  "version": "0.0.1",
+  "dependencies": {
+    "tslib": "^2.3.0",
+    "@dhruv-techapps/firebase-functions": "*",
+    "@dhruv-techapps/core-common": "*",
+    "@dhruv-techapps/core-service": "*"
+  },
+  "type": "commonjs",
+  "main": "./src/index.js",
+  "typings": "./src/index.d.ts"
+}

libs/shared/vision/project.json

@@ -0,0 +1,42 @@
+{
+  "name": "vision",
+  "$schema": "../../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "libs/shared/vision/src",
+  "projectType": "library",
+  "release": {
+    "version": {
+      "generatorOptions": {
+        "packageRoot": "dist/{projectRoot}",
+        "currentVersionResolver": "git-tag"
+      }
+    }
+  },
+  "tags": [],
+  "targets": {
+    "build": {
+      "executor": "@nx/js:tsc",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "outputPath": "dist/libs/shared/vision",
+        "main": "libs/shared/vision/src/index.ts",
+        "tsConfig": "libs/shared/vision/tsconfig.lib.json",
+        "assets": []
+      }
+    },
+    "nx-release-publish": {
+      "options": {
+        "packageRoot": "dist/{projectRoot}"
+      }
+    },
+    "lint": {
+      "executor": "@nx/eslint:lint"
+    },
+    "test": {
+      "executor": "@nx/jest:jest",
+      "outputs": ["{workspaceRoot}/coverage/{projectRoot}"],
+      "options": {
+        "jestConfig": "libs/shared/vision/jest.config.ts"
+      }
+    }
+  }
+}

libs/shared/vision/src/index.ts

@@ -0,0 +1,5 @@
+export * from './lib/vision-background';
+export * from './lib/vision-types';
+export * from './lib/vision-value';
+export * from './lib/vision.constant';
+export * from './lib/vision.service';

libs/shared/vision/src/lib/vision-background.ts

@@ -0,0 +1,13 @@
+import { FirebaseFunctionsBackground } from '@dhruv-techapps/firebase-functions';
+import { VisionImageRequest, VisionImageResponse } from './vision-types';
+
+export class VisionBackground extends FirebaseFunctionsBackground {
+  async imagesAnnotate({ content, imageUrl }: VisionImageRequest): Promise<string> {
+    if (!content && !imageUrl) {
+      throw new Error('No image data found');
+    }
+    const data: VisionImageRequest = { content, imageUrl };
+    const result = await this.visionImagesAnnotate<VisionImageRequest, VisionImageResponse>(data);
+    return result.responses[0].fullTextAnnotation.text;
+  }
+}

libs/shared/vision/src/lib/vision-types.ts

@@ -0,0 +1,8 @@
+export type VisionImageRequest = {
+  content: string;
+  imageUrl: string;
+};
+
+export type VisionImageResponse = {
+  responses: Array<{ fullTextAnnotation: { text: string } }>;
+};

libs/shared/vision/src/lib/vision-value.ts

@@ -0,0 +1,36 @@
+import { VisionImageRequest } from './vision-types';
+
+export class VisionValue {
+  static isBase64Image(dataURL: string) {
+    const regex = /^data:image\/(png|jpeg|jpg|gif|webp|bmp|svg\+xml|x-icon);base64,/;
+    return regex.test(dataURL);
+  }
+
+  static extractBase64Data(dataURL: string) {
+    // Split the data URL at the comma
+    const base64Data = dataURL.split(',')[1];
+    return base64Data;
+  }
+
+  static getImageSrc(elements: Array<HTMLElement>): VisionImageRequest {
+    const element = elements[0];
+    if (!element) {
+      throw new Error('No element found');
+    }
+    if (element.tagName !== 'IMG') {
+      throw new Error('Element is not an image');
+    }
+    const src = element.getAttribute('src');
+    if (!src) {
+      throw new Error('No image found');
+    }
+    let content = '';
+    let imageUrl = '';
+    if (this.isBase64Image(src)) {
+      content = this.extractBase64Data(src);
+    } else {
+      imageUrl = src;
+    }
+    return { content, imageUrl };
+  }
+}

libs/shared/vision/src/lib/vision.constant.ts

@@ -0,0 +1 @@
+export const RUNTIME_MESSAGE_VISION = 'vision';