Better orion language detection (#337)

* New model to track orion language processing

* New function to detect language

* New custom migration

* Add cursor tracker and video orion language to offchain export

* Running `custom-migration` command

* Add manger to trigger video language updates

* Adjust orion video language manager to support video update

* Revert "Running `custom-migration` command"

This reverts commit 71c4997.

* Second run of `create-migrations` command

* CR fixes

* move 'orion_offchain_cursor' table to admin schema

* bump package version and add change log

* fix: bug in case detected language is undefined

---------

Co-authored-by: Zeeshan Akram <97m.zeeshan@gmail.com>

.env

@@ -62,6 +62,9 @@ TRUST_PROXY=uniquelocal
 SENDGRID_API_KEY=
 SENDGRID_FROM_EMAIL=gateway@example.com
 
+# Detectlanguage 
+DETECTLANGUAGE_API_KEY=
+
 # Debug settings
 SQD_DEBUG=api:*
 

CHANGELOG.md

@@ -1,3 +1,8 @@
+# 4.0.4
+
+## Bug Fixes:
+- Fixed: improve the accuracy of `Video.orionLanguage` field by reworking the `predictVideoLanguage` function in `src/utils/language.ts`
+
 # 4.0.3
 
 ## Misc

db/migrations/1720623003671-Data.js

@@ -0,0 +1,11 @@
+module.exports = class Data1720623003671 {
+ name = 'Data1720623003671'
+
+ async up(db) {
+ await db.query(`CREATE TABLE "admin"."orion_offchain_cursor" ("cursor_name" character varying NOT NULL, "value" bigint NOT NULL, CONSTRAINT "PK_7083797352af5a21224b6c8ccbc" PRIMARY KEY ("cursor_name"))`)
+ }
+
+ async down(db) {
+ await db.query(`DROP TABLE "admin"."orion_offchain_cursor"`)
+ }
+}

db/migrations/1709641962433-Views.js → db/migrations/1720623003800-Views.js

@@ -1,8 +1,8 @@
 
 const { getViewDefinitions } = require('../viewDefinitions')
 
-module.exports = class Views1709641962433 {
- name = 'Views1709641962433'
+module.exports = class Views1720623003800 {
+ name = 'Views1720623003800'
 
  async up(db) {
  const viewDefinitions = getViewDefinitions(db);

package.json

@@ -1,6 +1,6 @@
 {
  "name": "orion",
- "version": "4.0.3",
+ "version": "4.0.4",
  "engines": {
  "node": ">=16"
  },
@@ -72,6 +72,7 @@
  "cookie-parser": "^1.4.6",
  "csv-stringify": "^6.3.0",
  "dayjs": "^1.11.7",
+ "detectlanguage": "^2.1.0",
  "dotenv": "^16.0.3",
  "dotenv-expand": "^10.0.0",
  "express-openapi-validator": "^5.0.3",
@@ -87,7 +88,6 @@
  "patch-package": "^6.5.0",
  "pg": "8.8.0",
  "swagger-ui-express": "^4.6.2",
- "tinyld": "^1.3.4",
  "type-graphql": "^1.2.0-rc.1",
  "typeorm": "^0.3.11",
  "ua-parser-js": "^1.0.34",
@@ -105,8 +105,8 @@
  "@subsquid/substrate-typegen": "^2.1.0",
  "@subsquid/typeorm-codegen": "0.3.1",
  "@types/async-lock": "^1.1.3",
- "@types/chai": "^4.3.11",
  "@types/big-json": "^3.2.4",
+ "@types/chai": "^4.3.11",
  "@types/cookie-parser": "^1.4.3",
  "@types/express-rate-limit": "^6.0.0",
  "@types/mocha": "^10.0.1",

src/mappings/content/video.ts

@@ -17,10 +17,10 @@ import {
  VideoViewEvent,
 } from '../../model'
 import { EventHandlerContext } from '../../utils/events'
-import { predictVideoLanguage } from '../../utils/language'
 import {
  deserializeMetadata,
  genericEventFields,
+ orionVideoLanguageManager,
  u8aToBytes,
  videoRelevanceManager,
 } from '../utils'
@@ -122,10 +122,7 @@ export async function processVideoCreatedEvent({
  }
  }
 
- video.orionLanguage = predictVideoLanguage({
- title: video.title ?? '',
- description: video.description ?? '',
- })
+ video.orionLanguage = null
 
  channel.totalVideosCreated += 1
 
@@ -192,10 +189,7 @@ export async function processVideoUpdatedEvent({
  )
  }
 
- video.orionLanguage = predictVideoLanguage({
- title: video.title ?? '',
- description: video.description ?? '',
- })
+ orionVideoLanguageManager.scheduleVideoForDetection(video.id)
 
  if (autoIssueNft) {
  await processNft(overlay, block, indexInBlock, extrinsicHash, video, contentActor, autoIssueNft)

src/mappings/utils.ts

@@ -10,10 +10,16 @@ import { Event, MetaprotocolTransactionResultFailed, NftActivity, NftHistoryEntr
 import { CommentCountersManager } from '../utils/CommentsCountersManager'
 import { VideoRelevanceManager } from '../utils/VideoRelevanceManager'
 import { EntityManagerOverlay } from '../utils/overlay'
+import { OrionVideoLanguageManager } from '../utils/OrionVideoLanguageManager'
 
+export const orionVideoLanguageManager = new OrionVideoLanguageManager()
 export const commentCountersManager = new CommentCountersManager()
 export const videoRelevanceManager = new VideoRelevanceManager()
 // eslint-disable-next-line no-void
+void orionVideoLanguageManager.init(
+ 1000 * 60 * 5 // 5 mins
+)
+// eslint-disable-next-line no-void
 void videoRelevanceManager.init({
  fullUpdateLoopTime: 1000 * 60 * 60 * 12, // 12 hrs
  scheduledUpdateLoopTime: 1000 * 60 * 10, // 10 mins

src/model/OrionOffchainCursor.ts

@@ -0,0 +1,20 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm'
+
+@Entity({ schema: 'admin' })
+export class OrionOffchainCursor {
+ constructor(props?: Partial<OrionOffchainCursor>) {
+ Object.assign(this, props)
+ }
+
+ /**
+ * Name of the offchain cursor
+ */
+ @PrimaryColumn()
+ cursorName!: string
+
+ /**
+ * Value of the cursor
+ */
+ @Column('int8', { nullable: false })
+ value!: number
+}

src/model/index.ts

@@ -1,2 +1,3 @@
 export * from './generated'
 export { NextEntityId } from './NextEntityId'
+export { OrionOffchainCursor } from './OrionOffchainCursor'

src/utils/OrionVideoLanguageManager.ts

@@ -0,0 +1,64 @@
+import { EntityManager } from 'typeorm'
+import {
+ detectVideoLanguageWithProvider,
+ updateVideoLanguages,
+ VIDEO_ORION_LANGUAGE_CURSOR_NAME,
+} from './customMigrations/setOrionLanguageProvider'
+import { globalEm } from './globalEm'
+
+export class OrionVideoLanguageManager {
+ private videoToDetect: Set<string> = new Set()
+
+ async init(intervalMs: number): Promise<void> {
+ if (!VIDEO_ORION_LANGUAGE_CURSOR_NAME) {
+ return
+ }
+
+ this.updateLoop(intervalMs)
+ .then(() => {
+ /* Do nothing */
+ })
+ .catch((err) => {
+ console.error(err)
+ process.exit(-1)
+ })
+ }
+
+ scheduleVideoForDetection(id: string | null | undefined) {
+ if (id) {
+ this.videoToDetect.add(id)
+ }
+ }
+
+ async updateScheduledVideoLanguage(em: EntityManager) {
+ if (!this.videoToDetect.size) {
+ return
+ }
+
+ const videos = await em.query(`
+ SELECT id, title, description
+ FROM admin.video
+ WHERE id in (${[...this.videoToDetect.values()].map((id) => `'${id}'`).join(',')})
+ `)
+
+ await updateVideoLanguages(em, videos)
+ this.videoToDetect.clear()
+ }
+
+ async updateOrionVideoLanguage() {
+ return detectVideoLanguageWithProvider()
+ }
+
+ private async updateLoop(intervalMs: number): Promise<void> {
+ const em = await globalEm
+ while (true) {
+ await this.updateScheduledVideoLanguage(em).catch((e) => {
+ console.log(`Updating scheduled videos Orion language with provider failed`, e)
+ })
+ await this.updateOrionVideoLanguage().catch((e) => {
+ console.log(`Updating Orion language with provider failed`, e)
+ })
+ await new Promise((resolve) => setTimeout(resolve, intervalMs))
+ }
+ }
+}