From c8b2c2e9fb7d962384142ce6d416aaca1611ac59 Mon Sep 17 00:00:00 2001 From: WRadoslaw <92513933+WRadoslaw@users.noreply.github.com> Date: Wed, 6 Dec 2023 10:27:03 +0100 Subject: [PATCH] Fix/homepage scoring (#251) * Update video relevance manager query * Remove single video schedules for relevance recalc * Fix namings * Reformat query * Revert "Remove single video schedules for relevance recalc" This reverts commit 3a685b97 * Adjust logic for single channel recalc * Unblock video relevance recalc only after processor reaches last exported block * Increase global recalc interval * Avoid query if set is empty * Introduce smaller interval for scheduled channels * Typo fix Co-authored-by: Zeeshan Akram <37098720+zeeshanakram3@users.noreply.github.com> * Linter * Add video recalc on channel weight change --------- Co-authored-by: Zeeshan Akram <37098720+zeeshanakram3@users.noreply.github.com> --- src/mappings/content/commentsAndReactions.ts | 6 +- src/mappings/content/metadata.ts | 2 - src/mappings/content/video.ts | 2 +- src/mappings/utils.ts | 6 +- src/processor.ts | 10 +- .../resolvers/AdminResolver/index.ts | 2 + .../resolvers/AdminResolver/utils.ts | 4 - .../resolvers/VideosResolver/index.ts | 3 +- src/utils/VideoRelevanceManager.ts | 135 ++++++++++++------ 9 files changed, 109 insertions(+), 61 deletions(-) diff --git a/src/mappings/content/commentsAndReactions.ts b/src/mappings/content/commentsAndReactions.ts index cd1dc16bb..12fbe909a 100644 --- a/src/mappings/content/commentsAndReactions.ts +++ b/src/mappings/content/commentsAndReactions.ts @@ -318,7 +318,7 @@ export async function processReactVideoMessage( existingReaction ) - videoRelevanceManager.scheduleRecalcForVideo(video.id) + videoRelevanceManager.scheduleRecalcForChannel(channelId) return new MetaprotocolTransactionResultOK() } @@ -497,7 +497,7 @@ export async function processCreateCommentMessage( // schedule comment counters update commentCountersManager.scheduleRecalcForComment(comment.parentCommentId) commentCountersManager.scheduleRecalcForVideo(comment.videoId) - videoRelevanceManager.scheduleRecalcForVideo(comment.videoId) + videoRelevanceManager.scheduleRecalcForChannel(video.channelId) const event = overlay.getRepository(Event).new({ ...genericEventFields(overlay, block, indexInBlock, txHash), @@ -661,7 +661,7 @@ export async function processDeleteCommentMessage( // schedule comment counters update commentCountersManager.scheduleRecalcForComment(comment.parentCommentId) commentCountersManager.scheduleRecalcForVideo(comment.videoId) - videoRelevanceManager.scheduleRecalcForVideo(comment.videoId) + videoRelevanceManager.scheduleRecalcForChannel(video.channelId) // update the comment comment.text = '' diff --git a/src/mappings/content/metadata.ts b/src/mappings/content/metadata.ts index 7d2f02ac8..a41b7bf10 100644 --- a/src/mappings/content/metadata.ts +++ b/src/mappings/content/metadata.ts @@ -65,7 +65,6 @@ import { genericEventFields, invalidMetadata, metaprotocolTransactionFailure, - videoRelevanceManager, } from '../utils' import { AsDecoded, @@ -582,7 +581,6 @@ export async function processModerateCommentMessage( // schedule comment counters updates commentCountersManager.scheduleRecalcForComment(comment.parentCommentId) commentCountersManager.scheduleRecalcForVideo(comment.videoId) - videoRelevanceManager.scheduleRecalcForVideo(comment.videoId) comment.text = '' comment.status = CommentStatus.MODERATED diff --git a/src/mappings/content/video.ts b/src/mappings/content/video.ts index ab704811d..185d7fba9 100644 --- a/src/mappings/content/video.ts +++ b/src/mappings/content/video.ts @@ -58,7 +58,7 @@ export async function processVideoCreatedEvent({ videoRelevance: 0, }) - videoRelevanceManager.scheduleRecalcForVideo(videoId) + videoRelevanceManager.scheduleRecalcForChannel(channelId.toString()) // fetch related channel and owner const channel = await overlay.getRepository(Channel).getByIdOrFail(channelId.toString()) diff --git a/src/mappings/utils.ts b/src/mappings/utils.ts index cc8a9e2aa..9e061c2a9 100644 --- a/src/mappings/utils.ts +++ b/src/mappings/utils.ts @@ -14,8 +14,12 @@ import { NextEntityIdManager } from '../utils/NextEntityIdManager' export const commentCountersManager = new CommentCountersManager() export const videoRelevanceManager = new VideoRelevanceManager() +// eslint-disable-next-line no-void +void videoRelevanceManager.init({ + fullUpdateLoopTime: 1000 * 60 * 60 * 12, // 12 hrs + scheduledUpdateLoopTime: 1000 * 60 * 10, // 10 mins +}) export const migrateCounters = new NextEntityIdManager() -videoRelevanceManager.init(1000 * 60 * 60) export const JOYSTREAM_SS58_PREFIX = 126 diff --git a/src/processor.ts b/src/processor.ts index 1a28835de..c0441abd4 100644 --- a/src/processor.ts +++ b/src/processor.ts @@ -284,7 +284,6 @@ async function processEvent( async function afterDbUpdate(em: EntityManager) { await commentCountersManager.updateVideoCommentsCounters(em) await commentCountersManager.updateParentRepliesCounters(em) - await videoRelevanceManager.updateVideoRelevanceValue(em) } processor.run(new TypeormDatabase({ isolationLevel: 'READ COMMITTED' }), async (ctx) => { @@ -314,9 +313,18 @@ processor.run(new TypeormDatabase({ isolationLevel: 'READ COMMITTED' }), async ( } } } + + if ( + !videoRelevanceManager.isVideoRelevanceEnabled && + block.header.height >= exportBlockNumber + ) { + videoRelevanceManager.turnOnVideoRelevanceManager() + } + // Importing exported offchain state if (block.header.height >= exportBlockNumber && !offchainState.isImported) { ctx.log.info(`Export block ${exportBlockNumber} reached, importing offchain state...`) + // there is no need to recalc video relevance before orion is synced await overlay.updateDatabase() const em = overlay.getEm() await offchainState.import(em) diff --git a/src/server-extension/resolvers/AdminResolver/index.ts b/src/server-extension/resolvers/AdminResolver/index.ts index d035e6836..54a4a0d02 100644 --- a/src/server-extension/resolvers/AdminResolver/index.ts +++ b/src/server-extension/resolvers/AdminResolver/index.ts @@ -216,6 +216,8 @@ export class AdminResolver { } ) + await videoRelevanceManager.updateVideoRelevanceValue(em, true) + // Push the result into the results array results.push({ channelId, diff --git a/src/server-extension/resolvers/AdminResolver/utils.ts b/src/server-extension/resolvers/AdminResolver/utils.ts index b39e331b0..91ac740ab 100644 --- a/src/server-extension/resolvers/AdminResolver/utils.ts +++ b/src/server-extension/resolvers/AdminResolver/utils.ts @@ -1,18 +1,14 @@ import { EntityManager, In } from 'typeorm' import { CommentCountersManager } from '../../../utils/CommentsCountersManager' import { Comment } from '../../../model' -import { VideoRelevanceManager } from '../../../utils/VideoRelevanceManager' export async function processCommentsCensorshipStatusUpdate(em: EntityManager, ids: string[]) { const manager = new CommentCountersManager() - const videoRelevanceManager = new VideoRelevanceManager() const comments = await em.getRepository(Comment).find({ where: { id: In(ids) } }) comments.forEach((c) => { manager.scheduleRecalcForComment(c.parentCommentId) manager.scheduleRecalcForVideo(c.videoId) - videoRelevanceManager.scheduleRecalcForVideo(c.videoId) }) await manager.updateVideoCommentsCounters(em) await manager.updateParentRepliesCounters(em) - await videoRelevanceManager.updateVideoRelevanceValue(em) } diff --git a/src/server-extension/resolvers/VideosResolver/index.ts b/src/server-extension/resolvers/VideosResolver/index.ts index 93f479f9f..b1e78b236 100644 --- a/src/server-extension/resolvers/VideosResolver/index.ts +++ b/src/server-extension/resolvers/VideosResolver/index.ts @@ -248,10 +248,9 @@ export class VideosResolver { const tick = await config.get(ConfigVariable.VideoRelevanceViewsTick, em) if (video.viewsNum % tick === 0) { - videoRelevanceManager.scheduleRecalcForVideo(videoId) + videoRelevanceManager.scheduleRecalcForChannel(video.channelId) } await em.save([video, video.channel, newView]) - await videoRelevanceManager.updateVideoRelevanceValue(em) return { videoId, viewsNum: video.viewsNum, diff --git a/src/utils/VideoRelevanceManager.ts b/src/utils/VideoRelevanceManager.ts index 7263f510c..a28e72bd4 100644 --- a/src/utils/VideoRelevanceManager.ts +++ b/src/utils/VideoRelevanceManager.ts @@ -5,11 +5,35 @@ import { globalEm } from './globalEm' // constant used to parse seconds from creation export const NEWNESS_SECONDS_DIVIDER = 60 * 60 * 24 +type VideoRelevanceManagerLoops = { + fullUpdateLoopTime: number + scheduledUpdateLoopTime: number +} + export class VideoRelevanceManager { - private videosToUpdate: Set = new Set() + private channelsToUpdate: Set = new Set() + private _isVideoRelevanceEnabled = false + + public get isVideoRelevanceEnabled(): boolean { + return this._isVideoRelevanceEnabled + } + + async init({ + fullUpdateLoopTime, + scheduledUpdateLoopTime, + }: VideoRelevanceManagerLoops): Promise { + const em = await globalEm + + this.updateScheduledLoop(em, scheduledUpdateLoopTime) + .then(() => { + /* Do nothing */ + }) + .catch((err) => { + console.error(err) + process.exit(-1) + }) - init(intervalMs: number): void { - this.updateLoop(intervalMs) + this.updateFullUpdateLoop(em, fullUpdateLoopTime) .then(() => { /* Do nothing */ }) @@ -19,68 +43,85 @@ export class VideoRelevanceManager { }) } - scheduleRecalcForVideo(id: string | null | undefined) { + turnOnVideoRelevanceManager() { + this._isVideoRelevanceEnabled = true + } + + scheduleRecalcForChannel(id: string | null | undefined) { if (id) { - this.videosToUpdate.add(id) + this.channelsToUpdate.add(id) } } async updateVideoRelevanceValue(em: EntityManager, forceUpdateAll?: boolean) { - if (this.videosToUpdate.size || forceUpdateAll) { - const [ - newnessWeight, - viewsWeight, - commentsWeight, - reactionsWeight, - [joystreamTimestampWeight, ytTimestampWeight] = [7, 3], - defaultChannelWeight, - ] = await config.get(ConfigVariable.RelevanceWeights, em) - const channelWeight = defaultChannelWeight ?? 1 - await em.query(` - WITH weighted_timestamp AS ( - SELECT - "video"."id", - ( + if (!this._isVideoRelevanceEnabled || !(this.channelsToUpdate.size || forceUpdateAll)) { + return + } + + const [ + newnessWeight, + viewsWeight, + commentsWeight, + reactionsWeight, + [joystreamTimestampWeight, ytTimestampWeight] = [7, 3], + defaultChannelWeight, + ] = await config.get(ConfigVariable.RelevanceWeights, em) + const channelWeight = defaultChannelWeight ?? 1 + const wtEpoch = `(( extract(epoch from video.created_at)*${joystreamTimestampWeight} + COALESCE(extract(epoch from video.published_before_joystream), extract(epoch from video.created_at))*${ytTimestampWeight} - ) / ${joystreamTimestampWeight} + ${ytTimestampWeight} as wtEpoch, - "channel"."channel_weight" as CW - FROM - "video" - INNER JOIN - "channel" ON "video"."channel_id" = "channel"."id" + ) / ${joystreamTimestampWeight} + ${ytTimestampWeight})` + + await em.query(` + WITH videos_with_weight AS ( + SELECT + video.id as videoId, + channel.id as channelId, + (ROUND(( + (extract(epoch from now()) - ${wtEpoch}) + / ${NEWNESS_SECONDS_DIVIDER} * ${newnessWeight * -1} + + (views_num * ${viewsWeight}) + + (comments_count * ${commentsWeight}) + + (reactions_count * ${reactionsWeight})) + * COALESCE(channel.channel_weight, ${channelWeight}),2)) as videoRelevance + FROM video + INNER JOIN channel ON video.channel_id = channel.id ${ forceUpdateAll ? '' - : `WHERE "video"."id" IN (${[...this.videosToUpdate.values()] + : `WHERE video.channel_id in (${[...this.channelsToUpdate.values()] .map((id) => `'${id}'`) .join(', ')})` } - ) - UPDATE - "video" - SET - "video_relevance" = ROUND( - ( - (extract(epoch from now()) - wtEpoch) / ${NEWNESS_SECONDS_DIVIDER} * ${newnessWeight * -1} + - (views_num * ${viewsWeight}) + - (comments_count * ${commentsWeight}) + - (reactions_count * ${reactionsWeight}) - ) * COALESCE(CW, ${channelWeight}), - 2) - FROM - weighted_timestamp - WHERE - "video".id = weighted_timestamp.id; + ORDER BY video.id), + + top_channel_score as ( + SELECT + channel.id as channelId, + MAX(videoCte.videoRelevance) as maxChannelRelevance + FROM channel + INNER JOIN videos_with_weight as videoCte on videoCte.channelId = channel.id + GROUP BY channel.id) + + UPDATE video + SET video_relevance = COALESCE(topChannelVideo.maxChannelRelevance, 1) + FROM videos_with_weight as videoCte + LEFT JOIN top_channel_score as topChannelVideo on topChannelVideo.channelId = videoCte.channelId and topChannelVideo.maxChannelRelevance = videoCte.videoRelevance + WHERE video.id = videoCte.videoId; `) - this.videosToUpdate.clear() + this.channelsToUpdate.clear() + } + + private async updateScheduledLoop(em: EntityManager, intervalMs: number): Promise { + while (true) { + await this.updateVideoRelevanceValue(em) + await new Promise((resolve) => setTimeout(resolve, intervalMs)) } } - private async updateLoop(intervalMs: number): Promise { - const em = await globalEm + private async updateFullUpdateLoop(em: EntityManager, intervalMs: number): Promise { while (true) { - await this.updateVideoRelevanceValue(em, true) + await this.updateVideoRelevanceValue(em) await new Promise((resolve) => setTimeout(resolve, intervalMs)) } }