Fix/introduce average newness for ypp (#137)

* Adjust logic at creation

* Adjust logic on manager

* Transfer sub-weights to env

* Adjust resolver

* CR fixes

* Update src/utils/VideoRelevanceManager.ts

* Update .env

---------

Co-authored-by: Leszek Wiesner <leszek@jsgenesis.com>

.env

@@ -20,7 +20,7 @@ SUPPORT_NEW_CATEGORIES=true
 KILL_SWITCH_ON=false
 VIDEO_VIEW_PER_IP_TIME_LIMIT=86400 # 86400 seconds = 24 hours
 VIDEO_RELEVANCE_VIEWS_TICK=50 # every 50 views video relevance score will be recalculated
-RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5]" # [newness (30 days - days since created) weight, views weight, comments weight, rections weights]
+RELEVANCE_WEIGHTS="[1, 0.03, 0.3, 0.5, [7,3]]" # [newness (negative number of days since created) weight, views weight, comments weight, rections weights, [joystream creation weight, YT creation weight]]
 
 # Operator API secret
 OPERATOR_SECRET=this-is-not-so-secret-change-it

src/mappings/content/video.ts

@@ -8,12 +8,10 @@ import { DecodedMetadataObject } from '@joystream/metadata-protobuf/types'
 import { integrateMeta } from '@joystream/metadata-protobuf/utils'
 import { Channel, Video, VideoViewEvent } from '../../model'
 import { EventHandlerContext } from '../../utils/events'
-import { deserializeMetadata, u8aToBytes } from '../utils'
+import { deserializeMetadata, u8aToBytes, videoRelevanceManager } from '../utils'
 import { processVideoMetadata } from './metadata'
 import { deleteVideo, encodeAssets, processAppActionMetadata, processNft } from './utils'
 import { generateAppActionCommitment } from '@joystream/js/utils'
-import { config, ConfigVariable } from '../../utils/config'
-import { NEWNESS_SECONDS_DIVIDER } from '../../utils/VideoRelevanceManager'
 
 export async function processVideoCreatedEvent({
  overlay,
@@ -28,10 +26,6 @@ export async function processVideoCreatedEvent({
 
  const videoId = contentId.toString()
  const viewsNum = await overlay.getEm().getRepository(VideoViewEvent).countBy({ videoId })
- const [newnessWeight, viewsWeight] = await config.get(
- ConfigVariable.RelevanceWeights,
- overlay.getEm()
- )
  const video = overlay.getRepository(Video).new({
  id: videoId,
  createdAt: new Date(block.timestamp),
@@ -46,13 +40,11 @@ export async function processVideoCreatedEvent({
  reactionsCount: 0,
  viewsNum,
  // First we need to dic by 1k to match postgres epoch (in seconds) then apply the further dividers
- videoRelevance: +(
- (30 - (Date.now() - new Date(block.timestamp).getTime()) / (1000 * NEWNESS_SECONDS_DIVIDER)) *
- newnessWeight +
- viewsNum * viewsWeight
- ).toFixed(2),
+ videoRelevance: 0,
  })
 
+ videoRelevanceManager.scheduleRecalcForVideo(videoId)
+
  // fetch related channel and owner
  const channel = await overlay.getRepository(Channel).getByIdOrFail(channelId.toString())
 
@@ -140,6 +132,10 @@ export async function processVideoUpdatedEvent({
  }
 
  if (videoMetadataUpdate) {
+ if ('publishedBeforeJoystream' in videoMetadataUpdate) {
+ delete videoMetadataUpdate.publishedBeforeJoystream
+ }
+
  await processVideoMetadata(
  overlay,
  block,

src/server-extension/resolvers/AdminResolver/index.ts

@@ -59,7 +59,13 @@ export class AdminResolver {
  const em = await this.em()
  await config.set(
  ConfigVariable.RelevanceWeights,
- [args.newnessWeight, args.viewsWeight, args.commentsWeight, args.reactionsWeight],
+ [
+ args.newnessWeight,
+ args.viewsWeight,
+ args.commentsWeight,
+ args.reactionsWeight,
+ [args.joysteamTimestampSubWeight, args.ytTimestampSubWeight],
+ ],
  em
  )
  await videoRelevanceManager.updateVideoRelevanceValue(em, true)

src/server-extension/resolvers/AdminResolver/types.ts

@@ -14,6 +14,12 @@ export class SetVideoWeightsInput {
 
  @Field(() => Float, { nullable: false })
  reactionsWeight!: number
+
+ @Field(() => Float, { nullable: false })
+ joysteamTimestampSubWeight!: number
+
+ @Field(() => Float, { nullable: false })
+ ytTimestampSubWeight!: number
 }
 
 @ObjectType()

src/utils/VideoRelevanceManager.ts

@@ -25,29 +25,45 @@ export class VideoRelevanceManager {
 
  async updateVideoRelevanceValue(em: EntityManager, forceUpdateAll?: boolean) {
  if (this.videosToUpdate.size || forceUpdateAll) {
- const [newnessWeight, viewsWeight, commentsWeight, reactionsWeight] = await config.get(
- ConfigVariable.RelevanceWeights,
- em
- )
+ const [
+ newnessWeight,
+ viewsWeight,
+ commentsWeight,
+ reactionsWeight,
+ [joystreamTimestampWeight, ytTimestampWeight] = [7, 3],
+ ] = await config.get(ConfigVariable.RelevanceWeights, em)
  await em.query(`
- UPDATE "video"
- SET
- "video_relevance" = ROUND(
- ((30 - (extract(epoch from now() - created_at) / ${NEWNESS_SECONDS_DIVIDER})) * ${newnessWeight}) +
- (views_num * ${viewsWeight}) +
- (
- comments_count * ${commentsWeight} 
- ) +
- (
- reactions_count * ${reactionsWeight} 
- ), 2)
+ WITH weighted_timestamp AS (
+ SELECT 
+ id,
+ (
+ extract(epoch from created_at)*${joystreamTimestampWeight} +
+ COALESCE(extract(epoch from published_before_joystream), extract(epoch from created_at))*${ytTimestampWeight}
+ ) / ${joystreamTimestampWeight + ytTimestampWeight} as wtEpoch
+ FROM 
+ "video" 
  ${
  forceUpdateAll
  ? ''
  : `WHERE "id" IN (${[...this.videosToUpdate.values()]
  .map((id) => `'${id}'`)
  .join(', ')})`
- }`)
+ }
+ )
+ UPDATE 
+ "video"
+ SET
+ "video_relevance" = ROUND(
+ (extract(epoch from now()) - wtEpoch) / (60 * 60 * 24) * ${newnessWeight * -1} +
+ (views_num * ${viewsWeight}) +
+ (comments_count * ${commentsWeight}) +
+ (reactions_count * ${reactionsWeight}), 
+ 2)
+ FROM
+ weighted_timestamp
+ WHERE
+ "video".id = weighted_timestamp.id;
+ `)
  this.videosToUpdate.clear()
  }
  }

src/utils/config.ts

@@ -37,7 +37,7 @@ export const configVariables = {
  [ConfigVariable.KillSwitch]: boolType,
  [ConfigVariable.VideoViewPerIpTimeLimit]: numberType,
  [ConfigVariable.VideoRelevanceViewsTick]: numberType,
- [ConfigVariable.RelevanceWeights]: jsonType<[number, number, number, number]>(),
+ [ConfigVariable.RelevanceWeights]: jsonType<[number, number, number, number, [number, number]]>(),
  [ConfigVariable.AppPrivateKey]: stringType,
 } as const