Merge pull request #1 from hanydd/dev-cid

Add support for video cid

.gitignore

@@ -1,3 +1,6 @@
+# vscode
+.vscode
+
 # Logs
 logs
 *.log

.prettierrc

@@ -1,5 +1,5 @@
 {
-    "printWidth": 120,
+    "printWidth": 140,
     "tabWidth": 4,
     "semi": true,
     "endOfLine": "auto"

databases/_private.db.sql

@@ -26,24 +26,6 @@ CREATE TABLE IF NOT EXISTS "config" (
 	"value" TEXT NOT NULL
 );
 
-CREATE TABLE IF NOT EXISTS "titleVotes" (
-	"id" SERIAL PRIMARY KEY,
-	"videoID"	TEXT NOT NULL,
-	"UUID"	TEXT NOT NULL,
-	"userID"	TEXT NOT NULL,
-	"hashedIP"	TEXT NOT NULL,
-	"type"	INTEGER NOT NULL
-);
-
-CREATE TABLE IF NOT EXISTS "thumbnailVotes" (
-	"id" SERIAL PRIMARY KEY,
-	"videoID"	TEXT NOT NULL,
-	"UUID"	TEXT NOT NULL,
-	"userID"	TEXT NOT NULL,
-	"hashedIP"	TEXT NOT NULL,
-	"type"	INTEGER NOT NULL
-);
-
 CREATE TABLE IF NOT EXISTS "portVideo" (
 	"bvID" TEXT NOT NULL,
 	"UUID" TEXT PRIMARY KEY,

databases/_private_indexes.sql

@@ -2,7 +2,7 @@
 
 CREATE INDEX IF NOT EXISTS "privateDB_sponsorTimes_v4"
     ON public."sponsorTimes" USING btree
-    ("videoID" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST);
+    ("videoID" ASC NULLS LAST, "cid" ASC NULLS LAST, service COLLATE pg_catalog."default" ASC NULLS LAST, "timeSubmitted" ASC NULLS LAST);
 
 -- votes
 

databases/_sponsorTimes.db.sql

@@ -44,49 +44,6 @@ CREATE TABLE IF NOT EXISTS "config" (
 	"value" TEXT NOT NULL
 );
 
-CREATE TABLE IF NOT EXISTS "titles" (
-	"videoID"	TEXT NOT NULL,
-	"title"	TEXT NOT NULL,
-	"original" INTEGER default 0,
-	"userID"	TEXT NOT NULL,
-	"service"	TEXT NOT NULL,
-	"hashedVideoID"	TEXT NOT NULL,
-	"timeSubmitted"	INTEGER NOT NULL,
-	"UUID"	TEXT NOT NULL PRIMARY KEY
-);
-
-CREATE TABLE IF NOT EXISTS "titleVotes" (
-	"UUID"	TEXT NOT NULL PRIMARY KEY,
-	"votes"	INTEGER NOT NULL default 0,
-	"locked"	INTEGER NOT NULL default 0,
-	"shadowHidden"	INTEGER NOT NULL default 0,
-	FOREIGN KEY("UUID") REFERENCES "titles"("UUID")
-);
-
-CREATE TABLE IF NOT EXISTS "thumbnails" (
-	"videoID"	TEXT NOT NULL,
-	"original" INTEGER default 0,
-	"userID"	TEXT NOT NULL,
-	"service"	TEXT NOT NULL,
-	"hashedVideoID"	TEXT NOT NULL,
-	"timeSubmitted"	INTEGER NOT NULL,
-	"UUID"	TEXT NOT NULL PRIMARY KEY
-);
-
-CREATE TABLE IF NOT EXISTS "thumbnailTimestamps" (
-	"UUID"	TEXT NOT NULL PRIMARY KEY,
-	"timestamp"	INTEGER NOT NULL default 0,
-	FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID")
-);
-
-CREATE TABLE IF NOT EXISTS "thumbnailVotes" (
-	"UUID"	TEXT NOT NULL PRIMARY KEY,
-	"votes"	INTEGER NOT NULL default 0,
-	"locked"	INTEGER NOT NULL default 0,
-	"shadowHidden"	INTEGER NOT NULL default 0,
-	FOREIGN KEY("UUID") REFERENCES "thumbnails"("UUID")
-);
-
 CREATE TABLE IF NOT EXISTS "portVideo" (
 	"bvID" TEXT NOT NULL,
 	"ytbID" TEXT NOT NULL,

databases/_upgrade_private_14.sql

@@ -0,0 +1,7 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE "sponsorTimes" ADD "cid" TEXT NOT NULL DEFAULT '';
+
+UPDATE "config" SET value = 14 WHERE key = 'version';
+
+COMMIT;

databases/_upgrade_private_4.sql

@@ -1,14 +1,5 @@
 BEGIN TRANSACTION;
 
-CREATE TABLE IF NOT EXISTS "ratings" (
-	"videoID"		TEXT NOT NULL,
-	"service"		TEXT NOT NULL default 'YouTube',
-	"type"			INTEGER NOT NULL,
-	"userID"		TEXT NOT NULL,
-	"timeSubmitted"	INTEGER NOT NULL,
-	"hashedIP"		TEXT NOT NULL
-);
-
 UPDATE "config" SET value = 4 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_28.sql

@@ -1,13 +1,5 @@
 BEGIN TRANSACTION;
 
-CREATE TABLE IF NOT EXISTS "ratings" (
-	"videoID"	    TEXT NOT NULL,
-	"service"	    TEXT NOT NULL default 'YouTube',
-	"type"	        INTEGER NOT NULL,
-	"count"	        INTEGER NOT NULL,
-	"hashedVideoID"	TEXT NOT NULL
-);
-
 UPDATE "config" SET value = 28 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_32.sql

@@ -12,7 +12,6 @@ ALTER TABLE "shadowBannedUsers" ADD PRIMARY KEY ("userID"); --!sqlite-ignore
 ALTER TABLE "unlistedVideos" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
 ALTER TABLE "config" ADD PRIMARY KEY ("key"); --!sqlite-ignore
 ALTER TABLE "archivedSponsorTimes" ADD PRIMARY KEY ("UUID"); --!sqlite-ignore
-ALTER TABLE "ratings" ADD "id" SERIAL PRIMARY KEY; --!sqlite-ignore
 
 UPDATE "config" SET value = 32 WHERE key = 'version';
 

databases/_upgrade_sponsorTimes_35.sql

@@ -1,7 +1,5 @@
 BEGIN TRANSACTION;
 
-ALTER TABLE "titleVotes" ADD "verification" INTEGER default 0;
-
 UPDATE "config" SET value = 35 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_37.sql

@@ -1,7 +1,5 @@
 BEGIN TRANSACTION;
 
-ALTER TABLE "titles" ADD UNIQUE ("videoID", "title"); --!sqlite-ignore
-
 UPDATE "config" SET value = 37 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_38.sql

@@ -1,11 +1,5 @@
 BEGIN TRANSACTION;
 
-UPDATE "titleVotes" SET "shadowHidden" = 1
-WHERE "UUID" IN (SELECT "UUID" FROM "titles" INNER JOIN "shadowBannedUsers" "bans" ON "titles"."userID" = "bans"."userID");
-
-UPDATE "thumbnailVotes" SET "shadowHidden" = 1
-WHERE "UUID" IN (SELECT "UUID" FROM "thumbnails" INNER JOIN "shadowBannedUsers" "bans" ON "thumbnails"."userID" = "bans"."userID");
-
 UPDATE "config" SET value = 38 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_39.sql

@@ -1,11 +1,5 @@
 BEGIN TRANSACTION;
 
-ALTER TABLE "titleVotes" ADD "downvotes" INTEGER default 0;
-ALTER TABLE "titleVotes" ADD "removed" INTEGER default 0;
-
-ALTER TABLE "thumbnailVotes" ADD "downvotes" INTEGER default 0;
-ALTER TABLE "thumbnailVotes" ADD "removed" INTEGER default 0;
-
 UPDATE "config" SET value = 39 WHERE key = 'version';
 
 COMMIT;

databases/_upgrade_sponsorTimes_43.sql

@@ -0,0 +1,30 @@
+BEGIN TRANSACTION;
+
+ALTER TABLE "sponsorTimes" ADD "cid" TEXT NOT NULL DEFAULT '';
+ALTER TABLE "portVideo" ADD "cid" TEXT NOT NULL DEFAULT '';
+ALTER TABLE "archivedSponsorTimes" ADD "cid" TEXT NOT NULL DEFAULT '';
+ALTER TABLE "lockCategories" ADD "cid" TEXT NOT NULL DEFAULT '';
+
+ALTER TABLE "archivedSponsorTimes" ADD "ytbID" TEXT;
+ALTER TABLE "archivedSponsorTimes" ADD "ytbSegmentUUID" TEXT;
+ALTER TABLE "archivedSponsorTimes" ADD "portUUID" TEXT;
+
+CREATE TABLE "sqlb_temp_table_43" (
+	"videoID"	TEXT NOT NULL,
+	"cid"	    TEXT NOT NULL DEFAULT '',
+	"channelID"	TEXT NOT NULL,
+	"title"	    TEXT NOT NULL,
+	"part"	    INTEGER NOT NULL DEFAULT 0,
+	"partTitile"	TEXT,
+	"published"	NUMERIC NOT NULL,
+	PRIMARY KEY("videoID", "cid")
+);
+
+INSERT INTO sqlb_temp_table_43 SELECT "videoID", '', "channelID", "title", 1, '', "published" FROM "videoInfo";
+
+DROP TABLE "videoInfo";
+ALTER TABLE sqlb_temp_table_43 RENAME TO "videoInfo";
+
+UPDATE "config" SET value = 43 WHERE key = 'version';
+
+COMMIT;

src/cronjob/index.ts

@@ -1,6 +1,7 @@
 import { config } from "../config";
 import { Logger } from "../utils/logger";
 import { dumpDatebaseJob } from "./dumpDatabase";
+import { refreshCidJob } from "./refreshCid";
 import refreshTopUserViewJob from "./refreshTopUserView";
 
 export function startAllCrons(): void {
@@ -9,6 +10,7 @@ export function startAllCrons(): void {
 
         refreshTopUserViewJob.start();
         dumpDatebaseJob.start();
+        refreshCidJob.start();
     } else {
         Logger.info("Crons dissabled");
     }

src/cronjob/refreshCid.ts

@@ -0,0 +1,78 @@
+import { CronJob } from "cron";
+import { saveVideoInfo } from "../dao/videoInfo";
+import { db } from "../databases/databases";
+import { DBSegment, HiddenType, SegmentUUID } from "../types/segments.model";
+import { durationEquals } from "../utils/durationUtil";
+import { getVideoDetails, VideoDetail } from "../utils/getVideoDetails";
+import { Logger } from "../utils/logger";
+import { sleep } from "../utils/timeUtil";
+
+export const refreshCidJob = new CronJob("*/1 * * * *", () => refreshCid());
+
+let isRunning = false;
+
+async function refreshCid() {
+    if (isRunning) {
+        Logger.info("refreshCid already running, skipping");
+        return;
+    }
+
+    isRunning = true;
+    const allSegments: DBSegment[] = await db.prepare("all", `SELECT * FROM "sponsorTimes" WHERE "cid" = NULL or "cid" = ''`, []);
+    const videoSegmentMap = new Map<string, DBSegment[]>();
+    for (const segment of allSegments) {
+        if (!videoSegmentMap.has(segment.videoID)) {
+            videoSegmentMap.set(segment.videoID, []);
+        }
+        videoSegmentMap.get(segment.videoID)?.push(segment);
+    }
+    Logger.info(`Found ${videoSegmentMap.size} videos with missing cids`);
+
+    for (const [videoID, segments] of videoSegmentMap) {
+        let biliVideoDetail: VideoDetail;
+        try {
+            biliVideoDetail = await getVideoDetails(videoID);
+            if (biliVideoDetail === null || biliVideoDetail === undefined) {
+                Logger.error(`Failed to get video detail for ${videoID}`);
+                continue;
+            }
+        } catch (e) {
+            Logger.error(`Failed to get video detail for ${videoID}`);
+            continue;
+        }
+
+        await saveVideoInfo(biliVideoDetail);
+
+        const invalidIDs: SegmentUUID[] = [];
+
+        if (biliVideoDetail.page.length === 1 || !!biliVideoDetail.page[0].cid) {
+            await db.prepare("run", `UPDATE "sponsorTimes" SET "cid" = ? WHERE "videoID" = ? AND "cid" = ''`, [
+                biliVideoDetail.page[0].cid,
+                videoID,
+            ]);
+            invalidIDs.push(
+                ...segments.filter((s) => !durationEquals(s.videoDuration, biliVideoDetail.page[0].duration)).map((s) => s.UUID)
+            );
+        } else {
+            // find a matching cid
+            for (const segment of segments) {
+                const possibleCids = biliVideoDetail.page.filter((p) => durationEquals(p.duration, segment.videoDuration));
+                if (possibleCids.length === 1) {
+                    await db.prepare("run", `UPDATE "sponsorTimes" SET "cid" = ? WHERE "UUID" = ?`, [possibleCids[0].cid, segment.UUID]);
+                }
+            }
+        }
+
+        // Hide segments with invalid cids
+        if (invalidIDs.length > 0) {
+            await db.prepare("run", `UPDATE "sponsorTimes" SET "hidden" = ? WHERE "UUID" IN (${invalidIDs.map(() => "?").join(",")})`, [
+                HiddenType.Hidden,
+                ...invalidIDs,
+            ]);
+        }
+
+        await sleep(10000);
+    }
+
+    isRunning = false;
+}