Add liquibase to perform database migrations

.github/workflows/docker-build-test.yml

@@ -9,34 +9,37 @@ on:
   pull_request:
 
 jobs:
+  build-jdk:
+    uses: ./.github/workflows/fat-build.yml
+
   build-test:
     runs-on: ubuntu-latest
+    needs: build-jdk
     strategy:
       matrix:
         docker-compose-file:
           - docker-compose.yml
-          - testing/docker-compose.hsqldb.yml
           - testing/docker-compose.cockroachdb.yml
           - testing/docker-compose.yugabytedb.yml
         dockerfile:
           - Dockerfile.ci
           - Dockerfile.azul.ci
           - Dockerfile.openj9.ci
           - Dockerfile.graalvm-jvm.ci
+        include:
+          - sleep: 20
+          - docker-compose-file: testing/docker-compose.cockroachdb.yml
+            sleep: 30
+          - docker-compose-file: testing/docker-compose.yugabytedb.yml
+            sleep: 120
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: piped.jar
       - name: Create Version File
         run: echo $(git log -1 --date=short --pretty=format:%cd)-$(git rev-parse --short HEAD) > VERSION
-      - name: set up JDK 17
-        uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-          cache: "gradle"
-      - name: Run Build
-        run: ./gradlew shadowJar
-      - run: mv build/libs/piped-*-all.jar piped.jar
       - name: Build Image Locally
         uses: docker/build-push-action@v4
         with:
@@ -45,7 +48,7 @@ jobs:
           file: ${{ matrix.dockerfile }}
           tags: 1337kavin/piped:latest
       - name: Start Docker-Compose services
-        run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep 20
+        run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }}
       - name: Run tests
         run: ./testing/api-test.sh
       - name: Collect services logs

.github/workflows/docker-build.yml

@@ -8,7 +8,11 @@ on:
       - master
 
 jobs:
+  build-jdk:
+    uses: ./.github/workflows/fat-build.yml
+
   build-docker:
+    needs: build-jdk
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -23,17 +27,11 @@ jobs:
             dockerfile: ./Dockerfile.graalvm-jvm.ci
     steps:
       - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: piped.jar
       - name: Create Version File
         run: echo $(git log -1 --date=short --pretty=format:%cd)-$(git rev-parse --short HEAD) > VERSION
-      - name: set up JDK 17
-        uses: actions/setup-java@v3
-        with:
-          java-version: 17
-          distribution: temurin
-          cache: "gradle"
-      - name: Run Build
-        run: ./gradlew shadowJar
-      - run: mv build/libs/piped-*-all.jar piped.jar
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
         with:

.github/workflows/docker-migrations-build-test.yml

@@ -0,0 +1,82 @@
+name: Docker-Compose Build and Test Migration
+
+on:
+  pull_request:
+    paths:
+      - "src/main/resources/changelog/**"
+      - "src/main/java/me/kavin/piped/utils/obj/db/**"
+
+jobs:
+  build-new:
+    uses: ./.github/workflows/fat-build.yml
+  build-old:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+      - name: set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: 17
+          distribution: temurin
+          cache: "gradle"
+      - name: Run Build
+        run: ./gradlew shadowJar
+      - run: mv build/libs/piped-*-all.jar piped.jar
+      - uses: actions/upload-artifact@v3
+        with:
+          name: piped-old.jar
+          path: piped.jar
+
+  docker-build-test:
+    needs: [ build-new, build-old ]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        docker-compose-file:
+          - docker-compose.yml
+          - testing/docker-compose.cockroachdb.yml
+          - testing/docker-compose.yugabytedb.yml
+        dockerfile:
+          - Dockerfile.ci
+        include:
+          - sleep: 20
+          - docker-compose-file: testing/docker-compose.cockroachdb.yml
+            sleep: 30
+          - docker-compose-file: testing/docker-compose.yugabytedb.yml
+            sleep: 120
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+      - run: echo "unknown" > VERSION
+      - uses: actions/download-artifact@v3
+        with:
+          name: piped-old.jar
+      - name: Build Old Image Locally
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          load: true
+          file: ${{ matrix.dockerfile }}
+          tags: 1337kavin/piped:latest
+      - name: Start Docker-Compose services
+        run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }}
+      - run: rm piped.jar
+      - uses: actions/download-artifact@v3
+        with:
+          name: piped.jar
+      - name: Build New Image Locally
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          load: true
+          file: ${{ matrix.dockerfile }}
+          tags: 1337kavin/piped:latest
+      - name: Start Docker-Compose services
+        run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }}
+      - name: Run tests
+        run: ./testing/api-test.sh
+      - name: Collect services logs
+        if: failure()
+        run: docker-compose -f ${{ matrix.docker-compose-file }} logs

.github/workflows/fat-build.yml

@@ -0,0 +1,24 @@
+name: Fat JAR Build
+
+on:
+  workflow_call:
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: set up JDK 17
+        uses: actions/setup-java@v3
+        with:
+          java-version: 17
+          distribution: temurin
+          cache: "gradle"
+      - name: Run Build
+        run: ./gradlew shadowJar
+      - run: mv build/libs/piped-*-all.jar piped.jar
+      - uses: actions/upload-artifact@v3
+        with:
+          name: piped.jar
+          path: piped.jar

build.gradle

@@ -29,10 +29,11 @@ dependencies {
     implementation 'io.activej:activej-boot:5.5'
     implementation 'io.activej:activej-specializer:5.5'
     implementation 'io.activej:activej-launchers-http:5.5'
-    implementation 'org.hsqldb:hsqldb:2.7.2'
     implementation 'org.postgresql:postgresql:42.6.0'
     implementation 'org.hibernate:hibernate-core:6.2.7.Final'
     implementation 'org.hibernate:hibernate-hikaricp:6.2.7.Final'
+    implementation 'org.liquibase:liquibase-core:4.23.1'
+    implementation('org.liquibase.ext:liquibase-yugabytedb:4.23.0') { exclude group: 'org.liquibase' }
     implementation 'com.zaxxer:HikariCP:5.0.1'
     implementation 'org.springframework.security:spring-security-crypto:6.1.2'
     implementation 'commons-logging:commons-logging:1.2'

src/main/java/me/kavin/piped/Main.java

@@ -20,7 +20,6 @@
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
-import rocks.kavin.reqwest4j.ReqwestUtils;
 
 import java.util.*;
 import java.util.concurrent.CompletableFuture;
@@ -47,6 +46,13 @@ public static void main(String[] args) throws Exception {
 
         Injector.useSpecializer();
 
+        try {
+            LiquibaseHelper.init();
+        } catch (Exception e) {
+            ExceptionHandler.handle(e);
+            System.exit(1);
+        }
+
         Multithreading.runAsync(() -> new Thread(new SyncRunner(
                 new OkHttpClient.Builder().readTimeout(60, TimeUnit.SECONDS).build(),
                 MATRIX_SERVER,

src/main/java/me/kavin/piped/utils/LiquibaseHelper.java

@@ -0,0 +1,50 @@
+package me.kavin.piped.utils;
+
+import liquibase.Liquibase;
+import liquibase.Scope;
+import liquibase.command.CommandScope;
+import liquibase.command.core.UpdateCommandStep;
+import liquibase.command.core.helpers.DbUrlConnectionCommandStep;
+import liquibase.database.Database;
+import liquibase.database.DatabaseFactory;
+import liquibase.database.jvm.JdbcConnection;
+import liquibase.resource.ClassLoaderResourceAccessor;
+import me.kavin.piped.consts.Constants;
+
+import java.sql.DriverManager;
+import java.util.HashMap;
+import java.util.Map;
+
+public class LiquibaseHelper {
+
+    public static void init() throws Exception {
+
+        String url = Constants.hibernateProperties.get("hibernate.connection.url");
+        String username = Constants.hibernateProperties.get("hibernate.connection.username");
+        String password = Constants.hibernateProperties.get("hibernate.connection.password");
+
+        // ensure postgres driver is loaded
+        DriverManager.registerDriver(new org.postgresql.Driver());
+
+        // register YugabyteDB database
+        DatabaseFactory.getInstance().register(new liquibase.ext.yugabytedb.database.YugabyteDBDatabase());
+
+        Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(DriverManager.getConnection(url, username, password)));
+
+        try (Liquibase liquibase = new Liquibase("changelog/db.changelog-master.xml", new ClassLoaderResourceAccessor(), database)) {
+
+            Map<String, Object> scopeObjects = new HashMap<>();
+            scopeObjects.put(Scope.Attr.database.name(), liquibase.getDatabase());
+            scopeObjects.put(Scope.Attr.resourceAccessor.name(), liquibase.getResourceAccessor());
+
+            Scope.child(scopeObjects, () -> {
+                CommandScope updateCommand = new CommandScope(UpdateCommandStep.COMMAND_NAME);
+                updateCommand.addArgumentValue(DbUrlConnectionCommandStep.DATABASE_ARG, liquibase.getDatabase());
+                updateCommand.addArgumentValue(UpdateCommandStep.CHANGELOG_FILE_ARG, liquibase.getChangeLogFile());
+                updateCommand.execute();
+            });
+
+        }
+    }
+
+}

src/main/java/me/kavin/piped/utils/obj/db/User.java

@@ -8,9 +8,10 @@
 import java.util.UUID;
 
 @Entity
-@Table(name = "users", indexes = {@Index(columnList = "id", name = "users_id_idx"),
+@Table(name = "users", indexes = {
         @Index(columnList = "username", name = "username_idx"),
-        @Index(columnList = "session_id", name = "users_session_id_idx")})
+        @Index(columnList = "session_id", name = "users_session_id_idx")
+})
 public class User implements Serializable {
 
     private static final long serialVersionUID = 1L;

src/main/resources/changelog/db.changelog-master.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                   https://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-latest.xsd">
+
+    <include file="version/0-init.xml" relativeToChangelogFile="true"/>
+
+</databaseChangeLog>

src/main/resources/changelog/version/0-0-init-yb.sql

@@ -0,0 +1,2 @@
+CREATE EXTENSION pgcrypto;
+--rollback DROP EXTENSION IF EXISTS pgcrypto;

src/main/resources/changelog/version/0-1-init-crdb.sql

@@ -0,0 +1,47 @@
+CREATE INDEX IF NOT EXISTS users_session_id_idx ON users (session_id ASC) STORING (password, username);
+
+--rollback DROP INDEX IF EXISTS users_session_id_idx;
+
+CREATE TABLE IF NOT EXISTS videos (
+    id VARCHAR(11) NOT NULL UNIQUE,
+    duration INT8 NULL,
+    thumbnail VARCHAR(400) NULL,
+    title VARCHAR(120) NULL,
+    uploaded INT8 NULL,
+    views INT8 NULL,
+    uploader_id VARCHAR(24) NOT NULL,
+    is_short BOOL NOT NULL DEFAULT false,
+    CONSTRAINT videos_pkey PRIMARY KEY (id ASC, uploader_id ASC) USING HASH,
+    CONSTRAINT fk_videos_channels_uploader_id FOREIGN KEY (uploader_id) REFERENCES channels(uploader_id),
+    INDEX videos_id_idx (id ASC),
+    INDEX video_uploaded_idx (uploaded ASC) USING HASH,
+    INDEX video_uploader_id_idx (uploader_id ASC) STORING (duration, thumbnail, title, uploaded, views, is_short),
+    UNIQUE INDEX videos_id_key (id ASC) STORING (duration, thumbnail, title, uploaded, views, is_short)
+);
+
+--rollback DROP TABLE IF EXISTS videos;
+
+CREATE TABLE IF NOT EXISTS users_subscribed (
+    subscriber INT8 NOT NULL,
+    channel VARCHAR(24) NOT NULL,
+    CONSTRAINT users_subscribed_pkey PRIMARY KEY (subscriber ASC, channel ASC) USING HASH,
+    CONSTRAINT fk_subscriber_users FOREIGN KEY (subscriber) REFERENCES users(id),
+    INDEX users_subscribed_subscriber_idx (subscriber ASC),
+    INDEX users_subscribed_channel_idx (channel ASC)
+);
+
+--rollback DROP TABLE IF EXISTS users_subscribed;
+
+CREATE INDEX IF NOT EXISTS pubsub_subbed_at_idx ON pubsub (subbed_at ASC) USING HASH;
+
+--rollback DROP INDEX IF EXISTS pubsub_subbed_at_idx;
+
+CREATE INDEX IF NOT EXISTS playlists_playlist_id_idx ON playlists (playlist_id ASC) STORING (name, short_description, thumbnail, owner);
+CREATE INDEX IF NOT EXISTS playlists_owner_idx ON playlists (owner ASC) STORING (name, short_description, thumbnail, playlist_id);
+
+--rollback DROP INDEX IF EXISTS playlists_playlist_id_idx;
+--rollback DROP INDEX IF EXISTS playlists_owner_idx;
+
+CREATE INDEX IF NOT EXISTS unauthenticated_subscriptions_id_idx ON unauthenticated_subscriptions (id ASC) USING HASH STORING (subscribed_at);
+
+--rollback DROP INDEX IF EXISTS unauthenticated_subscriptions_id_idx;