diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bc75c619..5edb614c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,15 +11,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [ 17 ] + java: [ 21 ] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: set up JDK ${{ matrix.java }} uses: actions/setup-java@v3 with: java-version: ${{ matrix.java }} - distribution: temurin + distribution: zulu cache: "gradle" - name: Run Build run: ./gradlew build diff --git a/.github/workflows/docker-build-test.yml b/.github/workflows/docker-build-test.yml index f58a6a4f..f3116305 100644 --- a/.github/workflows/docker-build-test.yml +++ b/.github/workflows/docker-build-test.yml @@ -9,43 +9,46 @@ on: pull_request: jobs: + build-jdk: + uses: ./.github/workflows/fat-build.yml + build-test: runs-on: ubuntu-latest + needs: build-jdk strategy: matrix: docker-compose-file: - docker-compose.yml - - testing/docker-compose.hsqldb.yml - testing/docker-compose.cockroachdb.yml - testing/docker-compose.yugabytedb.yml dockerfile: - Dockerfile.ci - Dockerfile.azul.ci - - Dockerfile.openj9.ci + #- Dockerfile.openj9.ci - Dockerfile.graalvm-jvm.ci + include: + - sleep: 20 + - docker-compose-file: testing/docker-compose.cockroachdb.yml + sleep: 30 + - docker-compose-file: testing/docker-compose.yugabytedb.yml + sleep: 120 fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v3 + with: + name: piped.jar - name: Create Version File run: echo $(git log -1 --date=short --pretty=format:%cd)-$(git rev-parse --short HEAD) > VERSION - - name: set up JDK 17 - uses: actions/setup-java@v3 - with: - java-version: 17 - distribution: temurin - cache: "gradle" - - name: Run Build - run: ./gradlew shadowJar - - run: mv build/libs/piped-*-all.jar piped.jar - name: Build Image Locally - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . load: true file: ${{ matrix.dockerfile }} tags: 1337kavin/piped:latest - name: Start Docker-Compose services - run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep 20 + run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }} - name: Run tests run: ./testing/api-test.sh - name: Collect services logs diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 3327e482..79ebb296 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -8,13 +8,17 @@ on: - master jobs: + build-jdk: + uses: ./.github/workflows/fat-build.yml + build-docker: + needs: build-jdk runs-on: ubuntu-latest strategy: matrix: include: - - image: 1337kavin/piped:openj9 - dockerfile: ./Dockerfile.openj9.ci +# - image: 1337kavin/piped:openj9 +# dockerfile: ./Dockerfile.openj9.ci - image: 1337kavin/piped:hotspot dockerfile: ./Dockerfile.ci - image: 1337kavin/piped:latest,1337kavin/piped:azul-zulu @@ -22,34 +26,28 @@ jobs: - image: 1337kavin/piped:graalvm-jvm dockerfile: ./Dockerfile.graalvm-jvm.ci steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v3 + with: + name: piped.jar - name: Create Version File run: echo $(git log -1 --date=short --pretty=format:%cd)-$(git rev-parse --short HEAD) > VERSION - - name: set up JDK 17 - uses: actions/setup-java@v3 - with: - java-version: 17 - distribution: temurin - cache: "gradle" - - name: Run Build - run: ./gradlew shadowJar - - run: mv build/libs/piped-*-all.jar piped.jar - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: all - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: latest - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build and push - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . file: ${{ matrix.dockerfile }} diff --git a/.github/workflows/docker-migrations-build-test.yml b/.github/workflows/docker-migrations-build-test.yml new file mode 100644 index 00000000..8290e96a --- /dev/null +++ b/.github/workflows/docker-migrations-build-test.yml @@ -0,0 +1,82 @@ +name: Docker-Compose Build and Test Migration + +on: + pull_request: + paths: + - "src/main/resources/changelog/**" + - "src/main/java/me/kavin/piped/utils/obj/db/**" + +jobs: + build-new: + uses: ./.github/workflows/fat-build.yml + build-old: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + - name: set up JDK 21 + uses: actions/setup-java@v3 + with: + java-version: 21 + distribution: zulu + cache: "gradle" + - name: Run Build + run: ./gradlew shadowJar + - run: mv build/libs/piped-*-all.jar piped.jar + - uses: actions/upload-artifact@v3 + with: + name: piped-old.jar + path: piped.jar + + docker-build-test: + needs: [ build-new, build-old ] + runs-on: ubuntu-latest + strategy: + matrix: + docker-compose-file: + - docker-compose.yml + - testing/docker-compose.cockroachdb.yml + - testing/docker-compose.yugabytedb.yml + dockerfile: + - Dockerfile.azul.ci + include: + - sleep: 20 + - docker-compose-file: testing/docker-compose.cockroachdb.yml + sleep: 30 + - docker-compose-file: testing/docker-compose.yugabytedb.yml + sleep: 120 + fail-fast: false + steps: + - uses: actions/checkout@v4 + - run: echo "unknown" > VERSION + - uses: actions/download-artifact@v3 + with: + name: piped-old.jar + - name: Build Old Image Locally + uses: docker/build-push-action@v5 + with: + context: . + load: true + file: ${{ matrix.dockerfile }} + tags: 1337kavin/piped:latest + - name: Start Docker-Compose services + run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }} + - run: rm piped.jar + - uses: actions/download-artifact@v3 + with: + name: piped.jar + - name: Build New Image Locally + uses: docker/build-push-action@v5 + with: + context: . + load: true + file: ${{ matrix.dockerfile }} + tags: 1337kavin/piped:latest + - name: Start Docker-Compose services + run: docker-compose -f ${{ matrix.docker-compose-file }} up -d && sleep ${{ matrix.sleep }} + - name: Run tests + run: ./testing/api-test.sh + - name: Collect services logs + if: failure() + run: docker-compose -f ${{ matrix.docker-compose-file }} logs diff --git a/.github/workflows/fat-build.yml b/.github/workflows/fat-build.yml new file mode 100644 index 00000000..6d3d85bf --- /dev/null +++ b/.github/workflows/fat-build.yml @@ -0,0 +1,24 @@ +name: Fat JAR Build + +on: + workflow_call: + +jobs: + build-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: set up JDK 21 + uses: actions/setup-java@v3 + with: + java-version: 21 + distribution: zulu + cache: "gradle" + - name: Run Build + run: ./gradlew shadowJar + - run: mv build/libs/piped-*-all.jar piped.jar + - uses: actions/upload-artifact@v3 + with: + name: piped.jar + path: piped.jar diff --git a/Dockerfile b/Dockerfile index 922e8316..039a7210 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM eclipse-temurin:17-jdk AS build +FROM eclipse-temurin:21-jdk AS build WORKDIR /app/ @@ -7,11 +7,19 @@ COPY . /app/ RUN --mount=type=cache,target=/root/.gradle/caches/ \ ./gradlew shadowJar -FROM eclipse-temurin:17-jre +FROM eclipse-temurin:21-jre + +RUN --mount=type=cache,target=/var/cache/apt/ \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* WORKDIR /app/ -COPY hotspot-entrypoint.sh / +COPY hotspot-entrypoint.sh docker-healthcheck.sh / COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar @@ -19,4 +27,5 @@ COPY VERSION . EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh ENTRYPOINT ["/hotspot-entrypoint.sh"] diff --git a/Dockerfile.azul b/Dockerfile.azul index b376720b..597aa35b 100644 --- a/Dockerfile.azul +++ b/Dockerfile.azul @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:17-latest AS build +FROM azul/zulu-openjdk:21-latest AS build WORKDIR /app/ @@ -7,11 +7,19 @@ COPY . /app/ RUN --mount=type=cache,target=/root/.gradle/caches/ \ ./gradlew shadowJar -FROM azul/zulu-openjdk:17-jre-headless-latest +FROM azul/zulu-openjdk:21-jre-headless-latest + +RUN --mount=type=cache,target=/var/cache/apt/ \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* WORKDIR /app/ -COPY hotspot-entrypoint.sh / +COPY hotspot-entrypoint.sh docker-healthcheck.sh / COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar @@ -19,4 +27,5 @@ COPY VERSION . EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh ENTRYPOINT ["/hotspot-entrypoint.sh"] diff --git a/Dockerfile.azul.ci b/Dockerfile.azul.ci index 6c7c24d2..1cb27a22 100644 --- a/Dockerfile.azul.ci +++ b/Dockerfile.azul.ci @@ -1,8 +1,16 @@ -FROM azul/zulu-openjdk:17-jre-headless-latest +FROM azul/zulu-openjdk:21-jre-headless-latest + +RUN --mount=type=cache,target=/var/cache/apt/ \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* WORKDIR /app/ -COPY hotspot-entrypoint.sh / +COPY hotspot-entrypoint.sh docker-healthcheck.sh / COPY ./piped.jar /app/piped.jar @@ -10,4 +18,5 @@ COPY VERSION . EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh ENTRYPOINT ["/hotspot-entrypoint.sh"] diff --git a/Dockerfile.ci b/Dockerfile.ci index 94acf719..43ce2e6c 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,8 +1,16 @@ -FROM eclipse-temurin:17-jre +FROM eclipse-temurin:21-jre + +RUN --mount=type=cache,target=/var/cache/apt/ \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* WORKDIR /app/ -COPY hotspot-entrypoint.sh / +COPY hotspot-entrypoint.sh docker-healthcheck.sh / COPY ./piped.jar /app/piped.jar @@ -10,4 +18,5 @@ COPY VERSION . EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh ENTRYPOINT ["/hotspot-entrypoint.sh"] diff --git a/Dockerfile.graalvm-jvm b/Dockerfile.graalvm-jvm index 1e07549e..c217048e 100644 --- a/Dockerfile.graalvm-jvm +++ b/Dockerfile.graalvm-jvm @@ -1,4 +1,4 @@ -FROM ghcr.io/graalvm/native-image:latest as build +FROM container-registry.oracle.com/graalvm/native-image:latest as build WORKDIR /app/ @@ -17,16 +17,27 @@ RUN jlink \ FROM debian:stable-slim +RUN --mount=type=cache,target=/var/cache/apt/ \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + ENV JAVA_HOME=/opt/java/openjdk ENV PATH "${JAVA_HOME}/bin:${PATH}" COPY --from=build /javaruntime $JAVA_HOME WORKDIR /app/ +COPY docker-healthcheck.sh / + COPY --from=build /app/build/libs/piped-1.0-all.jar /app/piped.jar COPY VERSION . EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh CMD java -jar /app/piped.jar diff --git a/Dockerfile.graalvm-jvm.ci b/Dockerfile.graalvm-jvm.ci index fd2d6ce5..6140fe4e 100644 --- a/Dockerfile.graalvm-jvm.ci +++ b/Dockerfile.graalvm-jvm.ci @@ -1,4 +1,4 @@ -FROM ghcr.io/graalvm/native-image:latest as build +FROM container-registry.oracle.com/graalvm/native-image:latest as build RUN jlink \ --add-modules java.base,java.logging,java.sql,java.management,java.xml,java.naming,java.desktop,jdk.crypto.ec \ @@ -10,16 +10,27 @@ RUN jlink \ FROM debian:stable-slim +RUN --mount=type=cache,target=/var/cache/apt/ \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + curl \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + ENV JAVA_HOME=/opt/java/openjdk ENV PATH "${JAVA_HOME}/bin:${PATH}" COPY --from=build /javaruntime $JAVA_HOME WORKDIR /app/ +COPY docker-healthcheck.sh / + COPY ./piped.jar /app/piped.jar COPY VERSION . EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 CMD /docker-healthcheck.sh CMD java -jar /app/piped.jar diff --git a/build.gradle b/build.gradle index d5cb1e72..3da01944 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,6 @@ plugins { id "com.github.johnrengelman.shadow" version "8.1.1" id "java" - id "io.freefair.lombok" version "8.1.0" id "eclipse" } @@ -13,36 +12,40 @@ repositories { dependencies { implementation 'org.apache.commons:commons-lang3:3.13.0' implementation 'org.apache.commons:commons-text:1.10.0' - implementation 'commons-io:commons-io:2.12.0' + implementation 'commons-io:commons-io:2.14.0' implementation 'it.unimi.dsi:fastutil-core:8.5.12' implementation 'commons-codec:commons-codec:1.16.0' implementation 'org.bouncycastle:bcprov-jdk15on:1.70' - implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:88ceba0da4a48b5f4ffecb3b5b2f36f95ec53afe' + implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:48beff184a9792c4787cfa05fce577c3adf89f56' implementation 'com.github.FireMasterK:nanojson:9f4af3b739cc13f3d0d9d4b758bbe2b2ae7119d7' + implementation 'com.nimbusds:oauth2-oidc-sdk:11.5.0' implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2' implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2' implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2' implementation 'com.rometools:rome:2.1.0' + implementation 'com.rometools:rome-modules:2.1.0' implementation 'org.jsoup:jsoup:1.16.1' implementation 'io.activej:activej-common:5.5' implementation 'io.activej:activej-http:5.5' implementation 'io.activej:activej-boot:5.5' implementation 'io.activej:activej-specializer:5.5' implementation 'io.activej:activej-launchers-http:5.5' - implementation 'org.hsqldb:hsqldb:2.7.2' implementation 'org.postgresql:postgresql:42.6.0' - implementation 'org.hibernate:hibernate-core:6.2.7.Final' - implementation 'org.hibernate:hibernate-hikaricp:6.2.7.Final' + implementation 'org.hibernate:hibernate-core:6.3.1.Final' + implementation 'org.hibernate:hibernate-hikaricp:6.3.1.Final' + implementation 'org.liquibase:liquibase-core:4.23.2' + implementation('org.liquibase.ext:liquibase-yugabytedb:4.23.2') { exclude group: 'org.liquibase' } implementation 'com.zaxxer:HikariCP:5.0.1' - implementation 'org.springframework.security:spring-security-crypto:6.1.2' + implementation 'org.springframework.security:spring-security-crypto:6.1.4' implementation 'commons-logging:commons-logging:1.2' implementation(platform("com.squareup.okhttp3:okhttp-bom:4.11.0")) implementation 'com.squareup.okhttp3:okhttp' implementation 'com.squareup.okhttp3:okhttp-brotli' - implementation 'com.nimbusds:oauth2-oidc-sdk:10.9.1' - implementation 'io.sentry:sentry:6.28.0' - implementation 'rocks.kavin:reqwest4j:1.0.7' - implementation 'io.minio:minio:8.5.4' + implementation 'io.sentry:sentry:6.30.0' + implementation 'rocks.kavin:reqwest4j:1.0.12' + implementation 'io.minio:minio:8.5.6' + compileOnly 'org.projectlombok:lombok:1.18.30' + annotationProcessor 'org.projectlombok:lombok:1.18.30' } shadowJar { @@ -59,5 +62,5 @@ jar { group = 'me.kavin.piped' version = '1.0' -sourceCompatibility = JavaVersion.VERSION_17 -targetCompatibility = JavaVersion.VERSION_17 +sourceCompatibility = JavaVersion.VERSION_21 +targetCompatibility = JavaVersion.VERSION_21 diff --git a/config.properties b/config.properties index 71df0fb1..f5387506 100644 --- a/config.properties +++ b/config.properties @@ -8,6 +8,9 @@ PROXY_PART:https://pipedproxy-cdg.kavin.rocks # Outgoing proxy to be used by reqwest4j - eg: socks5://127.0.0.1:1080 #REQWEST_PROXY: socks5://127.0.0.1:1080 +# Optional proxy username and password +#REQWEST_PROXY_USER: username +#REQWEST_PROXY_PASS: password # Captcha Parameters CAPTCHA_BASE_URL:https://api.capmonster.cloud/ diff --git a/docker-compose.yml b/docker-compose.yml index 9a98b246..01a83589 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,7 @@ services: depends_on: - postgres postgres: - image: postgres:15-alpine + image: postgres:16-alpine restart: unless-stopped volumes: - ./data/db:/var/lib/postgresql/data diff --git a/docker-healthcheck.sh b/docker-healthcheck.sh new file mode 100755 index 00000000..ab55be0d --- /dev/null +++ b/docker-healthcheck.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh + +# If PORT env var is set, use it, otherwise default to 8080 +PORT=${PORT:-8080} + +curl -f http://localhost:$PORT/healthcheck || exit 1 diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 033e24c4..7f93135c 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index c33e3020..a456744a 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.2.1-bin.zip +distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.4-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/gradlew b/gradlew index fcb6fca1..1aa94a42 100755 --- a/gradlew +++ b/gradlew @@ -83,7 +83,8 @@ done # This is normally unused # shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -144,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -201,11 +202,11 @@ fi # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ diff --git a/src/main/java/me/kavin/piped/Main.java b/src/main/java/me/kavin/piped/Main.java index 067f2296..b80d31f1 100644 --- a/src/main/java/me/kavin/piped/Main.java +++ b/src/main/java/me/kavin/piped/Main.java @@ -18,9 +18,8 @@ import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.localization.ContentCountry; import org.schabi.newpipe.extractor.localization.Localization; -import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter; +import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager; import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; -import rocks.kavin.reqwest4j.ReqwestUtils; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -47,17 +46,24 @@ public static void main(String[] args) throws Exception { Injector.useSpecializer(); - Multithreading.runAsync(() -> new Thread(new SyncRunner( + try { + LiquibaseHelper.init(); + } catch (Exception e) { + ExceptionHandler.handle(e); + System.exit(1); + } + + Multithreading.runAsync(() -> Thread.ofVirtual().start(new SyncRunner( new OkHttpClient.Builder().readTimeout(60, TimeUnit.SECONDS).build(), MATRIX_SERVER, MatrixHelper.MATRIX_TOKEN) - ).start()); + )); new Timer().scheduleAtFixedRate(new TimerTask() { @Override public void run() { - System.out.printf("ThrottlingCache: %o entries%n", YoutubeThrottlingDecrypter.getCacheSize()); - YoutubeThrottlingDecrypter.clearCache(); + System.out.printf("ThrottlingCache: %o entries%n", YoutubeJavaScriptPlayerManager.getThrottlingParametersCacheSize()); + YoutubeJavaScriptPlayerManager.clearThrottlingParametersCache(); } }, 0, TimeUnit.MINUTES.toMillis(60)); diff --git a/src/main/java/me/kavin/piped/consts/Constants.java b/src/main/java/me/kavin/piped/consts/Constants.java index 3b44b07b..07c5b81e 100644 --- a/src/main/java/me/kavin/piped/consts/Constants.java +++ b/src/main/java/me/kavin/piped/consts/Constants.java @@ -53,6 +53,8 @@ public class Constants { public static final String PUBSUB_HUB_URL; public static final String REQWEST_PROXY; + public static final String REQWEST_PROXY_USER; + public static final String REQWEST_PROXY_PASS; public static final String FRONTEND_URL; @@ -134,7 +136,9 @@ public class Constants { PUBSUB_URL = getProperty(prop, "PUBSUB_URL", PUBLIC_URL); PUBSUB_HUB_URL = getProperty(prop, "PUBSUB_HUB_URL", "https://pubsubhubbub.appspot.com/subscribe"); REQWEST_PROXY = getProperty(prop, "REQWEST_PROXY"); - ReqwestUtils.init(REQWEST_PROXY); + REQWEST_PROXY_USER = getProperty(prop, "REQWEST_PROXY_USER"); + REQWEST_PROXY_PASS = getProperty(prop, "REQWEST_PROXY_PASS"); + ReqwestUtils.init(REQWEST_PROXY, REQWEST_PROXY_USER, REQWEST_PROXY_PASS); FRONTEND_URL = getProperty(prop, "FRONTEND_URL", "https://piped.video"); COMPROMISED_PASSWORD_CHECK = Boolean.parseBoolean(getProperty(prop, "COMPROMISED_PASSWORD_CHECK", "true")); DISABLE_REGISTRATION = Boolean.parseBoolean(getProperty(prop, "DISABLE_REGISTRATION", "false")); diff --git a/src/main/java/me/kavin/piped/server/ServerLauncher.java b/src/main/java/me/kavin/piped/server/ServerLauncher.java index 676fc448..a089b396 100644 --- a/src/main/java/me/kavin/piped/server/ServerLauncher.java +++ b/src/main/java/me/kavin/piped/server/ServerLauncher.java @@ -97,53 +97,7 @@ AsyncServlet mainServlet(Executor executor) { })).map(POST, "/webhooks/pubsub", AsyncServlet.ofBlocking(executor, request -> { try { - SyndFeed feed = new SyndFeedInput().build( - new InputSource(new ByteArrayInputStream(request.loadBody().getResult().asArray()))); - - Multithreading.runAsyncLimited(() -> { - for (var entry : feed.getEntries()) { - String url = entry.getLinks().get(0).getHref(); - String videoId = StringUtils.substring(url, -11); - try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { - if (DatabaseHelper.doesVideoExist(s, videoId)) - continue; - } - Multithreading.runAsyncLimited(() -> { - try { - Sentry.setExtra("videoId", videoId); - var extractor = YOUTUBE_SERVICE.getStreamExtractor("https://youtube.com/watch?v=" + videoId); - extractor.fetchPage(); - - Multithreading.runAsync(() -> { - - DateWrapper uploadDate; - - try { - uploadDate = extractor.getUploadDate(); - } catch (ParsingException e) { - throw new RuntimeException(e); - } - - if (uploadDate != null && System.currentTimeMillis() - uploadDate.offsetDateTime().toInstant().toEpochMilli() < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) { - try { - MatrixHelper.sendEvent("video.piped.stream.info", new FederatedVideoInfo( - StringUtils.substring(extractor.getUrl(), -11), StringUtils.substring(extractor.getUploaderUrl(), -24), - extractor.getName(), - extractor.getLength(), extractor.getViewCount()) - ); - } catch (Exception e) { - ExceptionHandler.handle(e); - } - } - }); - - VideoHelpers.handleNewVideo(extractor, entry.getPublishedDate().getTime(), null); - } catch (Exception e) { - ExceptionHandler.handle(e); - } - }); - } - }); + PubSubHandlers.handlePubSub(request.loadBody().getResult().asArray()); return HttpResponse.ofCode(204); diff --git a/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java b/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java index e086df9e..1e43bc0e 100644 --- a/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/ChannelHandlers.java @@ -32,7 +32,7 @@ import static me.kavin.piped.consts.Constants.mapper; import static me.kavin.piped.utils.CollectionUtils.collectPreloadedTabs; import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems; -import static me.kavin.piped.utils.URLUtils.rewriteURL; +import static me.kavin.piped.utils.URLUtils.getLastThumbnail; public class ChannelHandlers { public static byte[] channelResponse(String channelPath) throws Exception { @@ -77,7 +77,7 @@ public static byte[] channelResponse(String channelPath) throws Exception { Multithreading.runAsync(() -> { try { MatrixHelper.sendEvent("video.piped.channel.info", new FederatedChannelInfo( - info.getId(), StringUtils.abbreviate(info.getName(), 100), info.getAvatarUrl(), info.isVerified()) + info.getId(), StringUtils.abbreviate(info.getName(), 100), info.getAvatars().isEmpty() ? null : info.getAvatars().getLast().getUrl(), info.isVerified()) ); } catch (IOException e) { throw new RuntimeException(e); @@ -93,7 +93,7 @@ public static byte[] channelResponse(String channelPath) throws Exception { if (channel != null) { - ChannelHelpers.updateChannel(s, channel, StringUtils.abbreviate(info.getName(), 100), info.getAvatarUrl(), info.isVerified()); + ChannelHelpers.updateChannel(s, channel, StringUtils.abbreviate(info.getName(), 100), info.getAvatars().isEmpty() ? null : info.getAvatars().getLast().getUrl(), info.isVerified()); Set ids = tabInfo.getRelatedItems() .stream() @@ -159,8 +159,8 @@ public static byte[] channelResponse(String channelPath) throws Exception { } }).toList(); - final Channel channel = new Channel(info.getId(), info.getName(), rewriteURL(info.getAvatarUrl()), - rewriteURL(info.getBannerUrl()), info.getDescription(), info.getSubscriberCount(), info.isVerified(), + final Channel channel = new Channel(info.getId(), info.getName(), getLastThumbnail(info.getAvatars()), + getLastThumbnail(info.getBanners()), info.getDescription(), info.getSubscriberCount(), info.isVerified(), nextpage, relatedStreams, tabs); return mapper.writeValueAsBytes(channel); @@ -210,6 +210,57 @@ public static byte[] channelTabResponse(String data) List items = collectRelatedItems(info.getRelatedItems()); + Multithreading.runAsync(() -> { + + var channel = DatabaseHelper.getChannelFromId(info.getId()); + + if (channel != null) { + try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { + var streamInfoItems = info.getRelatedItems() + .stream() + .parallel() + .filter(StreamInfoItem.class::isInstance) + .map(StreamInfoItem.class::cast) + .toList(); + + var channelIds = streamInfoItems + .stream() + .map(item -> { + try { + return YOUTUBE_SERVICE.getStreamLHFactory().getId(item.getUrl()); + } catch (ParsingException e) { + throw new RuntimeException(e); + } + }).collect(Collectors.toUnmodifiableSet()); + + List videoIdsPresent = DatabaseHelper.getVideosFromIds(s, channelIds) + .stream() + .map(Video::getId) + .toList(); + + streamInfoItems + .stream() + .parallel() + .forEach(item -> { + try { + String id = YOUTUBE_SERVICE.getStreamLHFactory().getId(item.getUrl()); + if (videoIdsPresent.contains(id)) + VideoHelpers.updateVideo(id, item); + else if (item.getUploadDate() != null) { + // shorts tab doesn't have upload date + // we don't want to fetch each video's upload date + long time = item.getUploadDate().offsetDateTime().toInstant().toEpochMilli(); + if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) + VideoHelpers.handleNewVideo(item.getUrl(), time, channel); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + } + }); + String nextpage = null; if (info.hasNextPage()) { Page page = info.getNextPage(); diff --git a/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java b/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java index 94d771e6..6b358b1b 100644 --- a/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/PlaylistHandlers.java @@ -30,8 +30,7 @@ import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; import static me.kavin.piped.consts.Constants.mapper; import static me.kavin.piped.utils.CollectionUtils.collectRelatedItems; -import static me.kavin.piped.utils.URLUtils.rewriteURL; -import static me.kavin.piped.utils.URLUtils.substringYouTube; +import static me.kavin.piped.utils.URLUtils.*; public class PlaylistHandlers { public static byte[] playlistResponse(String playlistId) throws Exception { @@ -60,10 +59,10 @@ private static byte[] playlistYouTubeResponse(String playlistId) nextpage = mapper.writeValueAsString(page); } - final Playlist playlist = new Playlist(info.getName(), rewriteURL(info.getThumbnailUrl()), - info.getDescription().getContent(), rewriteURL(info.getBannerUrl()), nextpage, + final Playlist playlist = new Playlist(info.getName(), getLastThumbnail(info.getThumbnails()), + info.getDescription().getContent(), getLastThumbnail(info.getBanners()), nextpage, info.getUploaderName().isEmpty() ? null : info.getUploaderName(), - substringYouTube(info.getUploaderUrl()), rewriteURL(info.getUploaderAvatarUrl()), + substringYouTube(info.getUploaderUrl()), getLastThumbnail(info.getUploaderAvatars()), (int) info.getStreamCount(), relatedStreams); return mapper.writeValueAsBytes(playlist); diff --git a/src/main/java/me/kavin/piped/server/handlers/PubSubHandlers.java b/src/main/java/me/kavin/piped/server/handlers/PubSubHandlers.java new file mode 100644 index 00000000..9738c224 --- /dev/null +++ b/src/main/java/me/kavin/piped/server/handlers/PubSubHandlers.java @@ -0,0 +1,100 @@ +package me.kavin.piped.server.handlers; + +import com.rometools.rome.feed.synd.SyndFeed; +import com.rometools.rome.io.SyndFeedInput; +import io.sentry.Sentry; +import me.kavin.piped.consts.Constants; +import me.kavin.piped.utils.*; +import me.kavin.piped.utils.obj.MatrixHelper; +import me.kavin.piped.utils.obj.federation.FederatedVideoInfo; +import org.apache.commons.lang3.StringUtils; +import org.hibernate.StatelessSession; +import org.schabi.newpipe.extractor.exceptions.ParsingException; +import org.schabi.newpipe.extractor.localization.DateWrapper; +import org.xml.sax.InputSource; + +import java.io.ByteArrayInputStream; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; + +public class PubSubHandlers { + + private static final LinkedBlockingQueue pubSubQueue = new LinkedBlockingQueue<>(); + + public static void handlePubSub(byte[] body) throws Exception { + SyndFeed feed = new SyndFeedInput().build(new InputSource(new ByteArrayInputStream(body))); + + + for (var entry : feed.getEntries()) { + String url = entry.getLinks().get(0).getHref(); + String videoId = StringUtils.substring(url, -11); + + long publishedDate = entry.getPublishedDate().getTime(); + + String str = videoId + ":" + publishedDate; + + if (pubSubQueue.contains(str)) + continue; + + pubSubQueue.put(str); + } + } + + static { + for (int i = 0; i < Runtime.getRuntime().availableProcessors(); i++) { + new Thread(() -> { + try { + while (true) { + String str = pubSubQueue.take(); + + String videoId = StringUtils.substringBefore(str, ":"); + long publishedDate = Long.parseLong(StringUtils.substringAfter(str, ":")); + + try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { + if (DatabaseHelper.doesVideoExist(s, videoId)) + continue; + } + + try { + Sentry.setExtra("videoId", videoId); + var extractor = YOUTUBE_SERVICE.getStreamExtractor("https://youtube.com/watch?v=" + videoId); + extractor.fetchPage(); + + Multithreading.runAsync(() -> { + + DateWrapper uploadDate; + + try { + uploadDate = extractor.getUploadDate(); + } catch (ParsingException e) { + throw new RuntimeException(e); + } + + if (uploadDate != null && System.currentTimeMillis() - uploadDate.offsetDateTime().toInstant().toEpochMilli() < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) { + try { + MatrixHelper.sendEvent("video.piped.stream.info", new FederatedVideoInfo( + StringUtils.substring(extractor.getUrl(), -11), StringUtils.substring(extractor.getUploaderUrl(), -24), + extractor.getName(), + extractor.getLength(), extractor.getViewCount()) + ); + } catch (Exception e) { + ExceptionHandler.handle(e); + } + } + }); + + VideoHelpers.handleNewVideo(extractor, publishedDate, null); + } catch (Exception e) { + ExceptionHandler.handle(e); + } + } + } catch (Exception e) { + ExceptionHandler.handle(e); + } + }, "PubSub-Worker-" + i).start(); + } + } + +} diff --git a/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java b/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java index a6e50e37..9dd6c6e0 100644 --- a/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java @@ -36,8 +36,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; import static me.kavin.piped.consts.Constants.mapper; -import static me.kavin.piped.utils.URLUtils.rewriteURL; -import static me.kavin.piped.utils.URLUtils.substringYouTube; +import static me.kavin.piped.utils.URLUtils.*; import static org.schabi.newpipe.extractor.NewPipe.getPreferredContentCountry; import static org.schabi.newpipe.extractor.NewPipe.getPreferredLocalization; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; @@ -342,10 +341,10 @@ public static byte[] commentsResponse(String videoId) throws Exception { if (comment.getReplies() != null) repliespage = mapper.writeValueAsString(comment.getReplies()); - comments.add(new Comment(comment.getUploaderName(), rewriteURL(comment.getUploaderAvatarUrl()), + comments.add(new Comment(comment.getUploaderName(), getLastThumbnail(comment.getUploaderAvatars()), comment.getCommentId(), Optional.ofNullable(comment.getCommentText()).map(Description::getContent).orElse(null), comment.getTextualUploadDate(), substringYouTube(comment.getUploaderUrl()), repliespage, comment.getLikeCount(), comment.getReplyCount(), - comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified())); + comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified(), comment.hasCreatorReply())); } catch (JsonProcessingException e) { ExceptionHandler.handle(e); } @@ -380,10 +379,10 @@ public static byte[] commentsPageResponse(String videoId, String prevpageStr) th if (comment.getReplies() != null) repliespage = mapper.writeValueAsString(comment.getReplies()); - comments.add(new Comment(comment.getUploaderName(), rewriteURL(comment.getUploaderAvatarUrl()), + comments.add(new Comment(comment.getUploaderName(), getLastThumbnail(comment.getUploaderAvatars()), comment.getCommentId(), Optional.ofNullable(comment.getCommentText()).map(Description::getContent).orElse(null), comment.getTextualUploadDate(), substringYouTube(comment.getUploaderUrl()), repliespage, comment.getLikeCount(), comment.getReplyCount(), - comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified())); + comment.isHeartedByUploader(), comment.isPinned(), comment.isUploaderVerified(), comment.hasCreatorReply())); } catch (JsonProcessingException e) { ExceptionHandler.handle(e); } diff --git a/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java b/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java index f6c6da3a..03829d9e 100644 --- a/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java +++ b/src/main/java/me/kavin/piped/server/handlers/auth/AuthPlaylistHandlers.java @@ -9,7 +9,6 @@ import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; import it.unimi.dsi.fastutil.objects.ObjectArrayList; import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; -import jakarta.persistence.criteria.JoinType; import me.kavin.piped.consts.Constants; import me.kavin.piped.utils.*; import me.kavin.piped.utils.obj.ContentItem; @@ -23,7 +22,7 @@ import me.kavin.piped.utils.resp.InvalidRequestResponse; import org.apache.commons.lang3.StringUtils; import org.hibernate.Session; -import org.hibernate.internal.util.ExceptionHelper; +import org.hibernate.StatelessSession; import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistInfo; @@ -269,7 +268,7 @@ public static byte[] addToPlaylistResponse(String session, String playlistId, Li channel = DatabaseHelper.saveChannel(channelId); } - video = new PlaylistVideo(videoId, info.getName(), info.getThumbnailUrl(), info.getDuration(), channel); + video = new PlaylistVideo(videoId, info.getName(), info.getThumbnails().getLast().getUrl(), info.getDuration(), channel); var tr = s.beginTransaction(); try { @@ -312,11 +311,16 @@ public static byte[] removeFromPlaylistResponse(String session, String playlistI if (StringUtils.isBlank(session) || StringUtils.isBlank(playlistId)) ExceptionHandler.throwErrorResponse(new InvalidRequestResponse("session and playlistId are required parameters")); - try (Session s = DatabaseSessionFactory.createSession()) { + if (index < 0) + return mapper.writeValueAsBytes(mapper.createObjectNode() + .put("error", "Video Index out of bounds")); + + long internalId; + + try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { var cb = s.getCriteriaBuilder(); var query = cb.createQuery(me.kavin.piped.utils.obj.db.Playlist.class); var root = query.from(me.kavin.piped.utils.obj.db.Playlist.class); - root.fetch("videos", JoinType.RIGHT); query.where(cb.equal(root.get("playlist_id"), UUID.fromString(playlistId))); var playlist = s.createQuery(query).uniqueResult(); @@ -327,19 +331,31 @@ public static byte[] removeFromPlaylistResponse(String session, String playlistI if (playlist.getOwner().getId() != DatabaseHelper.getUserFromSession(session).getId()) return mapper.writeValueAsBytes(mapper.createObjectNode() .put("error", "You are not the owner this playlist")); + internalId = playlist.getId(); + } - if (index < 0 || index >= playlist.getVideos().size()) - return mapper.writeValueAsBytes(mapper.createObjectNode() - .put("error", "Video Index out of bounds")); - - playlist.getVideos().remove(index); + try (Session s = DatabaseSessionFactory.createSession()) { var tr = s.beginTransaction(); - s.merge(playlist); - tr.commit(); - return mapper.writeValueAsBytes(new AcceptedResponse()); + var updated = s.createNativeMutationQuery("DELETE FROM playlists_videos_ids WHERE playlist_id = :playlistId AND videos_order = :index") + .setParameter("playlistId", internalId) + .setParameter("index", index) + .executeUpdate(); + + if (updated > 0) { + s.createNativeMutationQuery("UPDATE playlists_videos_ids SET videos_order = videos_order - 1 WHERE playlist_id = :playlistId AND videos_order > :index") + .setParameter("playlistId", internalId) + .setParameter("index", index) + .executeUpdate(); + } else + return mapper.writeValueAsBytes(mapper.createObjectNode() + .put("error", "Video Index not found")); + + tr.commit(); } + + return mapper.writeValueAsBytes(new AcceptedResponse()); } public static byte[] clearPlaylistResponse(String session, String playlistId) throws IOException { @@ -386,7 +402,7 @@ public static byte[] importPlaylistResponse(String session, String playlistId) t PlaylistInfo info = PlaylistInfo.getInfo(url); - var playlist = new me.kavin.piped.utils.obj.db.Playlist(info.getName(), user, info.getThumbnailUrl()); + var playlist = new me.kavin.piped.utils.obj.db.Playlist(info.getName(), user, info.getThumbnails().getLast().getUrl()); List videos = new ObjectArrayList<>(info.getRelatedItems()); @@ -435,7 +451,7 @@ public static byte[] importPlaylistResponse(String session, String playlistId) t var channel = channelMap.get(channelId); - playlist.getVideos().add(videoMap.computeIfAbsent(videoId, (key) -> new PlaylistVideo(videoId, video.getName(), video.getThumbnailUrl(), video.getDuration(), channel))); + playlist.getVideos().add(videoMap.computeIfAbsent(videoId, (key) -> new PlaylistVideo(videoId, video.getName(), video.getThumbnails().getLast().getUrl(), video.getDuration(), channel))); }); var tr = s.beginTransaction(); diff --git a/src/main/java/me/kavin/piped/utils/ChannelHelpers.java b/src/main/java/me/kavin/piped/utils/ChannelHelpers.java index 5f0a8e0b..a6495f5a 100644 --- a/src/main/java/me/kavin/piped/utils/ChannelHelpers.java +++ b/src/main/java/me/kavin/piped/utils/ChannelHelpers.java @@ -1,5 +1,7 @@ package me.kavin.piped.utils; +import com.rometools.modules.mediarss.MediaEntryModuleImpl; +import com.rometools.modules.mediarss.types.*; import com.rometools.rome.feed.synd.*; import me.kavin.piped.consts.Constants; import me.kavin.piped.utils.obj.db.Channel; @@ -11,6 +13,7 @@ import java.io.IOException; import java.net.MalformedURLException; +import java.net.URI; import java.net.URL; import java.util.Collections; import java.util.Date; @@ -79,6 +82,7 @@ public static SyndEntry createEntry(Video video, Channel channel) { entry.setAuthors(Collections.singletonList(person)); entry.setLink(Constants.FRONTEND_URL + "/watch?v=" + video.getId()); entry.setUri(Constants.FRONTEND_URL + "/watch?v=" + video.getId()); + entry.setTitle(video.getTitle()); entry.setPublishedDate(new Date(video.getUploaded())); @@ -95,6 +99,23 @@ public static SyndEntry createEntry(Video video, Channel channel) { entry.setContents(List.of(thumbnail, content)); + // the Media RSS content for embedding videos starts here + // see https://www.rssboard.org/media-rss#media-content + + String playerUrl = Constants.FRONTEND_URL + "/embed/" + video.getId(); + MediaContent media = new MediaContent(new PlayerReference(URI.create(playerUrl))); + media.setDuration(video.getDuration()); + + Metadata metadata = new Metadata(); + metadata.setTitle(video.getTitle()); + Thumbnail metadataThumbnail = new Thumbnail(URI.create(video.getThumbnail())); + metadata.setThumbnail(new Thumbnail[]{ metadataThumbnail }); + media.setMetadata(metadata); + + MediaEntryModuleImpl mediaModule = new MediaEntryModuleImpl(); + mediaModule.setMediaContents(new MediaContent[]{ media }); + entry.getModules().add(mediaModule); + return entry; } } diff --git a/src/main/java/me/kavin/piped/utils/CollectionUtils.java b/src/main/java/me/kavin/piped/utils/CollectionUtils.java index bd09132b..ea4efe8f 100644 --- a/src/main/java/me/kavin/piped/utils/CollectionUtils.java +++ b/src/main/java/me/kavin/piped/utils/CollectionUtils.java @@ -71,7 +71,7 @@ public static Streams collectStreamInfo(StreamInfo info) { return new Streams(info.getName(), info.getDescription().getContent(), info.getTextualUploadDate(), info.getUploaderName(), substringYouTube(info.getUploaderUrl()), - rewriteURL(info.getUploaderAvatarUrl()), rewriteURL(info.getThumbnailUrl()), info.getDuration(), + getLastThumbnail(info.getUploaderAvatars()), getLastThumbnail(info.getThumbnails()), info.getDuration(), info.getViewCount(), info.getLikeCount(), info.getDislikeCount(), info.getUploaderSubscriberCount(), info.isUploaderVerified(), audioStreams, videoStreams, relatedStreams, subtitles, livestream, rewriteVideoURL(info.getHlsUrl()), rewriteVideoURL(info.getDashMpdUrl()), null, info.getCategory(), info.getLicence(), @@ -101,9 +101,9 @@ private static StreamItem collectRelatedStream(Object o) { StreamInfoItem item = (StreamInfoItem) o; return new StreamItem(substringYouTube(item.getUrl()), item.getName(), - rewriteURL(item.getThumbnailUrl()), + getLastThumbnail(item.getThumbnails()), item.getUploaderName(), substringYouTube(item.getUploaderUrl()), - rewriteURL(item.getUploaderAvatarUrl()), item.getTextualUploadDate(), + getLastThumbnail(item.getUploaderAvatars()), item.getTextualUploadDate(), item.getShortDescription(), item.getDuration(), item.getViewCount(), item.getUploadDate() != null ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() : -1, @@ -115,7 +115,7 @@ private static PlaylistItem collectRelatedPlaylist(Object o) { PlaylistInfoItem item = (PlaylistInfoItem) o; return new PlaylistItem(substringYouTube(item.getUrl()), item.getName(), - rewriteURL(item.getThumbnailUrl()), + getLastThumbnail(item.getThumbnails()), item.getUploaderName(), substringYouTube(item.getUploaderUrl()), item.isUploaderVerified(), item.getPlaylistType().name(), item.getStreamCount()); @@ -126,7 +126,7 @@ private static ChannelItem collectRelatedChannel(Object o) { ChannelInfoItem item = (ChannelInfoItem) o; return new ChannelItem(substringYouTube(item.getUrl()), item.getName(), - rewriteURL(item.getThumbnailUrl()), + getLastThumbnail(item.getThumbnails()), item.getDescription(), item.getSubscriberCount(), item.getStreamCount(), item.isVerified()); } diff --git a/src/main/java/me/kavin/piped/utils/DatabaseHelper.java b/src/main/java/me/kavin/piped/utils/DatabaseHelper.java index 9b9ae913..a4af6272 100644 --- a/src/main/java/me/kavin/piped/utils/DatabaseHelper.java +++ b/src/main/java/me/kavin/piped/utils/DatabaseHelper.java @@ -192,7 +192,7 @@ public static Channel saveChannel(String channelId) { } var channel = new Channel(channelId, StringUtils.abbreviate(info.getName(), 100), - info.getAvatarUrl(), info.isVerified()); + info.getAvatars().isEmpty() ? null : info.getAvatars().getLast().getUrl(), info.isVerified()); try (StatelessSession s = DatabaseSessionFactory.createStatelessSession()) { var tr = s.beginTransaction(); @@ -214,9 +214,11 @@ public static Channel saveChannel(String channelId) { CollectionUtils.collectPreloadedTabs(info.getTabs()) .stream() .parallel() - .map(tab -> { + .mapMulti((tab, consumer) -> { try { - return ChannelTabInfo.getInfo(YOUTUBE_SERVICE, tab).getRelatedItems(); + ChannelTabInfo.getInfo(YOUTUBE_SERVICE, tab) + .getRelatedItems() + .forEach(consumer); } catch (ExtractionException | IOException e) { throw new RuntimeException(e); } @@ -224,11 +226,11 @@ public static Channel saveChannel(String channelId) { .filter(StreamInfoItem.class::isInstance) .map(StreamInfoItem.class::cast) .forEach(item -> { - long time = item.getUploadDate() != null - ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() - : System.currentTimeMillis(); - if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) - VideoHelpers.handleNewVideo(item.getUrl(), time, channel); + long time = item.getUploadDate() != null + ? item.getUploadDate().offsetDateTime().toInstant().toEpochMilli() + : System.currentTimeMillis(); + if ((System.currentTimeMillis() - time) < TimeUnit.DAYS.toMillis(Constants.FEED_RETENTION)) + VideoHelpers.handleNewVideo(item.getUrl(), time, channel); }); }); diff --git a/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java b/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java new file mode 100644 index 00000000..a97d09cc --- /dev/null +++ b/src/main/java/me/kavin/piped/utils/LiquibaseHelper.java @@ -0,0 +1,50 @@ +package me.kavin.piped.utils; + +import liquibase.Liquibase; +import liquibase.Scope; +import liquibase.command.CommandScope; +import liquibase.command.core.UpdateCommandStep; +import liquibase.command.core.helpers.DbUrlConnectionCommandStep; +import liquibase.database.Database; +import liquibase.database.DatabaseFactory; +import liquibase.database.jvm.JdbcConnection; +import liquibase.resource.ClassLoaderResourceAccessor; +import me.kavin.piped.consts.Constants; + +import java.sql.DriverManager; +import java.util.HashMap; +import java.util.Map; + +public class LiquibaseHelper { + + public static void init() throws Exception { + + String url = Constants.hibernateProperties.get("hibernate.connection.url"); + String username = Constants.hibernateProperties.get("hibernate.connection.username"); + String password = Constants.hibernateProperties.get("hibernate.connection.password"); + + // ensure postgres driver is loaded + DriverManager.registerDriver(new org.postgresql.Driver()); + + // register YugabyteDB database + DatabaseFactory.getInstance().register(new liquibase.ext.yugabytedb.database.YugabyteDBDatabase()); + + Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(DriverManager.getConnection(url, username, password))); + + try (Liquibase liquibase = new Liquibase("changelog/db.changelog-master.xml", new ClassLoaderResourceAccessor(), database)) { + + Map scopeObjects = new HashMap<>(); + scopeObjects.put(Scope.Attr.database.name(), liquibase.getDatabase()); + scopeObjects.put(Scope.Attr.resourceAccessor.name(), liquibase.getResourceAccessor()); + + Scope.child(scopeObjects, () -> { + CommandScope updateCommand = new CommandScope(UpdateCommandStep.COMMAND_NAME); + updateCommand.addArgumentValue(DbUrlConnectionCommandStep.DATABASE_ARG, liquibase.getDatabase()); + updateCommand.addArgumentValue(UpdateCommandStep.CHANGELOG_FILE_ARG, liquibase.getChangeLogFile()); + updateCommand.execute(); + }); + + } + } + +} diff --git a/src/main/java/me/kavin/piped/utils/Multithreading.java b/src/main/java/me/kavin/piped/utils/Multithreading.java index 2e4efd45..82c0b5ba 100644 --- a/src/main/java/me/kavin/piped/utils/Multithreading.java +++ b/src/main/java/me/kavin/piped/utils/Multithreading.java @@ -5,7 +5,7 @@ public class Multithreading { - private static final ExecutorService es = Executors.newCachedThreadPool(); + private static final ExecutorService es = Executors.newVirtualThreadPerTaskExecutor(); private static final ExecutorService esLimited = Executors .newFixedThreadPool(Runtime.getRuntime().availableProcessors() * 8); private static final ExecutorService esLimitedPubSub = Executors diff --git a/src/main/java/me/kavin/piped/utils/URLUtils.java b/src/main/java/me/kavin/piped/utils/URLUtils.java index 2ec0430d..a36f3e97 100644 --- a/src/main/java/me/kavin/piped/utils/URLUtils.java +++ b/src/main/java/me/kavin/piped/utils/URLUtils.java @@ -2,12 +2,14 @@ import me.kavin.piped.consts.Constants; import org.apache.commons.lang3.StringUtils; +import org.schabi.newpipe.extractor.Image; import java.net.MalformedURLException; import java.net.URL; import java.net.URLDecoder; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; +import java.util.List; public class URLUtils { @@ -37,6 +39,10 @@ public static String rewriteURL(final String old) { return rewriteURL(old, Constants.IMAGE_PROXY_PART); } + public static String getLastThumbnail(final List thumbnails) { + return thumbnails.isEmpty() ? null : rewriteURL(thumbnails.getLast().getUrl()); + } + public static String rewriteVideoURL(final String old) { return rewriteURL(old, Constants.PROXY_PART); } diff --git a/src/main/java/me/kavin/piped/utils/VideoHelpers.java b/src/main/java/me/kavin/piped/utils/VideoHelpers.java index 6c808345..a89c4130 100644 --- a/src/main/java/me/kavin/piped/utils/VideoHelpers.java +++ b/src/main/java/me/kavin/piped/utils/VideoHelpers.java @@ -6,6 +6,7 @@ import me.kavin.piped.utils.obj.db.Video; import org.apache.commons.lang3.StringUtils; import org.hibernate.StatelessSession; +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor; import org.schabi.newpipe.extractor.stream.StreamExtractor; import org.schabi.newpipe.extractor.stream.StreamInfo; import org.schabi.newpipe.extractor.stream.StreamInfoItem; @@ -14,6 +15,7 @@ import java.util.concurrent.TimeUnit; import static java.nio.charset.StandardCharsets.UTF_8; +import static me.kavin.piped.consts.Constants.YOUTUBE_SERVICE; import static org.schabi.newpipe.extractor.NewPipe.getPreferredContentCountry; import static org.schabi.newpipe.extractor.NewPipe.getPreferredLocalization; import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse; @@ -22,7 +24,9 @@ public class VideoHelpers { public static void handleNewVideo(String url, long time, me.kavin.piped.utils.obj.db.Channel channel) { try { - handleNewVideo(StreamInfo.getInfo(url), time, channel); + var extractor = YOUTUBE_SERVICE.getStreamExtractor(url); + extractor.fetchPage(); + handleNewVideo(extractor, time, channel); } catch (Exception e) { ExceptionHandler.handle(e); } @@ -46,7 +50,7 @@ public static void handleNewVideo(StreamInfo info, long time, me.kavin.piped.uti if (!DatabaseHelper.doesVideoExist(s, info.getId())) { Video video = new Video(info.getId(), info.getName(), info.getViewCount(), info.getDuration(), - Math.max(infoTime, time), info.getThumbnailUrl(), info.isShortFormContent(), channel); + Math.max(infoTime, time), info.getThumbnails().getLast().getUrl(), info.isShortFormContent(), channel); insertVideo(video); return; @@ -77,7 +81,7 @@ public static void handleNewVideo(StreamExtractor extractor, long time, me.kavin boolean isShort = extractor.isShortFormContent() || isShort(extractor.getId()); Video video = new Video(extractor.getId(), extractor.getName(), extractor.getViewCount(), extractor.getLength(), - Math.max(infoTime, time), extractor.getThumbnailUrl(), isShort, channel); + Math.max(infoTime, time), extractor.getThumbnails().getLast().getUrl(), isShort, channel); insertVideo(video); diff --git a/src/main/java/me/kavin/piped/utils/matrix/SyncRunner.java b/src/main/java/me/kavin/piped/utils/matrix/SyncRunner.java index 58377a24..091f41ee 100644 --- a/src/main/java/me/kavin/piped/utils/matrix/SyncRunner.java +++ b/src/main/java/me/kavin/piped/utils/matrix/SyncRunner.java @@ -129,7 +129,7 @@ public void run() { var type = event.get("type").asText(); var content = event.at("/content/content"); - if (type.startsWith("video.piped.stream.bypass.")) { + if (!UNAUTHENTICATED && type.startsWith("video.piped.stream.bypass.")) { switch (type) { case "video.piped.stream.bypass.request" -> { FederatedGeoBypassRequest bypassRequest = mapper.treeToValue(content, FederatedGeoBypassRequest.class); diff --git a/src/main/java/me/kavin/piped/utils/obj/Comment.java b/src/main/java/me/kavin/piped/utils/obj/Comment.java index ab68b6c9..53bdda28 100644 --- a/src/main/java/me/kavin/piped/utils/obj/Comment.java +++ b/src/main/java/me/kavin/piped/utils/obj/Comment.java @@ -4,10 +4,10 @@ public class Comment { public String author, thumbnail, commentId, commentText, commentedTime, commentorUrl, repliesPage; public int likeCount, replyCount; - public boolean hearted, pinned, verified; + public boolean hearted, pinned, verified, creatorReplied; public Comment(String author, String thumbnail, String commentId, String commentText, String commentedTime, - String commentorUrl, String repliesPage, int likeCount, int replyCount, boolean hearted, boolean pinned, boolean verified) { + String commentorUrl, String repliesPage, int likeCount, int replyCount, boolean hearted, boolean pinned, boolean verified, boolean creatorReplied) { this.author = author; this.thumbnail = thumbnail; this.commentId = commentId; @@ -20,5 +20,6 @@ public Comment(String author, String thumbnail, String commentId, String comment this.hearted = hearted; this.pinned = pinned; this.verified = verified; + this.creatorReplied = creatorReplied; } } diff --git a/src/main/java/me/kavin/piped/utils/obj/db/User.java b/src/main/java/me/kavin/piped/utils/obj/db/User.java index 1bf1b42b..4a3e343c 100644 --- a/src/main/java/me/kavin/piped/utils/obj/db/User.java +++ b/src/main/java/me/kavin/piped/utils/obj/db/User.java @@ -8,9 +8,10 @@ import java.util.UUID; @Entity -@Table(name = "users", indexes = {@Index(columnList = "id", name = "users_id_idx"), +@Table(name = "users", indexes = { @Index(columnList = "username", name = "username_idx"), - @Index(columnList = "session_id", name = "users_session_id_idx")}) + @Index(columnList = "session_id", name = "users_session_id_idx") +}) public class User implements Serializable { private static final long serialVersionUID = 1L; diff --git a/src/main/resources/changelog/db.changelog-master.xml b/src/main/resources/changelog/db.changelog-master.xml new file mode 100644 index 00000000..68fb5f2b --- /dev/null +++ b/src/main/resources/changelog/db.changelog-master.xml @@ -0,0 +1,10 @@ + + + + + + + diff --git a/src/main/resources/changelog/version/0-0-init-yb.sql b/src/main/resources/changelog/version/0-0-init-yb.sql new file mode 100644 index 00000000..7b3c88bd --- /dev/null +++ b/src/main/resources/changelog/version/0-0-init-yb.sql @@ -0,0 +1,2 @@ +CREATE EXTENSION pgcrypto; +--rollback DROP EXTENSION IF EXISTS pgcrypto; diff --git a/src/main/resources/changelog/version/0-1-init-crdb.sql b/src/main/resources/changelog/version/0-1-init-crdb.sql new file mode 100644 index 00000000..20b5879b --- /dev/null +++ b/src/main/resources/changelog/version/0-1-init-crdb.sql @@ -0,0 +1,47 @@ +CREATE INDEX IF NOT EXISTS users_session_id_idx ON users (session_id ASC) STORING (password, username); + +--rollback DROP INDEX IF EXISTS users_session_id_idx; + +CREATE TABLE IF NOT EXISTS videos ( + id VARCHAR(11) NOT NULL UNIQUE, + duration INT8 NULL, + thumbnail VARCHAR(400) NULL, + title VARCHAR(120) NULL, + uploaded INT8 NULL, + views INT8 NULL, + uploader_id VARCHAR(24) NOT NULL, + is_short BOOL NOT NULL DEFAULT false, + CONSTRAINT videos_pkey PRIMARY KEY (id ASC, uploader_id ASC) USING HASH, + CONSTRAINT fk_videos_channels_uploader_id FOREIGN KEY (uploader_id) REFERENCES channels(uploader_id), + INDEX videos_id_idx (id ASC), + INDEX video_uploaded_idx (uploaded ASC) USING HASH, + INDEX video_uploader_id_idx (uploader_id ASC) STORING (duration, thumbnail, title, uploaded, views, is_short), + UNIQUE INDEX videos_id_key (id ASC) STORING (duration, thumbnail, title, uploaded, views, is_short) +); + +--rollback DROP TABLE IF EXISTS videos; + +CREATE TABLE IF NOT EXISTS users_subscribed ( + subscriber INT8 NOT NULL, + channel VARCHAR(24) NOT NULL, + CONSTRAINT users_subscribed_pkey PRIMARY KEY (subscriber ASC, channel ASC) USING HASH, + CONSTRAINT fk_subscriber_users FOREIGN KEY (subscriber) REFERENCES users(id), + INDEX users_subscribed_subscriber_idx (subscriber ASC), + INDEX users_subscribed_channel_idx (channel ASC) +); + +--rollback DROP TABLE IF EXISTS users_subscribed; + +CREATE INDEX IF NOT EXISTS pubsub_subbed_at_idx ON pubsub (subbed_at ASC) USING HASH; + +--rollback DROP INDEX IF EXISTS pubsub_subbed_at_idx; + +CREATE INDEX IF NOT EXISTS playlists_playlist_id_idx ON playlists (playlist_id ASC) STORING (name, short_description, thumbnail, owner); +CREATE INDEX IF NOT EXISTS playlists_owner_idx ON playlists (owner ASC) STORING (name, short_description, thumbnail, playlist_id); + +--rollback DROP INDEX IF EXISTS playlists_playlist_id_idx; +--rollback DROP INDEX IF EXISTS playlists_owner_idx; + +CREATE INDEX IF NOT EXISTS unauthenticated_subscriptions_id_idx ON unauthenticated_subscriptions (id ASC) USING HASH STORING (subscribed_at); + +--rollback DROP INDEX IF EXISTS unauthenticated_subscriptions_id_idx; diff --git a/src/main/resources/changelog/version/0-1-init-pg.sql b/src/main/resources/changelog/version/0-1-init-pg.sql new file mode 100644 index 00000000..1fc4351f --- /dev/null +++ b/src/main/resources/changelog/version/0-1-init-pg.sql @@ -0,0 +1,48 @@ +CREATE INDEX IF NOT EXISTS users_session_id_idx ON users (session_id ASC); + +--rollback DROP INDEX IF EXISTS users_session_id_idx; + +CREATE TABLE IF NOT EXISTS videos ( + id VARCHAR(11) NOT NULL UNIQUE, + duration INT8 NULL, + thumbnail VARCHAR(400) NULL, + title VARCHAR(120) NULL, + uploaded INT8 NULL, + views INT8 NULL, + uploader_id VARCHAR(24) NOT NULL, + is_short BOOL NOT NULL DEFAULT false, + CONSTRAINT videos_pkey PRIMARY KEY (id, uploader_id), + CONSTRAINT fk_videos_channels_uploader_id FOREIGN KEY (uploader_id) REFERENCES channels(uploader_id) +); + +CREATE UNIQUE INDEX IF NOT EXISTS videos_id_idx ON videos (id ASC); +CREATE INDEX IF NOT EXISTS video_uploaded_idx ON videos (uploaded ASC); +CREATE INDEX IF NOT EXISTS video_uploader_id_idx ON videos (uploader_id ASC); + +--rollback DROP TABLE IF EXISTS videos; + +CREATE TABLE IF NOT EXISTS users_subscribed ( + subscriber INT8 NOT NULL, + channel VARCHAR(24) NOT NULL, + CONSTRAINT users_subscribed_pkey PRIMARY KEY (subscriber, channel), + CONSTRAINT fk_subscriber_users FOREIGN KEY (subscriber) REFERENCES users(id) +); + +CREATE INDEX IF NOT EXISTS users_subscribed_subscriber_idx ON users_subscribed (subscriber ASC); +CREATE INDEX IF NOT EXISTS users_subscribed_channel_idx ON users_subscribed (channel ASC); + +--rollback DROP TABLE IF EXISTS users_subscribed; + +CREATE INDEX IF NOT EXISTS pubsub_subbed_at_idx ON pubsub (subbed_at ASC); + +--rollback DROP INDEX IF EXISTS pubsub_subbed_at_idx; + +CREATE INDEX IF NOT EXISTS playlists_playlist_id_idx ON playlists (playlist_id ASC); +CREATE INDEX IF NOT EXISTS playlists_owner_idx ON playlists (owner ASC); + +--rollback DROP INDEX IF EXISTS playlists_playlist_id_idx; +--rollback DROP INDEX IF EXISTS playlists_owner_idx; + +CREATE INDEX IF NOT EXISTS unauthenticated_subscriptions_id_idx ON unauthenticated_subscriptions (id ASC); + +--rollback DROP INDEX IF EXISTS unauthenticated_subscriptions_id_idx; diff --git a/src/main/resources/changelog/version/0-1-init.sql b/src/main/resources/changelog/version/0-1-init.sql new file mode 100644 index 00000000..0accbb91 --- /dev/null +++ b/src/main/resources/changelog/version/0-1-init.sql @@ -0,0 +1,87 @@ +CREATE TABLE IF NOT EXISTS users ( + id BIGSERIAL NOT NULL, + password TEXT NULL, + session_id VARCHAR(36) NULL, + username VARCHAR(24) NULL UNIQUE, + CONSTRAINT users_pkey PRIMARY KEY (id) +); + +DROP INDEX IF EXISTS users_id_idx; + +CREATE INDEX IF NOT EXISTS username_idx ON users (username ASC); + +--rollback DROP TABLE IF EXISTS users; + +CREATE TABLE IF NOT EXISTS channels ( + uploader_id VARCHAR(24) NOT NULL, + uploader VARCHAR(100) NULL, + uploader_avatar VARCHAR(150) NULL, + verified BOOL NULL, + CONSTRAINT channels_pkey PRIMARY KEY (uploader_id) +); + +CREATE INDEX IF NOT EXISTS channels_uploader_idx ON channels (uploader ASC); + +--rollback DROP TABLE IF EXISTS channels; + +CREATE TABLE IF NOT EXISTS pubsub ( + id VARCHAR(24) NOT NULL, + subbed_at INT8 NULL, + CONSTRAINT pubsub_pkey PRIMARY KEY (id) +); + +CREATE INDEX IF NOT EXISTS pubsub_id_idx ON pubsub (id ASC); + +--rollback DROP TABLE IF EXISTS pubsub; + +CREATE TABLE IF NOT EXISTS playlists ( + id BIGSERIAL NOT NULL, + name VARCHAR(200) NULL, + playlist_id UUID NOT NULL UNIQUE DEFAULT gen_random_uuid(), + short_description VARCHAR(100) NULL, + thumbnail VARCHAR(300) NULL, + owner INT8 NOT NULL, + CONSTRAINT playlists_pkey PRIMARY KEY (id), + CONSTRAINT fk_playlists_owner FOREIGN KEY (owner) REFERENCES users(id) +); + +--rollback DROP TABLE IF EXISTS playlists; + +CREATE TABLE IF NOT EXISTS playlist_videos ( + id VARCHAR(11) NOT NULL, + duration INT8 NULL, + thumbnail VARCHAR(400) NULL, + title VARCHAR(120) NULL, + uploader_id VARCHAR(24) NOT NULL, + CONSTRAINT playlist_videos_pkey PRIMARY KEY (id), + CONSTRAINT fk_playlist_video_uploader_id FOREIGN KEY (uploader_id) REFERENCES channels(uploader_id) +); + +CREATE INDEX IF NOT EXISTS playlist_videos_id_idx ON playlist_videos (id ASC); +CREATE INDEX IF NOT EXISTS playlist_videos_uploader_id_idx ON playlist_videos (uploader_id ASC); + +--rollback DROP TABLE IF EXISTS playlist_videos; + +CREATE TABLE IF NOT EXISTS playlists_videos_ids ( + playlist_id INT8 NOT NULL, + videos_id VARCHAR(11) NOT NULL, + videos_order INT4 NOT NULL, + CONSTRAINT playlists_videos_ids_pkey PRIMARY KEY (playlist_id, videos_order), + CONSTRAINT fk_playlists_videos_video_id_playlist_video FOREIGN KEY (videos_id) REFERENCES playlist_videos(id), + CONSTRAINT fk_playlists_videos_playlist_id_playlist FOREIGN KEY (playlist_id) REFERENCES playlists(id) +); + +CREATE INDEX IF NOT EXISTS playlists_videos_ids_playlist_id_idx ON playlists_videos_ids (playlist_id ASC); + +--rollback DROP TABLE IF EXISTS playlists_videos_ids; + +CREATE TABLE IF NOT EXISTS unauthenticated_subscriptions ( + id VARCHAR(24) NOT NULL, + subscribed_at INT8 NOT NULL, + CONSTRAINT unauthenticated_subscriptions_pkey PRIMARY KEY (id), + CONSTRAINT fk_unauthenticated_subscriptions_id_channels FOREIGN KEY (id) REFERENCES channels(uploader_id) +); + +CREATE INDEX IF NOT EXISTS unauthenticated_subscriptions_subscribed_at_idx ON unauthenticated_subscriptions (subscribed_at ASC); + +--rollback DROP TABLE IF EXISTS unauthenticated_subscriptions; diff --git a/src/main/resources/changelog/version/0-init.xml b/src/main/resources/changelog/version/0-init.xml new file mode 100644 index 00000000..649566a9 --- /dev/null +++ b/src/main/resources/changelog/version/0-init.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + diff --git a/src/main/resources/changelog/version/1-fix-subs.xml b/src/main/resources/changelog/version/1-fix-subs.xml new file mode 100644 index 00000000..e3f44cc7 --- /dev/null +++ b/src/main/resources/changelog/version/1-fix-subs.xml @@ -0,0 +1,15 @@ + + + + + + ALTER TABLE unauthenticated_subscriptions DROP CONSTRAINT IF EXISTS fk_unauthenticated_subscriptions_id_channels; + + ALTER TABLE unauthenticated_subscriptions ADD CONSTRAINT fk_unauthenticated_subscriptions_id_channels FOREIGN KEY (id) REFERENCES channels(uploader_id); + + + + diff --git a/src/main/resources/hibernate.cfg.xml b/src/main/resources/hibernate.cfg.xml index a0953d00..63747373 100644 --- a/src/main/resources/hibernate.cfg.xml +++ b/src/main/resources/hibernate.cfg.xml @@ -4,11 +4,11 @@ "http://www.hibernate.org/dtd/hibernate-configuration-3.0.dtd"> - update + validate false true - + org.hibernate.hikaricp.internal.HikariCPConnectionProvider DELAYED_ACQUISITION_AND_RELEASE_AFTER_STATEMENT 50 diff --git a/testing/api-test.sh b/testing/api-test.sh index df3b9be1..1f045253 100755 --- a/testing/api-test.sh +++ b/testing/api-test.sh @@ -1,6 +1,6 @@ #!/bin/bash -CURLOPTS=(-i -s -S -o /dev/null -f -w "%{http_code}\tTime:\t%{time_starttransfer}\t%{url_effective}\n") +CURLOPTS=(-i -s -S --max-time 60 -o /dev/null -f -w "%{http_code}\tTime:\t%{time_starttransfer}\t%{url_effective}\n") HOST="127.0.0.1:8080" # Healthcheck Test diff --git a/testing/config.hsqldb.properties b/testing/config.hsqldb.properties deleted file mode 100644 index 4c8ebe27..00000000 --- a/testing/config.hsqldb.properties +++ /dev/null @@ -1,18 +0,0 @@ -# The port to Listen on. -PORT: 8080 - -# Proxy -PROXY_PART: https://pipedproxy-ams.kavin.rocks - -# Public API URL -API_URL: https://pipedapi.kavin.rocks - -# Public Frontend URL -FRONTEND_URL: https://piped.video - -# Hibernate properties -hibernate.connection.url: jdbc:hsqldb:mem:memdb;sql.syntax_pgs=true -hibernate.connection.driver_class: org.hsqldb.jdbcDriver -hibernate.dialect: org.hibernate.dialect.HSQLDialect -hibernate.connection.username: piped -hibernate.connection.password: changeme diff --git a/testing/docker-compose.hsqldb.yml b/testing/docker-compose.hsqldb.yml deleted file mode 100644 index 31f40700..00000000 --- a/testing/docker-compose.hsqldb.yml +++ /dev/null @@ -1,8 +0,0 @@ -services: - piped: - image: 1337kavin/piped:latest - restart: unless-stopped - ports: - - "127.0.0.1:8080:8080" - volumes: - - ./config.hsqldb.properties:/app/config.properties