Merge branch 'master' into oidc

# Conflicts:
#	build.gradle

build.gradle

@@ -1,7 +1,7 @@
 plugins {
  id "com.github.johnrengelman.shadow" version "8.1.1"
  id "java"
- id "io.freefair.lombok" version "8.0.1"
+ id "io.freefair.lombok" version "8.1.0"
  id "eclipse"
 }
 
@@ -15,9 +15,9 @@ dependencies {
  implementation 'org.apache.commons:commons-text:1.10.0'
  implementation 'commons-io:commons-io:2.12.0'
  implementation 'it.unimi.dsi:fastutil-core:8.5.12'
- implementation 'commons-codec:commons-codec:1.15'
+ implementation 'commons-codec:commons-codec:1.16.0'
  implementation 'org.bouncycastle:bcprov-jdk15on:1.70'
- implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:4ab4bf51ca6e924945a071d359b1ee5cdf9c7605'
+ implementation 'com.github.FireMasterK.NewPipeExtractor:NewPipeExtractor:bdd1366285614f56a245e3baceb26fb8864bda27'
  implementation 'com.github.FireMasterK:nanojson:01934924442edda6952f3bedf80ba9e969cba8bc'
  implementation 'com.fasterxml.jackson.core:jackson-core:2.15.2'
  implementation 'com.fasterxml.jackson.core:jackson-annotations:2.15.2'
@@ -34,15 +34,15 @@ dependencies {
  implementation 'org.hibernate:hibernate-core:6.2.5.Final'
  implementation 'org.hibernate:hibernate-hikaricp:6.2.5.Final'
  implementation 'com.zaxxer:HikariCP:5.0.1'
- implementation 'org.springframework.security:spring-security-crypto:6.1.0'
+ implementation 'org.springframework.security:spring-security-crypto:6.1.1'
  implementation 'commons-logging:commons-logging:1.2'
  implementation(platform("com.squareup.okhttp3:okhttp-bom:4.11.0"))
  implementation 'com.squareup.okhttp3:okhttp'
  implementation 'com.squareup.okhttp3:okhttp-brotli'
- implementation 'io.sentry:sentry:6.23.0'
- implementation 'rocks.kavin:reqwest4j:1.0.4'
- implementation 'io.minio:minio:8.5.3'
  implementation 'com.nimbusds:oauth2-oidc-sdk:10.9.1'
+ implementation 'io.sentry:sentry:6.25.0'
+ implementation 'rocks.kavin:reqwest4j:1.0.6'
+ implementation 'io.minio:minio:8.5.4'
 }
 
 shadowJar {

config.properties

@@ -6,8 +6,8 @@ HTTP_WORKERS:2
 # Proxy
 PROXY_PART:https://pipedproxy-cdg.kavin.rocks
 
-# Outgoing HTTP Proxy - eg: 127.0.0.1:8118
-#HTTP_PROXY: 127.0.0.1:8118
+# Outgoing proxy to be used by reqwest4j - eg: socks5://127.0.0.1:1080
+#REQWEST_PROXY: socks5://127.0.0.1:1080
 
 # Captcha Parameters
 CAPTCHA_BASE_URL:https://api.capmonster.cloud/

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.1.1-bin.zip
+distributionUrl=https\://downloads.gradle.org/distributions/gradle-8.2-bin.zip
 networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradlew

@@ -130,10 +130,13 @@ location of your Java installation."
  fi
 else
  JAVACMD=java
- which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+ if ! command -v java >/dev/null 2>&1
+ then
+ die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+ fi
 fi
 
 # Increase the maximum file descriptors if we can.

src/main/java/me/kavin/piped/Main.java

@@ -20,6 +20,7 @@
 import org.schabi.newpipe.extractor.localization.Localization;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeThrottlingDecrypter;
 import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
+import rocks.kavin.reqwest4j.ReqwestUtils;
 
 import java.util.*;
 import java.util.concurrent.TimeUnit;
@@ -35,6 +36,8 @@ public static void main(String[] args) throws Exception {
  YoutubeStreamExtractor.forceFetchAndroidClient(true);
  YoutubeStreamExtractor.forceFetchIosClient(true);
 
+ ReqwestUtils.init(Constants.REQWEST_PROXY);
+
  Sentry.init(options -> {
  options.setDsn(Constants.SENTRY_DSN);
  options.setRelease(Constants.VERSION);

src/main/java/me/kavin/piped/consts/Constants.java

@@ -52,7 +52,7 @@ public class Constants {
 
  public static final String PUBSUB_HUB_URL;
 
- public static final String HTTP_PROXY;
+ public static final String REQWEST_PROXY;
 
  public static final String FRONTEND_URL;
 
@@ -133,7 +133,7 @@ public class Constants {
  PUBLIC_URL = getProperty(prop, "API_URL");
  PUBSUB_URL = getProperty(prop, "PUBSUB_URL", PUBLIC_URL);
  PUBSUB_HUB_URL = getProperty(prop, "PUBSUB_HUB_URL", "https://pubsubhubbub.appspot.com/subscribe");
- HTTP_PROXY = getProperty(prop, "HTTP_PROXY");
+ REQWEST_PROXY = getProperty(prop, "REQWEST_PROXY");
  FRONTEND_URL = getProperty(prop, "FRONTEND_URL", "https://piped.video");
  COMPROMISED_PASSWORD_CHECK = Boolean.parseBoolean(getProperty(prop, "COMPROMISED_PASSWORD_CHECK", "true"));
  DISABLE_REGISTRATION = Boolean.parseBoolean(getProperty(prop, "DISABLE_REGISTRATION", "false"));
@@ -208,18 +208,11 @@ else if (key.startsWith("oidc.provider")) {
  OkHttpClient.Builder builder_noredir = new OkHttpClient.Builder()
  .followRedirects(false)
  .addInterceptor(BrotliInterceptor.INSTANCE);
- if (HTTP_PROXY != null && HTTP_PROXY.contains(":")) {
- String host = StringUtils.substringBefore(HTTP_PROXY, ":");
- String port = StringUtils.substringAfter(HTTP_PROXY, ":");
- InetSocketAddress sa = new InetSocketAddress(host, Integer.parseInt(port));
- ProxySelector ps = ProxySelector.of(sa);
- ProxySelector.setDefault(ps);
- }
  h2client = builder.build();
  h2_no_redir_client = builder_noredir.build();
  String temp = null;
  try {
- var html = RequestUtils.sendGet("https://www.youtube.com/");
+ var html = RequestUtils.sendGet("https://www.youtube.com/").get();
  var regex = Pattern.compile("GL\":\"([A-Z]{2})\"", Pattern.MULTILINE);
  var matcher = regex.matcher(html);
  if (matcher.find()) {

src/main/java/me/kavin/piped/server/handlers/StreamHandlers.java

@@ -150,7 +150,7 @@ public static byte[] streamsResponse(String videoId) throws Exception {
  List<String> allowedCountries = new ObjectArrayList<>();
 
  {
- var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId);
+ var restrictedTree = RequestUtils.sendGetJson(Constants.GEO_RESTRICTION_CHECKER_URL + "/api/region/check?video_id=" + videoId).get();
  if (!restrictedTree.get("restricted").asBoolean()) {
  assert exception != null;
  throw (Exception) exception;
@@ -357,7 +357,7 @@ public static byte[] commentsResponse(String videoId) throws Exception {
  nextpage = mapper.writeValueAsString(page);
  }
 
- CommentsPage commentsItem = new CommentsPage(comments, nextpage, info.isCommentsDisabled());
+ CommentsPage commentsItem = new CommentsPage(comments, nextpage, info.isCommentsDisabled(), info.getCommentsCount());
 
  return mapper.writeValueAsBytes(commentsItem);
 
@@ -395,7 +395,7 @@ public static byte[] commentsPageResponse(String videoId, String prevpageStr) th
  nextpage = mapper.writeValueAsString(page);
  }
 
- CommentsPage commentsItem = new CommentsPage(comments, nextpage, false);
+ CommentsPage commentsItem = new CommentsPage(comments, nextpage, false, -1);
 
  return mapper.writeValueAsBytes(commentsItem);
 

src/main/java/me/kavin/piped/server/handlers/auth/UserHandlers.java

@@ -28,7 +28,7 @@ public class UserHandlers {
  private static final Argon2PasswordEncoder argon2PasswordEncoder = Argon2PasswordEncoder.defaultsForSpringSecurity_v5_8();
  private static final BCryptPasswordEncoder bcryptPasswordEncoder = new BCryptPasswordEncoder();
 
- public static byte[] registerResponse(String user, String pass) throws IOException {
+ public static byte[] registerResponse(String user, String pass) throws Exception {
 
  if (Constants.DISABLE_REGISTRATION)
  ExceptionHandler.throwErrorResponse(new DisabledRegistrationResponse());
@@ -57,7 +57,8 @@ public static byte[] registerResponse(String user, String pass) throws IOExcepti
  String suffix = sha1Hash.substring(5);
  String[] entries = RequestUtils
  .sendGet("https://api.pwnedpasswords.com/range/" + prefix, "github.com/TeamPiped/Piped-Backend")
- .split("\n");
+ .thenApplyAsync(str -> str.split("\n"))
+ .get();
  for (String entry : entries)
  if (StringUtils.substringBefore(entry, ":").equals(suffix))
  ExceptionHandler.throwErrorResponse(new CompromisedPasswordResponse());

src/main/java/me/kavin/piped/utils/DownloaderImpl.java

@@ -1,13 +1,6 @@
 package me.kavin.piped.utils;
 
-import com.grack.nanojson.JsonParserException;
 import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap;
-import me.kavin.piped.consts.Constants;
-import me.kavin.piped.utils.obj.SolvedCaptcha;
-import okhttp3.FormBody;
-import org.apache.commons.lang3.StringUtils;
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Element;
 import org.schabi.newpipe.extractor.downloader.Downloader;
 import org.schabi.newpipe.extractor.downloader.Request;
 import org.schabi.newpipe.extractor.downloader.Response;
@@ -18,7 +11,7 @@
 import java.net.HttpCookie;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ExecutionException;
 
 public class DownloaderImpl extends Downloader {
 
@@ -41,70 +34,85 @@ public Response execute(Request request) throws IOException, ReCaptchaException
 
  request.headers().forEach((name, values) -> values.forEach(value -> headers.put(name, value)));
 
- var resp = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
-
- if (resp.status() == 429) {
-
- synchronized (cookie_lock) {
-
- if (saved_cookie != null && saved_cookie.hasExpired()
- || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
- saved_cookie = null;
-
- String redir_url = String.valueOf(resp.finalUrl());
-
- if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
-
- var formBuilder = new FormBody.Builder();
- String sitekey = null, data_s = null;
-
- for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
- String name;
- if (!(name = el.tagName()).equals("script")) {
- if (name.equals("input"))
- formBuilder.add(el.attr("name"), el.attr("value"));
- else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
- sitekey = el.attr("data-sitekey");
- data_s = el.attr("data-s");
- }
- }
- }
- if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
- throw new ReCaptchaException("Could not get recaptcha", redir_url);
-
- SolvedCaptcha solved = null;
-
- try {
- solved = CaptchaSolver.solve(redir_url, sitekey, data_s);
- } catch (JsonParserException | InterruptedException e) {
- e.printStackTrace();
- }
-
- formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
-
- var formReqBuilder = new okhttp3.Request.Builder()
- .url("https://www.google.com/sorry/index")
- .header("User-Agent", Constants.USER_AGENT)
- .post(formBuilder.build());
-
- var formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
-
- saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
- .substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
- .get(0);
- cookie_received = System.currentTimeMillis();
- }
-
- if (saved_cookie != null) // call again as captcha has been solved or cookie has not expired.
- execute(request);
- }
-
+ var future = ReqwestUtils.fetch(request.url(), request.httpMethod(), bytes, headers);
+
+ // Recaptcha solver code
+ // Commented out, as it hasn't been ported to reqwest4j yet
+ // Also, this was last seen a long time back
+
+// future.thenAcceptAsync(resp -> {
+// if (resp.status() == 429) {
+// synchronized (cookie_lock) {
+//
+// if (saved_cookie != null && saved_cookie.hasExpired()
+// || (System.currentTimeMillis() - cookie_received > TimeUnit.MINUTES.toMillis(30)))
+// saved_cookie = null;
+//
+// String redir_url = String.valueOf(resp.finalUrl());
+//
+// if (saved_cookie == null && redir_url.startsWith("https://www.google.com/sorry")) {
+//
+// var formBuilder = new FormBody.Builder();
+// String sitekey = null, data_s = null;
+//
+// for (Element el : Jsoup.parse(new String(resp.body())).selectFirst("form").children()) {
+// String name;
+// if (!(name = el.tagName()).equals("script")) {
+// if (name.equals("input"))
+// formBuilder.add(el.attr("name"), el.attr("value"));
+// else if (name.equals("div") && el.attr("id").equals("recaptcha")) {
+// sitekey = el.attr("data-sitekey");
+// data_s = el.attr("data-s");
+// }
+// }
+// }
+//
+// if (StringUtils.isEmpty(sitekey) || StringUtils.isEmpty(data_s))
+// ExceptionHandler.handle(new ReCaptchaException("Could not get recaptcha", redir_url));
+//
+// SolvedCaptcha solved = null;
+//
+// try {
+// solved = CaptchaSolver.solve(redir_url, sitekey, data_s);
+// } catch (JsonParserException | InterruptedException | IOException e) {
+// e.printStackTrace();
+// }
+//
+// formBuilder.add("g-recaptcha-response", solved.getRecaptchaResponse());
+//
+// var formReqBuilder = new okhttp3.Request.Builder()
+// .url("https://www.google.com/sorry/index")
+// .header("User-Agent", Constants.USER_AGENT)
+// .post(formBuilder.build());
+//
+// okhttp3.Response formResponse;
+// try {
+// formResponse = Constants.h2_no_redir_client.newCall(formReqBuilder.build()).execute();
+// } catch (IOException e) {
+// throw new RuntimeException(e);
+// }
+//
+// saved_cookie = HttpCookie.parse(URLUtils.silentDecode(StringUtils
+// .substringAfter(formResponse.headers().get("Location"), "google_abuse=")))
+// .get(0);
+// cookie_received = System.currentTimeMillis();
+// }
+// }
+// }
+// }, Multithreading.getCachedExecutor());
+
+ var responseFuture = future.thenApplyAsync(resp -> {
+ Map<String, List<String>> headerMap = resp.headers().entrySet().stream()
+ .collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll);
+
+ return new Response(resp.status(), null, headerMap, new String(resp.body()),
+ resp.finalUrl());
+ }, Multithreading.getCachedExecutor());
+
+ try {
+ return responseFuture.get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new IOException(e);
  }
-
- Map<String, List<String>> headerMap = resp.headers().entrySet().stream()
- .collect(Object2ObjectOpenHashMap::new, (m, e) -> m.put(e.getKey(), List.of(e.getValue())), Map::putAll);
-
- return new Response(resp.status(), null, headerMap, new String(resp.body()),
- resp.finalUrl());
  }
 }