Merge branch 'master' into feat/max_bytes

Cargo.toml

@@ -21,5 +21,6 @@ dotenv = "0.15"
 tempfile = "3.2.0"
 ffprobe = "0.3.0"
 encoding_rs = "0.8"
+moka = { version = "0.10", features = ["future"]}
 
 validator = { version = "0.15.0", features = ["derive"] }

src/routes/embed.rs

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use actix_web::{
     web::{self, Query},
     Responder,
@@ -13,14 +15,20 @@ use crate::{
     util::{request::consume_size, result::Error},
 };
 
+lazy_static! {
+    static ref CACHE: moka::future::Cache<String, Result<Embed, Error>> =
+        moka::future::Cache::builder()
+            .max_capacity(1_000)
+            .time_to_live(Duration::from_secs(60))
+            .build();
+}
+
 #[derive(Deserialize)]
 pub struct Parameters {
     url: String,
 }
 
-pub async fn get(info: Query<Parameters>) -> Result<impl Responder, Error> {
-    let mut url = info.into_inner().url;
-
+async fn embed(mut url: String) -> Result<Embed, Error> {
     // Twitter is a piece of shit and does not
     // provide metadata in an easily consumable format.
     //
@@ -53,30 +61,38 @@ pub async fn get(info: Query<Parameters>) -> Result<impl Responder, Error> {
             metadata.resolve_external().await;
 
             if metadata.is_none() {
-                return Ok(web::Json(Embed::None));
+                return Ok(Embed::None);
             }
 
-            Ok(web::Json(Embed::Website(metadata)))
+            Ok(Embed::Website(metadata))
         }
         (mime::IMAGE, _) => {
             if let Ok((width, height)) = consume_size(resp, mime).await {
-                Ok(web::Json(Embed::Image(Image {
+                Ok(Embed::Image(Image {
                     url,
                     width,
                     height,
                     size: ImageSize::Large,
-                })))
+                }))
             } else {
-                Ok(web::Json(Embed::None))
+                Ok(Embed::None)
             }
         }
         (mime::VIDEO, _) => {
             if let Ok((width, height)) = consume_size(resp, mime).await {
-                Ok(web::Json(Embed::Video(Video { url, width, height })))
+                Ok(Embed::Video(Video { url, width, height }))
             } else {
-                Ok(web::Json(Embed::None))
+                Ok(Embed::None)
             }
         }
-        _ => Ok(web::Json(Embed::None)),
+        _ => Ok(Embed::None),
     }
 }
+
+pub async fn get(Query(info): Query<Parameters>) -> Result<impl Responder, Error> {
+    let url = info.url;
+    let result = CACHE
+        .get_with(url.clone(), async { embed(url).await })
+        .await;
+    result.map(web::Json)
+}

src/routes/proxy.rs

@@ -1,22 +1,43 @@
+use std::time::Duration;
+
+use actix_web::web::Bytes;
 use actix_web::{web::Query, HttpResponse, Responder};
 use serde::Deserialize;
 
 use crate::util::request::{fetch, get_bytes};
 use crate::util::result::Error;
 
+lazy_static! {
+    static ref CACHE: moka::future::Cache<String, Result<Bytes, Error>> =
+        moka::future::Cache::builder()
+            .weigher(|_key, value: &Result<Bytes, Error>| {
+                value.as_ref().map(|bytes| bytes.len() as u32).unwrap_or(1)
+            })
+            .max_capacity(1024 * 1024 * 1024)
+            .time_to_live(Duration::from_secs(60))
+            .build();
+}
+
 #[derive(Deserialize)]
 pub struct Parameters {
     url: String,
 }
 
-pub async fn get(info: Query<Parameters>) -> Result<impl Responder, Error> {
-    let url = info.into_inner().url;
+async fn proxy(url: String) -> Result<Bytes, Error> {
     let (mut resp, mime) = fetch(&url).await?;
 
     if matches!(mime.type_(), mime::IMAGE | mime::VIDEO) {
         let bytes = get_bytes(&mut resp).await?;
-        Ok(HttpResponse::Ok().body(bytes))
+        Ok(bytes)
     } else {
         Err(Error::NotAllowedToProxy)
     }
 }
+
+pub async fn get(Query(info): Query<Parameters>) -> Result<impl Responder, Error> {
+    let url = info.url;
+    let result = CACHE
+        .get_with(url.clone(), async { proxy(url).await })
+        .await;
+    result.map(|b| HttpResponse::Ok().body(b))
+}

src/structs/embed.rs

@@ -1,8 +1,11 @@
 use serde::Serialize;
 
-use super::{media::{Image, Video}, metadata::Metadata};
+use super::{
+    media::{Image, Video},
+    metadata::Metadata,
+};
 
-#[derive(Debug, Serialize)]
+#[derive(Clone, Debug, Serialize)]
 #[serde(tag = "type")]
 #[allow(clippy::large_enum_variant)]
 pub enum Embed {

src/structs/media.rs

@@ -1,13 +1,13 @@
 use serde::Serialize;
 use validator::Validate;
 
-#[derive(Debug, Serialize)]
+#[derive(Clone, Debug, Serialize)]
 pub enum ImageSize {
     Large,
     Preview,
 }
 
-#[derive(Validate, Debug, Serialize)]
+#[derive(Clone, Validate, Debug, Serialize)]
 pub struct Image {
     #[validate(length(min = 1, max = 512))]
     pub url: String,
@@ -16,7 +16,7 @@ pub struct Image {
     pub size: ImageSize,
 }
 
-#[derive(Validate, Debug, Serialize)]
+#[derive(Clone, Validate, Debug, Serialize)]
 pub struct Video {
     #[validate(length(min = 1, max = 512))]
     pub url: String,

src/structs/metadata.rs

@@ -18,7 +18,7 @@ use super::{
     special::Special,
 };
 
-#[derive(Validate, Debug, Serialize)]
+#[derive(Clone, Validate, Debug, Serialize)]
 pub struct Metadata {
     #[validate(length(min = 1, max = 256))]
     url: String,
@@ -52,7 +52,7 @@ pub struct Metadata {
 }
 
 impl Metadata {
-    pub async fn from(resp: Response, url: String) -> Result<Metadata, Error> {
+    pub async fn from(resp: Response, original_url: String) -> Result<Metadata, Error> {
         let fragment = consume_fragment(resp).await?;
 
         let meta_selector = Selector::parse("meta").map_err(|_| Error::MetaSelectionFailed)?;
@@ -92,14 +92,19 @@ impl Metadata {
                 .or_else(|| meta.remove("og:image:secure_url"))
                 .or_else(|| meta.remove("twitter:image"))
                 .or_else(|| meta.remove("twitter:image:src"))
-                .map(|url| {
+                .map(|mut url| {
+                    // If relative URL, prepend root URL. Also if root URL ends with a slash, remove it.
+                    if let Some(ch) = url.chars().next() {
+                        if ch == '/' {
+                            url = format!("{}{}", &original_url.trim_end_matches('/'), &url);
+                        }
+                    }
                     let mut size = ImageSize::Preview;
                     if let Some(card) = meta.remove("twitter:card") {
                         if &card == "summary_large_image" {
                             size = ImageSize::Large;
                         }
                     }
-
                     Image {
                         url,
                         width: meta
@@ -119,18 +124,26 @@ impl Metadata {
                 .remove("og:video")
                 .or_else(|| meta.remove("og:video:url"))
                 .or_else(|| meta.remove("og:video:secure_url"))
-                .map(|url| Video {
-                    url,
-                    width: meta
-                        .remove("og:video:width")
-                        .unwrap_or_else(|| "0".to_string())
-                        .parse()
-                        .unwrap_or(0),
-                    height: meta
-                        .remove("og:video:height")
-                        .unwrap_or_else(|| "0".to_string())
-                        .parse()
-                        .unwrap_or(0),
+                .map(|mut url| {
+                    // If relative URL, prepend root URL. Also if root URL ends with a slash, remove it.
+                    if let Some(ch) = url.chars().next() {
+                        if ch == '/' {
+                            url = format!("{}{}", &original_url.trim_end_matches('/'), &url);
+                        }
+                    }
+                    Video {
+                        url,
+                        width: meta
+                            .remove("og:video:width")
+                            .unwrap_or_else(|| "0".to_string())
+                            .parse()
+                            .unwrap_or(0),
+                        height: meta
+                            .remove("og:video:height")
+                            .unwrap_or_else(|| "0".to_string())
+                            .parse()
+                            .unwrap_or(0),
+                    }
                 }),
             icon_url: link
                 .remove("apple-touch-icon")
@@ -139,7 +152,7 @@ impl Metadata {
                     // If relative URL, prepend root URL.
                     if let Some(ch) = v.chars().next() {
                         if ch == '/' {
-                            v = format!("{}{}", &url, v);
+                            v = format!("{}{}", &original_url.trim_end_matches('/'), v);
                         }
                     }
 
@@ -148,8 +161,10 @@ impl Metadata {
             colour: meta.remove("theme-color"),
             opengraph_type: meta.remove("og:type"),
             site_name: meta.remove("og:site_name"),
-            url: meta.remove("og:url").unwrap_or_else(|| url.clone()),
-            original_url: url,
+            url: meta
+                .remove("og:url")
+                .unwrap_or_else(|| original_url.clone()),
+            original_url,
             special: None,
         };
 

src/structs/special.rs

@@ -1,24 +1,24 @@
 use serde::Serialize;
 
-#[derive(Debug, Serialize)]
+#[derive(Clone, Debug, Serialize)]
 pub enum TwitchType {
     Channel,
     Video,
     Clip,
 }
 
-#[derive(Debug, Serialize)]
+#[derive(Clone, Debug, Serialize)]
 pub enum LightspeedType {
     Channel,
 }
 
-#[derive(Debug, Serialize)]
+#[derive(Clone, Debug, Serialize)]
 pub enum BandcampType {
     Album,
     Track,
 }
 
-#[derive(Debug, Serialize)]
+#[derive(Clone, Debug, Serialize)]
 #[serde(tag = "type")]
 pub enum Special {
     None,

src/util/request.rs

@@ -16,7 +16,8 @@ use super::{result::Error, variables::MAX_BYTES};
 lazy_static! {
     static ref CLIENT: Client = reqwest::Client::builder()
         .user_agent("Mozilla/5.0 (compatible; January/1.0; +https://github.com/revoltchat/january)")
-        .timeout(Duration::from_secs(2))
+        .timeout(Duration::from_secs(15))
+        .connect_timeout(Duration::from_secs(5))
         .build()
         .expect("reqwest Client");
 }

src/util/result.rs

@@ -5,7 +5,7 @@ use serde_json;
 use std::fmt::Display;
 use validator::ValidationErrors;
 
-#[derive(Serialize, Debug)]
+#[derive(Clone, Serialize, Debug)]
 #[serde(tag = "type")]
 pub enum Error {
     CouldNotDetermineImageSize,