diff --git a/Cargo.toml b/Cargo.toml index 7cb4757..96e3bd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,5 +21,6 @@ dotenv = "0.15" tempfile = "3.2.0" ffprobe = "0.3.0" encoding_rs = "0.8" +moka = { version = "0.10", features = ["future"]} validator = { version = "0.15.0", features = ["derive"] } diff --git a/src/routes/embed.rs b/src/routes/embed.rs index 116ed1f..911ff83 100644 --- a/src/routes/embed.rs +++ b/src/routes/embed.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use actix_web::{ web::{self, Query}, Responder, @@ -13,14 +15,20 @@ use crate::{ util::{request::consume_size, result::Error}, }; +lazy_static! { + static ref CACHE: moka::future::Cache> = + moka::future::Cache::builder() + .max_capacity(1_000) + .time_to_live(Duration::from_secs(60)) + .build(); +} + #[derive(Deserialize)] pub struct Parameters { url: String, } -pub async fn get(info: Query) -> Result { - let mut url = info.into_inner().url; - +async fn embed(mut url: String) -> Result { // Twitter is a piece of shit and does not // provide metadata in an easily consumable format. // @@ -53,30 +61,38 @@ pub async fn get(info: Query) -> Result { metadata.resolve_external().await; if metadata.is_none() { - return Ok(web::Json(Embed::None)); + return Ok(Embed::None); } - Ok(web::Json(Embed::Website(metadata))) + Ok(Embed::Website(metadata)) } (mime::IMAGE, _) => { if let Ok((width, height)) = consume_size(resp, mime).await { - Ok(web::Json(Embed::Image(Image { + Ok(Embed::Image(Image { url, width, height, size: ImageSize::Large, - }))) + })) } else { - Ok(web::Json(Embed::None)) + Ok(Embed::None) } } (mime::VIDEO, _) => { if let Ok((width, height)) = consume_size(resp, mime).await { - Ok(web::Json(Embed::Video(Video { url, width, height }))) + Ok(Embed::Video(Video { url, width, height })) } else { - Ok(web::Json(Embed::None)) + Ok(Embed::None) } } - _ => Ok(web::Json(Embed::None)), + _ => Ok(Embed::None), } } + +pub async fn get(Query(info): Query) -> Result { + let url = info.url; + let result = CACHE + .get_with(url.clone(), async { embed(url).await }) + .await; + result.map(web::Json) +} diff --git a/src/routes/proxy.rs b/src/routes/proxy.rs index 78b67b0..f53e163 100644 --- a/src/routes/proxy.rs +++ b/src/routes/proxy.rs @@ -1,22 +1,43 @@ +use std::time::Duration; + +use actix_web::web::Bytes; use actix_web::{web::Query, HttpResponse, Responder}; use serde::Deserialize; use crate::util::request::{fetch, get_bytes}; use crate::util::result::Error; +lazy_static! { + static ref CACHE: moka::future::Cache> = + moka::future::Cache::builder() + .weigher(|_key, value: &Result| { + value.as_ref().map(|bytes| bytes.len() as u32).unwrap_or(1) + }) + .max_capacity(1024 * 1024 * 1024) + .time_to_live(Duration::from_secs(60)) + .build(); +} + #[derive(Deserialize)] pub struct Parameters { url: String, } -pub async fn get(info: Query) -> Result { - let url = info.into_inner().url; +async fn proxy(url: String) -> Result { let (mut resp, mime) = fetch(&url).await?; if matches!(mime.type_(), mime::IMAGE | mime::VIDEO) { let bytes = get_bytes(&mut resp).await?; - Ok(HttpResponse::Ok().body(bytes)) + Ok(bytes) } else { Err(Error::NotAllowedToProxy) } } + +pub async fn get(Query(info): Query) -> Result { + let url = info.url; + let result = CACHE + .get_with(url.clone(), async { proxy(url).await }) + .await; + result.map(|b| HttpResponse::Ok().body(b)) +} diff --git a/src/structs/embed.rs b/src/structs/embed.rs index f48af8e..d0be001 100644 --- a/src/structs/embed.rs +++ b/src/structs/embed.rs @@ -1,8 +1,11 @@ use serde::Serialize; -use super::{media::{Image, Video}, metadata::Metadata}; +use super::{ + media::{Image, Video}, + metadata::Metadata, +}; -#[derive(Debug, Serialize)] +#[derive(Clone, Debug, Serialize)] #[serde(tag = "type")] #[allow(clippy::large_enum_variant)] pub enum Embed { diff --git a/src/structs/media.rs b/src/structs/media.rs index eaebb32..0ce68ab 100644 --- a/src/structs/media.rs +++ b/src/structs/media.rs @@ -1,13 +1,13 @@ use serde::Serialize; use validator::Validate; -#[derive(Debug, Serialize)] +#[derive(Clone, Debug, Serialize)] pub enum ImageSize { Large, Preview, } -#[derive(Validate, Debug, Serialize)] +#[derive(Clone, Validate, Debug, Serialize)] pub struct Image { #[validate(length(min = 1, max = 512))] pub url: String, @@ -16,7 +16,7 @@ pub struct Image { pub size: ImageSize, } -#[derive(Validate, Debug, Serialize)] +#[derive(Clone, Validate, Debug, Serialize)] pub struct Video { #[validate(length(min = 1, max = 512))] pub url: String, diff --git a/src/structs/metadata.rs b/src/structs/metadata.rs index 0d7d669..dd3083e 100644 --- a/src/structs/metadata.rs +++ b/src/structs/metadata.rs @@ -18,7 +18,7 @@ use super::{ special::Special, }; -#[derive(Validate, Debug, Serialize)] +#[derive(Clone, Validate, Debug, Serialize)] pub struct Metadata { #[validate(length(min = 1, max = 256))] url: String, @@ -52,7 +52,7 @@ pub struct Metadata { } impl Metadata { - pub async fn from(resp: Response, url: String) -> Result { + pub async fn from(resp: Response, original_url: String) -> Result { let fragment = consume_fragment(resp).await?; let meta_selector = Selector::parse("meta").map_err(|_| Error::MetaSelectionFailed)?; @@ -92,14 +92,19 @@ impl Metadata { .or_else(|| meta.remove("og:image:secure_url")) .or_else(|| meta.remove("twitter:image")) .or_else(|| meta.remove("twitter:image:src")) - .map(|url| { + .map(|mut url| { + // If relative URL, prepend root URL. Also if root URL ends with a slash, remove it. + if let Some(ch) = url.chars().next() { + if ch == '/' { + url = format!("{}{}", &original_url.trim_end_matches('/'), &url); + } + } let mut size = ImageSize::Preview; if let Some(card) = meta.remove("twitter:card") { if &card == "summary_large_image" { size = ImageSize::Large; } } - Image { url, width: meta @@ -119,18 +124,26 @@ impl Metadata { .remove("og:video") .or_else(|| meta.remove("og:video:url")) .or_else(|| meta.remove("og:video:secure_url")) - .map(|url| Video { - url, - width: meta - .remove("og:video:width") - .unwrap_or_else(|| "0".to_string()) - .parse() - .unwrap_or(0), - height: meta - .remove("og:video:height") - .unwrap_or_else(|| "0".to_string()) - .parse() - .unwrap_or(0), + .map(|mut url| { + // If relative URL, prepend root URL. Also if root URL ends with a slash, remove it. + if let Some(ch) = url.chars().next() { + if ch == '/' { + url = format!("{}{}", &original_url.trim_end_matches('/'), &url); + } + } + Video { + url, + width: meta + .remove("og:video:width") + .unwrap_or_else(|| "0".to_string()) + .parse() + .unwrap_or(0), + height: meta + .remove("og:video:height") + .unwrap_or_else(|| "0".to_string()) + .parse() + .unwrap_or(0), + } }), icon_url: link .remove("apple-touch-icon") @@ -139,7 +152,7 @@ impl Metadata { // If relative URL, prepend root URL. if let Some(ch) = v.chars().next() { if ch == '/' { - v = format!("{}{}", &url, v); + v = format!("{}{}", &original_url.trim_end_matches('/'), v); } } @@ -148,8 +161,10 @@ impl Metadata { colour: meta.remove("theme-color"), opengraph_type: meta.remove("og:type"), site_name: meta.remove("og:site_name"), - url: meta.remove("og:url").unwrap_or_else(|| url.clone()), - original_url: url, + url: meta + .remove("og:url") + .unwrap_or_else(|| original_url.clone()), + original_url, special: None, }; diff --git a/src/structs/special.rs b/src/structs/special.rs index 80103cb..27906a8 100644 --- a/src/structs/special.rs +++ b/src/structs/special.rs @@ -1,24 +1,24 @@ use serde::Serialize; -#[derive(Debug, Serialize)] +#[derive(Clone, Debug, Serialize)] pub enum TwitchType { Channel, Video, Clip, } -#[derive(Debug, Serialize)] +#[derive(Clone, Debug, Serialize)] pub enum LightspeedType { Channel, } -#[derive(Debug, Serialize)] +#[derive(Clone, Debug, Serialize)] pub enum BandcampType { Album, Track, } -#[derive(Debug, Serialize)] +#[derive(Clone, Debug, Serialize)] #[serde(tag = "type")] pub enum Special { None, diff --git a/src/util/request.rs b/src/util/request.rs index f89ce38..eb3af26 100644 --- a/src/util/request.rs +++ b/src/util/request.rs @@ -16,7 +16,8 @@ use super::{result::Error, variables::MAX_BYTES}; lazy_static! { static ref CLIENT: Client = reqwest::Client::builder() .user_agent("Mozilla/5.0 (compatible; January/1.0; +https://github.com/revoltchat/january)") - .timeout(Duration::from_secs(2)) + .timeout(Duration::from_secs(15)) + .connect_timeout(Duration::from_secs(5)) .build() .expect("reqwest Client"); } diff --git a/src/util/result.rs b/src/util/result.rs index 0dbe4f6..1c107cf 100644 --- a/src/util/result.rs +++ b/src/util/result.rs @@ -5,7 +5,7 @@ use serde_json; use std::fmt::Display; use validator::ValidationErrors; -#[derive(Serialize, Debug)] +#[derive(Clone, Serialize, Debug)] #[serde(tag = "type")] pub enum Error { CouldNotDetermineImageSize,