diff --git a/Cargo.lock b/Cargo.lock index 230c616..cde64e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -72,7 +72,7 @@ dependencies = [ [[package]] name = "aurorium" -version = "1.1.0" +version = "2.0.0" dependencies = [ "axum", "bpaf", @@ -86,9 +86,11 @@ dependencies = [ "quick-xml 0.31.0", "quickxml_to_serde", "rayon", + "regex", "reqwest", "serde", "serde_json", + "thiserror", "tokio", "tokio-util", "tower", @@ -194,9 +196,9 @@ dependencies = [ [[package]] name = "bpaf" -version = "0.9.6" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "565688a36ddfcfc189cc927c94a6b69cc96c9f8a69030151ae8f0ed2b54a2ad3" +checksum = "19232d7d855392d993f6dabd8dea40a457a6d24ef679fe98f5edca811bb11e21" [[package]] name = "bumpalo" @@ -835,9 +837,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "wasi", @@ -898,15 +900,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl" -version = "0.10.60" +version = "0.10.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" +checksum = "6b8419dc8cc6d866deb801274bba2e6f8f6108c1bb7fcc10ee5ab864931dbb45" dependencies = [ "bitflags 2.4.1", "cfg-if", @@ -936,9 +938,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.96" +version = "0.9.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" +checksum = "c3eaad34cdd97d81de97964fc7f29e2d104f483840d906ef56daa1912338460b" dependencies = [ "cc", "libc", @@ -1015,9 +1017,9 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "portable-atomic" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bccab0e7fd7cc19f820a1c8c91720af652d0c88dc9664dd72aef2614f04af3b" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" [[package]] name = "proc-macro2" @@ -1401,6 +1403,26 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1523,9 +1545,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" @@ -1535,9 +1557,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" [[package]] name = "unicode-ident" @@ -1924,18 +1946,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.28" +version = "0.7.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e" +checksum = "5d075cf85bbb114e933343e087b92f2146bac0d55b534cbb8188becf0039948e" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.28" +version = "0.7.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b" +checksum = "86cd5ca076997b97ef09d3ad65efe811fa68c9e874cb636ccb211223a813b0c2" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 3b3b5cc..04ea62a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,25 +1,28 @@ [package] name = "aurorium" authors = ["Phill030"] -version = "1.1.0" +version = "2.0.0" edition = "2021" [dependencies] axum = { version = "0.6.20", features = ["headers"] } -bpaf = "0.9.6" +# axum-extra = { version = "0.9.0", features = ["typed-header"] } # SOON +bpaf = "0.9.8" chrono = "0.4.31" +console = "0.15.7" env_logger = "0.10.1" +futures = "0.3.29" +indicatif = "0.17.7" +lazy_static = "1.4.0" log = "0.4.20" quick-xml = { version = "0.31.0", features = ["serialize"] } quickxml_to_serde = { version = "0.5", features = ["json_types"] } +regex = "1.10.2" reqwest = "0.11.22" rayon = "1.8.0" serde = { version = "1.0.193", features = ["derive"] } serde_json = "1.0.108" +thiserror = "1.0.50" tokio-util = { version = "0.7.10", features = ["io-util", "rt"] } tokio = { version = "1.34.0", features = ["full"] } -futures = "0.3.29" -console = "0.15.7" -indicatif = "0.17.7" -lazy_static = "1.4.0" tower = { version = "0.4.13", features = ["limit"] } diff --git a/README.md b/README.md index babbb68..e115ecf 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ git clone https://github.com/Revive101/Aurorium.git ### Usage To run the executable, use `cargo run` alternatively you can build it using `cargo build` or in release `cargo build --release`. -If you just want to host already fetched revisions of the game, simply start the executable. In order to fetch a revision, you need to provide the executable with the `--revision` / `-r` argument. +Since Version 2.0, Aurorium automatically fetches the newest Revision from their server and downloads all resources associated with it. (In Version < 2.0 you need to provide the executable with `--revision` or `-r`) If you get an `error: linker link.exe not found` error, that means you are missing the buildtools from Microsoft for C++. To solve this issue you can either install `Build Tools for Visual Studio 2019/2022` or use following commands on windows: @@ -49,7 +49,6 @@ rustup default stable-x86_64-pc-windows-msvc ``` to mark this toolchain as default, then try to build the project again. - For 64-Bit Linux-Based systems, you need to use `x86_64-unknown-linux-gnu` as target. The structure of a revision looks like following: `V_r[Major].[Minor]`. Sometimes, the Minor version can even be `WizardDev`. Example: `V_r746756.WizardDev`. @@ -59,14 +58,14 @@ The structure of a revision looks like following: `V_r[Major].[Minor]`. Sometime You can provide the (built) executable with following parameters: ``` --v, --verbose Activate verbosity (Default: warn) --r, --revision= Fetch from a revision string (Example V_r740872.Wizard_1_520) --i, --ip= Override the default endpoint IP (Default: 0.0.0.0:12369) +-v, --verbose Activate verbosity (Default: warn) +-i, --ip= Override the default endpoint IP (Default: 0.0.0.0:12369) -c, --concurrent_downloads= Override the count of concurrent downloads at once (Default: 8) - --max_requests= Change the amount of requests a user can send before getting rate-limited by the server + --max_requests= Change the amount of requests a user can send before getting rate-limited by the server --reset_duration= Change the duration for the interval in which the rate-limit list get's cleared (In seconds) - --disable_ratelimit Disable ratelimits --h, --help Prints help information + --disable_ratelimit Disable ratelimits + --revision_check_interval= Change the interval for checking for new revisions (In minutes) +-h, --help Prints help information ``` ## Contributing diff --git a/src/errors.rs b/src/errors.rs new file mode 100644 index 0000000..67218fd --- /dev/null +++ b/src/errors.rs @@ -0,0 +1,11 @@ +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum RevisionError { + #[error("IO Error while reading/writing TcpStream")] + IO(#[from] std::io::Error), + #[error("Received invalid MagicHeader sequence")] + InvalidMagicHeader, + #[error("Expected SERVICE_ID (8) & MESSAGE_ID (2) but got {0} & {1}")] + InvalidProtocol(u8, u8), +} diff --git a/src/http/http_request.rs b/src/http/http_request.rs index 660f3c2..0e3d6a3 100644 --- a/src/http/http_request.rs +++ b/src/http/http_request.rs @@ -5,17 +5,19 @@ use futures::StreamExt; use quickxml_to_serde::Config; use serde::Deserialize; +use crate::revision_checker::revision_checker::Revision; + #[derive(Debug, Clone)] pub struct HttpRequest { - pub filelist_url: String, - pub file_url: String, + pub list_file_url: String, + pub url_prefix: String, pub files: FileList, + pub revision: String, max_concurrent_downloads: usize, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct FileList { - pub revision: String, pub wad_list: Vec, pub util_list: Vec, } @@ -40,8 +42,7 @@ static LINK: Emoji<'_, '_> = Emoji("🔗 ", ""); static BOX: Emoji<'_, '_> = Emoji("📦 ", ""); impl HttpRequest { - pub async fn new(revision: String, concurrent: usize) -> HttpRequest { - const BASE_URL: &str = "http://versionec.us.wizard101.com/WizPatcher"; + pub async fn new(revision: Revision, max_concurrent_downloads: usize) -> Self { println!( "{} {}Resolving revision...", style("[1/6]").bold().dim(), @@ -49,14 +50,11 @@ impl HttpRequest { ); HttpRequest { - filelist_url: format!("{BASE_URL}/{revision}/Windows"), - file_url: format!("{BASE_URL}/{revision}/LatestBuild"), - max_concurrent_downloads: concurrent, - files: FileList { - wad_list: Vec::new(), - util_list: Vec::new(), - revision: revision.replace("\r\n", ""), - }, + revision: revision.revision, + url_prefix: revision.url_prefix, + list_file_url: revision.list_file_url, + files: FileList::default(), + max_concurrent_downloads, } } @@ -67,11 +65,9 @@ impl HttpRequest { TRUCK ); - let save_path = PathBuf::from("files").join(&self.files.revision); - let bin_url = &format!("{}/LatestFileList.bin", &self.filelist_url); + let save_path = PathBuf::from("files").join(&self.revision); - // LatestFileList.bin TODO: Move this into their own functions!! - if let Ok(res) = request_file(&bin_url).await { + if let Ok(res) = request_file(&self.list_file_url).await { if !save_path.join("utils").join("LatestFileList.bin").exists() { if let Err(_) = write_to_file( &save_path.join("utils").join("LatestFileList.bin"), @@ -86,9 +82,10 @@ impl HttpRequest { log::error!("Could not fetch LatestFileList.bin") }; - let xml_url = &format!("{}/LatestFileList.xml", &self.filelist_url); - // LatestFileList.xml TODO: Move this into their own functions!! - match request_file(&xml_url).await { + let xml_url = &self + .list_file_url + .replace("LatestFileList.bin", "LatestFileList.xml"); + match request_file(xml_url).await { Ok(res) => { let xml_text = res.text().await.unwrap_or(String::new()); @@ -175,7 +172,7 @@ impl HttpRequest { }); println!( - "{} {}Inserted {} wad files & {} util files...", + "{} {}Inserted {} wad files & {} util files into list...", style("[3/6]").bold().dim(), LINK, &self.files.wad_list.len(), @@ -190,7 +187,7 @@ impl HttpRequest { /// This is pure 🌟 Magic 🌟 async fn fetch_wads(&mut self, save_path: PathBuf) { - let url = self.file_url.clone(); + let url = self.url_prefix.clone(); futures::stream::iter(self.files.wad_list.clone().into_iter().map(|wad| { let url_cloned = url.clone(); @@ -238,7 +235,7 @@ impl HttpRequest { /// This is pure 🌟 Magic 🌟 async fn fetch_utils(&mut self, save_path: PathBuf) { println!("{} {}fetching util files", style("[5/6]").bold().dim(), BOX); - let url = self.file_url.clone(); + let url = self.url_prefix.clone(); futures::stream::iter(self.files.util_list.clone().into_iter().map(|util| { let url_cloned = url.clone(); diff --git a/src/main.rs b/src/main.rs index a726887..6de23c0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,20 +1,22 @@ +use crate::{ + http::http_request::HttpRequest, + routes::{get_revisions, get_util, get_wad, get_xml_filelist}, +}; +use axum::{routing::get, Extension, Router}; +use bpaf::{construct, long, short, OptionParser, Parser}; +use lazy_static::lazy_static; +use revision_checker::revision_checker::Revision; use std::{ net::SocketAddr, - process, sync::{Arc, Mutex, RwLock}, time::Duration, }; - -use axum::{routing::get, Extension, Router}; -use bpaf::{construct, long, short, OptionParser, Parser}; -use lazy_static::lazy_static; - use util::explore_revisions; -use crate::routes::{get_revisions, get_util, get_wad, get_xml_filelist}; - +pub mod errors; mod http; mod rate_limit; +mod revision_checker; mod routes; mod util; @@ -26,12 +28,12 @@ lazy_static! { #[derive(Debug, Clone)] struct Opt { verbose: bool, - revision: Option, ip: SocketAddr, concurrent_downloads: usize, rl_max_requests: u32, rl_reset_duration: u32, rl_disable: bool, + rc_interval: u64, } fn opts() -> OptionParser { @@ -40,12 +42,6 @@ fn opts() -> OptionParser { .help("Activate verbosity (Default: warn)") .switch(); - let revision = short('r') - .long("revision") - .help("Fetch from a revision string (Example V_r740872.Wizard_1_520)") - .argument::("String") - .optional(); - let ip = short('i') .long("ip") .help("Override the default endpoint IP (Default: 0.0.0.0:12369)") @@ -72,7 +68,12 @@ fn opts() -> OptionParser { .help("Disable ratelimits") .switch(); - construct!(Opt { verbose, revision, ip, concurrent_downloads, rl_max_requests, rl_reset_duration, rl_disable }) + let rc_interval = long("revision_check_interval") + .help("Change the interval for checking for new revisions (In minutes)") + .argument::("u64") + .fallback(0); + + construct!(Opt { verbose, ip, concurrent_downloads, rl_max_requests, rl_reset_duration, rl_disable, rc_interval }) .to_options() .footer("Copyright (c) 2023 Phill030") .descr("By default, only the webserver will start. If you want to fetch from a revision, use the --revision or -r parameter.") @@ -85,17 +86,15 @@ async fn main() { let filter = if opts.verbose { "info" } else { "warn" }; env_logger::init_from_env(env_logger::Env::new().default_filter_or(filter)); - if opts.revision.is_some() { - let mut req = - http::http_request::HttpRequest::new(opts.revision.unwrap(), opts.concurrent_downloads) - .await; - req.propogate_filelist().await; - } + check_revision(opts.concurrent_downloads).await; - // If there are no files to host, why have the server running anyways? 🤓☝ - if (explore_revisions().await).is_err() { - log::error!("There are no revisions for the server to host! (Quitting)"); - process::exit(0); + if opts.rc_interval > 0 { + tokio::spawn(async move { + loop { + tokio::time::sleep(Duration::from_secs(60 * opts.rc_interval)).await; + check_revision(opts.concurrent_downloads).await; + } + }); } let state = Arc::new(Mutex::new(rate_limit::rate_limiter::RateLimiter::new( @@ -121,3 +120,19 @@ async fn main() { Err(why) => log::error!("Could not start Axum server! {}", why), } } + +async fn check_revision(concurrent_downloads: usize) { + let fetched_revision = Revision::check::<256>().await.unwrap(); + if explore_revisions().await.is_err() + || !REVISIONS + .read() + .unwrap() + .to_vec() + .contains(&fetched_revision.revision) + { + let mut req = HttpRequest::new(fetched_revision, concurrent_downloads).await; + req.propogate_filelist().await; + } else { + log::warn!("Newest revision is already fetched!") + } +} diff --git a/src/revision_checker/mod.rs b/src/revision_checker/mod.rs new file mode 100644 index 0000000..cb6bcb7 --- /dev/null +++ b/src/revision_checker/mod.rs @@ -0,0 +1 @@ +pub mod revision_checker; diff --git a/src/revision_checker/revision_checker.rs b/src/revision_checker/revision_checker.rs new file mode 100644 index 0000000..a0ea2a4 --- /dev/null +++ b/src/revision_checker/revision_checker.rs @@ -0,0 +1,113 @@ +use regex::Regex; +use std::{ + io::{Cursor, Read, Write}, + net::{TcpStream, ToSocketAddrs}, + time::Duration, +}; +use tokio::io::AsyncReadExt; + +use crate::{ + errors::RevisionError, + util::{hex_decode, Endianness}, +}; + +const URL: &str = "patch.us.wizard101.com"; +const PORT: &str = "12500"; +const MAGIC_HEADER: [u8; 2] = [0x0D, 0xF0]; +const SESSION_ACCEPT: &str = + "0DF02700000000000802220000000000000000000000000000000000000000000000000000000000000000"; +const SERVICE_ID: u8 = 8; // PATCH +const MESSAGE_ID: u8 = 2; // MSG_LATEST_FILE_LIST_V2 + +pub struct Revision { + pub list_file_url: String, + pub url_prefix: String, + pub revision: String, +} +impl Revision { + fn create_stream() -> std::io::Result { + let mut ip = format!("{URL}:{PORT}").to_socket_addrs()?; + log::info!("Successfully connected to {URL}"); + + Ok(TcpStream::connect_timeout( + &ip.next().unwrap(), + Duration::from_secs(20), + )?) + } + + pub async fn check() -> Result { + let mut stream = Self::create_stream()?; + + let mut buffer = [0u8; N]; + stream.read(&mut buffer)?; // We don't need the SessionOffer + buffer = [0u8; N]; + + stream.write_all(&hex_decode(SESSION_ACCEPT, Endianness::Little).unwrap()[..])?; + + stream.read(&mut buffer)?; + let mut cursor = Cursor::new(buffer); + + if !Self::is_magic_header(&mut cursor).await { + log::error!("Received invalid MagicHeader sequence"); + return Err(RevisionError::InvalidMagicHeader); + } + + let _ = cursor.read_u16_le().await?; + let _ = cursor.read_u32_le().await?; + + let service_id = cursor.read_u8().await?; + let message_id = cursor.read_u8().await?; + + if service_id != SERVICE_ID || message_id != MESSAGE_ID { + log::error!( + "Expected SERVICE_ID (8) & MESSAGE_ID (2) but got {service_id} & {message_id}" + ); + return Err(RevisionError::InvalidProtocol(service_id, message_id)); + } + + let _dml_length = cursor.read_u16_le().await?; + let _latest_version = cursor.read_u32_le().await?; + let _list_file_name = Self::read_bytestring(&mut cursor).await; + let _ = cursor.read_u128_le().await?; + let list_file_url = Self::read_bytestring(&mut cursor).await; + let url_prefix = Self::read_bytestring(&mut cursor).await; + + stream.shutdown(std::net::Shutdown::Both)?; + + Ok(Revision { + list_file_url: list_file_url.clone(), + url_prefix, + revision: Self::parse_revision(&list_file_url), + }) + } + + async fn is_magic_header(cursor: &mut Cursor<[u8; N]>) -> bool { + let magic_header = cursor.read_u16_le().await.unwrap(); + magic_header.to_le_bytes().eq(&MAGIC_HEADER) + } + + async fn read_bytestring(cursor: &mut Cursor<[u8; N]>) -> String { + let len = cursor.read_u16_le().await.unwrap(); + let mut buff = vec![0u8; len as usize]; + + tokio::io::AsyncReadExt::read_exact(cursor, &mut buff) + .await + .unwrap(); + + String::from_utf8_lossy(&buff).to_string() + } + + pub fn parse_revision(url: &String) -> String { + let reg = Regex::new(r"/V_([^/]+)/").unwrap(); + + if let Some(captures) = reg.captures(&url) { + if let Some(version) = captures.get(1) { + return format!("V_{}", version.as_str()); + } else { + return String::from(""); + } + } else { + return String::from(""); + } + } +} diff --git a/src/util.rs b/src/util.rs index bbb03da..75b6597 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,9 +1,7 @@ -use std::net::SocketAddr; - +use crate::REVISIONS; use axum::headers::UserAgent; use chrono::Local; - -use crate::REVISIONS; +use std::net::SocketAddr; pub fn log_access(addr: SocketAddr, header: &UserAgent, route: &str) { const REQUIRED_USER_AGENT: &str = "KingsIsle Patcher"; @@ -31,3 +29,29 @@ pub async fn explore_revisions() -> std::io::Result<()> { Ok(()) } + +pub enum Endianness { + Little, + Big, +} + +pub fn hex_decode(hex_string: &str, endianness: Endianness) -> Option> { + // Check if the hex string is a valid length + if hex_string.len() % 2 != 0 { + return None; + } + + // Iterate over pairs of characters and convert them to u8 + let bytes: Option> = (0..hex_string.len()) + .step_by(2) + .map(|i| { + let byte = u8::from_str_radix(&hex_string[i..i + 2], 16).ok()?; + match endianness { + Endianness::Little => Some(byte), + Endianness::Big => Some(byte.reverse_bits()), // For Big Endian, reverse the bits + } + }) + .collect(); + + bytes +}