From a44ddf44251319ba260b25b27b9af17779051796 Mon Sep 17 00:00:00 2001 From: circlesabound Date: Mon, 15 Apr 2024 12:02:04 +1000 Subject: [PATCH] Implemented download savefile --- Cargo.lock | 93 ++++++++++++------- Cargo.toml | 3 +- src/mgmt-server/link_download.rs | 49 ++++++++-- src/mgmt-server/main.rs | 2 +- src/mgmt-server/routes/download.rs | 43 ++++++++- src/mgmt-server/routes/mod.rs | 36 +++++-- src/mgmt-server/routes/server.rs | 45 +-------- .../oauth-redirect.component.ts | 3 - .../app/dashboard2/dashboard2.component.html | 9 +- .../app/dashboard2/dashboard2.component.ts | 33 +++---- 10 files changed, 201 insertions(+), 115 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6eece1f..606c7b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.13" @@ -290,16 +305,17 @@ checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" dependencies = [ - "libc", - "num-integer", + "android-tzdata", + "iana-time-zone", + "js-sys", "num-traits", "serde", - "time 0.1.43", - "winapi", + "wasm-bindgen", + "windows-targets 0.52.0", ] [[package]] @@ -332,7 +348,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cd91cf61412820176e137621345ee43b3f4423e589e7ae4e50d601d93e35ef8" dependencies = [ "percent-encoding", - "time 0.3.31", + "time", "version_check", ] @@ -348,9 +364,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.2" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" @@ -576,6 +592,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-tungstenite 0.21.0", + "tokio-util 0.7.3", "toml", "url", "urlencoding", @@ -1026,6 +1043,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "idna" version = "0.5.0" @@ -1329,16 +1369,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "num-integer" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" -dependencies = [ - "autocfg", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.14" @@ -1840,7 +1870,7 @@ dependencies = [ "serde_json", "state", "tempfile", - "time 0.3.31", + "time", "tokio", "tokio-stream", "tokio-util 0.7.3", @@ -1888,7 +1918,7 @@ dependencies = [ "smallvec", "stable-pattern", "state", - "time 0.3.31", + "time", "tokio", "uncased", ] @@ -2110,7 +2140,7 @@ dependencies = [ "secrecy", "serde", "serde_json", - "time 0.3.31", + "time", "tokio", "tokio-tungstenite 0.20.1", "tracing", @@ -2351,16 +2381,6 @@ dependencies = [ "syn 1.0.74", ] -[[package]] -name = "time" -version = "0.1.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "time" version = "0.3.31" @@ -2935,6 +2955,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.0", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 3c2209f..59b4323 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ publish = false base64 = "0.22" bincode = "1.3" bytes = "1.0" -chrono = { version = "0.4", features = [ "serde" ] } +chrono = { version = "0.4.37", features = [ "serde" ] } derive_more = "0.99" env_logger = "0.11.3" factorio-mod-settings-parser = { git = "https://github.com/circlesabound/factorio-mod-settings-parser", rev = "a9fecd6" } @@ -36,6 +36,7 @@ tar = "0.4" tokio = { version = "1.5", features = [ "full" ] } tokio-stream = { version = "0.1", features = [ "sync" ] } tokio-tungstenite = "0.21" +tokio-util = "0.7" toml = "0.8.8" url = "2" urlencoding = "2.1.0" diff --git a/src/mgmt-server/link_download.rs b/src/mgmt-server/link_download.rs index 6fb30ad..cd1ff65 100644 --- a/src/mgmt-server/link_download.rs +++ b/src/mgmt-server/link_download.rs @@ -1,11 +1,18 @@ -use std::collections::HashMap; -use chrono::{DateTime, Utc}; +use std::{collections::HashMap, sync::Arc}; +use chrono::{DateTime, Duration, Utc}; use log::info; -use tokio::sync::RwLock; +use tokio::{select, sync::RwLock}; +use tokio_util::sync::CancellationToken; use uuid::Uuid; +const CLEANUP_INTERVAL: Duration = Duration::minutes(15); +const LINK_EXPIRY: Duration = Duration::minutes(60); + +type LinkMap = Arc)>>>; + pub struct LinkDownloadManager { - links: RwLock)>>, + links: LinkMap, + _cleanup_task_ct: CancellationToken, } #[derive(Clone, Debug)] @@ -14,10 +21,17 @@ pub enum LinkDownloadTarget { } impl LinkDownloadManager { - pub fn new() -> LinkDownloadManager { - // TODO periodic job to expire out links + pub async fn new() -> LinkDownloadManager { + let links = LinkMap::default(); + let links_clone = Arc::clone(&links); + let cancellation_token = CancellationToken::new(); + let _cleanup_task_ct = cancellation_token.clone(); + tokio::spawn(async move { + Self::cleanup_job(links_clone, cancellation_token).await; + }); LinkDownloadManager { - links: RwLock::new(HashMap::new()), + links, + _cleanup_task_ct, } } @@ -33,4 +47,25 @@ impl LinkDownloadManager { let r_guard = self.links.read().await; r_guard.get(&link).map(|(target, _dt)| target.clone()) } + + async fn cleanup_job(links: LinkMap, cancellation_token: CancellationToken) { + loop { + select! { + _ = cancellation_token.cancelled() => { + break; + } + _ = tokio::time::sleep(CLEANUP_INTERVAL.to_std().unwrap()) => { + let mut w_guard = links.write().await; + let now = Utc::now(); + w_guard.retain(|link, (target, dt)| { + let should_remove = now - *dt > LINK_EXPIRY; + if should_remove { + info!("Expiring download link: {} -> {:?}", link, target); + } + !should_remove + }); + } + } + } + } } diff --git a/src/mgmt-server/main.rs b/src/mgmt-server/main.rs index 4b4023a..6ba7910 100644 --- a/src/mgmt-server/main.rs +++ b/src/mgmt-server/main.rs @@ -123,7 +123,7 @@ async fn main() -> std::result::Result<(), Box> { .await?; info!("Creating link download manager"); - let link_download_manager = Arc::new(LinkDownloadManager::new()); + let link_download_manager = Arc::new(LinkDownloadManager::new().await); let ws_port = std::env::var("MGMT_SERVER_WS_PORT")?.parse()?; let ws_addr = std::env::var("MGMT_SERVER_WS_ADDRESS")?.parse()?; diff --git a/src/mgmt-server/routes/download.rs b/src/mgmt-server/routes/download.rs index da2fad8..3d488a0 100644 --- a/src/mgmt-server/routes/download.rs +++ b/src/mgmt-server/routes/download.rs @@ -2,17 +2,22 @@ use std::sync::Arc; use crate::{clients::AgentApiClient, error::{Error, Result}, link_download::{LinkDownloadManager, LinkDownloadTarget}}; +use fctrl::schema::{AgentOutMessage, AgentResponseWithId}; +use log::{error, info}; use rocket::{get, response::stream::ByteStream, State}; +use tokio_stream::StreamExt; + +use super::DownloadResponder; #[get("/")] pub async fn download( agent_client: &State>, link_download_manager: &State>, link_id: String, -) -> Result]> { +) -> Result]>> { match link_download_manager.get_link(link_id).await { Some(target) => match target { - LinkDownloadTarget::Savefile { id } => crate::routes::server::get_savefile_real( + LinkDownloadTarget::Savefile { id } => download_savefile( agent_client, id, ).await, @@ -21,3 +26,37 @@ pub async fn download( } } +pub async fn download_savefile( + agent_client: &State>, + id: String, +) -> Result]>> { + let (_operation_id, sub) = agent_client.save_get(id.clone()).await?; + let download_filename = format!("{}.zip", &id); + // TODO figure out how to properly handle errors + let s = sub.filter_map(|event| { + match serde_json::from_str::(&event.content) { + Ok(m) => { + match m.content { + AgentOutMessage::SaveFile(sb) => { + if sb.bytes.len() == 0 { + info!("get_savefile completed with total multiparts = {:?}", sb.multipart_seqnum); + None + } else { + Some(sb.bytes) + } + } + c => { + error!("Expected AgentOutMessage::SaveFile during get_savefile, got something else: {:?}", c); + None + }, + } + } + Err(e) => { + error!("Error deserialising event content during get_savefile: {:?}", e); + None + } + } + }); + + Ok(DownloadResponder::new(ByteStream::from(s), download_filename)) +} diff --git a/src/mgmt-server/routes/mod.rs b/src/mgmt-server/routes/mod.rs index fdf06ad..17fefe1 100644 --- a/src/mgmt-server/routes/mod.rs +++ b/src/mgmt-server/routes/mod.rs @@ -18,20 +18,16 @@ pub mod proxy; pub mod server; pub struct LinkDownloadResponder { - pub link_id: String, - full_uri: String, + path: String, } impl LinkDownloadResponder { fn new( - host: HostHeader, link_id: String, ) -> LinkDownloadResponder { let path = format!("/download/{}", link_id); - let full_uri = format!("{}{}", host.hostname, path); LinkDownloadResponder { - link_id, - full_uri, + path, } } } @@ -40,11 +36,37 @@ impl<'r> Responder<'r, 'static> for LinkDownloadResponder { fn respond_to(self, _: &'r rocket::Request<'_>) -> rocket::response::Result<'static> { Response::build() .status(Status::Accepted) - .header(Header::new("Location", self.full_uri)) + .header(Header::new("Location", self.path)) .ok() } } +#[derive(Responder)] +pub struct DownloadResponder { + inner: T, + content_disposition: ContentDisposition, +} + +impl DownloadResponder { + pub fn new(content: T, download_filename: String) -> DownloadResponder { + DownloadResponder { + inner: content, + content_disposition: ContentDisposition(download_filename), + } + } +} + +struct ContentDisposition(String); + +impl From for Header<'static> { + fn from(value: ContentDisposition) -> Self { + Header::new( + "Content-Disposition", + format!("attachment; filename={}", value.0) + ) + } +} + pub struct WsStreamingResponder { pub path: String, full_uri: String, diff --git a/src/mgmt-server/routes/server.rs b/src/mgmt-server/routes/server.rs index 38b6dcc..c39e3d2 100644 --- a/src/mgmt-server/routes/server.rs +++ b/src/mgmt-server/routes/server.rs @@ -6,16 +6,14 @@ use std::{ use factorio_mod_settings_parser::ModSettings; use fctrl::schema::{ - mgmt_server_rest::*, AgentOutMessage, AgentResponseWithId, FactorioVersion, ModSettingsBytes, RconConfig, SecretsObject, ServerSettingsConfig, ServerStartSaveFile, ServerStatus + mgmt_server_rest::*, FactorioVersion, ModSettingsBytes, RconConfig, SecretsObject, ServerSettingsConfig, ServerStartSaveFile, ServerStatus }; -use log::{error, info}; -use rocket::{response::stream::ByteStream, serde::json::Json}; +use rocket::serde::json::Json; use rocket::{get, post, put}; use rocket::{http::Status, State}; -use tokio_stream::StreamExt; use crate::{ - auth::AuthorizedUser, clients::AgentApiClient, guards::HostHeader, link_download::{LinkDownloadManager, LinkDownloadTarget}, ws::WebSocketServer + auth::AuthorizedUser, clients::AgentApiClient, guards::HostHeader, link_download::{LinkDownloadManager, LinkDownloadTarget}, routes::ContentDisposition, ws::WebSocketServer }; use crate::{error::Result, routes::WsStreamingResponder}; @@ -138,47 +136,12 @@ pub async fn get_savefiles( #[get("/server/savefiles/")] pub async fn get_savefile<'a>( - host: HostHeader<'a>, _a: AuthorizedUser, link_download_manager: &State>, id: String, ) -> Result { let link_id = link_download_manager.create_link(LinkDownloadTarget::Savefile { id }).await; - Ok(LinkDownloadResponder::new(host, link_id)) -} - -pub async fn get_savefile_real( - agent_client: &State>, - id: String, -) -> Result]> { - let (_operation_id, sub) = agent_client.save_get(id).await?; - // TODO figure out how to properly handle errors - let s = sub.filter_map(|event| { - match serde_json::from_str::(&event.content) { - Ok(m) => { - match m.content { - AgentOutMessage::SaveFile(sb) => { - if sb.bytes.len() == 0 { - info!("get_savefile completed with total multiparts = {:?}", sb.multipart_seqnum); - None - } else { - Some(sb.bytes) - } - } - c => { - error!("Expected AgentOutMessage::SaveFile during get_savefile, got something else: {:?}", c); - None - }, - } - } - Err(e) => { - error!("Error deserialising event content during get_savefile: {:?}", e); - None - } - } - }); - - Ok(ByteStream::from(s)) + Ok(LinkDownloadResponder::new(link_id)) } #[get("/server/config/adminlist")] diff --git a/web/src/app/auth/oauth-redirect/oauth-redirect.component.ts b/web/src/app/auth/oauth-redirect/oauth-redirect.component.ts index a755fcd..a7e4d87 100644 --- a/web/src/app/auth/oauth-redirect/oauth-redirect.component.ts +++ b/web/src/app/auth/oauth-redirect/oauth-redirect.component.ts @@ -16,13 +16,10 @@ export class OauthRedirectComponent implements OnInit { ) { } ngOnInit(): void { - // TODO clean out logs let ss = this.route.snapshot; if (ss.queryParamMap.has('code')) { let code = ss.queryParamMap.get('code')!; - console.log(`code is ${code}`); this.authDiscordService.codeToToken(code).subscribe(s => { - console.log(`Got token = ${s}. Redirecting to dashboard`); this.router.navigate(['dashboard']); }); } diff --git a/web/src/app/dashboard2/dashboard2.component.html b/web/src/app/dashboard2/dashboard2.component.html index bae2419..271ebdc 100644 --- a/web/src/app/dashboard2/dashboard2.component.html +++ b/web/src/app/dashboard2/dashboard2.component.html @@ -14,15 +14,14 @@

Saves:

Existing save files - + {{name}} .zip - - - - + + +

Create new save file

diff --git a/web/src/app/dashboard2/dashboard2.component.ts b/web/src/app/dashboard2/dashboard2.component.ts index cf817e4..3f2c5d5 100644 --- a/web/src/app/dashboard2/dashboard2.component.ts +++ b/web/src/app/dashboard2/dashboard2.component.ts @@ -2,6 +2,7 @@ import { Component, OnInit } from '@angular/core'; import { MgmtServerRestApiService } from '../mgmt-server-rest-api/services'; import { OperationService } from '../operation.service'; import { MatSelectChange } from '@angular/material/select'; +import { environment } from 'src/environments/environment'; @Component({ selector: 'app-dashboard2', @@ -15,9 +16,6 @@ export class Dashboard2Component implements OnInit { selectedSave: string; createSaveName: string; installVersionString: string; - saveDownloadHref: string; - saveDownloadName: string; - saveIsSelected: boolean; downloadAvailableVersions: string[] = []; saves: string[] = []; @@ -32,9 +30,6 @@ export class Dashboard2Component implements OnInit { this.selectedSave = ''; this.createSaveName = ''; this.installVersionString = ''; - this.saveDownloadHref = ''; - this.saveDownloadName = ''; - this.saveIsSelected = false; } ngOnInit(): void { @@ -62,16 +57,6 @@ export class Dashboard2Component implements OnInit { // TODO } - saveSelectionChange(selectChangeEvent: MatSelectChange): void { - // update download link - this.saveIsSelected = true; - this.saveDownloadName = selectChangeEvent.value + ".zip"; - this.saveDownloadHref = "api/v0/server/savefiles/" + selectChangeEvent.value; - // this.apiClient.serverSavefilesSavefileIdGet({ savefile_id: selectChangeEvent.value }).subscribe(s => { - // - // }) - } - startServer(): void { if (this.selectedSave) { const payload = { @@ -114,6 +99,22 @@ export class Dashboard2Component implements OnInit { }); } + downloadSave(): void { + // 2-part download process to allow us to use native browser download experience + // first, we generate a temporary download link that does not require authentication token + this.apiClient.serverSavefilesSavefileIdGet$Response({ savefile_id: this.selectedSave }).subscribe(resp => { + const location = resp.headers.get('Location'); + if (location !== null) { + // then we create an invisible element with that link destination and trigger a click on it + const a = document.createElement('a'); + a.href = window.location.protocol + '//' + environment.backendHost + location; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + } + }) + } + deleteSave(): void { // TODO no endpoint for this }