From a491b423aa01eae819aa407ac881fe5794d62b3c Mon Sep 17 00:00:00 2001 From: circlesabound Date: Tue, 23 Apr 2024 15:24:49 +1000 Subject: [PATCH] Upload mod-settings-dat, chunked upload for saves --- local_build.sh | 2 +- local_down.sh | 2 +- local_up.sh | 2 +- openapi/mgmt-server-rest.yaml | 5 ++ src/agent/main.rs | 21 ++--- src/agent/util/saves.rs | 40 +++++++--- src/mgmt-server/guards.rs | 76 ++++++++++++++++++ src/mgmt-server/routes/download.rs | 4 +- src/mgmt-server/routes/server.rs | 13 ++-- src/schema.rs | 25 ++++-- .../app/dashboard2/dashboard2.component.html | 16 ++++ .../app/dashboard2/dashboard2.component.ts | 77 ++++++++++++++++++- .../mod-settings/mod-settings.component.html | 8 +- 13 files changed, 248 insertions(+), 43 deletions(-) diff --git a/local_build.sh b/local_build.sh index 3da5477..c185b39 100755 --- a/local_build.sh +++ b/local_build.sh @@ -6,4 +6,4 @@ else GIT_COMMIT_HASH="$(git rev-parse HEAD)-dirty" fi -env DOCKER_BUILDKIT=1 docker-compose -f docker-compose.yml -f docker-compose.local.yml build --build-arg GIT_COMMIT_HASH=$GIT_COMMIT_HASH +env DOCKER_BUILDKIT=1 docker compose -f docker-compose.yml -f docker-compose.local.yml build --build-arg GIT_COMMIT_HASH=$GIT_COMMIT_HASH diff --git a/local_down.sh b/local_down.sh index 5e1f5d1..a830c81 100755 --- a/local_down.sh +++ b/local_down.sh @@ -1,3 +1,3 @@ #!/bin/sh -docker-compose -f docker-compose.yml -f docker-compose.local.yml down \ No newline at end of file +docker compose -f docker-compose.yml -f docker-compose.local.yml down \ No newline at end of file diff --git a/local_up.sh b/local_up.sh index 6e3b887..3babed9 100755 --- a/local_up.sh +++ b/local_up.sh @@ -1,3 +1,3 @@ #!/bin/sh -docker-compose $@ -f docker-compose.yml -f docker-compose.local.yml up -d \ No newline at end of file +docker compose $@ -f docker-compose.yml -f docker-compose.local.yml up -d \ No newline at end of file diff --git a/openapi/mgmt-server-rest.yaml b/openapi/mgmt-server-rest.yaml index d238b8f..312eca5 100644 --- a/openapi/mgmt-server-rest.yaml +++ b/openapi/mgmt-server-rest.yaml @@ -163,6 +163,11 @@ paths: required: true schema: type: string + - name: Content-Range + in: header + required: true + schema: + type: string requestBody: required: true content: diff --git a/src/agent/main.rs b/src/agent/main.rs index 63b74b5..6273144 100644 --- a/src/agent/main.rs +++ b/src/agent/main.rs @@ -1035,12 +1035,13 @@ impl AgentController { let chunks = savebytes.bytes.chunks(MAX_WS_PAYLOAD_BYTES); let mut i = 0; for chunk in chunks { + let chunk_len = chunk.len(); let msg = AgentOutMessage::SaveFile(SaveBytes { - multipart_seqnum: Some(i), + multipart_start: Some(i), bytes: chunk.to_vec(), }); self.reply(msg, &operation_id).await; - i += 1; + i += chunk_len; } self.reply_success(AgentOutMessage::SaveFile(SaveBytes::sentinel(i)), operation_id).await; }, @@ -1070,18 +1071,10 @@ impl AgentController { } async fn save_set(&self, save_name: String, savebytes: SaveBytes, operation_id: OperationId) { - match util::saves::exists_savefile(&save_name).await { - Ok(false) => { - if let Err(e) = util::saves::set_savefile(&save_name, savebytes).await { - self.reply_failed(AgentOutMessage::Error(format!("Failed to set savefile with name `{}`: {:?}", &save_name, e)), operation_id).await - } else { - self.reply_success(AgentOutMessage::Ok, operation_id).await - } - }, - Ok(true) => self.reply_failed(AgentOutMessage::Error(format!("Savefile with name {} already exists", &save_name)), operation_id).await, - Err(e) => self.reply_failed( - AgentOutMessage::Error(format!("Failed to read saves: {:?}", e)), - operation_id).await, + if let Err(e) = util::saves::set_savefile(&save_name, savebytes).await { + self.reply_failed(AgentOutMessage::Error(format!("Failed to set savefile with name `{}`: {:?}", &save_name, e)), operation_id).await + } else { + self.reply_success(AgentOutMessage::Ok, operation_id).await } } diff --git a/src/agent/util/saves.rs b/src/agent/util/saves.rs index ec0d9be..3f58358 100644 --- a/src/agent/util/saves.rs +++ b/src/agent/util/saves.rs @@ -1,8 +1,8 @@ -use std::path::{Path, PathBuf}; +use std::{io::SeekFrom, path::{Path, PathBuf}}; use fctrl::schema::{Save, SaveBytes}; use log::{error, info, warn}; -use tokio::fs; +use tokio::{fs::{self, OpenOptions}, io::{AsyncSeekExt, AsyncWriteExt}}; use crate::{consts::*, error::Result}; @@ -63,14 +63,34 @@ pub async fn list_savefiles() -> Result> { pub async fn set_savefile(save_name: impl AsRef, savebytes: SaveBytes) -> Result<()> { let bytes_length = savebytes.bytes.len(); - match fs::write(get_savefile_path(save_name.as_ref()), savebytes.bytes).await { - Ok(()) => { - info!("Successfully set savefile `{}`, wrote {} bytes", save_name.as_ref(), bytes_length); - Ok(()) - }, - Err(e) => { - error!("Failed to set savefile `{}`: {:?}", save_name.as_ref(), e); - Err(e.into()) + if let Some(start_byte) = savebytes.multipart_start { + // partial file write + let filename = get_savefile_path(save_name.as_ref()); + // create if not exist + let mut file = OpenOptions::new().write(true).create(true).open(filename).await?; + if savebytes.is_sentinel() { + // finalise and trim down to size + file.set_len(start_byte as u64).await?; + info!("Successfully finalised savefile `{}`, final length {} bytes", save_name.as_ref(), start_byte); + } else { + // seek to correct write location before writing + file.seek(SeekFrom::Start(start_byte as u64)).await?; + file.write_all(&savebytes.bytes).await?; + file.flush().await?; + info!("Successfully wrote to savefile `{}`, wrote {} bytes from offset {}", save_name.as_ref(), savebytes.bytes.len(), start_byte); + } + Ok(()) + } else { + // write the whole file + match fs::write(get_savefile_path(save_name.as_ref()), savebytes.bytes).await { + Ok(()) => { + info!("Successfully set savefile `{}`, wrote {} bytes", save_name.as_ref(), bytes_length); + Ok(()) + }, + Err(e) => { + error!("Failed to set savefile `{}`: {:?}", save_name.as_ref(), e); + Err(e.into()) + } } } } diff --git a/src/mgmt-server/guards.rs b/src/mgmt-server/guards.rs index 1bde9cf..fb98612 100644 --- a/src/mgmt-server/guards.rs +++ b/src/mgmt-server/guards.rs @@ -1,3 +1,4 @@ +use fctrl::schema::regex::CONTENT_RANGE_RE; use log::error; use rocket::{ http::Status, @@ -31,6 +32,81 @@ impl<'r> FromRequest<'r> for HostHeader<'r> { } } +pub struct ContentLengthHeader { + pub length: usize, +} + +#[derive(Debug)] +pub enum ContentLengthHeaderError { + Missing, + Format, +} + +#[rocket::async_trait] +impl<'r> FromRequest<'r> for ContentLengthHeader { + type Error = ContentLengthHeaderError; + + async fn from_request(request: &'r rocket::Request<'_>) -> Outcome { + match request.headers().get_one("Content-Length") { + Some(h) => { + if let Ok(length) = h.parse::() { + Outcome::Success(ContentLengthHeader { + length, + }) + } else { + Outcome::Error((Status::BadRequest, ContentLengthHeaderError::Format)) + } + }, + None => Outcome::Error((Status::BadRequest, ContentLengthHeaderError::Missing)), + } + } +} + +pub struct ContentRangeHeader { + pub start: usize, + pub end: usize, + pub length: usize, +} + +#[derive(Debug)] +pub enum ContentRangeHeaderError { + Missing, + Format, +} + +#[rocket::async_trait] +impl<'r> FromRequest<'r> for ContentRangeHeader { + type Error = ContentRangeHeaderError; + + async fn from_request(request: &'r rocket::Request<'_>) -> Outcome { + match request.headers().get_one("Content-Range") { + Some(h) => { + if let Some(captures) = CONTENT_RANGE_RE.captures(h) { + if let Some(start) = captures.get(1) { + if let Ok(start) = start.as_str().parse::() { + if let Some(end) = captures.get(2) { + if let Ok(end) = end.as_str().parse::() { + if let Some(length) = captures.get(3) { + if let Ok(length) = length.as_str().parse::() { + return Outcome::Success(ContentRangeHeader { + start, + end, + length, + }); + } + } + } + } + } + } + } + Outcome::Error((Status::BadRequest, ContentRangeHeaderError::Format)) + }, + None => Outcome::Error((Status::BadRequest, ContentRangeHeaderError::Missing)), + } + } +} + #[derive(Debug)] pub enum AuthError { Missing, diff --git a/src/mgmt-server/routes/download.rs b/src/mgmt-server/routes/download.rs index 2dfb1f6..221aab7 100644 --- a/src/mgmt-server/routes/download.rs +++ b/src/mgmt-server/routes/download.rs @@ -49,8 +49,8 @@ async fn download_save( Ok(m) => { match m.content { AgentOutMessage::SaveFile(sb) => { - if sb.bytes.len() == 0 { - info!("get_savefile completed with total multiparts = {:?}", sb.multipart_seqnum); + if sb.is_sentinel() { + info!("get_savefile completed with total multipart length = {:?}", sb.multipart_start); None } else { Some(sb.bytes) diff --git a/src/mgmt-server/routes/server.rs b/src/mgmt-server/routes/server.rs index a241a08..d300157 100644 --- a/src/mgmt-server/routes/server.rs +++ b/src/mgmt-server/routes/server.rs @@ -8,12 +8,12 @@ use factorio_file_parser::ModSettings; use fctrl::schema::{ mgmt_server_rest::*, FactorioVersion, ModSettingsBytes, RconConfig, SaveBytes, SecretsObject, ServerSettingsConfig, ServerStartSaveFile, ServerStatus }; -use rocket::{delete, serde::json::Json}; +use rocket::{data::ToByteUnit, delete, serde::json::Json, Data}; use rocket::{get, post, put}; use rocket::{http::Status, State}; use crate::{ - auth::AuthorizedUser, clients::AgentApiClient, guards::HostHeader, link_download::{LinkDownloadManager, LinkDownloadTarget}, ws::WebSocketServer + auth::AuthorizedUser, clients::AgentApiClient, guards::{ContentLengthHeader, ContentRangeHeader, HostHeader}, link_download::{LinkDownloadManager, LinkDownloadTarget}, ws::WebSocketServer }; use crate::{error::Result, routes::WsStreamingResponder}; @@ -158,11 +158,14 @@ pub async fn put_savefile( _a: AuthorizedUser, agent_client: &State>, id: String, - body: Vec, + body: Data<'_>, + content_length: ContentLengthHeader, + content_range: ContentRangeHeader, ) -> Result<()> { + let chunk_stream = body.open(content_length.length.bytes()); let savebytes = SaveBytes { - multipart_seqnum: None, - bytes: body, + multipart_start: Some(content_range.start), + bytes: chunk_stream.into_bytes().await?.into_inner(), }; agent_client.save_put(id, savebytes).await?; Ok(()) diff --git a/src/schema.rs b/src/schema.rs index b7791c2..5349620 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -231,7 +231,7 @@ pub struct Save { #[derive(Deserialize, Serialize)] pub struct SaveBytes { - pub multipart_seqnum: Option, + pub multipart_start: Option, #[serde(with = "base64")] pub bytes: Vec, } @@ -239,17 +239,21 @@ pub struct SaveBytes { impl SaveBytes { pub fn new(bytes: Vec) -> SaveBytes { SaveBytes { - multipart_seqnum: None, + multipart_start: None, bytes, } } - pub fn sentinel(total_num_parts: u32) -> SaveBytes { + pub fn sentinel(total_length: usize) -> SaveBytes { SaveBytes { - multipart_seqnum: Some(total_num_parts), + multipart_start: Some(total_length), bytes: vec![], } } + + pub fn is_sentinel(&self) -> bool { + self.bytes.len() == 0 + } } impl std::fmt::Debug for SaveBytes { @@ -257,12 +261,12 @@ impl std::fmt::Debug for SaveBytes { if self.bytes.len() > 16 { let debug_bytes = format!("{:?}...", &self.bytes[..16]); f.debug_struct("SaveBytes") - .field("multipart_seqnum", &self.multipart_seqnum) + .field("multipart_start", &self.multipart_start) .field("bytes", &debug_bytes) .finish() } else { f.debug_struct("SaveBytes") - .field("multipart_seqnum", &self.multipart_seqnum) + .field("multipart_start", &self.multipart_start) .field("bytes", &self.bytes) .finish() } @@ -393,6 +397,7 @@ pub mod regex { use lazy_static::lazy_static; use regex::Regex; + // ***** agent stream message regular expressions ***** lazy_static! { // echo from achievement-preserve setting discord chat link pub static ref CHAT_DISCORD_ECHO_RE: Regex = Regex::new( @@ -426,4 +431,12 @@ pub mod regex { r"changing state from\(([a-zA-Z]+)\) to\(([a-zA-Z]+)\)" ).unwrap(); } + + // ***** other misc expressions ***** + lazy_static! { + // Content-Range header value + pub static ref CONTENT_RANGE_RE: Regex = Regex::new( + r"^bytes (\d+)-(\d+)/(\d+)$" + ).unwrap(); + } } diff --git a/web/src/app/dashboard2/dashboard2.component.html b/web/src/app/dashboard2/dashboard2.component.html index 271ebdc..93ae250 100644 --- a/web/src/app/dashboard2/dashboard2.component.html +++ b/web/src/app/dashboard2/dashboard2.component.html @@ -1,3 +1,5 @@ + +

Status: {{status}} @@ -31,3 +33,17 @@

Create new save file

+ + + + + + diff --git a/web/src/app/dashboard2/dashboard2.component.ts b/web/src/app/dashboard2/dashboard2.component.ts index 5591665..e8c6f27 100644 --- a/web/src/app/dashboard2/dashboard2.component.ts +++ b/web/src/app/dashboard2/dashboard2.component.ts @@ -2,6 +2,9 @@ import { Component, OnInit } from '@angular/core'; import { MgmtServerRestApiService } from '../mgmt-server-rest-api/services'; import { OperationService } from '../operation.service'; import { environment } from 'src/environments/environment'; +import { faCheck, faUpload } from '@fortawesome/free-solid-svg-icons'; +import { delay, switchMap, tap } from 'rxjs/operators'; +import { concat, of } from 'rxjs'; @Component({ selector: 'app-dashboard2', @@ -16,9 +19,16 @@ export class Dashboard2Component implements OnInit { createSaveName: string; installVersionString: string; + uploadSavefileButtonLoading = false; + uploadSavefileButtonShowTickIcon = false; + uploadIcon = faUpload; + tickIcon = faCheck; + downloadAvailableVersions: string[] = []; saves: string[] = []; + savefileToUpload: File | null; + constructor( private apiClient: MgmtServerRestApiService, private operationService: OperationService, @@ -29,6 +39,7 @@ export class Dashboard2Component implements OnInit { this.selectedSave = ''; this.createSaveName = ''; this.installVersionString = ''; + this.savefileToUpload = null; } ngOnInit(): void { @@ -57,7 +68,7 @@ export class Dashboard2Component implements OnInit { } private internalUpdateAvailableVersions(): void { - // TODO + // TODO implement for install version dropdown } startServer(): void { @@ -158,4 +169,68 @@ export class Dashboard2Component implements OnInit { }); } + uploadSavefile(): void { + if (this.savefileToUpload === null) { + return; + } + + this.uploadSavefileButtonLoading = true; + + // trim ".zip" from end of filename + let savefile_id = this.savefileToUpload.name.split('.').slice(0, -1).join('.') + + let totalSize = this.savefileToUpload.size; + let chunkSizeBytes = 2 * 1000 * 1000; // 2 MB + let offset = 0; + let offsetsObservableArray = []; + + while (offset <= totalSize) { + console.debug("preparing observable for chunk from " + offset); + let currentChunkSize = Math.min(chunkSizeBytes, totalSize - offset); + + if (currentChunkSize === 0) { + // finalise with sentinel + offsetsObservableArray.push(this.apiClient.serverSavefilesSavefileIdPut({ + body: new Blob(), + savefile_id, + "Content-Range": `bytes ${offset}-${offset}/${totalSize}`, + }).pipe( + tap({ + complete: () => this.uploadSavefileButtonShowTickIcon = true, + error: e => alert('Error uploading save: ' + JSON.stringify(e)), + finalize: () => this.uploadSavefileButtonLoading = false, + }), + )); + console.debug("prepared sentinel"); + break; + } + + let observable = of([offset, currentChunkSize]).pipe( + switchMap(([offset, chunkSize]) => { + let chunk = this.savefileToUpload!.slice(offset, offset + chunkSize); + return this.apiClient.serverSavefilesSavefileIdPut({ + body: chunk, + savefile_id, + "Content-Range": `bytes ${offset}-${offset + chunkSize}/${totalSize}`, + }); + }), + tap({ + complete: () => this.uploadSavefileButtonShowTickIcon = true, + error: e => alert('Error uploading save: ' + JSON.stringify(e)), + finalize: () => this.uploadSavefileButtonLoading = false, + }), + ); + offsetsObservableArray.push(observable); + offset += currentChunkSize; + } + + // do them in order + concat(...offsetsObservableArray).pipe( + delay(3000), + ).subscribe(() => { + this.uploadSavefileButtonShowTickIcon = false + this.internalUpdateSaves(); + }); + + } } diff --git a/web/src/app/mods/mod-settings/mod-settings.component.html b/web/src/app/mods/mod-settings/mod-settings.component.html index 9f1278a..fe89456 100644 --- a/web/src/app/mods/mod-settings/mod-settings.component.html +++ b/web/src/app/mods/mod-settings/mod-settings.component.html @@ -1,4 +1,5 @@ +

Mod settings

@@ -12,9 +13,12 @@

Mod settings

-