Upload mod-settings-dat, chunked upload for saves

local_build.sh

@@ -6,4 +6,4 @@ else
     GIT_COMMIT_HASH="$(git rev-parse HEAD)-dirty"
 fi
 
-env DOCKER_BUILDKIT=1 docker-compose -f docker-compose.yml -f docker-compose.local.yml build --build-arg GIT_COMMIT_HASH=$GIT_COMMIT_HASH
+env DOCKER_BUILDKIT=1 docker compose -f docker-compose.yml -f docker-compose.local.yml build --build-arg GIT_COMMIT_HASH=$GIT_COMMIT_HASH

local_down.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-docker-compose -f docker-compose.yml -f docker-compose.local.yml down
+docker compose -f docker-compose.yml -f docker-compose.local.yml down

local_up.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
 
-docker-compose $@ -f docker-compose.yml -f docker-compose.local.yml up -d
+docker compose $@ -f docker-compose.yml -f docker-compose.local.yml up -d

openapi/mgmt-server-rest.yaml

@@ -163,6 +163,11 @@ paths:
           required: true
           schema:
             type: string
+        - name: Content-Range
+          in: header
+          required: true
+          schema:
+            type: string
       requestBody:
         required: true
         content:

src/agent/main.rs

@@ -1035,12 +1035,13 @@ impl AgentController {
                 let chunks = savebytes.bytes.chunks(MAX_WS_PAYLOAD_BYTES);
                 let mut i = 0;
                 for chunk in chunks {
+                    let chunk_len = chunk.len();
                     let msg = AgentOutMessage::SaveFile(SaveBytes {
-                        multipart_seqnum: Some(i),
+                        multipart_start: Some(i),
                         bytes: chunk.to_vec(),
                     });
                     self.reply(msg, &operation_id).await;
-                    i += 1;
+                    i += chunk_len;
                 }
                 self.reply_success(AgentOutMessage::SaveFile(SaveBytes::sentinel(i)), operation_id).await;
             },
@@ -1070,18 +1071,10 @@ impl AgentController {
     }
 
     async fn save_set(&self, save_name: String, savebytes: SaveBytes, operation_id: OperationId) {
-        match util::saves::exists_savefile(&save_name).await {
-            Ok(false) => {
-                if let Err(e) = util::saves::set_savefile(&save_name, savebytes).await {
-                    self.reply_failed(AgentOutMessage::Error(format!("Failed to set savefile with name `{}`: {:?}", &save_name, e)), operation_id).await
-                } else {
-                    self.reply_success(AgentOutMessage::Ok, operation_id).await
-                }
-            },
-            Ok(true) => self.reply_failed(AgentOutMessage::Error(format!("Savefile with name {} already exists", &save_name)), operation_id).await,
-            Err(e) => self.reply_failed(
-                AgentOutMessage::Error(format!("Failed to read saves: {:?}", e)),
-                operation_id).await,
+        if let Err(e) = util::saves::set_savefile(&save_name, savebytes).await {
+            self.reply_failed(AgentOutMessage::Error(format!("Failed to set savefile with name `{}`: {:?}", &save_name, e)), operation_id).await
+        } else {
+            self.reply_success(AgentOutMessage::Ok, operation_id).await
         }
     }
 

src/agent/util/saves.rs

@@ -1,8 +1,8 @@
-use std::path::{Path, PathBuf};
+use std::{io::SeekFrom, path::{Path, PathBuf}};
 
 use fctrl::schema::{Save, SaveBytes};
 use log::{error, info, warn};
-use tokio::fs;
+use tokio::{fs::{self, OpenOptions}, io::{AsyncSeekExt, AsyncWriteExt}};
 
 use crate::{consts::*, error::Result};
 
@@ -63,14 +63,34 @@ pub async fn list_savefiles() -> Result<Vec<Save>> {
 
 pub async fn set_savefile(save_name: impl AsRef<str>, savebytes: SaveBytes) -> Result<()> {
     let bytes_length = savebytes.bytes.len();
-    match fs::write(get_savefile_path(save_name.as_ref()), savebytes.bytes).await {
-        Ok(()) => {
-            info!("Successfully set savefile `{}`, wrote {} bytes", save_name.as_ref(), bytes_length);
-            Ok(())
-        },
-        Err(e) => {
-            error!("Failed to set savefile `{}`: {:?}", save_name.as_ref(), e);
-            Err(e.into())
+    if let Some(start_byte) = savebytes.multipart_start {
+        // partial file write
+        let filename = get_savefile_path(save_name.as_ref());
+        // create if not exist
+        let mut file = OpenOptions::new().write(true).create(true).open(filename).await?;
+        if savebytes.is_sentinel() {
+            // finalise and trim down to size
+            file.set_len(start_byte as u64).await?;
+            info!("Successfully finalised savefile `{}`, final length {} bytes", save_name.as_ref(), start_byte);
+        } else {
+            // seek to correct write location before writing
+            file.seek(SeekFrom::Start(start_byte as u64)).await?;
+            file.write_all(&savebytes.bytes).await?;
+            file.flush().await?;
+            info!("Successfully wrote to savefile `{}`, wrote {} bytes from offset {}", save_name.as_ref(), savebytes.bytes.len(), start_byte);
+        }
+        Ok(())
+    } else {
+        // write the whole file
+        match fs::write(get_savefile_path(save_name.as_ref()), savebytes.bytes).await {
+            Ok(()) => {
+                info!("Successfully set savefile `{}`, wrote {} bytes", save_name.as_ref(), bytes_length);
+                Ok(())
+            },
+            Err(e) => {
+                error!("Failed to set savefile `{}`: {:?}", save_name.as_ref(), e);
+                Err(e.into())
+            }
         }
     }
 }

src/mgmt-server/guards.rs

@@ -1,3 +1,4 @@
+use fctrl::schema::regex::CONTENT_RANGE_RE;
 use log::error;
 use rocket::{
     http::Status,
@@ -31,6 +32,81 @@ impl<'r> FromRequest<'r> for HostHeader<'r> {
     }
 }
 
+pub struct ContentLengthHeader {
+    pub length: usize,
+}
+
+#[derive(Debug)]
+pub enum ContentLengthHeaderError {
+    Missing,
+    Format,
+}
+
+#[rocket::async_trait]
+impl<'r> FromRequest<'r> for ContentLengthHeader {
+    type Error = ContentLengthHeaderError;
+
+    async fn from_request(request: &'r rocket::Request<'_>) -> Outcome<Self, Self::Error> {
+        match request.headers().get_one("Content-Length") {
+            Some(h) => {
+                if let Ok(length) = h.parse::<usize>() {
+                    Outcome::Success(ContentLengthHeader {
+                        length,
+                    })
+                } else {
+                    Outcome::Error((Status::BadRequest, ContentLengthHeaderError::Format))
+                }
+            },
+            None => Outcome::Error((Status::BadRequest, ContentLengthHeaderError::Missing)),
+        }
+    }
+}
+
+pub struct ContentRangeHeader {
+    pub start: usize,
+    pub end: usize,
+    pub length: usize,
+}
+
+#[derive(Debug)]
+pub enum ContentRangeHeaderError {
+    Missing,
+    Format,
+}
+
+#[rocket::async_trait]
+impl<'r> FromRequest<'r> for ContentRangeHeader {
+    type Error = ContentRangeHeaderError;
+
+    async fn from_request(request: &'r rocket::Request<'_>) -> Outcome<Self, Self::Error> {
+        match request.headers().get_one("Content-Range") {
+            Some(h) => {
+                if let Some(captures) = CONTENT_RANGE_RE.captures(h) {
+                    if let Some(start) = captures.get(1) {
+                        if let Ok(start) = start.as_str().parse::<usize>() {
+                            if let Some(end) = captures.get(2) {
+                                if let Ok(end) = end.as_str().parse::<usize>() {
+                                    if let Some(length) = captures.get(3) {
+                                        if let Ok(length) = length.as_str().parse::<usize>() {
+                                            return Outcome::Success(ContentRangeHeader {
+                                                start,
+                                                end,
+                                                length,
+                                            });
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                Outcome::Error((Status::BadRequest, ContentRangeHeaderError::Format))
+            },
+            None => Outcome::Error((Status::BadRequest, ContentRangeHeaderError::Missing)),
+        }
+    }
+}
+
 #[derive(Debug)]
 pub enum AuthError {
     Missing,

src/mgmt-server/routes/download.rs

@@ -49,8 +49,8 @@ async fn download_save(
             Ok(m) => {
                 match m.content {
                     AgentOutMessage::SaveFile(sb) => {
-                        if sb.bytes.len() == 0 {
-                            info!("get_savefile completed with total multiparts = {:?}", sb.multipart_seqnum);
+                        if sb.is_sentinel() {
+                            info!("get_savefile completed with total multipart length = {:?}", sb.multipart_start);
                             None
                         } else {
                             Some(sb.bytes)

src/mgmt-server/routes/server.rs

@@ -8,12 +8,12 @@ use factorio_file_parser::ModSettings;
 use fctrl::schema::{
     mgmt_server_rest::*, FactorioVersion, ModSettingsBytes, RconConfig, SaveBytes, SecretsObject, ServerSettingsConfig, ServerStartSaveFile, ServerStatus
 };
-use rocket::{delete, serde::json::Json};
+use rocket::{data::ToByteUnit, delete, serde::json::Json, Data};
 use rocket::{get, post, put};
 use rocket::{http::Status, State};
 
 use crate::{
-    auth::AuthorizedUser, clients::AgentApiClient, guards::HostHeader, link_download::{LinkDownloadManager, LinkDownloadTarget}, ws::WebSocketServer
+    auth::AuthorizedUser, clients::AgentApiClient, guards::{ContentLengthHeader, ContentRangeHeader, HostHeader}, link_download::{LinkDownloadManager, LinkDownloadTarget}, ws::WebSocketServer
 };
 use crate::{error::Result, routes::WsStreamingResponder};
 
@@ -158,11 +158,14 @@ pub async fn put_savefile(
     _a: AuthorizedUser,
     agent_client: &State<Arc<AgentApiClient>>,
     id: String,
-    body: Vec<u8>,
+    body: Data<'_>,
+    content_length: ContentLengthHeader,
+    content_range: ContentRangeHeader,
 ) -> Result<()> {
+    let chunk_stream = body.open(content_length.length.bytes());
     let savebytes = SaveBytes {
-        multipart_seqnum: None,
-        bytes: body,
+        multipart_start: Some(content_range.start),
+        bytes: chunk_stream.into_bytes().await?.into_inner(),
     };
     agent_client.save_put(id, savebytes).await?;
     Ok(())

src/schema.rs

@@ -231,38 +231,42 @@ pub struct Save {
 
 #[derive(Deserialize, Serialize)]
 pub struct SaveBytes {
-    pub multipart_seqnum: Option<u32>,
+    pub multipart_start: Option<usize>,
     #[serde(with = "base64")]
     pub bytes: Vec<u8>,
 }
 
 impl SaveBytes {
     pub fn new(bytes: Vec<u8>) -> SaveBytes {
         SaveBytes {
-            multipart_seqnum: None,
+            multipart_start: None,
             bytes,
         }
     }
 
-    pub fn sentinel(total_num_parts: u32) -> SaveBytes {
+    pub fn sentinel(total_length: usize) -> SaveBytes {
         SaveBytes {
-            multipart_seqnum: Some(total_num_parts),
+            multipart_start: Some(total_length),
             bytes: vec![],
         }
     }
+
+    pub fn is_sentinel(&self) -> bool {
+        self.bytes.len() == 0
+    }
 }
 
 impl std::fmt::Debug for SaveBytes {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         if self.bytes.len() > 16 {
             let debug_bytes = format!("{:?}...", &self.bytes[..16]);
             f.debug_struct("SaveBytes")
-                .field("multipart_seqnum", &self.multipart_seqnum)
+                .field("multipart_start", &self.multipart_start)
                 .field("bytes", &debug_bytes)
                 .finish()
         } else {
             f.debug_struct("SaveBytes")
-                .field("multipart_seqnum", &self.multipart_seqnum)
+                .field("multipart_start", &self.multipart_start)
                 .field("bytes", &self.bytes)
                 .finish()
         }
@@ -393,6 +397,7 @@ pub mod regex {
     use lazy_static::lazy_static;
     use regex::Regex;
 
+    // ***** agent stream message regular expressions *****
     lazy_static! {
         // echo from achievement-preserve setting discord chat link
         pub static ref CHAT_DISCORD_ECHO_RE: Regex = Regex::new(
@@ -426,4 +431,12 @@ pub mod regex {
             r"changing state from\(([a-zA-Z]+)\) to\(([a-zA-Z]+)\)"
         ).unwrap();
     }
+
+    // ***** other misc expressions *****
+    lazy_static! {
+        // Content-Range header value
+        pub static ref CONTENT_RANGE_RE: Regex = Regex::new(
+            r"^bytes (\d+)-(\d+)/(\d+)$"
+        ).unwrap();
+    }
 }

web/src/app/dashboard2/dashboard2.component.html

@@ -1,3 +1,5 @@
+<link href="https://fonts.googleapis.com/icon?family=Material+Icons&display=block" rel="stylesheet">
+
 <p>
   Status: {{status}}
   <button (click)="stopServer()">Stop</button>
@@ -31,3 +33,17 @@ <h4>Create new save file</h4>
 </mat-form-field>
 
 <button mat-raised-button (click)="createSave()">Create</button>
+
+<mat-form-field>
+  <ngx-mat-file-input [(ngModel)]="savefileToUpload" [multiple]="false" [accept]="'.zip'">
+  </ngx-mat-file-input>
+</mat-form-field>
+<button id="upload-dat-button" class="button"
+  (click)="uploadSavefile()"
+  [disabled]="savefileToUpload === null"
+  [ngClass]="{'is-loading': uploadSavefileButtonLoading}">
+  <span class="icon is-small">
+    <fa-icon class="fa-fw" [icon]="uploadSavefileButtonShowTickIcon ? tickIcon : uploadIcon"></fa-icon>
+  </span>
+  <span>Upload save</span>
+</button>