From a9ba7eadbd4b87e321dc8f5ee3c7f39a2ad68a5c Mon Sep 17 00:00:00 2001 From: circlesabound Date: Mon, 21 Oct 2024 11:48:23 +1100 Subject: [PATCH] Add system resource statistics #4 --- Cargo.lock | 142 ++++++++++++++++-- Cargo.toml | 7 +- openapi/mgmt-server-rest.yaml | 32 ++++ src/agent/error.rs | 1 + src/agent/main.rs | 21 +++ src/agent/server/proc.rs | 32 ++++ src/mgmt-server/clients.rs | 12 ++ src/mgmt-server/discord.rs | 26 +++- src/mgmt-server/main.rs | 1 + src/mgmt-server/routes/mod.rs | 1 + src/mgmt-server/routes/system.rs | 25 +++ src/schema.rs | 17 +++ .../app/dashboard2/dashboard2.component.html | 4 + .../app/dashboard2/dashboard2.component.ts | 16 +- 14 files changed, 320 insertions(+), 17 deletions(-) create mode 100644 src/mgmt-server/routes/system.rs diff --git a/Cargo.lock b/Cargo.lock index c89d46d..5eaebda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -445,6 +445,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crypto-common" version = "0.1.6" @@ -737,6 +762,7 @@ dependencies = [ "stream-cancel", "strum", "strum_macros", + "sysinfo", "tar", "tokio", "tokio-stream", @@ -938,7 +964,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows", + "windows 0.48.0", ] [[package]] @@ -1237,7 +1263,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core", + "windows-core 0.52.0", ] [[package]] @@ -1578,6 +1604,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1680,9 +1715,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.103" +version = "0.9.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" dependencies = [ "cc", "libc", @@ -1893,6 +1928,26 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "rcon" version = "0.6.0" @@ -2401,9 +2456,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -2663,6 +2718,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "sysinfo" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ae3f4f7d64646c46c4cae4e3f01d1c5d255c7406fdd7c7f999a94e488791" +dependencies = [ + "core-foundation-sys", + "libc", + "memchr", + "ntapi", + "rayon", + "windows 0.57.0", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -3194,9 +3263,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", "serde", @@ -3409,6 +3478,16 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core 0.57.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" version = "0.52.0" @@ -3418,17 +3497,60 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.79", +] + [[package]] name = "windows-registry" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-strings", "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.2.0" @@ -3444,7 +3566,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-targets 0.52.6", ] diff --git a/Cargo.toml b/Cargo.toml index 172a980..92dc7cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,11 +29,12 @@ reqwest = { version = "0.12.8", features = [ "json" ] } rocksdb = "0.22" rocket = { version = "0.5.1", features = [ "json" ] } serde = { version = "1.0.210", features = [ "derive" ] } -serde_json = "1.0.128" +serde_json = "1.0.132" serenity = { version = "0.12.2", default-features = false, features = [ "client", "gateway", "rustls_backend", "model", "cache" ] } stream-cancel = "0.8.2" strum = "0.26.3" strum_macros = "0.26.4" +sysinfo = "0.32.0" tar = "0.4.42" tokio = { version = "1.40.0", features = [ "full" ] } tokio-stream = { version = "0.1.16", features = [ "sync" ] } @@ -43,7 +44,7 @@ toml = "0.8.19" unicode-xid = "0.2.6" url = "2.5.2" urlencoding = "2.1.3" -uuid = { version = "1.10.0", features = [ "serde", "v4" ] } +uuid = { version = "1.11.0", features = [ "serde", "v4" ] } xz2 = "0.1.7" [build-dependencies] @@ -53,7 +54,7 @@ vergen-gitcl = { version = "1.0", features = [ "build" ] } serial_test = "3.1.1" [target.'cfg(not(windows))'.dependencies] -openssl-sys = { version = "0.9.103", features = [ "vendored" ] } +openssl-sys = { version = "0.9.104", features = [ "vendored" ] } [[bin]] name = "agent" diff --git a/openapi/mgmt-server-rest.yaml b/openapi/mgmt-server-rest.yaml index d6c1221..151f057 100644 --- a/openapi/mgmt-server-rest.yaml +++ b/openapi/mgmt-server-rest.yaml @@ -531,6 +531,16 @@ paths: application/json: schema: $ref: '#/components/schemas/MetricsPaginationObject' + /system/monitor: + get: + summary: Get system resource utilisation stats + responses: + '200': + description: System resource utilisation stats + content: + application/json: + schema: + $ref: '#/components/schemas/SystemResources' /buildinfo: get: summary: Gets build information for all components @@ -1046,6 +1056,28 @@ components: type: array items: $ref: '#/components/schemas/MetricsDataPoint' + SystemResources: + type: object + required: + - cpu_total + - cpus + - mem_total_bytes + - mem_used_bytes + properties: + cpu_total: + type: number + cpus: + type: array + items: + type: number + mem_total_bytes: + type: integer + minimum: 0 + format: int64 + mem_used_bytes: + type: integer + minimum: 0 + format: int64 BuildInfoObject: properties: agent: diff --git a/src/agent/error.rs b/src/agent/error.rs index 5361cdf..e29fa0a 100644 --- a/src/agent/error.rs +++ b/src/agent/error.rs @@ -26,6 +26,7 @@ pub enum Error { // Generic Aggregate(Vec), + Timeout, // Generic wrappers around external error types FactorioDatFileSerde(factorio_file_parser::Error), diff --git a/src/agent/main.rs b/src/agent/main.rs index 50f3ad5..a289cc2 100644 --- a/src/agent/main.rs +++ b/src/agent/main.rs @@ -281,6 +281,13 @@ impl AgentController { self.build_version(operation_id).await; } + // ***************** + // System monitoring + // ***************** + AgentRequest::SystemResources => { + self.system_resources(operation_id).await; + } + // *********************** // Installation management // *********************** @@ -541,6 +548,20 @@ impl AgentController { .await; } + async fn system_resources(&self, operation_id: OperationId) { + match self.proc_manager.system_resources().await { + Ok(system_resources) => { + self.reply_success(AgentOutMessage::SystemResources(system_resources), operation_id).await; + }, + Err(e) => { + self.reply_failed( + AgentOutMessage::Error(format!("Failed to fetch system resource statistics: {:?}", e)), + operation_id + ).await; + }, + } + } + async fn version_install( &self, version_to_install: FactorioVersion, diff --git a/src/agent/server/proc.rs b/src/agent/server/proc.rs index aa230dd..c3916d7 100644 --- a/src/agent/server/proc.rs +++ b/src/agent/server/proc.rs @@ -3,6 +3,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; use log::debug; +use sysinfo::{CpuRefreshKind, MemoryRefreshKind, RefreshKind, System}; use tokio::{ io::{AsyncBufRead, AsyncBufReadExt}, sync::{Mutex, RwLock}, @@ -18,16 +19,47 @@ use crate::{ use fctrl::schema::regex::*; pub struct ProcessManager { + sysinfo: Arc>, running_instance: Arc>>, } impl ProcessManager { pub fn new() -> Self { + let sysinfo_refresh_specifics = RefreshKind::new() + .with_cpu(CpuRefreshKind::new().with_cpu_usage()) + .with_memory(MemoryRefreshKind::new().with_ram()); + let sysinfo = Arc::new(RwLock::new(System::new_with_specifics(sysinfo_refresh_specifics))); + let sysinfo_arc = Arc::clone(&sysinfo); + tokio::spawn(async move { + loop { + // refresh system stats every 10 seconds + tokio::time::sleep(Duration::from_secs(10)).await; + if let Ok(mut sysinfo) = tokio::time::timeout(Duration::from_millis(250), sysinfo_arc.write()).await { + sysinfo.refresh_specifics(sysinfo_refresh_specifics); + } else { + warn!("Unable to acquire write lock for sysinfo, skipping this cycle"); + } + } + }); ProcessManager { + sysinfo, running_instance: Arc::new(Mutex::new(None)), } } + pub async fn system_resources(&self) -> Result { + if let Ok(sysinfo) = tokio::time::timeout(Duration::from_millis(250), self.sysinfo.read()).await { + Ok(SystemResources { + cpu_total: sysinfo.global_cpu_usage(), + cpus: sysinfo.cpus().into_iter().map(|cpu| cpu.cpu_usage()).collect(), + mem_total_bytes: sysinfo.total_memory(), + mem_used_bytes: sysinfo.used_memory(), + }) + } else { + Err(Error::Timeout) + } + } + pub async fn status(&self) -> ProcessStatus { if !self.instance_is_running_or_cleanup().await { ProcessStatus::NotRunning diff --git a/src/mgmt-server/clients.rs b/src/mgmt-server/clients.rs index f4011a0..fcc952a 100644 --- a/src/mgmt-server/clients.rs +++ b/src/mgmt-server/clients.rs @@ -76,6 +76,17 @@ impl AgentApiClient { .await } + pub async fn system_resources(&self) -> Result { + let request = AgentRequest::SystemResources; + let (_id, sub) = self.send_request_and_subscribe(request).await?; + + response_or_timeout(sub, Duration::from_millis(500), |r| match r.content { + AgentOutMessage::SystemResources(s) => Ok(s), + m => Err(default_message_handler(m)), + }) + .await + } + pub async fn version_install( &self, version: FactorioVersion, @@ -494,6 +505,7 @@ fn default_message_handler(agent_message: AgentOutMessage) -> Error { | AgentOutMessage::SaveFile(_) | AgentOutMessage::SaveList(_) | AgentOutMessage::ServerStatus(_) + | AgentOutMessage::SystemResources(_) | AgentOutMessage::Ok => Error::AgentCommunicationError, AgentOutMessage::Error(e) => Error::AgentInternalError(e), AgentOutMessage::ConflictingOperation => { diff --git a/src/mgmt-server/discord.rs b/src/mgmt-server/discord.rs index 8453a3e..b14f855 100644 --- a/src/mgmt-server/discord.rs +++ b/src/mgmt-server/discord.rs @@ -345,10 +345,11 @@ impl EventHandler for Handler { if let Interaction::Command(command) = interaction { let response = match command.data.name.as_str() { "server-save" => Some(commands::server_save(self.agent_client.as_ref()).await), + "system-resources" => Some(commands::system_resources(self.agent_client.as_ref()).await), _ => { warn!("unimplemented interaction command"); None - }, + } }; if let Some(response) = response { if let Err(e) = command.create_response(&ctx.http, response).await { @@ -361,6 +362,7 @@ impl EventHandler for Handler { async fn ready(&self, ctx: Context, _ready: Ready) { if let Err(e) = self.guild_id.set_commands(&ctx.http, vec![ CreateCommand::new("server-save").description("Trigger a server-side save"), + CreateCommand::new("system-resources").description("Get system resource usage statistics") ]).await { error!("Error creating slash commands: {:?}", e); } @@ -396,8 +398,8 @@ impl EventHandler for Handler { } mod commands { - use log::error; - use serenity::all::{CreateInteractionResponse, CreateInteractionResponseMessage}; + use log::{error, info}; + use serenity::all::{CreateEmbed, CreateInteractionResponse, CreateInteractionResponseMessage}; use crate::clients::AgentApiClient; @@ -411,4 +413,22 @@ mod commands { CreateInteractionResponse::Message(data) } } + + pub async fn system_resources(agent_client: &AgentApiClient) -> CreateInteractionResponse { + match agent_client.system_resources().await { + Ok(system_resources) => { + info!("{:?}", system_resources); + let embed = CreateEmbed::new() + .title("System resource statistics") + .field("CPU total", format!("{:.2}%", system_resources.cpu_total), false) + .fields(system_resources.cpus.iter().enumerate().map(|(i, cpu)| (format!("cpu{}", i), format!("{:.2}%", cpu), true))) + .field("Memory used", format!("{:.2}%", (system_resources.mem_used_bytes as f64 / system_resources.mem_total_bytes as f64) * 100 as f64), false); + CreateInteractionResponse::Message(CreateInteractionResponseMessage::new().embed(embed)) + }, + Err(e) => { + let data = CreateInteractionResponseMessage::new().content(format!("Failed to get system resource statistics: {:?}", e)); + CreateInteractionResponse::Message(data) + }, + } + } } diff --git a/src/mgmt-server/main.rs b/src/mgmt-server/main.rs index 5a4310e..d5d1192 100644 --- a/src/mgmt-server/main.rs +++ b/src/mgmt-server/main.rs @@ -187,6 +187,7 @@ async fn main() -> std::result::Result<(), Box> { routes::server::get_mod_settings_dat, routes::server::put_mod_settings_dat, routes::server::send_rcon_command, + routes::system::monitor, routes::logs::get, routes::logs::stream, routes::metrics::get, diff --git a/src/mgmt-server/routes/mod.rs b/src/mgmt-server/routes/mod.rs index 223ea64..7837e2a 100644 --- a/src/mgmt-server/routes/mod.rs +++ b/src/mgmt-server/routes/mod.rs @@ -17,6 +17,7 @@ pub mod metrics; pub mod options; pub mod proxy; pub mod server; +pub mod system; pub struct LinkDownloadResponder { path: String, diff --git a/src/mgmt-server/routes/system.rs b/src/mgmt-server/routes/system.rs new file mode 100644 index 0000000..5f47c98 --- /dev/null +++ b/src/mgmt-server/routes/system.rs @@ -0,0 +1,25 @@ +use std::sync::Arc; + +use log::error; +use rocket::{get, serde::json::Json, State}; + +use crate::clients::AgentApiClient; +use crate::error::Result; + +#[get("/system/monitor")] +pub async fn monitor( + agent_client: &State>, +) -> Result> { + match agent_client.system_resources().await { + Ok(s) => Ok(Json(fctrl::schema::mgmt_server_rest::SystemResources { + cpu_total: s.cpu_total, + cpus: s.cpus, + mem_total_bytes: s.mem_total_bytes as i64, + mem_used_bytes: s.mem_used_bytes as i64, + })), + Err(e) => { + error!("Error retrieving agent build version: {:?}", e); + Err(e) + }, + } +} diff --git a/src/schema.rs b/src/schema.rs index 7cdc8d0..eabdaeb 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -43,6 +43,14 @@ pub enum AgentRequest { /// Get the build info for the agent BuildVersion, + // ********************************* + // * System monitoring * + // ********************************* + // + // + /// Get system resource statistics + SystemResources, + // ********************************* // * Installation management * // ********************************* @@ -205,6 +213,7 @@ pub enum AgentOutMessage { SaveList(Vec), SaveNotFound, ServerStatus(ServerStatus), + SystemResources(SystemResources), } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -403,6 +412,14 @@ pub enum InternalServerState { Closed, } +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct SystemResources { + pub cpu_total: f32, + pub cpus: Vec, + pub mem_total_bytes: u64, + pub mem_used_bytes: u64, +} + /// module for serde to handle binary fields mod base64 { use base64::Engine; diff --git a/web/src/app/dashboard2/dashboard2.component.html b/web/src/app/dashboard2/dashboard2.component.html index 74886e4..1bd3720 100644 --- a/web/src/app/dashboard2/dashboard2.component.html +++ b/web/src/app/dashboard2/dashboard2.component.html @@ -56,6 +56,10 @@

Saves:

Upload save +

CPU total: {{cpuTotal ?? "n/a"}}%

+

CPU{{i}}: {{cpu}}%

+

Memory used: {{mem_used_prct ?? "n/a"}}%

+
diff --git a/web/src/app/dashboard2/dashboard2.component.ts b/web/src/app/dashboard2/dashboard2.component.ts index 10daede..d7f970a 100644 --- a/web/src/app/dashboard2/dashboard2.component.ts +++ b/web/src/app/dashboard2/dashboard2.component.ts @@ -4,7 +4,7 @@ import { OperationService } from '../operation.service'; import { environment } from 'src/environments/environment'; import { faCheck, faFileCirclePlus, faUpload } from '@fortawesome/free-solid-svg-icons'; import { delay, switchMap, tap } from 'rxjs/operators'; -import { concat, of } from 'rxjs'; +import { concat, interval, of } from 'rxjs'; @Component({ selector: 'app-dashboard2', @@ -32,6 +32,10 @@ export class Dashboard2Component implements OnInit { createSaveModalActive = false; createSaveName: string | null; + cpuTotal: number | null = null; + cpus: number[] = []; + mem_used_prct: number | null = null; + constructor( private apiClient: MgmtServerRestApiService, private operationService: OperationService, @@ -47,12 +51,14 @@ export class Dashboard2Component implements OnInit { ngOnInit(): void { this.internalUpdateAll(); + interval(15000).subscribe(_ => this.internalUpdateSystemResources()); } private internalUpdateAll(): void { this.internalUpdateVersion(); this.internalUpdateSaves(); this.internalUpdateGameStatus(); + this.internalUpdateSystemResources(); } private internalUpdateGameStatus(): void { @@ -78,6 +84,14 @@ export class Dashboard2Component implements OnInit { // TODO implement for install version dropdown } + private internalUpdateSystemResources(): void { + this.apiClient.systemMonitorGet().subscribe(s => { + this.cpuTotal = s.cpu_total; + this.cpus = s.cpus; + this.mem_used_prct = s.mem_used_bytes / s.mem_total_bytes * 100; + }); + } + startServer(): void { if (this.selectedSave) { const payload = {