From 52180b9b4e8b9e269eb891ee31c327b851bf42fc Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Mon, 18 Dec 2023 09:20:30 -0800 Subject: [PATCH 01/31] config demo v1 --- .../constants.default.yaml | 0 .../pub_sub_service_settings.default.yaml | 0 Cargo.lock | 20 +++++++++++++++++++ Cargo.toml | 1 + common/Cargo.toml | 3 ++- common/src/config_utils.rs | 17 ++++++++++++---- 6 files changed, 36 insertions(+), 5 deletions(-) rename .agemo/{config => default}/constants.default.yaml (100%) rename .agemo/{config => default}/pub_sub_service_settings.default.yaml (100%) diff --git a/.agemo/config/constants.default.yaml b/.agemo/default/constants.default.yaml similarity index 100% rename from .agemo/config/constants.default.yaml rename to .agemo/default/constants.default.yaml diff --git a/.agemo/config/pub_sub_service_settings.default.yaml b/.agemo/default/pub_sub_service_settings.default.yaml similarity index 100% rename from .agemo/config/pub_sub_service_settings.default.yaml rename to .agemo/default/pub_sub_service_settings.default.yaml diff --git a/Cargo.lock b/Cargo.lock index fc48c2c..c27602f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -411,6 +411,7 @@ version = "0.1.0" dependencies = [ "config", "home", + "include_dir", "log", "serde", "serde_derive", @@ -911,6 +912,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "include_dir" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18762faeff7122e89e0857b02f7ce6fcc0d101d5e9ad2ad7846cc01d61b7f19e" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b139284b5cf57ecfa712bcc66950bb635b31aff41c188e8a4cfc758eca374a3f" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "1.9.3" diff --git a/Cargo.toml b/Cargo.toml index 9810deb..14f361c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ ctrlc = { version = "3.0", features = ["termination"] } env_logger = "0.10" futures = "0.3" home = "0.5.5" +include_dir = "0.7.3" log = "^0.4" paho-mqtt = "0.12" prost = "0.12" diff --git a/common/Cargo.toml b/common/Cargo.toml index a5a68a9..844eb52 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -11,7 +11,8 @@ license = "MIT" [dependencies] config = { workspace = true } home = { workspace = true } +include_dir = { workspace = true } log = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true } \ No newline at end of file diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 4b15b54..10fe928 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -4,9 +4,10 @@ use std::{env, path::Path}; -use config::File; +use config::{File, FileFormat}; use home::home_dir; use serde::Deserialize; +use include_dir::{include_dir, Dir}; pub const YAML_EXT: &str = "yaml"; @@ -14,6 +15,8 @@ const CONFIG_DIR: &str = "config"; const DOT_AGEMO_DIR: &str = ".agemo"; const AGEMO_HOME: &str = "AGEMO_HOME"; +const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../.agemo/default"); + /// Read config from layered configuration files. /// Searches for `{config_file_name}.default.{config_file_ext}` as the base configuration in `$AGEMO_HOME`, /// then searches for overrides named `{config_file_name}.{config_file_ext}` in the current directory and `$AGEMO_HOME`. @@ -29,7 +32,12 @@ pub fn read_from_files<T>( where T: for<'a> Deserialize<'a>, { - let default_config_file = format!("{config_file_name}.default.{config_file_ext}"); + // Get default config. + let default_config_filename = format!("{config_file_name}.default.{config_file_ext}"); + let default_config_file = DEFAULT_DIR.get_file(default_config_filename).unwrap(); + let default_config_contents_str = default_config_file.contents_utf8().unwrap(); + + // Get override_files let overrides_file = format!("{config_file_name}.{config_file_ext}"); let config_path = match env::var(AGEMO_HOME) { @@ -52,7 +60,7 @@ where }; // The path below resolves to {config_path}/{default_config_file}. - let default_config_file_path = config_path.join(default_config_file); + // let default_config_file_path = config_path.join(default_config_file); // The path below resolves to {current_dir}/{overrides_file}. let current_dir_config_file_path = env::current_dir()?.join(overrides_file.clone()); @@ -61,7 +69,8 @@ where let overrides_config_file_path = config_path.join(overrides_file); let config_store = config::Config::builder() - .add_source(File::from(default_config_file_path)) + .add_source(File::from_str(default_config_contents_str, FileFormat::Yaml)) + //.add_source(File::from(default_config_file_path)) .add_source(File::from(current_dir_config_file_path).required(false)) .add_source(File::from(overrides_config_file_path).required(false)) .build()?; From 33efa4b9cb86d81bde0f0a6a8ac31a34ce44b96d Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:13:12 -0800 Subject: [PATCH 02/31] Add command line configuration --- .../pub_sub_service_settings.default.yaml | 11 -- Cargo.lock | 122 +++++++++++++++++- Cargo.toml | 5 + .../config/default.yaml | 16 +++ common/src/config_utils.rs | 34 +++-- proc-macros/Cargo.toml | 18 +++ proc-macros/src/config_source.rs | 95 ++++++++++++++ proc-macros/src/lib.rs | 17 +++ pub-sub-service/Cargo.toml | 2 + pub-sub-service/src/load_config.rs | 56 +++++++- pub-sub-service/src/main.rs | 16 ++- 11 files changed, 356 insertions(+), 36 deletions(-) delete mode 100644 .agemo/default/pub_sub_service_settings.default.yaml rename .agemo/default/constants.default.yaml => common/config/default.yaml (52%) create mode 100644 proc-macros/Cargo.toml create mode 100644 proc-macros/src/config_source.rs create mode 100644 proc-macros/src/lib.rs diff --git a/.agemo/default/pub_sub_service_settings.default.yaml b/.agemo/default/pub_sub_service_settings.default.yaml deleted file mode 100644 index c65099a..0000000 --- a/.agemo/default/pub_sub_service_settings.default.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# -# Pub Sub Service Settings -# - -# The IP address and port number that the Pub Sub Service listens on for requests. -# Example: "0.0.0.0:50051" -pub_sub_authority: "0.0.0.0:50051" - -# The URI of the messaging service used to facilitate publish and subscribe functionality. -# Example: "mqtt://0.0.0.0:1883" -messaging_uri: "mqtt://0.0.0.0:1883" diff --git a/Cargo.lock b/Cargo.lock index c27602f..2d29e73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,54 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "anyhow" version = "1.0.75" @@ -396,6 +444,46 @@ dependencies = [ "uuid", ] +[[package]] +name = "clap" +version = "4.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + [[package]] name = "cmake" version = "0.1.50" @@ -405,6 +493,12 @@ dependencies = [ "cc", ] +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "common" version = "0.1.0" @@ -1396,6 +1490,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macros" +version = "0.1.0" +dependencies = [ + "config", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "prost" version = "0.12.3" @@ -1467,12 +1571,14 @@ version = "0.1.0" dependencies = [ "async-std", "async-trait", + "clap", "common", "config", "env_logger", "futures", "log", "paho-mqtt", + "proc-macros", "proto", "serde", "serde_derive", @@ -1767,6 +1873,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strum" version = "0.25.0" @@ -1788,9 +1900,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" dependencies = [ "proc-macro2", "quote", @@ -2085,6 +2197,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "uuid" version = "1.6.1" diff --git a/Cargo.toml b/Cargo.toml index 14f361c..45138eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ resolver = "2" members = [ "common", + "proc-macros", "pub-sub-service", "samples/chariott-publisher", "samples/chariott-subscriber", @@ -17,6 +18,7 @@ members = [ [workspace.dependencies] async-std = "1" async-trait = "0.1.64" +clap = { version = "4.4.11" } config = "0.13.3" ctrlc = { version = "3.0", features = ["termination"] } env_logger = "0.10" @@ -25,13 +27,16 @@ home = "0.5.5" include_dir = "0.7.3" log = "^0.4" paho-mqtt = "0.12" +proc-macro2 = "1.0.70" prost = "0.12" prost-types = "0.12" +quote = "1.0.23" serde = "1.0.160" serde_derive = "1.0.163" serde_json = "^1.0" strum = "0.25" strum_macros = "0.25" +syn = { version = "2.0.40", features = ["extra-traits", "full"] } tokio = { version = "1.18.0", features = ["time"] } tonic = "0.10" tonic-build = "0.10" diff --git a/.agemo/default/constants.default.yaml b/common/config/default.yaml similarity index 52% rename from .agemo/default/constants.default.yaml rename to common/config/default.yaml index 969a653..b30aa58 100644 --- a/.agemo/default/constants.default.yaml +++ b/common/config/default.yaml @@ -20,3 +20,19 @@ pub_sub_reference: "pubsub.v1.pubsub.proto" retry_interval_secs: 5 ### + +# +# Pub Sub Service Settings +# + +### Standalone Settings + +# The IP address and port number that the Pub Sub Service listens on for requests. +# Example: "0.0.0.0:50051" +pub_sub_authority: "0.0.0.0:50051" + +# The URI of the messaging service used to facilitate publish and subscribe functionality. +# Example: "mqtt://0.0.0.0:1883" +messaging_uri: "mqtt://0.0.0.0:1883" + +### diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 10fe928..be529d2 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -4,18 +4,19 @@ use std::{env, path::Path}; -use config::{File, FileFormat}; +use config::{File, FileFormat, Source}; use home::home_dir; -use serde::Deserialize; use include_dir::{include_dir, Dir}; +use serde::Deserialize; pub const YAML_EXT: &str = "yaml"; const CONFIG_DIR: &str = "config"; +const DEFAULT_FILE_STEM: &str = "default"; const DOT_AGEMO_DIR: &str = ".agemo"; const AGEMO_HOME: &str = "AGEMO_HOME"; -const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../.agemo/default"); +const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/config"); /// Read config from layered configuration files. /// Searches for `{config_file_name}.default.{config_file_ext}` as the base configuration in `$AGEMO_HOME`, @@ -25,15 +26,17 @@ const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../.agemo/default"); /// # Arguments /// - `config_file_name`: The config file name. This is used to construct the file names to search for. /// - `config_file_ext`: The config file extension. This is used to construct the file names to search for. -pub fn read_from_files<T>( +pub fn read_from_files<T, A>( config_file_name: &str, config_file_ext: &str, + args: Option<A>, ) -> Result<T, Box<dyn std::error::Error + Send + Sync>> where T: for<'a> Deserialize<'a>, + A: Source + Send + Sync + 'static + Clone, { // Get default config. - let default_config_filename = format!("{config_file_name}.default.{config_file_ext}"); + let default_config_filename = format!("{DEFAULT_FILE_STEM}.{config_file_ext}"); let default_config_file = DEFAULT_DIR.get_file(default_config_filename).unwrap(); let default_config_contents_str = default_config_file.contents_utf8().unwrap(); @@ -59,21 +62,26 @@ where } }; - // The path below resolves to {config_path}/{default_config_file}. - // let default_config_file_path = config_path.join(default_config_file); - // The path below resolves to {current_dir}/{overrides_file}. let current_dir_config_file_path = env::current_dir()?.join(overrides_file.clone()); // The path below resolves to {config_path}/{overrides_file} let overrides_config_file_path = config_path.join(overrides_file); - let config_store = config::Config::builder() - .add_source(File::from_str(default_config_contents_str, FileFormat::Yaml)) - //.add_source(File::from(default_config_file_path)) + let mut config_sources = config::Config::builder() + .add_source(File::from_str( + default_config_contents_str, + FileFormat::Yaml, + )) .add_source(File::from(current_dir_config_file_path).required(false)) - .add_source(File::from(overrides_config_file_path).required(false)) - .build()?; + .add_source(File::from(overrides_config_file_path).required(false)); + + // Adds command line arguments if there are any. + if let Some(args) = args { + config_sources = config_sources.add_source(args); + } + + let config_store = config_sources.build()?; config_store.try_deserialize().map_err(|e| e.into()) } diff --git a/proc-macros/Cargo.toml b/proc-macros/Cargo.toml new file mode 100644 index 0000000..5b569a0 --- /dev/null +++ b/proc-macros/Cargo.toml @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +# SPDX-License-Identifier: MIT + +[package] +name = "proc-macros" +version = "0.1.0" +edition = "2021" +license = "MIT" + +[lib] +proc-macro = true + +[dependencies] +config = { workspace = true } +proc-macro2 = { workspace = true } +quote = { workspace = true } +syn = { workspace = true } diff --git a/proc-macros/src/config_source.rs b/proc-macros/src/config_source.rs new file mode 100644 index 0000000..bf62797 --- /dev/null +++ b/proc-macros/src/config_source.rs @@ -0,0 +1,95 @@ +use proc_macro2::TokenStream; +use quote::quote; +use syn::{Data, DataStruct, Fields}; +use syn::{DeriveInput, GenericArgument, Path, PathArguments, Type}; + +pub fn expand_struct_to_source(input: DeriveInput) -> TokenStream { + let struct_name = input.ident; + let fields = match input.data { + Data::Struct(DataStruct { + fields: Fields::Named(fields), + .. + }) => fields.named, + _ => panic!("this derive macro only works on structs with named fields"), + }; + + let entries = fields.into_iter().map(|field| { + let field_name = field.ident; + let field_name_str = field_name.clone().unwrap().to_string(); + let field_type = field.ty; + + // Need to check if the value is an option and if so get the inner value if it is Some + match option_inner_type(&field_type) { + None => { + quote! { + (String::from(#field_name_str), Some(Value::from((&self.#field_name).clone()))), + } + }, + Some(Some(_type)) => { + quote! { + (String::from(#field_name_str), (&self.#field_name).clone().map(|v| Value::from(v))), + } + }, + _ => { + quote! { + (String::from(#field_name_str), None), + } + } + } + }); + + let (impl_generics, type_generics, where_clause) = input.generics.split_for_impl(); + + quote! { + #[automatically_derived] + impl #impl_generics config::Source #type_generics for #struct_name #where_clause { + fn clone_into_box(&self) -> Box<dyn config::Source + Send + Sync> { + Box::new((*self).clone()) + } + + fn collect(&self) -> Result<config::Map<String, config::Value>, config::ConfigError> { + let mut entries: config::Map::<String, Option<config::Value>> = config::Map::from([#(#entries)*]); + + entries.retain(|_, v| v.is_some()); + let entries_w_values = entries.clone(); + let valid_entries: config::Map::<String, config::Value> = entries_w_values.iter().map(|(k, v)| (k.clone(), v.clone().unwrap())).collect(); + + Ok(valid_entries.clone()) + } + } + } +} + +// Checks if path is option +fn path_is_option(path: &Path) -> bool { + path.leading_colon.is_none() && path.segments.len() == 1 && path.segments[0].ident == "Option" +} + +// Gets the option argument within the angle brackets. +fn option_arg(path_args: &PathArguments) -> Option<&Type> { + match path_args { + PathArguments::AngleBracketed(bracket) => { + if bracket.args.len() == 1 { + return match &bracket.args[0] { + GenericArgument::Type(t) => Some(t), + _ => None, + }; + } + None + } + _ => None, + } +} + +// Gets the inner option type if the type is option. +fn option_inner_type(ty: &Type) -> Option<Option<&Type>> { + match ty { + Type::Path(typepath) if typepath.qself.is_none() && path_is_option(&typepath.path) => { + let path_args = &typepath.path.segments[0].arguments; + + Some(option_arg(path_args)) + } + + _ => None, + } +} diff --git a/proc-macros/src/lib.rs b/proc-macros/src/lib.rs new file mode 100644 index 0000000..4675851 --- /dev/null +++ b/proc-macros/src/lib.rs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +mod config_source; +use config_source::expand_struct_to_source; + +use proc_macro::TokenStream; +use syn::{parse_macro_input, DeriveInput}; + +/// Implements Config Source +#[proc_macro_derive(ConfigSource)] +pub fn config_source(input: TokenStream) -> TokenStream { + let input: DeriveInput = parse_macro_input!(input); + + expand_struct_to_source(input).into() +} diff --git a/pub-sub-service/Cargo.toml b/pub-sub-service/Cargo.toml index 8a3380f..88b8ddd 100644 --- a/pub-sub-service/Cargo.toml +++ b/pub-sub-service/Cargo.toml @@ -11,12 +11,14 @@ license = "MIT" [dependencies] async-std = { workspace = true } async-trait = { workspace = true } +clap = { workspace = true, features = [ "derive" ] } common = { path = "../common" } config = { workspace = true } env_logger = { workspace = true } futures = { workspace = true } log = { workspace = true } paho-mqtt = { workspace = true } +proc-macros = { path = "../proc-macros"} proto = { path = "../proto-build" } serde = { workspace = true } serde_derive = { workspace = true } diff --git a/pub-sub-service/src/load_config.rs b/pub-sub-service/src/load_config.rs index c1f1f80..2481c9a 100644 --- a/pub-sub-service/src/load_config.rs +++ b/pub-sub-service/src/load_config.rs @@ -6,13 +6,45 @@ use std::env; +use clap::Parser; use common::config_utils; +use config::Value; use log::error; +use proc_macros::ConfigSource; use serde_derive::{Deserialize, Serialize}; const CONFIG_FILE_NAME: &str = "pub_sub_service_settings"; const CONSTANTS_FILE_NAME: &str = "constants"; +/// Object containing commandline config options for the Pub Sub service. +#[derive(Clone, Debug, Parser, Serialize, Deserialize, ConfigSource)] +#[command(author, about, long_about = None)] +pub struct CmdConfigOptions { + /// The IP address and port number that the Pub Sub service listens on for requests. + /// Required if not set in configuration files. (eg. "0.0.0.0:50051"). + #[arg(short, long)] + pub pub_sub_authority: Option<String>, + /// The URI of the messaging service used to facilitate publish and subscribe functionality. + /// Required if not set in configuration files. (eg. "mqtt://0.0.0.0:1883"). + #[arg(short, long)] + pub messaging_uri: Option<String>, + /// The URI that the Chariott service listens on for requests. (eg. "http://0.0.0.0:50000"). + #[arg(short, long)] + pub chariott_uri: Option<String>, + /// The namespace of the Pub Sub service. + #[arg(short = 's', long)] + pub namespace: Option<String>, + /// The name of the Pub Sub service. + #[arg(short, long)] + pub name: Option<String>, + /// The current version of the Pub Sub Service. + #[arg(short, long)] + pub version: Option<String>, + /// The log level of the program. + #[arg(short, long, default_value = "info")] + pub log_level: String, +} + /// Object that contains constants used for establishing connection between services. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct CommunicationConstants { @@ -29,7 +61,7 @@ pub struct CommunicationConstants { } /// Object containing configuration settings to run the Pub Sub service. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct Settings { /// The IP address and port number that the Pub Sub service listens on for requests. pub pub_sub_authority: String, @@ -49,19 +81,31 @@ pub struct Settings { /// /// # Arguments /// * `config_file_name` - Name of the config file to load settings from. -pub fn load_config<T>(config_file_name: &str) -> Result<T, Box<dyn std::error::Error + Send + Sync>> +pub fn load_config<T>( + config_file_name: &str, + args: Option<CmdConfigOptions>, +) -> Result<T, Box<dyn std::error::Error + Send + Sync>> where T: for<'de> serde::Deserialize<'de>, { - config_utils::read_from_files(config_file_name, config_utils::YAML_EXT) + config_utils::read_from_files(config_file_name, config_utils::YAML_EXT, args) } /// Load the settings. /// /// Will attempt to load the settings from the service configuration file. If the necessary config /// is set will run in Chariott enabled mode, otherwise the service will run in standalone mode. -pub fn load_settings() -> Result<Settings, Box<dyn std::error::Error + Send + Sync>> { - let mut settings: Settings = load_config(CONFIG_FILE_NAME)?; +pub fn load_settings( + args: CmdConfigOptions, +) -> Result<Settings, Box<dyn std::error::Error + Send + Sync>> { + let mut settings: Settings = load_config(CONFIG_FILE_NAME, Some(args)) + .map_err(|e| { + format!( + "Failed to load required configuration settings with error: {e}. See --help for more details." + ) + })?; + + println!("after config: {:?}", settings); if settings.chariott_uri.is_some() { // Get version of the service for Chariott registration if not defined. @@ -96,5 +140,5 @@ pub fn load_constants<T>() -> Result<T, Box<dyn std::error::Error + Send + Sync> where T: for<'de> serde::Deserialize<'de>, { - load_config(CONSTANTS_FILE_NAME) + load_config(CONSTANTS_FILE_NAME, None) } diff --git a/pub-sub-service/src/main.rs b/pub-sub-service/src/main.rs index 1101dce..de1474b 100644 --- a/pub-sub-service/src/main.rs +++ b/pub-sub-service/src/main.rs @@ -13,8 +13,9 @@ // Tells cargo to warn if a doc comment is missing and should be provided. #![warn(missing_docs)] -use std::sync::mpsc; +use std::{str::FromStr, sync::mpsc}; +use clap::Parser; use env_logger::{Builder, Target}; use log::{error, info, warn, LevelFilter}; use pubsub_connector::PubSubConnector; @@ -25,7 +26,7 @@ use proto::pubsub::v1::pub_sub_server::PubSubServer; use crate::{ connectors::chariott_connector::{self, ServiceIdentifier}, - load_config::CommunicationConstants, + load_config::{CmdConfigOptions, CommunicationConstants}, pubsub_connector::MonitorMessage, }; @@ -37,14 +38,21 @@ pub mod topic_manager; #[tokio::main] async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> { + // Load command line arguments if any. + let parsed_args = CmdConfigOptions::parse(); + + // Get log level. Defaults to info. + let log_level = + LevelFilter::from_str(&parsed_args.log_level).expect("Could not parse log level"); + // Setup logging. Builder::new() - .filter(None, LevelFilter::Info) + .filter(None, log_level) .target(Target::Stdout) .init(); // Load settings in from config file. - let settings = load_config::load_settings()?; + let settings = load_config::load_settings(parsed_args)?; let communication_consts = load_config::load_constants::<CommunicationConstants>()?; // Initialize pub sub service From ccd1e130f79a082114af46f095c6da352476b56f Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:39:35 -0800 Subject: [PATCH 03/31] order config overrides --- common/src/config_utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index be529d2..20ac8ff 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -73,8 +73,8 @@ where default_config_contents_str, FileFormat::Yaml, )) - .add_source(File::from(current_dir_config_file_path).required(false)) - .add_source(File::from(overrides_config_file_path).required(false)); + .add_source(File::from(overrides_config_file_path).required(false)) + .add_source(File::from(current_dir_config_file_path).required(false)); // Adds command line arguments if there are any. if let Some(args) = args { From b7c6438e356e1ad67cf335ccb10d6d6e91e1fca7 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Tue, 2 Jan 2024 11:26:09 -0800 Subject: [PATCH 04/31] Remove curr dir config --- common/src/config_utils.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 20ac8ff..83a8190 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -62,9 +62,6 @@ where } }; - // The path below resolves to {current_dir}/{overrides_file}. - let current_dir_config_file_path = env::current_dir()?.join(overrides_file.clone()); - // The path below resolves to {config_path}/{overrides_file} let overrides_config_file_path = config_path.join(overrides_file); @@ -73,8 +70,7 @@ where default_config_contents_str, FileFormat::Yaml, )) - .add_source(File::from(overrides_config_file_path).required(false)) - .add_source(File::from(current_dir_config_file_path).required(false)); + .add_source(File::from(overrides_config_file_path).required(false)); // Adds command line arguments if there are any. if let Some(args) = args { From 9881694d24047a7a293300efa5d17c14cf667aaf Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Wed, 3 Jan 2024 08:02:38 -0800 Subject: [PATCH 05/31] Update docs and modify default files --- README.md | 2 +- common/src/config_utils.rs | 6 ++-- .../constants.default.yaml | 18 ------------ config/pub_sub_service_settings.default.yaml | 17 +++++++++++ .../template/pub_sub_service_settings.yaml | 0 docs/config-overrides.md | 28 +++++++++++++------ pub-sub-service/README.md | 2 +- 7 files changed, 41 insertions(+), 32 deletions(-) rename common/config/default.yaml => config/constants.default.yaml (51%) create mode 100644 config/pub_sub_service_settings.default.yaml rename {.agemo/config => config}/template/pub_sub_service_settings.yaml (100%) diff --git a/README.md b/README.md index 994f0ad..6b1ac36 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ cargo test ## Configuration Setup -The service configuration is defined in [.agemo/config](.agemo/config/). The default configuration +The service configuration is defined in [config](config/). The default configuration files will allow a user to run the service without any modification. Please read [config_overrides](./docs/config-overrides.md) for more information on how to modify the service's configuration. diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 83a8190..0bb6a4e 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -12,11 +12,11 @@ use serde::Deserialize; pub const YAML_EXT: &str = "yaml"; const CONFIG_DIR: &str = "config"; -const DEFAULT_FILE_STEM: &str = "default"; +const DEFAULT: &str = "default"; const DOT_AGEMO_DIR: &str = ".agemo"; const AGEMO_HOME: &str = "AGEMO_HOME"; -const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/config"); +const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../config"); /// Read config from layered configuration files. /// Searches for `{config_file_name}.default.{config_file_ext}` as the base configuration in `$AGEMO_HOME`, @@ -36,7 +36,7 @@ where A: Source + Send + Sync + 'static + Clone, { // Get default config. - let default_config_filename = format!("{DEFAULT_FILE_STEM}.{config_file_ext}"); + let default_config_filename = format!("{config_file_name}.{DEFAULT}.{config_file_ext}"); let default_config_file = DEFAULT_DIR.get_file(default_config_filename).unwrap(); let default_config_contents_str = default_config_file.contents_utf8().unwrap(); diff --git a/common/config/default.yaml b/config/constants.default.yaml similarity index 51% rename from common/config/default.yaml rename to config/constants.default.yaml index b30aa58..08c5d33 100644 --- a/common/config/default.yaml +++ b/config/constants.default.yaml @@ -18,21 +18,3 @@ pub_sub_reference: "pubsub.v1.pubsub.proto" # Retry interval for connections. retry_interval_secs: 5 - -### - -# -# Pub Sub Service Settings -# - -### Standalone Settings - -# The IP address and port number that the Pub Sub Service listens on for requests. -# Example: "0.0.0.0:50051" -pub_sub_authority: "0.0.0.0:50051" - -# The URI of the messaging service used to facilitate publish and subscribe functionality. -# Example: "mqtt://0.0.0.0:1883" -messaging_uri: "mqtt://0.0.0.0:1883" - -### diff --git a/config/pub_sub_service_settings.default.yaml b/config/pub_sub_service_settings.default.yaml new file mode 100644 index 0000000..7db8a9a --- /dev/null +++ b/config/pub_sub_service_settings.default.yaml @@ -0,0 +1,17 @@ +### + +# +# Pub Sub Service Settings +# + +### Standalone Settings + +# The IP address and port number that the Pub Sub Service listens on for requests. +# Example: "0.0.0.0:50051" +pub_sub_authority: "0.0.0.0:50051" + +# The URI of the messaging service used to facilitate publish and subscribe functionality. +# Example: "mqtt://0.0.0.0:1883" +messaging_uri: "mqtt://0.0.0.0:1883" + +### diff --git a/.agemo/config/template/pub_sub_service_settings.yaml b/config/template/pub_sub_service_settings.yaml similarity index 100% rename from .agemo/config/template/pub_sub_service_settings.yaml rename to config/template/pub_sub_service_settings.yaml diff --git a/docs/config-overrides.md b/docs/config-overrides.md index dcf114d..08a5aee 100644 --- a/docs/config-overrides.md +++ b/docs/config-overrides.md @@ -2,26 +2,36 @@ The pub sub service supports configuration overrides that enable users to override the default settings and provide custom configuration. This is achieved with configuration layering. Default -configuration files are defined in `.agemo/config` at the root of the project, and this is often -suitable for basic scenarios or getting started quickly. The service relies on the environment -variable `$AGEMO_HOME` to find the default configuration files. This variable is set by default to -point to `{path_to_project_root}/.agemo` when running the service with `cargo run`. The default +configuration files are defined in `config` at the root of the project, and this is often +suitable for basic scenarios or getting started quickly. The service includes the default +configuration files at build time. The service relies on the environment variable `$AGEMO_HOME` to +find any override configuration files. This variable is set by default to point to +`{path_to_project_root}/.agemo` when running the service with `cargo run`. Template configuration +files to use to override can be found under [config\template](../config/template/). The default configuration files can be overridden at runtime using custom values. When loading configuration, the service will probe for and unify config in the following order, with values near the end of the list taking higher precedence: -- The default config -- A config file in the working directory of the executable (for example, the directory you were in -when you ran the `cargo run` command) - `$AGEMO_HOME/config/{config_name}.yaml`. If you have not set a `$AGEMO_HOME` directory or are not running the service with `cargo run`, this defaults to: - Unix: `$HOME/.agemo/config/{config_name}.yaml` - Windows: `%USERPROFILE%\.agemo\config\{config_name}.yaml` (note that Windows support is not guaranteed by Agemo) +- Command line arguments. Because the config is layered, the overrides can be partially defined and only specify the top-level configuration fields that should be overridden. Anything not specified in an override file will use the default value. -Samples handles configuration in the same way, except it utilizes the `.agemo-samples` directory -and the `$AGEMO_SAMPLES_HOME` env variable to point to that directory. +Samples handles configuration in the same way, except it utilizes the `$AGEMO_SAMPLES_HOME` env +variable to point to the `.agemo-samples` directory at the project root. + +## Command Line Arguments + +The service leverages [clap (command line argument parser)](https://github.com/clap-rs/clap) to +override individual configuration values through command line arguments when starting the service. +To see the list of possible parameters, run: + +```shell +cargo run -p pub-sub-service -- --help +``` diff --git a/pub-sub-service/README.md b/pub-sub-service/README.md index 7a469e1..fc542a2 100644 --- a/pub-sub-service/README.md +++ b/pub-sub-service/README.md @@ -67,7 +67,7 @@ service and directly communicate. file does not already exist. From the enlistment root, run: ```shell - cp ./.agemo/config/template/pub_sub_service_settings.yaml ./.agemo/config/ + cp ./config/template/pub_sub_service_settings.yaml ./.agemo/config/ ``` 2. Uncomment and set the following values: From 84a3e96b98f88d768a2213809f81fae41ce3bd86 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Wed, 3 Jan 2024 08:13:53 -0800 Subject: [PATCH 06/31] fix doc errs --- docs/containers.md | 3 --- pub-sub-service/README.md | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/containers.md b/docs/containers.md index 8c5139e..50040d0 100644 --- a/docs/containers.md +++ b/docs/containers.md @@ -14,9 +14,6 @@ x86-64 architecture. - [Dockerfile.arm64](../Dockerfile.arm64) - Dockerfile used to build the `Pub Sub Service` for the aarch64 architecture. ->Note: The default configuration files are cloned from [.agemo/config](../.agemo/config/), defined -in the project's root. - #### Mosquitto MQTT Broker - [Dockerfile.mosquitto.amd64](../Dockerfile.mosquitto.amd64) - Dockerfile used to build the diff --git a/pub-sub-service/README.md b/pub-sub-service/README.md index fc542a2..250768e 100644 --- a/pub-sub-service/README.md +++ b/pub-sub-service/README.md @@ -63,8 +63,8 @@ service and directly communicate. ### Configure Pub Sub Service to use Chariott -1. Copy the `pub_sub_service_settings.yaml` template to [.agemo/config](../.agemo/config/) if the -file does not already exist. From the enlistment root, run: +1. Copy the `pub_sub_service_settings.yaml` template to `.agemo/config` if the file does not +already exist. From the enlistment root, run: ```shell cp ./config/template/pub_sub_service_settings.yaml ./.agemo/config/ From 17de42fdbf9abf63b318b685c0581340827928e9 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Wed, 3 Jan 2024 08:17:27 -0800 Subject: [PATCH 07/31] remove dockerfile cp as default config is now part of executable --- Dockerfile.amd64 | 3 --- Dockerfile.arm64 | 3 --- 2 files changed, 6 deletions(-) diff --git a/Dockerfile.amd64 b/Dockerfile.amd64 index 7700aa5..0265284 100644 --- a/Dockerfile.amd64 +++ b/Dockerfile.amd64 @@ -90,9 +90,6 @@ ENV AGEMO_HOME=/sdv/.agemo # Copy the executable from the "build" stage. COPY --from=build /sdv/service /sdv/ -# Copy default configs. -COPY --from=build /sdv/.agemo/config/ /sdv/.agemo/config/ - # Expose the port that the application listens on. EXPOSE 50051 diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index b28e72a..94e7e7f 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -93,9 +93,6 @@ ENV AGEMO_HOME=/sdv/.agemo # Copy the executable from the "build" stage. COPY --from=build /sdv/service /sdv/ -# Copy default configs. -COPY --from=build /sdv/.agemo/config/ /sdv/.agemo/config/ - # Expose the port that the application listens on. EXPOSE 50051 From fbed66e33118efa16bce21fa16659611d83a78ef Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Thu, 4 Jan 2024 02:52:46 -0800 Subject: [PATCH 08/31] Refactored proc macro, added detailed comments --- proc-macros/src/config_source.rs | 95 ----------------------- proc-macros/src/config_source/generate.rs | 58 ++++++++++++++ proc-macros/src/config_source/mod.rs | 34 ++++++++ proc-macros/src/config_source/parse.rs | 42 ++++++++++ proc-macros/src/config_source/process.rs | 79 +++++++++++++++++++ proc-macros/src/lib.rs | 56 +++++++++++-- pub-sub-service/src/load_config.rs | 1 - 7 files changed, 262 insertions(+), 103 deletions(-) delete mode 100644 proc-macros/src/config_source.rs create mode 100644 proc-macros/src/config_source/generate.rs create mode 100644 proc-macros/src/config_source/mod.rs create mode 100644 proc-macros/src/config_source/parse.rs create mode 100644 proc-macros/src/config_source/process.rs diff --git a/proc-macros/src/config_source.rs b/proc-macros/src/config_source.rs deleted file mode 100644 index bf62797..0000000 --- a/proc-macros/src/config_source.rs +++ /dev/null @@ -1,95 +0,0 @@ -use proc_macro2::TokenStream; -use quote::quote; -use syn::{Data, DataStruct, Fields}; -use syn::{DeriveInput, GenericArgument, Path, PathArguments, Type}; - -pub fn expand_struct_to_source(input: DeriveInput) -> TokenStream { - let struct_name = input.ident; - let fields = match input.data { - Data::Struct(DataStruct { - fields: Fields::Named(fields), - .. - }) => fields.named, - _ => panic!("this derive macro only works on structs with named fields"), - }; - - let entries = fields.into_iter().map(|field| { - let field_name = field.ident; - let field_name_str = field_name.clone().unwrap().to_string(); - let field_type = field.ty; - - // Need to check if the value is an option and if so get the inner value if it is Some - match option_inner_type(&field_type) { - None => { - quote! { - (String::from(#field_name_str), Some(Value::from((&self.#field_name).clone()))), - } - }, - Some(Some(_type)) => { - quote! { - (String::from(#field_name_str), (&self.#field_name).clone().map(|v| Value::from(v))), - } - }, - _ => { - quote! { - (String::from(#field_name_str), None), - } - } - } - }); - - let (impl_generics, type_generics, where_clause) = input.generics.split_for_impl(); - - quote! { - #[automatically_derived] - impl #impl_generics config::Source #type_generics for #struct_name #where_clause { - fn clone_into_box(&self) -> Box<dyn config::Source + Send + Sync> { - Box::new((*self).clone()) - } - - fn collect(&self) -> Result<config::Map<String, config::Value>, config::ConfigError> { - let mut entries: config::Map::<String, Option<config::Value>> = config::Map::from([#(#entries)*]); - - entries.retain(|_, v| v.is_some()); - let entries_w_values = entries.clone(); - let valid_entries: config::Map::<String, config::Value> = entries_w_values.iter().map(|(k, v)| (k.clone(), v.clone().unwrap())).collect(); - - Ok(valid_entries.clone()) - } - } - } -} - -// Checks if path is option -fn path_is_option(path: &Path) -> bool { - path.leading_colon.is_none() && path.segments.len() == 1 && path.segments[0].ident == "Option" -} - -// Gets the option argument within the angle brackets. -fn option_arg(path_args: &PathArguments) -> Option<&Type> { - match path_args { - PathArguments::AngleBracketed(bracket) => { - if bracket.args.len() == 1 { - return match &bracket.args[0] { - GenericArgument::Type(t) => Some(t), - _ => None, - }; - } - None - } - _ => None, - } -} - -// Gets the inner option type if the type is option. -fn option_inner_type(ty: &Type) -> Option<Option<&Type>> { - match ty { - Type::Path(typepath) if typepath.qself.is_none() && path_is_option(&typepath.path) => { - let path_args = &typepath.path.segments[0].arguments; - - Some(option_arg(path_args)) - } - - _ => None, - } -} diff --git a/proc-macros/src/config_source/generate.rs b/proc-macros/src/config_source/generate.rs new file mode 100644 index 0000000..f59f2e1 --- /dev/null +++ b/proc-macros/src/config_source/generate.rs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +use proc_macro2::TokenStream; +use quote::quote; + +use super::process::StructDataOutput; + +/// Generate code for the ConfigSource derive macro. +/// +/// # Arguments +/// * `struct_data` - Data gathered from a Struct. +pub(crate) fn generate(struct_data: StructDataOutput) -> TokenStream { + // Define values for the code generation. + let struct_name = struct_data.struct_name; + let struct_entries = struct_data.struct_fields; + + // Define generics information for the code generation. + let (impl_generics, type_generics, where_clause) = struct_data.struct_generics.split_for_impl(); + + // Construct a list of entries from the fields of the Struct. + let entries = struct_entries.into_iter().map(|entry| { + let field_name = entry.name; + let field_name_str = entry.name_str; + + // Code snippet changes based on whether the entry is an optional field. + if entry.is_optional { + quote! { + (String::from(#field_name_str), (&self.#field_name).clone().map(|v| config::Value::from(v))), + } + } else { + quote! { + (String::from(#field_name_str), Some(config::Value::from((&self.#field_name).clone()))), + } + } + }); + + // Construct a code snippet that implements the `Source` Trait. + quote! { + #[automatically_derived] + impl #impl_generics config::Source #type_generics for #struct_name #where_clause { + fn clone_into_box(&self) -> Box<dyn config::Source + Send + Sync> { + Box::new((*self).clone()) + } + + fn collect(&self) -> Result<config::Map<String, config::Value>, config::ConfigError> { + let mut entries: config::Map::<String, Option<config::Value>> = config::Map::from([#(#entries)*]); + + entries.retain(|_, v| v.is_some()); + let entries_w_values = entries.clone(); + let valid_entries: config::Map::<String, config::Value> = entries_w_values.iter().map(|(k, v)| (k.clone(), v.clone().unwrap())).collect(); + + Ok(valid_entries.clone()) + } + } + } +} diff --git a/proc-macros/src/config_source/mod.rs b/proc-macros/src/config_source/mod.rs new file mode 100644 index 0000000..b905f5b --- /dev/null +++ b/proc-macros/src/config_source/mod.rs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +mod generate; +mod parse; +mod process; + +use proc_macro::TokenStream; +use syn::{parse_macro_input, DeriveInput}; + +use generate::generate; +use process::process; + +use self::parse::parse_input; + +/// Implements the ConfigSource derive macro +/// +/// # Arguments: +/// +/// - `ts`: The token stream input +pub fn config_source(ts: TokenStream) -> TokenStream { + // Parse token stream into input. + let input: DeriveInput = parse_macro_input!(ts); + + // Parse input into Struct data. + let data = parse_input(input); + + // Process the Struct data. + let processed_data = process(data); + + // Generate the output code. + generate(processed_data).into() +} diff --git a/proc-macros/src/config_source/parse.rs b/proc-macros/src/config_source/parse.rs new file mode 100644 index 0000000..c1fc296 --- /dev/null +++ b/proc-macros/src/config_source/parse.rs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +use proc_macro2::Ident; +use syn::{punctuated::Punctuated, token::Comma, Field}; +use syn::{Data, DataStruct, DeriveInput, Fields, Generics}; + +/// Represents a Struct. +pub(crate) struct StructData { + /// The identifier of the Struct. + pub struct_name: Ident, + /// List of fields of the Struct. + pub struct_fields: Punctuated<Field, Comma>, + /// The generics associated with the Struct. + pub struct_generics: Generics, +} + +/// Parse input data for the ConfigSource derive macro. +/// Will panic if input is not gathered from a Struct. +/// +/// # Arguments +/// * `input` - Parsed derive macro input. +pub(crate) fn parse_input(input: DeriveInput) -> StructData { + let struct_name = input.ident; + let struct_generics = input.generics; + + // Processes input data into Struct Fields. Panics if data is not from a Struct. + let struct_fields = match input.data { + Data::Struct(DataStruct { + fields: Fields::Named(fields), + .. + }) => fields.named, + _ => panic!("this derive macro only works on structs with named fields"), + }; + + StructData { + struct_name, + struct_fields, + struct_generics, + } +} diff --git a/proc-macros/src/config_source/process.rs b/proc-macros/src/config_source/process.rs new file mode 100644 index 0000000..7470ce6 --- /dev/null +++ b/proc-macros/src/config_source/process.rs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +use proc_macro2::Ident; +use syn::{Generics, Path, Type}; + +use super::parse::StructData; + +/// Represents data gathered from a Struct. +pub(crate) struct StructDataOutput { + /// The identifier of the Struct. + pub struct_name: Ident, + /// A vector of fields for the Struct. + pub struct_fields: Vec<FieldEntry>, + /// The generics associated with the Struct. + pub struct_generics: Generics, +} + +/// Represents a single named field in a Struct. +pub(crate) struct FieldEntry { + /// The identifier of the field. + pub name: Ident, + /// The identifier of the field as a string. + pub name_str: String, + /// Whether the field is optional. + pub is_optional: bool, +} + +/// Process the data for the ConfigSource derive macro. +/// This method collects the relevent struct values for generation. +/// +/// # Arguments +/// * `data` - Parsed Struct data. +pub(crate) fn process(data: StructData) -> StructDataOutput { + // Process fields from Struct. + let struct_fields: Vec<FieldEntry> = data + .struct_fields + .into_iter() + .map(|field| { + let field_name = field.ident.unwrap(); + // Get the field name as a string. Will be used as a key in the code generation step. + let field_name_str = field_name.clone().to_string(); + // Determine if field is optional. Relevant for the code generation step. + let is_optional = is_option(&field.ty); + + FieldEntry { + name: field_name, + name_str: field_name_str, + is_optional, + } + }) + .collect(); + + StructDataOutput { + struct_name: data.struct_name, + struct_fields, + struct_generics: data.struct_generics, + } +} + +/// Helper method to determine if a Type is of type `Option`. +/// +/// # Arguments +/// * `path` - Path to check. +fn path_is_option(path: &Path) -> bool { + path.leading_colon.is_none() && path.segments.len() == 1 && path.segments[0].ident == "Option" +} + +/// Determines if the provided Type is of type `Option`. +/// +/// # Arguments +/// * `ty` - Struct field type to check. +fn is_option(ty: &Type) -> bool { + match ty { + Type::Path(typepath) if typepath.qself.is_none() && path_is_option(&typepath.path) => true, + _ => false, + } +} diff --git a/proc-macros/src/lib.rs b/proc-macros/src/lib.rs index 4675851..afe673b 100644 --- a/proc-macros/src/lib.rs +++ b/proc-macros/src/lib.rs @@ -3,15 +3,57 @@ // SPDX-License-Identifier: MIT mod config_source; -use config_source::expand_struct_to_source; use proc_macro::TokenStream; -use syn::{parse_macro_input, DeriveInput}; -/// Implements Config Source +/// Derives `config::Source` Trait (from the config crate) for a Struct. +/// +/// Note: The Struct must have named fields and the Type for each field must be convertable into a +/// `config::Value` from the `config` crate. +/// +/// # Arguments +/// * `ts`: A token stream. +/// +/// # Examples +/// +/// Given a Struct: +/// +/// #[[derive(ConfigSource)]] +/// pub struct CmdOptions { +/// pub endpoint: String, +/// pub log_level: Option<String>, +/// } +/// +/// The `ConfigSource` derive macro will implement `config::Source`: +/// +/// impl config::Source for CmdOptions { +/// +/// fn clone_into_box(&self) -> Box<dyn config::Source + Send + Sync> { +/// Box::new((*self).clone()) +/// } +/// +/// fn collect(&self) -> Result<config::Map<String, config::Value>, config::ConfigError> { +/// let mut entries: config::Map::<String, Option<config::Value>> = config::Map::from([ +/// ( +/// String::from("endpoint"), +/// (&self.endpoint).clone().map(|v| config::Value::from(v)) +/// ), +/// ( +/// String::from("log_level"), +/// Some(config::Value::from((&self.log_level).clone())) +/// ), +/// ]); +/// +/// entries.retain(|_, v| v.is_some()); +/// let entries_w_values = entries.clone(); +/// let valid_entries: config::Map::<String, config::Value> = entries_w_values.iter().map(|(k, v)| (k.clone(), v.clone().unwrap())).collect(); +/// +/// Ok(valid_entries.clone()) +/// } +/// } +/// +/// This allows for Structs to be used as a `Source` for configuration through the `config` crate. #[proc_macro_derive(ConfigSource)] -pub fn config_source(input: TokenStream) -> TokenStream { - let input: DeriveInput = parse_macro_input!(input); - - expand_struct_to_source(input).into() +pub fn config_source(ts: TokenStream) -> TokenStream { + config_source::config_source(ts).into() } diff --git a/pub-sub-service/src/load_config.rs b/pub-sub-service/src/load_config.rs index 2481c9a..bbbec21 100644 --- a/pub-sub-service/src/load_config.rs +++ b/pub-sub-service/src/load_config.rs @@ -8,7 +8,6 @@ use std::env; use clap::Parser; use common::config_utils; -use config::Value; use log::error; use proc_macros::ConfigSource; use serde_derive::{Deserialize, Serialize}; From 99630167077de5294c7ad0d4721ebfb9409ea344 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Thu, 4 Jan 2024 02:57:05 -0800 Subject: [PATCH 09/31] fixed clippy warnings --- proc-macros/src/config_source/mod.rs | 3 +-- proc-macros/src/config_source/process.rs | 5 +---- proc-macros/src/lib.rs | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/proc-macros/src/config_source/mod.rs b/proc-macros/src/config_source/mod.rs index b905f5b..359201c 100644 --- a/proc-macros/src/config_source/mod.rs +++ b/proc-macros/src/config_source/mod.rs @@ -10,10 +10,9 @@ use proc_macro::TokenStream; use syn::{parse_macro_input, DeriveInput}; use generate::generate; +use parse::parse_input; use process::process; -use self::parse::parse_input; - /// Implements the ConfigSource derive macro /// /// # Arguments: diff --git a/proc-macros/src/config_source/process.rs b/proc-macros/src/config_source/process.rs index 7470ce6..95ce0a5 100644 --- a/proc-macros/src/config_source/process.rs +++ b/proc-macros/src/config_source/process.rs @@ -72,8 +72,5 @@ fn path_is_option(path: &Path) -> bool { /// # Arguments /// * `ty` - Struct field type to check. fn is_option(ty: &Type) -> bool { - match ty { - Type::Path(typepath) if typepath.qself.is_none() && path_is_option(&typepath.path) => true, - _ => false, - } + matches!(ty, Type::Path(typepath) if typepath.qself.is_none() && path_is_option(&typepath.path)) } diff --git a/proc-macros/src/lib.rs b/proc-macros/src/lib.rs index afe673b..5543436 100644 --- a/proc-macros/src/lib.rs +++ b/proc-macros/src/lib.rs @@ -55,5 +55,5 @@ use proc_macro::TokenStream; /// This allows for Structs to be used as a `Source` for configuration through the `config` crate. #[proc_macro_derive(ConfigSource)] pub fn config_source(ts: TokenStream) -> TokenStream { - config_source::config_source(ts).into() + config_source::config_source(ts) } From 1ba0b4019188b6a512d361b56562b8b182a70b3b Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Thu, 4 Jan 2024 03:00:48 -0800 Subject: [PATCH 10/31] fix unexpected delimiter --- proc-macros/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proc-macros/src/lib.rs b/proc-macros/src/lib.rs index 5543436..78e44b8 100644 --- a/proc-macros/src/lib.rs +++ b/proc-macros/src/lib.rs @@ -18,7 +18,7 @@ use proc_macro::TokenStream; /// /// Given a Struct: /// -/// #[[derive(ConfigSource)]] +/// #[derive(ConfigSource)] /// pub struct CmdOptions { /// pub endpoint: String, /// pub log_level: Option<String>, From f874053bcf4e5f9418ada67db0ac21c2c404789f Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Thu, 4 Jan 2024 03:14:05 -0800 Subject: [PATCH 11/31] fixed doctest errs --- proc-macros/src/lib.rs | 59 ++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/proc-macros/src/lib.rs b/proc-macros/src/lib.rs index 78e44b8..cd6e94d 100644 --- a/proc-macros/src/lib.rs +++ b/proc-macros/src/lib.rs @@ -18,39 +18,42 @@ use proc_macro::TokenStream; /// /// Given a Struct: /// -/// #[derive(ConfigSource)] -/// pub struct CmdOptions { -/// pub endpoint: String, -/// pub log_level: Option<String>, -/// } +/// ```rust +/// #[derive(ConfigSource)] +/// pub struct CmdOptions { +/// pub endpoint: String, +/// pub log_level: Option<String>, +/// } +/// ``` /// /// The `ConfigSource` derive macro will implement `config::Source`: /// -/// impl config::Source for CmdOptions { -/// -/// fn clone_into_box(&self) -> Box<dyn config::Source + Send + Sync> { -/// Box::new((*self).clone()) -/// } -/// -/// fn collect(&self) -> Result<config::Map<String, config::Value>, config::ConfigError> { -/// let mut entries: config::Map::<String, Option<config::Value>> = config::Map::from([ -/// ( -/// String::from("endpoint"), -/// (&self.endpoint).clone().map(|v| config::Value::from(v)) -/// ), -/// ( -/// String::from("log_level"), -/// Some(config::Value::from((&self.log_level).clone())) -/// ), -/// ]); -/// -/// entries.retain(|_, v| v.is_some()); -/// let entries_w_values = entries.clone(); -/// let valid_entries: config::Map::<String, config::Value> = entries_w_values.iter().map(|(k, v)| (k.clone(), v.clone().unwrap())).collect(); -/// -/// Ok(valid_entries.clone()) +/// ```rust +/// impl config::Source for CmdOptions { +/// fn clone_into_box(&self) -> Box<dyn config::Source + Send + Sync> { +/// Box::new((*self).clone()) +/// } +/// +/// fn collect(&self) -> Result<config::Map<String, config::Value>, config::ConfigError> { +/// let mut entries: config::Map::<String, Option<config::Value>> = config::Map::from([ +/// ( +/// String::from("endpoint"), +/// (&self.endpoint).clone().map(|v| config::Value::from(v)) +/// ), +/// ( +/// String::from("log_level"), +/// Some(config::Value::from((&self.log_level).clone())) +/// ), +/// ]); +/// +/// entries.retain(|_, v| v.is_some()); +/// let entries_w_values = entries.clone(); +/// let valid_entries: config::Map::<String, config::Value> = entries_w_values.iter().map(|(k, v)| (k.clone(), v.clone().unwrap())).collect(); +/// +/// Ok(valid_entries.clone()) /// } /// } +/// ``` /// /// This allows for Structs to be used as a `Source` for configuration through the `config` crate. #[proc_macro_derive(ConfigSource)] From 87d1117c9167f4bb9ff802ff58c82a9daf7f58d8 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Thu, 4 Jan 2024 03:16:27 -0800 Subject: [PATCH 12/31] second attempt --- proc-macros/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/proc-macros/src/lib.rs b/proc-macros/src/lib.rs index cd6e94d..c84383c 100644 --- a/proc-macros/src/lib.rs +++ b/proc-macros/src/lib.rs @@ -19,7 +19,6 @@ use proc_macro::TokenStream; /// Given a Struct: /// /// ```rust -/// #[derive(ConfigSource)] /// pub struct CmdOptions { /// pub endpoint: String, /// pub log_level: Option<String>, From 94ee888600aaf78ff9f58450498684bac8209047 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Thu, 4 Jan 2024 03:18:59 -0800 Subject: [PATCH 13/31] remove example as it is over complicated --- proc-macros/src/lib.rs | 42 ------------------------------------------ 1 file changed, 42 deletions(-) diff --git a/proc-macros/src/lib.rs b/proc-macros/src/lib.rs index c84383c..624bc18 100644 --- a/proc-macros/src/lib.rs +++ b/proc-macros/src/lib.rs @@ -13,48 +13,6 @@ use proc_macro::TokenStream; /// /// # Arguments /// * `ts`: A token stream. -/// -/// # Examples -/// -/// Given a Struct: -/// -/// ```rust -/// pub struct CmdOptions { -/// pub endpoint: String, -/// pub log_level: Option<String>, -/// } -/// ``` -/// -/// The `ConfigSource` derive macro will implement `config::Source`: -/// -/// ```rust -/// impl config::Source for CmdOptions { -/// fn clone_into_box(&self) -> Box<dyn config::Source + Send + Sync> { -/// Box::new((*self).clone()) -/// } -/// -/// fn collect(&self) -> Result<config::Map<String, config::Value>, config::ConfigError> { -/// let mut entries: config::Map::<String, Option<config::Value>> = config::Map::from([ -/// ( -/// String::from("endpoint"), -/// (&self.endpoint).clone().map(|v| config::Value::from(v)) -/// ), -/// ( -/// String::from("log_level"), -/// Some(config::Value::from((&self.log_level).clone())) -/// ), -/// ]); -/// -/// entries.retain(|_, v| v.is_some()); -/// let entries_w_values = entries.clone(); -/// let valid_entries: config::Map::<String, config::Value> = entries_w_values.iter().map(|(k, v)| (k.clone(), v.clone().unwrap())).collect(); -/// -/// Ok(valid_entries.clone()) -/// } -/// } -/// ``` -/// -/// This allows for Structs to be used as a `Source` for configuration through the `config` crate. #[proc_macro_derive(ConfigSource)] pub fn config_source(ts: TokenStream) -> TokenStream { config_source::config_source(ts) From 6c40e8a0e7449e7d5b05186518ff516c2c875d79 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Thu, 4 Jan 2024 14:11:20 -0800 Subject: [PATCH 14/31] add params to comments --- common/src/config_utils.rs | 9 +++++---- pub-sub-service/src/load_config.rs | 10 +++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 0bb6a4e..3f7b47c 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -19,13 +19,14 @@ const AGEMO_HOME: &str = "AGEMO_HOME"; const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../config"); /// Read config from layered configuration files. -/// Searches for `{config_file_name}.default.{config_file_ext}` as the base configuration in `$AGEMO_HOME`, -/// then searches for overrides named `{config_file_name}.{config_file_ext}` in the current directory and `$AGEMO_HOME`. +/// Searches for `{config_file_name}.default.{config_file_ext}` as the base configuration, +/// then searches for overrides named `{config_file_name}.{config_file_ext}` in `$AGEMO_HOME`. /// If `$AGEMO_HOME` is not set, it defaults to `$HOME/.agemo`. /// /// # Arguments -/// - `config_file_name`: The config file name. This is used to construct the file names to search for. -/// - `config_file_ext`: The config file extension. This is used to construct the file names to search for. +/// * `config_file_name` - The config file name. This is used to construct the file names to search for. +/// * `config_file_ext` - The config file extension. This is used to construct the file names to search for. +/// * `args` - Optional commandline arguments. Any values set will override values gathered from config files. pub fn read_from_files<T, A>( config_file_name: &str, config_file_ext: &str, diff --git a/pub-sub-service/src/load_config.rs b/pub-sub-service/src/load_config.rs index bbbec21..53517ab 100644 --- a/pub-sub-service/src/load_config.rs +++ b/pub-sub-service/src/load_config.rs @@ -8,7 +8,7 @@ use std::env; use clap::Parser; use common::config_utils; -use log::error; +use log::{debug, error}; use proc_macros::ConfigSource; use serde_derive::{Deserialize, Serialize}; @@ -80,6 +80,7 @@ pub struct Settings { /// /// # Arguments /// * `config_file_name` - Name of the config file to load settings from. +/// * `args` - Optional commandline config arguments. pub fn load_config<T>( config_file_name: &str, args: Option<CmdConfigOptions>, @@ -94,17 +95,20 @@ where /// /// Will attempt to load the settings from the service configuration file. If the necessary config /// is set will run in Chariott enabled mode, otherwise the service will run in standalone mode. +/// +/// # Arguments +/// * `args` - Commandline config arguments. pub fn load_settings( args: CmdConfigOptions, ) -> Result<Settings, Box<dyn std::error::Error + Send + Sync>> { let mut settings: Settings = load_config(CONFIG_FILE_NAME, Some(args)) .map_err(|e| { format!( - "Failed to load required configuration settings with error: {e}. See --help for more details." + "Failed to load required configuration settings due to error: {e}. See --help for more details." ) })?; - println!("after config: {:?}", settings); + debug!("settings config: {:?}", settings); if settings.chariott_uri.is_some() { // Get version of the service for Chariott registration if not defined. From de215b200ad098a77db6a3641915779a3fa8e20f Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Thu, 4 Jan 2024 18:23:01 -0800 Subject: [PATCH 15/31] remove unecessary clonse --- proc-macros/src/config_source/generate.rs | 3 +-- proc-macros/src/config_source/process.rs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/proc-macros/src/config_source/generate.rs b/proc-macros/src/config_source/generate.rs index f59f2e1..dc5e2ab 100644 --- a/proc-macros/src/config_source/generate.rs +++ b/proc-macros/src/config_source/generate.rs @@ -48,8 +48,7 @@ pub(crate) fn generate(struct_data: StructDataOutput) -> TokenStream { let mut entries: config::Map::<String, Option<config::Value>> = config::Map::from([#(#entries)*]); entries.retain(|_, v| v.is_some()); - let entries_w_values = entries.clone(); - let valid_entries: config::Map::<String, config::Value> = entries_w_values.iter().map(|(k, v)| (k.clone(), v.clone().unwrap())).collect(); + let valid_entries: config::Map::<String, config::Value> = entries.iter().map(|(k, v)| (k.clone(), v.clone().unwrap())).collect(); Ok(valid_entries.clone()) } diff --git a/proc-macros/src/config_source/process.rs b/proc-macros/src/config_source/process.rs index 95ce0a5..a55b7fb 100644 --- a/proc-macros/src/config_source/process.rs +++ b/proc-macros/src/config_source/process.rs @@ -40,7 +40,7 @@ pub(crate) fn process(data: StructData) -> StructDataOutput { .map(|field| { let field_name = field.ident.unwrap(); // Get the field name as a string. Will be used as a key in the code generation step. - let field_name_str = field_name.clone().to_string(); + let field_name_str = field_name.to_string(); // Determine if field is optional. Relevant for the code generation step. let is_optional = is_option(&field.ty); From 02002e0f7a727200d5ab08ab144bc634a99a009e Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Fri, 5 Jan 2024 00:55:22 -0800 Subject: [PATCH 16/31] add comment about required cmdline params --- pub-sub-service/src/load_config.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pub-sub-service/src/load_config.rs b/pub-sub-service/src/load_config.rs index 53517ab..b9ec89a 100644 --- a/pub-sub-service/src/load_config.rs +++ b/pub-sub-service/src/load_config.rs @@ -16,6 +16,8 @@ const CONFIG_FILE_NAME: &str = "pub_sub_service_settings"; const CONSTANTS_FILE_NAME: &str = "constants"; /// Object containing commandline config options for the Pub Sub service. +/// Non-optional fields must be passed in via the commandline and will override any values from +/// configuration files. #[derive(Clone, Debug, Parser, Serialize, Deserialize, ConfigSource)] #[command(author, about, long_about = None)] pub struct CmdConfigOptions { From 148a04a6c114cf0f468f8982bb9be3e39ca4ec4f Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Fri, 5 Jan 2024 00:57:13 -0800 Subject: [PATCH 17/31] Add tests for config_source macro --- proc-macros/src/config_source/parse.rs | 58 +++++ proc-macros/src/config_source/process.rs | 283 +++++++++++++++++++++++ 2 files changed, 341 insertions(+) diff --git a/proc-macros/src/config_source/parse.rs b/proc-macros/src/config_source/parse.rs index c1fc296..6819988 100644 --- a/proc-macros/src/config_source/parse.rs +++ b/proc-macros/src/config_source/parse.rs @@ -40,3 +40,61 @@ pub(crate) fn parse_input(input: DeriveInput) -> StructData { struct_generics, } } + +#[cfg(test)] +mod config_source_parse_tests { + use quote::quote; + use std::panic::catch_unwind; + + use super::*; + + #[test] + fn can_parse_struct() { + let struct_tok = quote! { + pub struct Foo { + pub bar: String, + pub baz: Option<String>, + } + } + .into(); + + // Parses token stream into DeriveInput for test. + let derive_input = syn::parse2::<DeriveInput>(struct_tok).unwrap(); + + let output = parse_input(derive_input.clone()); + + assert_eq!(output.struct_name, derive_input.ident); + assert_eq!(output.struct_generics, derive_input.generics); + } + + #[test] + fn parse_panics_with_non_struct_type() { + let enum_tok = quote! { + pub enum Foo { + Bar(String), + Baz(Option<String>), + } + } + .into(); + + // Parses token stream into DeriveInput for test. + let derive_input = syn::parse2::<DeriveInput>(enum_tok).unwrap(); + + let result = catch_unwind(|| parse_input(derive_input)); + assert!(result.is_err()); + } + + #[test] + fn parse_panics_with_non_named_fields() { + let unit_struct_tok = quote! { + pub struct Foo; + } + .into(); + + // Parses token stream into DeriveInput for test. + let derive_input = syn::parse2::<DeriveInput>(unit_struct_tok).unwrap(); + + let result = catch_unwind(|| parse_input(derive_input)); + assert!(result.is_err()); + } +} diff --git a/proc-macros/src/config_source/process.rs b/proc-macros/src/config_source/process.rs index a55b7fb..2452291 100644 --- a/proc-macros/src/config_source/process.rs +++ b/proc-macros/src/config_source/process.rs @@ -74,3 +74,286 @@ fn path_is_option(path: &Path) -> bool { fn is_option(ty: &Type) -> bool { matches!(ty, Type::Path(typepath) if typepath.qself.is_none() && path_is_option(&typepath.path)) } + +#[cfg(test)] +mod config_source_process_tests { + use quote::format_ident; + use std::panic::catch_unwind; + use syn::{parse_quote, punctuated::Punctuated, token::Comma, Field, TypePath}; + + use crate::config_source::process::path_is_option; + + use super::*; + + #[test] + fn path_is_option_type() { + let option_string: TypePath = parse_quote!(Option<String>); + assert!(path_is_option(&option_string.path)); + + let option_u64: TypePath = parse_quote!(Option<u64>); + assert!(path_is_option(&option_u64.path)); + + let option_bool: TypePath = parse_quote!(Option<bool>); + assert!(path_is_option(&option_bool.path)); + } + + #[test] + fn path_is_not_option_type() { + let string_type: TypePath = parse_quote!(String); + assert!(!path_is_option(&string_type.path)); + + let u64_type: TypePath = parse_quote!(u64); + assert!(!path_is_option(&u64_type.path)); + + let bool_type: TypePath = parse_quote!(bool); + assert!(!path_is_option(&bool_type.path)); + } + + #[test] + fn type_is_option() { + let option_string_type: Type = parse_quote!(Option<String>); + assert!(is_option(&option_string_type)); + + let option_u64_type: Type = parse_quote!(Option<u64>); + assert!(is_option(&option_u64_type)); + + let option_bool_type: Type = parse_quote!(Option<bool>); + assert!(is_option(&option_bool_type)); + } + + #[test] + fn type_is_not_option() { + let string_type: Type = parse_quote!(String); + assert!(!is_option(&string_type)); + + let u64_type: Type = parse_quote!(u64); + assert!(!is_option(&u64_type)); + + let bool_type: Type = parse_quote!(bool); + assert!(!is_option(&bool_type)); + } + + #[test] + fn can_process_struct_data_with_optional_fields() { + let struct_name = format_ident!("Foo"); + let struct_generics = Generics::default(); + + let field_a: Field = parse_quote!(field_a: Option<String>); + let field_b: Field = parse_quote!(field_b: Option<u64>); + let field_c: Field = parse_quote!(field_c: Option<bool>); + + // Create Punctuated list for input data. + let mut fields = Punctuated::<Field, Comma>::new(); + fields.push_value(field_a.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_b.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_c.clone()); + fields.push_punct(Comma::default()); + + let struct_data = StructData { + struct_name: struct_name.clone(), + struct_fields: fields, + struct_generics: struct_generics.clone(), + }; + + let output = process(struct_data); + + assert_eq!(output.struct_name, struct_name); + assert_eq!(output.struct_generics, struct_generics); + assert_eq!(output.struct_fields.len(), 3); + + // Check that each of the fields is present. + let mut field_iter = output.struct_fields.into_iter(); + let expected_field_a_name = field_a.ident.expect("Field_A ident should be present."); + let expected_field_b_name = field_b.ident.expect("Field_B ident should be present."); + let expected_field_c_name = field_c.ident.expect("Field_C ident should be present."); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_a_name) + && field.name_str.eq(&expected_field_a_name.to_string()) + && field.is_optional + }), + "expected Field_A did not match processed Field_A" + ); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_b_name) + && field.name_str.eq(&expected_field_b_name.to_string()) + && field.is_optional + }), + "expected Field_B did not match processed Field_B" + ); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_c_name) + && field.name_str.eq(&expected_field_c_name.to_string()) + && field.is_optional + }), + "expected Field_C did not match processed Field_C" + ); + } + + #[test] + fn can_process_struct_data_with_non_optional_fields() { + let struct_name = format_ident!("Foo"); + let struct_generics = Generics::default(); + + let field_a: Field = parse_quote!(field_a: String); + let field_b: Field = parse_quote!(field_b: u64); + let field_c: Field = parse_quote!(field_c: bool); + + // Create Punctuated list for input data. + let mut fields = Punctuated::<Field, Comma>::new(); + fields.push_value(field_a.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_b.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_c.clone()); + fields.push_punct(Comma::default()); + + let struct_data = StructData { + struct_name: struct_name.clone(), + struct_fields: fields, + struct_generics: struct_generics.clone(), + }; + + let output = process(struct_data); + + assert_eq!(output.struct_name, struct_name); + assert_eq!(output.struct_generics, struct_generics); + assert_eq!(output.struct_fields.len(), 3); + + // Check that each of the fields is present. + let mut field_iter = output.struct_fields.into_iter(); + let expected_field_a_name = field_a.ident.expect("Field_A ident should be present."); + let expected_field_b_name = field_b.ident.expect("Field_B ident should be present."); + let expected_field_c_name = field_c.ident.expect("Field_C ident should be present."); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_a_name) + && field.name_str.eq(&expected_field_a_name.to_string()) + && !field.is_optional + }), + "expected Field_A did not match processed Field_A" + ); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_b_name) + && field.name_str.eq(&expected_field_b_name.to_string()) + && !field.is_optional + }), + "expected Field_B did not match processed Field_B" + ); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_c_name) + && field.name_str.eq(&expected_field_c_name.to_string()) + && !field.is_optional + }), + "expected Field_C did not match processed Field_C" + ); + } + + #[test] + fn can_process_struct_data_with_mixed_fields() { + let struct_name = format_ident!("Foo"); + let struct_generics = Generics::default(); + + let field_a: Field = parse_quote!(field_a: String); + let field_b: Field = parse_quote!(field_b: Option<u64>); + let field_c: Field = parse_quote!(field_c: bool); + + // Create Punctuated list for input data. + let mut fields = Punctuated::<Field, Comma>::new(); + fields.push_value(field_a.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_b.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_c.clone()); + fields.push_punct(Comma::default()); + + let struct_data = StructData { + struct_name: struct_name.clone(), + struct_fields: fields, + struct_generics: struct_generics.clone(), + }; + + let output = process(struct_data); + + assert_eq!(output.struct_name, struct_name); + assert_eq!(output.struct_generics, struct_generics); + assert_eq!(output.struct_fields.len(), 3); + + // Check that each of the fields is present. + let mut field_iter = output.struct_fields.into_iter(); + let expected_field_a_name = field_a.ident.expect("Field_A ident should be present."); + let expected_field_b_name = field_b.ident.expect("Field_B ident should be present."); + let expected_field_c_name = field_c.ident.expect("Field_C ident should be present."); + + // Is a non optional field. + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_a_name) + && field.name_str.eq(&expected_field_a_name.to_string()) + && !field.is_optional + }), + "expected Field_A did not match processed Field_A" + ); + + // Is an optional field. + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_b_name) + && field.name_str.eq(&expected_field_b_name.to_string()) + && field.is_optional + }), + "expected Field_B did not match processed Field_B" + ); + + // Is a non optional field. + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_c_name) + && field.name_str.eq(&expected_field_c_name.to_string()) + && !field.is_optional + }), + "expected Field_C did not match processed Field_C" + ); + } + + #[test] + fn panic_with_malformed_field_data() { + let struct_name = format_ident!("Foo"); + let struct_generics = Generics::default(); + + let field_a: Field = parse_quote!(field_a: String); + // Malformed Field entry with no name. + let field_b: Field = parse_quote!(Option<u64>); + let field_c: Field = parse_quote!(field_c: bool); + + // Create Punctuated list for input data. + let mut fields = Punctuated::<Field, Comma>::new(); + fields.push_value(field_a.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_b.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_c.clone()); + fields.push_punct(Comma::default()); + + let struct_data = StructData { + struct_name: struct_name.clone(), + struct_fields: fields, + struct_generics: struct_generics.clone(), + }; + + let result = catch_unwind(|| process(struct_data)); + assert!(result.is_err()); + } +} From e671b0608b8a9efedc166697f8ef5bd9fa27f64a Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Fri, 5 Jan 2024 01:58:47 -0800 Subject: [PATCH 18/31] fix clippy warnings --- proc-macros/src/config_source/parse.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/proc-macros/src/config_source/parse.rs b/proc-macros/src/config_source/parse.rs index 6819988..f6f73ca 100644 --- a/proc-macros/src/config_source/parse.rs +++ b/proc-macros/src/config_source/parse.rs @@ -55,8 +55,7 @@ mod config_source_parse_tests { pub bar: String, pub baz: Option<String>, } - } - .into(); + }; // Parses token stream into DeriveInput for test. let derive_input = syn::parse2::<DeriveInput>(struct_tok).unwrap(); @@ -74,8 +73,7 @@ mod config_source_parse_tests { Bar(String), Baz(Option<String>), } - } - .into(); + }; // Parses token stream into DeriveInput for test. let derive_input = syn::parse2::<DeriveInput>(enum_tok).unwrap(); @@ -88,8 +86,7 @@ mod config_source_parse_tests { fn parse_panics_with_non_named_fields() { let unit_struct_tok = quote! { pub struct Foo; - } - .into(); + }; // Parses token stream into DeriveInput for test. let derive_input = syn::parse2::<DeriveInput>(unit_struct_tok).unwrap(); From df650011bf089192836479ad98f4d22ab3982c40 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Fri, 5 Jan 2024 02:18:32 -0800 Subject: [PATCH 19/31] change to forward slash --- docs/config-overrides.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/config-overrides.md b/docs/config-overrides.md index 08a5aee..cd520f0 100644 --- a/docs/config-overrides.md +++ b/docs/config-overrides.md @@ -7,7 +7,7 @@ suitable for basic scenarios or getting started quickly. The service includes th configuration files at build time. The service relies on the environment variable `$AGEMO_HOME` to find any override configuration files. This variable is set by default to point to `{path_to_project_root}/.agemo` when running the service with `cargo run`. Template configuration -files to use to override can be found under [config\template](../config/template/). The default +files to use to override can be found under [config/template](../config/template/). The default configuration files can be overridden at runtime using custom values. When loading configuration, the service will probe for and unify config in the following order, with values near the end of the list taking higher precedence: From 1fae48d6a56da2dda04fd42a2ed13a928f481580 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Fri, 5 Jan 2024 02:20:52 -0800 Subject: [PATCH 20/31] Capitalized err messages --- proc-macros/src/config_source/parse.rs | 2 +- proc-macros/src/config_source/process.rs | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/proc-macros/src/config_source/parse.rs b/proc-macros/src/config_source/parse.rs index f6f73ca..5d024c5 100644 --- a/proc-macros/src/config_source/parse.rs +++ b/proc-macros/src/config_source/parse.rs @@ -31,7 +31,7 @@ pub(crate) fn parse_input(input: DeriveInput) -> StructData { fields: Fields::Named(fields), .. }) => fields.named, - _ => panic!("this derive macro only works on structs with named fields"), + _ => panic!("This derive macro only works on structs with named fields"), }; StructData { diff --git a/proc-macros/src/config_source/process.rs b/proc-macros/src/config_source/process.rs index 2452291..7fec5da 100644 --- a/proc-macros/src/config_source/process.rs +++ b/proc-macros/src/config_source/process.rs @@ -175,7 +175,7 @@ mod config_source_process_tests { && field.name_str.eq(&expected_field_a_name.to_string()) && field.is_optional }), - "expected Field_A did not match processed Field_A" + "Expected Field_A did not match processed Field_A" ); assert!( @@ -184,7 +184,7 @@ mod config_source_process_tests { && field.name_str.eq(&expected_field_b_name.to_string()) && field.is_optional }), - "expected Field_B did not match processed Field_B" + "Expected Field_B did not match processed Field_B" ); assert!( @@ -193,7 +193,7 @@ mod config_source_process_tests { && field.name_str.eq(&expected_field_c_name.to_string()) && field.is_optional }), - "expected Field_C did not match processed Field_C" + "Expected Field_C did not match processed Field_C" ); } @@ -239,7 +239,7 @@ mod config_source_process_tests { && field.name_str.eq(&expected_field_a_name.to_string()) && !field.is_optional }), - "expected Field_A did not match processed Field_A" + "Expected Field_A did not match processed Field_A" ); assert!( @@ -248,7 +248,7 @@ mod config_source_process_tests { && field.name_str.eq(&expected_field_b_name.to_string()) && !field.is_optional }), - "expected Field_B did not match processed Field_B" + "Expected Field_B did not match processed Field_B" ); assert!( @@ -257,7 +257,7 @@ mod config_source_process_tests { && field.name_str.eq(&expected_field_c_name.to_string()) && !field.is_optional }), - "expected Field_C did not match processed Field_C" + "Expected Field_C did not match processed Field_C" ); } @@ -304,7 +304,7 @@ mod config_source_process_tests { && field.name_str.eq(&expected_field_a_name.to_string()) && !field.is_optional }), - "expected Field_A did not match processed Field_A" + "Expected Field_A did not match processed Field_A" ); // Is an optional field. @@ -314,7 +314,7 @@ mod config_source_process_tests { && field.name_str.eq(&expected_field_b_name.to_string()) && field.is_optional }), - "expected Field_B did not match processed Field_B" + "Expected Field_B did not match processed Field_B" ); // Is a non optional field. @@ -324,7 +324,7 @@ mod config_source_process_tests { && field.name_str.eq(&expected_field_c_name.to_string()) && !field.is_optional }), - "expected Field_C did not match processed Field_C" + "Expected Field_C did not match processed Field_C" ); } From 88d17e0ae2fef732c07dbdc199d3dbb84a6eb8a0 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Fri, 5 Jan 2024 02:53:59 -0800 Subject: [PATCH 21/31] simplify collect generated code --- proc-macros/src/config_source/generate.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/proc-macros/src/config_source/generate.rs b/proc-macros/src/config_source/generate.rs index dc5e2ab..dfc34a7 100644 --- a/proc-macros/src/config_source/generate.rs +++ b/proc-macros/src/config_source/generate.rs @@ -45,12 +45,15 @@ pub(crate) fn generate(struct_data: StructDataOutput) -> TokenStream { } fn collect(&self) -> Result<config::Map<String, config::Value>, config::ConfigError> { - let mut entries: config::Map::<String, Option<config::Value>> = config::Map::from([#(#entries)*]); + let entries: config::Map::<String, Option<config::Value>> = config::Map::from([#(#entries)*]); - entries.retain(|_, v| v.is_some()); - let valid_entries: config::Map::<String, config::Value> = entries.iter().map(|(k, v)| (k.clone(), v.clone().unwrap())).collect(); + // Filters out entries with value of None. + let valid_entries: config::Map::<String, config::Value> = entries + .into_iter() + .filter_map(|(k, v)| v.map(|val| (k, val))) + .collect(); - Ok(valid_entries.clone()) + Ok(valid_entries) } } } From faef371e3cb17eee00d344aa30d6d2c551112de0 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Thu, 11 Jan 2024 09:19:32 -0800 Subject: [PATCH 22/31] remove agemo specifics from config_utils --- Cargo.lock | 2 + Cargo.toml | 1 + common/Cargo.toml | 1 + common/src/config_utils.rs | 195 +++++++++++++++++++++-------- pub-sub-service/Cargo.toml | 1 + pub-sub-service/src/load_config.rs | 35 +++++- 6 files changed, 181 insertions(+), 54 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2de78fb..9fcebbc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -506,6 +506,7 @@ dependencies = [ "config", "home", "include_dir", + "lazy_static", "log", "serde", "serde_derive", @@ -1576,6 +1577,7 @@ dependencies = [ "config", "env_logger", "futures", + "include_dir", "log", "paho-mqtt", "proc-macros", diff --git a/Cargo.toml b/Cargo.toml index 119b7c1..8ac3084 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ env_logger = "0.10" futures = "0.3" home = "0.5.9" include_dir = "0.7.3" +lazy_static = "1.0" log = "^0.4" paho-mqtt = "0.12" proc-macro2 = "1.0.70" diff --git a/common/Cargo.toml b/common/Cargo.toml index 844eb52..df11886 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -12,6 +12,7 @@ license = "MIT" config = { workspace = true } home = { workspace = true } include_dir = { workspace = true } +lazy_static = { workspace = true } log = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 3f7b47c..a4f9eee 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -2,55 +2,101 @@ // Licensed under the MIT license. // SPDX-License-Identifier: MIT -use std::{env, path::Path}; +use std::{ + collections::HashMap, + env, io, + path::{Path, PathBuf}, +}; -use config::{File, FileFormat, Source}; +use config::{Config, File, FileFormat, FileStoredFormat, Map, Source}; use home::home_dir; -use include_dir::{include_dir, Dir}; +use include_dir::Dir; +use lazy_static::lazy_static; use serde::Deserialize; -pub const YAML_EXT: &str = "yaml"; +lazy_static! { + /// Creates a map between the file extensions as a str and FileFormat enum. + /// This is used to convert a string into a FileFormat enum. + static ref FILE_EXTS: HashMap<FileFormat, &'static [&'static str]> = { + let mut format_map = Map::<FileFormat, &'static [&'static str]>::default(); -const CONFIG_DIR: &str = "config"; -const DEFAULT: &str = "default"; -const DOT_AGEMO_DIR: &str = ".agemo"; -const AGEMO_HOME: &str = "AGEMO_HOME"; + format_map.insert(FileFormat::Ini, FileFormat::Ini.file_extensions()); + format_map.insert(FileFormat::Json, FileFormat::Json.file_extensions()); + format_map.insert(FileFormat::Json5, FileFormat::Json5.file_extensions()); + format_map.insert(FileFormat::Ron, FileFormat::Ron.file_extensions()); + format_map.insert(FileFormat::Toml, FileFormat::Toml.file_extensions()); + format_map.insert(FileFormat::Yaml, FileFormat::Yaml.file_extensions()); -const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../config"); + format_map + }; +} + +/// Attempts to convert an extension in str format into a FileFormat enum. +/// Throws an error if the extension is unknown. +/// +/// # Arguments +/// * `ext` - extension str to convert. +fn try_into_format(ext: &str) -> Result<FileFormat, Box<dyn std::error::Error + Send + Sync>> { + for (format, extensions) in FILE_EXTS.iter() { + if extensions.contains(&ext) { + return Ok(*format); + } + } + + Err(Box::new(io::Error::new( + io::ErrorKind::NotFound, + "No Supported format found.", + ))) +} -/// Read config from layered configuration files. -/// Searches for `{config_file_name}.default.{config_file_ext}` as the base configuration, -/// then searches for overrides named `{config_file_name}.{config_file_ext}` in `$AGEMO_HOME`. -/// If `$AGEMO_HOME` is not set, it defaults to `$HOME/.agemo`. +/// Loads default config for the given configuration file. +/// Extracts configuration parameters from the provided directory object. /// /// # Arguments -/// * `config_file_name` - The config file name. This is used to construct the file names to search for. -/// * `config_file_ext` - The config file extension. This is used to construct the file names to search for. -/// * `args` - Optional commandline arguments. Any values set will override values gathered from config files. -pub fn read_from_files<T, A>( - config_file_name: &str, +/// * `config_file_stem` - The default config file name without an extension. This is used to +/// construct the file name to search for. +/// * `config_file_ext` - The config file extension. This is used to construct the file name to +/// search for. +/// * `default_dir` - Object that represents directory to pull default config file from. Generated +/// by the `include_dir!` macro. +pub fn load_default_config_from_file( + config_file_stem: &str, config_file_ext: &str, - args: Option<A>, -) -> Result<T, Box<dyn std::error::Error + Send + Sync>> -where - T: for<'a> Deserialize<'a>, - A: Source + Send + Sync + 'static + Clone, -{ - // Get default config. - let default_config_filename = format!("{config_file_name}.{DEFAULT}.{config_file_ext}"); - let default_config_file = DEFAULT_DIR.get_file(default_config_filename).unwrap(); - let default_config_contents_str = default_config_file.contents_utf8().unwrap(); + default_dir: Dir, +) -> Result<Config, Box<dyn std::error::Error + Send + Sync>> { + // Get appropriate default config. + let filename = format!("{config_file_stem}.{config_file_ext}"); + let file = default_dir.get_file(filename).unwrap(); + let file_contents = file.contents_utf8().unwrap(); + let file_format = try_into_format(config_file_ext)?; - // Get override_files - let overrides_file = format!("{config_file_name}.{config_file_ext}"); + Config::builder() + .add_source(File::from_str(file_contents, file_format)) + .build() + .map_err(|e| e.into()) +} - let config_path = match env::var(AGEMO_HOME) { +/// Retrieve configuration home path from the service's HOME dir environment variable. +/// Attempts to construct a path from the provided service HOME env var. If the given env var is +/// not set, it defaults a path under the $HOME directory. +/// +/// # Arguments +/// * `svc_home_env_var` - Name of the environment variable used to set the service's HOME dir. +/// * `svc_home_dir_name` - Default name for the service's HOME dir, used if `svc_home_env_var` is +/// not set. +/// * `config_dir_name` - Name of the config directory where configuration files should live. +pub fn get_config_home_path_from_env( + svc_home_env_var: &str, + svc_home_dir_name: &str, + config_dir_name: &str, +) -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> { + let config_path = match env::var(svc_home_env_var) { Ok(agemo_home) => { - // The path below resolves to $AGEMO_HOME/config/ - Path::new(&agemo_home).join(CONFIG_DIR) + // The path below resolves to $SVC_HOME/{config_dir_name}/ + Path::new(&agemo_home).join(config_dir_name) } Err(_) => { - // The path below resolves to $HOME/.agemo/config/ + // The path below resolves to $HOME/{svc_home_dir_name}/{config_dir_name}/ home_dir() .ok_or_else(|| { std::io::Error::new( @@ -58,27 +104,76 @@ where "Could not retrieve home directory", ) })? - .join(DOT_AGEMO_DIR) - .join(CONFIG_DIR) + .join(svc_home_dir_name) + .join(config_dir_name) } }; - // The path below resolves to {config_path}/{overrides_file} - let overrides_config_file_path = config_path.join(overrides_file); + Ok(config_path) +} - let mut config_sources = config::Config::builder() - .add_source(File::from_str( - default_config_contents_str, - FileFormat::Yaml, - )) - .add_source(File::from(overrides_config_file_path).required(false)); +/// Read config from a configuration file located at the given config path. +/// +/// # Arguments +/// * `config_file_stem` - The config file name without an extension. This is used to construct the +/// file name to search for. +/// * `config_file_ext` - The config file extension. This is used to construct the file name to +/// search for. +/// * `config_path` - The path to the directory containing the config. +pub fn read_from_file<TPath>( + config_file_stem: &str, + config_file_ext: &str, + config_path: TPath, +) -> Result<Config, Box<dyn std::error::Error + Send + Sync>> +where + TPath: AsRef<Path>, +{ + let config_file_name = format!("{config_file_stem}.{config_file_ext}"); - // Adds command line arguments if there are any. - if let Some(args) = args { - config_sources = config_sources.add_source(args); - } + // The path below resolves to {config_path}/{config_file_name} + let config_file_path = config_path.as_ref().join(config_file_name); + + Config::builder() + .add_source(File::from(config_file_path).required(false)) + .build() + .map_err(|e| e.into()) +} + +/// Builds unified config from provided configuration sources. +/// Config will be compiled in the following order, with values from sources near the end of the +/// list taking higher precedence: +/// +/// - default source +/// - file source +/// - commandline source +/// +/// Since the configuration is layered, config can be partially defined. Any unspecified +/// configuration will use the default value from the default config source. +/// +/// # Arguments +/// * `default_source` - Config gathered from default configuration sources. +/// * `file_source` - Config read in from configuration files. +/// * `cmdline_source` - Optional config gathered from commandline parameters. +pub fn build_config<TConfig, TSource>( + default_source: Config, + file_source: Config, + cmdline_source: Option<TSource>, +) -> Result<TConfig, Box<dyn std::error::Error + Send + Sync>> +where + TConfig: for<'a> Deserialize<'a>, + TSource: Source + Send + Sync + 'static, +{ + let mut config_sources = Config::builder() + .add_source(default_source) + .add_source(file_source); - let config_store = config_sources.build()?; + // Adds command line arguments if there are any. + if let Some(cmdline_args) = cmdline_source { + config_sources = config_sources.add_source(cmdline_args); + }; - config_store.try_deserialize().map_err(|e| e.into()) + config_sources + .build()? + .try_deserialize() + .map_err(|e| e.into()) } diff --git a/pub-sub-service/Cargo.toml b/pub-sub-service/Cargo.toml index 88b8ddd..f2410fc 100644 --- a/pub-sub-service/Cargo.toml +++ b/pub-sub-service/Cargo.toml @@ -16,6 +16,7 @@ common = { path = "../common" } config = { workspace = true } env_logger = { workspace = true } futures = { workspace = true } +include_dir = { workspace = true } log = { workspace = true } paho-mqtt = { workspace = true } proc-macros = { path = "../proc-macros"} diff --git a/pub-sub-service/src/load_config.rs b/pub-sub-service/src/load_config.rs index b9ec89a..c3daa6c 100644 --- a/pub-sub-service/src/load_config.rs +++ b/pub-sub-service/src/load_config.rs @@ -8,6 +8,7 @@ use std::env; use clap::Parser; use common::config_utils; +use include_dir::{include_dir, Dir}; use log::{debug, error}; use proc_macros::ConfigSource; use serde_derive::{Deserialize, Serialize}; @@ -15,6 +16,14 @@ use serde_derive::{Deserialize, Serialize}; const CONFIG_FILE_NAME: &str = "pub_sub_service_settings"; const CONSTANTS_FILE_NAME: &str = "constants"; +const YAML_EXT: &str = "yaml"; +const CONFIG_DIR: &str = "config"; +const DOT_AGEMO_DIR: &str = ".agemo"; +const AGEMO_HOME_ENV_VAR: &str = "AGEMO_HOME"; +const DEFAULT: &str = "default"; + +const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../config"); + /// Object containing commandline config options for the Pub Sub service. /// Non-optional fields must be passed in via the commandline and will override any values from /// configuration files. @@ -78,19 +87,37 @@ pub struct Settings { pub version: Option<String>, } -/// Load a configuration file. +/// Load configuration given a file and commandline arguments. /// /// # Arguments -/// * `config_file_name` - Name of the config file to load settings from. +/// * `config_file_stem` - Name of the config file to load settings from. /// * `args` - Optional commandline config arguments. pub fn load_config<T>( - config_file_name: &str, + config_file_stem: &str, args: Option<CmdConfigOptions>, ) -> Result<T, Box<dyn std::error::Error + Send + Sync>> where T: for<'de> serde::Deserialize<'de>, { - config_utils::read_from_files(config_file_name, config_utils::YAML_EXT, args) + let config_file_ext = YAML_EXT; + let default_file_stem = format!("{config_file_stem}.{DEFAULT}"); + + // Load default configuration for the given configuration file. + let default_source = config_utils::load_default_config_from_file( + &default_file_stem, + config_file_ext, + DEFAULT_DIR, + )?; + + // Get configuration path from environment variable. + let config_path = + config_utils::get_config_home_path_from_env(AGEMO_HOME_ENV_VAR, DOT_AGEMO_DIR, CONFIG_DIR)?; + + // Read configuration file for any overrides. + let file_source = config_utils::read_from_file(config_file_stem, config_file_ext, config_path)?; + + // Build config object from gathered sources. + config_utils::build_config(default_source, file_source, args) } /// Load the settings. From efedf3a795cdcd06f00a6a18de1ea52fe0377986 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Fri, 12 Jan 2024 05:26:24 -0800 Subject: [PATCH 23/31] further improvements to config_utils --- Cargo.lock | 1 - Cargo.toml | 1 - common/Cargo.toml | 1 - common/src/config_utils.rs | 215 +++++++++++++++++------------ pub-sub-service/src/load_config.rs | 67 +++++---- 5 files changed, 163 insertions(+), 122 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9fcebbc..3844f7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -506,7 +506,6 @@ dependencies = [ "config", "home", "include_dir", - "lazy_static", "log", "serde", "serde_derive", diff --git a/Cargo.toml b/Cargo.toml index 8ac3084..119b7c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,6 @@ env_logger = "0.10" futures = "0.3" home = "0.5.9" include_dir = "0.7.3" -lazy_static = "1.0" log = "^0.4" paho-mqtt = "0.12" proc-macro2 = "1.0.70" diff --git a/common/Cargo.toml b/common/Cargo.toml index df11886..844eb52 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -12,7 +12,6 @@ license = "MIT" config = { workspace = true } home = { workspace = true } include_dir = { workspace = true } -lazy_static = { workspace = true } log = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index a4f9eee..c3924c3 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -3,33 +3,16 @@ // SPDX-License-Identifier: MIT use std::{ - collections::HashMap, env, io, path::{Path, PathBuf}, }; -use config::{Config, File, FileFormat, FileStoredFormat, Map, Source}; +use config::{Config, File, FileFormat, FileStoredFormat, Source}; use home::home_dir; use include_dir::Dir; -use lazy_static::lazy_static; use serde::Deserialize; -lazy_static! { - /// Creates a map between the file extensions as a str and FileFormat enum. - /// This is used to convert a string into a FileFormat enum. - static ref FILE_EXTS: HashMap<FileFormat, &'static [&'static str]> = { - let mut format_map = Map::<FileFormat, &'static [&'static str]>::default(); - - format_map.insert(FileFormat::Ini, FileFormat::Ini.file_extensions()); - format_map.insert(FileFormat::Json, FileFormat::Json.file_extensions()); - format_map.insert(FileFormat::Json5, FileFormat::Json5.file_extensions()); - format_map.insert(FileFormat::Ron, FileFormat::Ron.file_extensions()); - format_map.insert(FileFormat::Toml, FileFormat::Toml.file_extensions()); - format_map.insert(FileFormat::Yaml, FileFormat::Yaml.file_extensions()); - - format_map - }; -} +pub const FILE_SEPARATOR: &str = "."; /// Attempts to convert an extension in str format into a FileFormat enum. /// Throws an error if the extension is unknown. @@ -37,43 +20,74 @@ lazy_static! { /// # Arguments /// * `ext` - extension str to convert. fn try_into_format(ext: &str) -> Result<FileFormat, Box<dyn std::error::Error + Send + Sync>> { - for (format, extensions) in FILE_EXTS.iter() { - if extensions.contains(&ext) { - return Ok(*format); - } + match ext { + ext if FileFormat::Ini.file_extensions().contains(&ext) => Ok(FileFormat::Ini), + ext if FileFormat::Json.file_extensions().contains(&ext) => Ok(FileFormat::Json), + ext if FileFormat::Json5.file_extensions().contains(&ext) => Ok(FileFormat::Json5), + ext if FileFormat::Ron.file_extensions().contains(&ext) => Ok(FileFormat::Ron), + ext if FileFormat::Toml.file_extensions().contains(&ext) => Ok(FileFormat::Toml), + ext if FileFormat::Yaml.file_extensions().contains(&ext) => Ok(FileFormat::Yaml), + _ => Err(Box::new(io::Error::new( + io::ErrorKind::NotFound, + "No Supported format found.", + ))), } +} + +/// Service's home directory metadata. +pub struct SvcConfigHomeMetadata { + /// Name of the environment variable used to set the service's HOME dir. + pub home_env_var: String, + /// Default name for the service's HOME dir, used if `home_env_var` is not set. + pub home_dir: String, + /// Name of the config directory where configuration files should live. + pub config_dir: String, +} + +/// Metadata for a config file. +pub struct ConfigFileMetadata { + /// Config file name with extension. + pub name: String, + /// File extension. + pub ext: FileFormat, +} - Err(Box::new(io::Error::new( - io::ErrorKind::NotFound, - "No Supported format found.", - ))) +impl ConfigFileMetadata { + pub fn new(file_name: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> { + let name = file_name.to_string(); + let mut split_name: Vec<&str> = file_name.split(FILE_SEPARATOR).collect(); + + if split_name.len() <= 1 { + return Err(Box::new(io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid file name format. Expected an extension in name '{file_name}'."), + ))); + } + + let parsed_ext = split_name.pop().unwrap(); + let ext = try_into_format(parsed_ext)?; + + Ok(ConfigFileMetadata { name, ext }) + } } -/// Loads default config for the given configuration file. -/// Extracts configuration parameters from the provided directory object. +/// Loads default config source for the given configuration file. +/// Extracts configuration parameters from the provided directory object which pulled in default +/// configuration files in at build time. /// /// # Arguments -/// * `config_file_stem` - The default config file name without an extension. This is used to -/// construct the file name to search for. -/// * `config_file_ext` - The config file extension. This is used to construct the file name to -/// search for. +/// * `config_file` - The default config file to load. /// * `default_dir` - Object that represents directory to pull default config file from. Generated /// by the `include_dir!` macro. pub fn load_default_config_from_file( - config_file_stem: &str, - config_file_ext: &str, - default_dir: Dir, -) -> Result<Config, Box<dyn std::error::Error + Send + Sync>> { + config_file: &ConfigFileMetadata, + default_dir: &Dir, +) -> Result<Box<dyn Source + Send + Sync>, Box<dyn std::error::Error + Send + Sync>> { // Get appropriate default config. - let filename = format!("{config_file_stem}.{config_file_ext}"); - let file = default_dir.get_file(filename).unwrap(); + let file = default_dir.get_file(&config_file.name).unwrap(); let file_contents = file.contents_utf8().unwrap(); - let file_format = try_into_format(config_file_ext)?; - Config::builder() - .add_source(File::from_str(file_contents, file_format)) - .build() - .map_err(|e| e.into()) + Ok(File::from_str(file_contents, config_file.ext).clone_into_box()) } /// Retrieve configuration home path from the service's HOME dir environment variable. @@ -81,19 +95,14 @@ pub fn load_default_config_from_file( /// not set, it defaults a path under the $HOME directory. /// /// # Arguments -/// * `svc_home_env_var` - Name of the environment variable used to set the service's HOME dir. -/// * `svc_home_dir_name` - Default name for the service's HOME dir, used if `svc_home_env_var` is -/// not set. -/// * `config_dir_name` - Name of the config directory where configuration files should live. +/// * `svc_home_metadata` - Metadata related to the service's home and config directories. pub fn get_config_home_path_from_env( - svc_home_env_var: &str, - svc_home_dir_name: &str, - config_dir_name: &str, + svc_home_metadata: &SvcConfigHomeMetadata, ) -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> { - let config_path = match env::var(svc_home_env_var) { + let config_path = match env::var(&svc_home_metadata.home_env_var) { Ok(agemo_home) => { // The path below resolves to $SVC_HOME/{config_dir_name}/ - Path::new(&agemo_home).join(config_dir_name) + Path::new(&agemo_home).join(&svc_home_metadata.config_dir) } Err(_) => { // The path below resolves to $HOME/{svc_home_dir_name}/{config_dir_name}/ @@ -104,76 +113,98 @@ pub fn get_config_home_path_from_env( "Could not retrieve home directory", ) })? - .join(svc_home_dir_name) - .join(config_dir_name) + .join(&svc_home_metadata.home_dir) + .join(&svc_home_metadata.config_dir) } }; Ok(config_path) } -/// Read config from a configuration file located at the given config path. +/// Read config from a configuration file located at the given config path at runtime. /// /// # Arguments -/// * `config_file_stem` - The config file name without an extension. This is used to construct the -/// file name to search for. -/// * `config_file_ext` - The config file extension. This is used to construct the file name to -/// search for. +/// * `config_file` - The config file to read. /// * `config_path` - The path to the directory containing the config. pub fn read_from_file<TPath>( - config_file_stem: &str, - config_file_ext: &str, + config_file: &ConfigFileMetadata, config_path: TPath, -) -> Result<Config, Box<dyn std::error::Error + Send + Sync>> +) -> Result<Box<dyn Source + Send + Sync>, Box<dyn std::error::Error + Send + Sync>> where TPath: AsRef<Path>, { - let config_file_name = format!("{config_file_stem}.{config_file_ext}"); + let config_file_path = config_path.as_ref().join(&config_file.name); - // The path below resolves to {config_path}/{config_file_name} - let config_file_path = config_path.as_ref().join(config_file_name); + Ok(File::from(config_file_path) + .required(false) + .clone_into_box()) +} +/// Builds unified config from provided configuration sources. +/// +/// # Arguments +/// * `sources` - List of sources to build configuration from. Sources towards the end of the list +/// take higher precedence over sources at the beginning of the list. +pub fn build_config_from_sources<TConfig>( + sources: Vec<Box<dyn Source + Send + Sync>>, +) -> Result<TConfig, Box<dyn std::error::Error + Send + Sync>> +where + TConfig: for<'a> Deserialize<'a>, +{ Config::builder() - .add_source(File::from(config_file_path).required(false)) - .build() + .add_source(sources) + .build()? + .try_deserialize() .map_err(|e| e.into()) } -/// Builds unified config from provided configuration sources. +/// Load a unified configuration given a file and commandline arguments. /// Config will be compiled in the following order, with values from sources near the end of the /// list taking higher precedence: /// -/// - default source -/// - file source -/// - commandline source +/// - default config file +/// - config file +/// - commandline args /// /// Since the configuration is layered, config can be partially defined. Any unspecified /// configuration will use the default value from the default config source. /// /// # Arguments -/// * `default_source` - Config gathered from default configuration sources. -/// * `file_source` - Config read in from configuration files. -/// * `cmdline_source` - Optional config gathered from commandline parameters. -pub fn build_config<TConfig, TSource>( - default_source: Config, - file_source: Config, - cmdline_source: Option<TSource>, +/// * `config_file` - The config file to load configuration from. +/// * `default_config_file` - The default config file to load default config from. +/// * `default_dir` - Object that represents directory to pull default config file from. Generated +/// by the `include_dir!` macro. +/// * `svc_home_metadata` - Metadata related to the service's home and config directories. Used to +/// get path to provided config file. +/// * `cmdline_args` - Optional commandline config arguments. +pub fn load_config<TConfig, TArgs>( + config_file: &ConfigFileMetadata, + default_config_file: &ConfigFileMetadata, + default_dir: &Dir, + svc_home_metadata: &SvcConfigHomeMetadata, + cmdline_args: Option<TArgs>, ) -> Result<TConfig, Box<dyn std::error::Error + Send + Sync>> where - TConfig: for<'a> Deserialize<'a>, - TSource: Source + Send + Sync + 'static, + TConfig: for<'de> serde::Deserialize<'de>, + TArgs: Source + Send + Sync, { - let mut config_sources = Config::builder() - .add_source(default_source) - .add_source(file_source); + // Load default configuration for the given configuration file. + let default_source = load_default_config_from_file(default_config_file, default_dir)?; - // Adds command line arguments if there are any. - if let Some(cmdline_args) = cmdline_source { - config_sources = config_sources.add_source(cmdline_args); - }; + // Get configuration path from environment variable. + let config_path = get_config_home_path_from_env(svc_home_metadata)?; - config_sources - .build()? - .try_deserialize() - .map_err(|e| e.into()) + // Read configuration file for any overrides. + let file_source = read_from_file(config_file, config_path)?; + + // Create source list from lowest to highest priority. + let mut sources = vec![default_source, file_source]; + + // If commandline args are present, add them to the source list. + if let Some(args) = cmdline_args { + sources.push(args.clone_into_box()); + } + + // Build config from source list. + build_config_from_sources(sources) } diff --git a/pub-sub-service/src/load_config.rs b/pub-sub-service/src/load_config.rs index c3daa6c..cf152d7 100644 --- a/pub-sub-service/src/load_config.rs +++ b/pub-sub-service/src/load_config.rs @@ -7,21 +7,28 @@ use std::env; use clap::Parser; -use common::config_utils; +use common::config_utils::{self, ConfigFileMetadata, SvcConfigHomeMetadata}; use include_dir::{include_dir, Dir}; use log::{debug, error}; use proc_macros::ConfigSource; use serde_derive::{Deserialize, Serialize}; -const CONFIG_FILE_NAME: &str = "pub_sub_service_settings"; -const CONSTANTS_FILE_NAME: &str = "constants"; +// Config file stems +const CONFIG_FILE_STEM: &str = "pub_sub_service_settings"; +const CONSTANTS_FILE_STEM: &str = "constants"; +// Config file extensions const YAML_EXT: &str = "yaml"; + +// Default config file marker +const DEFAULT: &str = "default"; + +// Config directory consts const CONFIG_DIR: &str = "config"; const DOT_AGEMO_DIR: &str = ".agemo"; const AGEMO_HOME_ENV_VAR: &str = "AGEMO_HOME"; -const DEFAULT: &str = "default"; +// Default directory struct const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../config"); /// Object containing commandline config options for the Pub Sub service. @@ -90,34 +97,34 @@ pub struct Settings { /// Load configuration given a file and commandline arguments. /// /// # Arguments -/// * `config_file_stem` - Name of the config file to load settings from. +/// * `config_file_name` - Name of the config file to load settings from. /// * `args` - Optional commandline config arguments. pub fn load_config<T>( - config_file_stem: &str, + config_file_name: &str, + default_file_name: &str, args: Option<CmdConfigOptions>, ) -> Result<T, Box<dyn std::error::Error + Send + Sync>> where T: for<'de> serde::Deserialize<'de>, { - let config_file_ext = YAML_EXT; - let default_file_stem = format!("{config_file_stem}.{DEFAULT}"); - - // Load default configuration for the given configuration file. - let default_source = config_utils::load_default_config_from_file( - &default_file_stem, - config_file_ext, - DEFAULT_DIR, - )?; - - // Get configuration path from environment variable. - let config_path = - config_utils::get_config_home_path_from_env(AGEMO_HOME_ENV_VAR, DOT_AGEMO_DIR, CONFIG_DIR)?; - - // Read configuration file for any overrides. - let file_source = config_utils::read_from_file(config_file_stem, config_file_ext, config_path)?; - - // Build config object from gathered sources. - config_utils::build_config(default_source, file_source, args) + let config_file = ConfigFileMetadata::new(config_file_name)?; + let default_config_file = ConfigFileMetadata::new(default_file_name)?; + + let default_dir = DEFAULT_DIR; + + let svc_home_metadata = SvcConfigHomeMetadata { + home_env_var: AGEMO_HOME_ENV_VAR.to_string(), + home_dir: DOT_AGEMO_DIR.to_string(), + config_dir: CONFIG_DIR.to_string(), + }; + + config_utils::load_config( + &config_file, + &default_config_file, + &default_dir, + &svc_home_metadata, + args, + ) } /// Load the settings. @@ -130,7 +137,10 @@ where pub fn load_settings( args: CmdConfigOptions, ) -> Result<Settings, Box<dyn std::error::Error + Send + Sync>> { - let mut settings: Settings = load_config(CONFIG_FILE_NAME, Some(args)) + let file_name = format!("{CONFIG_FILE_STEM}.{YAML_EXT}"); + let default_file_name = format!("{CONFIG_FILE_STEM}.{DEFAULT}.{YAML_EXT}"); + + let mut settings: Settings = load_config(&file_name, &default_file_name, Some(args)) .map_err(|e| { format!( "Failed to load required configuration settings due to error: {e}. See --help for more details." @@ -172,5 +182,8 @@ pub fn load_constants<T>() -> Result<T, Box<dyn std::error::Error + Send + Sync> where T: for<'de> serde::Deserialize<'de>, { - load_config(CONSTANTS_FILE_NAME, None) + let file_name = format!("{CONSTANTS_FILE_STEM}.{YAML_EXT}"); + let default_file_name = format!("{CONSTANTS_FILE_STEM}.{DEFAULT}.{YAML_EXT}"); + + load_config(&file_name, &default_file_name, None) } From 31b343b5158fe3fcc9026b2f16e40475939d730e Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Fri, 12 Jan 2024 05:36:19 -0800 Subject: [PATCH 24/31] Add comment to configFileMetadata new func --- common/src/config_utils.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index c3924c3..6a25a48 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -53,6 +53,11 @@ pub struct ConfigFileMetadata { } impl ConfigFileMetadata { + /// Create a new instance of ConfigFileMetadata. + /// Will result in an error if provided file does not have a valid extension. + /// + /// # Arguments + /// * `file_name` - Name of the file including the extension. pub fn new(file_name: &str) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> { let name = file_name.to_string(); let mut split_name: Vec<&str> = file_name.split(FILE_SEPARATOR).collect(); From cf167fb20f73b944df0b0f03bf25ecfa430ed377 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Wed, 17 Jan 2024 11:11:52 -0800 Subject: [PATCH 25/31] Add tests for config utils --- common/src/config_utils.rs | 271 +++++++++++++++++++++++++++++++++++-- 1 file changed, 262 insertions(+), 9 deletions(-) diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 6a25a48..88aa7fe 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: MIT use std::{ - env, io, + env, path::{Path, PathBuf}, }; @@ -27,8 +27,8 @@ fn try_into_format(ext: &str) -> Result<FileFormat, Box<dyn std::error::Error + ext if FileFormat::Ron.file_extensions().contains(&ext) => Ok(FileFormat::Ron), ext if FileFormat::Toml.file_extensions().contains(&ext) => Ok(FileFormat::Toml), ext if FileFormat::Yaml.file_extensions().contains(&ext) => Ok(FileFormat::Yaml), - _ => Err(Box::new(io::Error::new( - io::ErrorKind::NotFound, + _ => Err(Box::new(std::io::Error::new( + std::io::ErrorKind::NotFound, "No Supported format found.", ))), } @@ -45,6 +45,7 @@ pub struct SvcConfigHomeMetadata { } /// Metadata for a config file. +#[derive(Debug, PartialEq, Eq)] pub struct ConfigFileMetadata { /// Config file name with extension. pub name: String, @@ -63,8 +64,8 @@ impl ConfigFileMetadata { let mut split_name: Vec<&str> = file_name.split(FILE_SEPARATOR).collect(); if split_name.len() <= 1 { - return Err(Box::new(io::Error::new( - io::ErrorKind::InvalidData, + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidInput, format!("Invalid file name format. Expected an extension in name '{file_name}'."), ))); } @@ -72,6 +73,15 @@ impl ConfigFileMetadata { let parsed_ext = split_name.pop().unwrap(); let ext = try_into_format(parsed_ext)?; + if split_name.join(FILE_SEPARATOR).is_empty() { + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "Invalid file name format. File cannot have an empty file_stem '{file_name}'." + ), + ))); + } + Ok(ConfigFileMetadata { name, ext }) } } @@ -89,8 +99,20 @@ pub fn load_default_config_from_file( default_dir: &Dir, ) -> Result<Box<dyn Source + Send + Sync>, Box<dyn std::error::Error + Send + Sync>> { // Get appropriate default config. - let file = default_dir.get_file(&config_file.name).unwrap(); - let file_contents = file.contents_utf8().unwrap(); + let file = default_dir + .get_file(&config_file.name) + .ok_or(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("Unable to find default file '{}'.", &config_file.name), + ))?; + + let file_contents = file.contents_utf8().ok_or(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "Unable to parse default file '{}' contents.", + &config_file.name + ), + ))?; Ok(File::from_str(file_contents, config_file.ext).clone_into_box()) } @@ -105,9 +127,9 @@ pub fn get_config_home_path_from_env( svc_home_metadata: &SvcConfigHomeMetadata, ) -> Result<PathBuf, Box<dyn std::error::Error + Send + Sync>> { let config_path = match env::var(&svc_home_metadata.home_env_var) { - Ok(agemo_home) => { + Ok(svc_home) => { // The path below resolves to $SVC_HOME/{config_dir_name}/ - Path::new(&agemo_home).join(&svc_home_metadata.config_dir) + Path::new(&svc_home).join(&svc_home_metadata.config_dir) } Err(_) => { // The path below resolves to $HOME/{svc_home_dir_name}/{config_dir_name}/ @@ -213,3 +235,234 @@ where // Build config from source list. build_config_from_sources(sources) } + +#[cfg(test)] +mod pubsub_impl_tests { + use config::Value; + use include_dir::{DirEntry, File}; + + use super::*; + + #[test] + fn try_into_format_success() { + // Ini format + let ini_ext = "ini"; + let ini_format = try_into_format(ini_ext).unwrap(); + assert_eq!(ini_format, FileFormat::Ini); + + // Json format + let json_ext = "json"; + let json_format = try_into_format(json_ext).unwrap(); + assert_eq!(json_format, FileFormat::Json); + + // Json5 format + let json5_ext = "json5"; + let json5_format = try_into_format(json5_ext).unwrap(); + assert_eq!(json5_format, FileFormat::Json5); + + // Ron format + let ron_ext = "ron"; + let ron_format = try_into_format(ron_ext).unwrap(); + assert_eq!(ron_format, FileFormat::Ron); + + // Toml format + let toml_ext = "toml"; + let toml_format = try_into_format(toml_ext).unwrap(); + assert_eq!(toml_format, FileFormat::Toml); + + // Yaml format + let yaml_ext = "yaml"; + let yaml_format = try_into_format(yaml_ext).unwrap(); + assert_eq!(yaml_format, FileFormat::Yaml); + + let yaml_ext_2 = "yml"; + let yaml_format_2 = try_into_format(yaml_ext_2).unwrap(); + assert_eq!(yaml_format_2, FileFormat::Yaml); + } + + #[test] + fn try_into_format_invalid_err() { + let ext_1 = "invalid"; + let result_1 = try_into_format(ext_1); + assert!(result_1.is_err()); + + let ext_2 = ""; + let result_2 = try_into_format(ext_2); + assert!(result_2.is_err()); + + let ext_3 = "123@"; + let result_3 = try_into_format(ext_3); + assert!(result_3.is_err()); + } + + #[test] + fn new_config_metadata_from_file_name() { + let expected_name = "test.yaml"; + let expected_metadata = ConfigFileMetadata { + name: expected_name.to_string(), + ext: FileFormat::Yaml, + }; + + let metadata = ConfigFileMetadata::new(expected_name).unwrap(); + assert_eq!(metadata, expected_metadata); + + let expected_name_2 = "test.default.json"; + let expected_metadata_2 = ConfigFileMetadata { + name: expected_name_2.to_string(), + ext: FileFormat::Json, + }; + + let metadata_2 = ConfigFileMetadata::new(expected_name_2).unwrap(); + assert_eq!(metadata_2, expected_metadata_2); + } + + #[test] + fn new_config_metadata_from_invalid_str_err() { + let result = ConfigFileMetadata::new("no_extension"); + let err = result.err().unwrap().downcast::<std::io::Error>().unwrap(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidInput); + + let result_2 = ConfigFileMetadata::new(""); + let err_2 = result_2 + .err() + .unwrap() + .downcast::<std::io::Error>() + .unwrap(); + assert_eq!(err_2.kind(), std::io::ErrorKind::InvalidInput); + + let result_3 = ConfigFileMetadata::new(".yaml"); + let err_3 = result_3 + .err() + .unwrap() + .downcast::<std::io::Error>() + .unwrap(); + assert_eq!(err_3.kind(), std::io::ErrorKind::InvalidInput); + + let result_4 = ConfigFileMetadata::new("test.bad_extension"); + let err_4 = result_4 + .err() + .unwrap() + .downcast::<std::io::Error>() + .unwrap(); + assert_eq!(err_4.kind(), std::io::ErrorKind::NotFound); + } + + #[test] + fn load_default_config_from_file_success() { + let file_name = "config.yaml"; + let config_file = ConfigFileMetadata { + name: file_name.to_string(), + ext: FileFormat::Yaml, + }; + + // Expected property to be returned. + let expected_property_name = "test"; + let expected_property_value = Value::new(None, config::ValueKind::I64(1)); + let expected_properties_list_len = 1; + + // u8 representation of the expected property in yaml: "test: 1". + let contents: &[u8] = &[116, 101, 115, 116, 58, 32, 49]; + + // Create directory object. + let expected_file = File::new(file_name, contents); + let entry = DirEntry::File(expected_file); + let entries = &[entry]; + let dir = Dir::new("", entries); + + let source = load_default_config_from_file(&config_file, &dir).unwrap(); + let properties = source.collect().unwrap(); + + assert_eq!(properties.len(), expected_properties_list_len); + assert!(properties.contains_key(expected_property_name)); + assert_eq!( + properties.get(expected_property_name).unwrap(), + &expected_property_value + ); + } + + #[test] + fn load_default_config_from_file_non_existent() { + let non_existent_file = ConfigFileMetadata { + name: "non_existent.yaml".to_string(), + ext: FileFormat::Yaml, + }; + + let dir = Dir::new("", &[]); + + let result = load_default_config_from_file(&non_existent_file, &dir); + let err = result.err().unwrap().downcast::<std::io::Error>().unwrap(); + assert_eq!(err.kind(), std::io::ErrorKind::NotFound); + } + + #[test] + fn load_default_config_from_file_malformed_contents() { + let file_name = "config.yaml"; + let config_file = ConfigFileMetadata { + name: file_name.to_string(), + ext: FileFormat::Yaml, + }; + + // Malformed bytes. + let contents: &[u8] = &[0, 159, 146, 150]; + + // Create directory object. + let expected_file = File::new(file_name, contents); + let entry = DirEntry::File(expected_file); + let entries = &[entry]; + let dir = Dir::new("", entries); + + let result = load_default_config_from_file(&config_file, &dir); + let err = result.err().unwrap().downcast::<std::io::Error>().unwrap(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); + } + + #[test] + fn get_config_home_path_from_env_success() { + let env_var_key = "TEST_ENV_VAR"; + let env_var_value = "test_dir"; + let svc_home_dir = ".svc"; + let config_dir = "config"; + + let expected_path = Path::new(env_var_value).join(config_dir); + + // Set the test environment variable. + env::set_var(env_var_key, env_var_value); + + let svc_home_metadata = SvcConfigHomeMetadata { + home_env_var: env_var_key.to_string(), + home_dir: svc_home_dir.to_string(), + config_dir: config_dir.to_string(), + }; + + let path = get_config_home_path_from_env(&svc_home_metadata); + + // Unset the environment variable. + env::remove_var(env_var_key); + + assert_eq!(path.unwrap(), expected_path); + } + + #[test] + fn get_config_home_path_from_env_no_svc_home() { + let env_var_key = "TEST_ENV_VAR"; + let home_dir = home_dir(); + let svc_home_dir = ".svc"; + let config_dir = "config"; + + // If the environment variable happens to be set, we want to unset it. + if env::var(env_var_key).is_ok() { + env::remove_var(env_var_key); + } + + let expected_path = home_dir.unwrap().join(svc_home_dir).join(config_dir); + + let svc_home_metadata = SvcConfigHomeMetadata { + home_env_var: env_var_key.to_string(), + home_dir: svc_home_dir.to_string(), + config_dir: config_dir.to_string(), + }; + + let path = get_config_home_path_from_env(&svc_home_metadata).unwrap(); + assert_eq!(path, expected_path); + } +} From fe0bc3dac2295a2f519bd45151bb168c16d473d4 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:13:53 -0800 Subject: [PATCH 26/31] fixed capitalization --- common/src/config_utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 88aa7fe..430b946 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -29,7 +29,7 @@ fn try_into_format(ext: &str) -> Result<FileFormat, Box<dyn std::error::Error + ext if FileFormat::Yaml.file_extensions().contains(&ext) => Ok(FileFormat::Yaml), _ => Err(Box::new(std::io::Error::new( std::io::ErrorKind::NotFound, - "No Supported format found.", + "No supported format found.", ))), } } @@ -137,7 +137,7 @@ pub fn get_config_home_path_from_env( .ok_or_else(|| { std::io::Error::new( std::io::ErrorKind::Other, - "Could not retrieve home directory", + "Could not retrieve home directory.", ) })? .join(&svc_home_metadata.home_dir) From e6bb529d8fb4eddf8ac3302990a1fbb75a820ec0 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:17:37 -0800 Subject: [PATCH 27/31] renamed func and removed some comments. --- common/src/config_utils.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 430b946..5578f1d 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -153,7 +153,7 @@ pub fn get_config_home_path_from_env( /// # Arguments /// * `config_file` - The config file to read. /// * `config_path` - The path to the directory containing the config. -pub fn read_from_file<TPath>( +pub fn read_config_from_file<TPath>( config_file: &ConfigFileMetadata, config_path: TPath, ) -> Result<Box<dyn Source + Send + Sync>, Box<dyn std::error::Error + Send + Sync>> @@ -215,24 +215,21 @@ where TConfig: for<'de> serde::Deserialize<'de>, TArgs: Source + Send + Sync, { - // Load default configuration for the given configuration file. let default_source = load_default_config_from_file(default_config_file, default_dir)?; - // Get configuration path from environment variable. + // Find and read configuration file for any overrides. let config_path = get_config_home_path_from_env(svc_home_metadata)?; - - // Read configuration file for any overrides. - let file_source = read_from_file(config_file, config_path)?; + let file_source = read_config_from_file(config_file, config_path)?; // Create source list from lowest to highest priority. let mut sources = vec![default_source, file_source]; // If commandline args are present, add them to the source list. + // Commandline args will override any config from file sources. if let Some(args) = cmdline_args { sources.push(args.clone_into_box()); } - // Build config from source list. build_config_from_sources(sources) } From 151cbca37e12dd42eba8d9afc9053a11627a19c8 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:19:15 -0800 Subject: [PATCH 28/31] add arg comment --- pub-sub-service/src/load_config.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pub-sub-service/src/load_config.rs b/pub-sub-service/src/load_config.rs index cf152d7..71e3b7d 100644 --- a/pub-sub-service/src/load_config.rs +++ b/pub-sub-service/src/load_config.rs @@ -97,7 +97,8 @@ pub struct Settings { /// Load configuration given a file and commandline arguments. /// /// # Arguments -/// * `config_file_name` - Name of the config file to load settings from. +/// * `config_file_name` - Name of the config file to load override settings from. +/// * `default_file_name` - Name of default config file to load settings from. /// * `args` - Optional commandline config arguments. pub fn load_config<T>( config_file_name: &str, From a445026dfbb8c73d84e49d1d7667ca72d3c6d013 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:40:33 -0800 Subject: [PATCH 29/31] updated crates to fix security audit --- Cargo.lock | 341 ++++++++++++++++++------------------- common/src/config_utils.rs | 2 +- 2 files changed, 169 insertions(+), 174 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3844f7b..528b17b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,9 +39,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.5" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +checksum = "628a8f9bd1e24b4e0db2b4bc2d000b001e7dd032d54afa60a68836aeec5aa54a" dependencies = [ "anstyle", "anstyle-parse", @@ -87,9 +87,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "async-channel" @@ -109,7 +109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" dependencies = [ "concurrent-queue", - "event-listener 4.0.0", + "event-listener 4.0.3", "event-listener-strategy", "futures-core", "pin-project-lite", @@ -121,26 +121,26 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ae5ebefcc48e7452b4987947920dac9450be1110cadf34d1b8c116bdbaf97c" dependencies = [ - "async-lock 3.1.2", + "async-lock 3.3.0", "async-task", "concurrent-queue", "fastrand 2.0.1", - "futures-lite 2.0.1", + "futures-lite 2.2.0", "slab", ] [[package]] name = "async-global-executor" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4353121d5644cdf2beb5726ab752e79a8db1ebb52031770ec47db31d245526" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ "async-channel 2.1.1", "async-executor", - "async-io 2.2.1", - "async-lock 3.1.2", + "async-io 2.3.0", + "async-lock 3.3.0", "blocking", - "futures-lite 2.0.1", + "futures-lite 2.2.0", "once_cell", ] @@ -166,18 +166,18 @@ dependencies = [ [[package]] name = "async-io" -version = "2.2.1" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6d3b15875ba253d1110c740755e246537483f152fa334f91abd7fe84c88b3ff" +checksum = "fb41eb19024a91746eba0773aa5e16036045bbf45733766661099e182ea6a744" dependencies = [ - "async-lock 3.1.2", + "async-lock 3.3.0", "cfg-if", "concurrent-queue", "futures-io", - "futures-lite 2.0.1", + "futures-lite 2.2.0", "parking", - "polling 3.3.1", - "rustix 0.38.25", + "polling 3.3.2", + "rustix 0.38.30", "slab", "tracing", "windows-sys 0.52.0", @@ -194,11 +194,11 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.1.2" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea8b3453dd7cc96711834b75400d671b73e3656975fa68d9f277163b7f7e316" +checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" dependencies = [ - "event-listener 4.0.0", + "event-listener 4.0.3", "event-listener-strategy", "pin-project-lite", ] @@ -253,15 +253,15 @@ dependencies = [ [[package]] name = "async-task" -version = "4.5.0" +version = "4.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eb2cdb97421e01129ccb49169d8279ed21e829929144f4a22a6e54ac549ca1" +checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799" [[package]] name = "async-trait" -version = "0.1.76" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531b97fb4cd3dfdce92c35dedbfdc1f0b9d8091c8ca943d6dae340ef5012d514" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", @@ -348,9 +348,9 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bitflags" @@ -360,9 +360,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "block-buffer" @@ -380,11 +380,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ "async-channel 2.1.1", - "async-lock 3.1.2", + "async-lock 3.3.0", "async-task", "fastrand 2.0.1", "futures-io", - "futures-lite 2.0.1", + "futures-lite 2.2.0", "piper", "tracing", ] @@ -446,9 +446,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.11" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" dependencies = [ "clap_builder", "clap_derive", @@ -456,9 +456,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.11" +version = "4.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" dependencies = [ "anstream", "anstyle", @@ -514,9 +514,9 @@ dependencies = [ [[package]] name = "concurrent-queue" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f057a694a54f12365049b0958a1685bb52d567f5593b355fbf685838e873d400" +checksum = "d16048cd947b08fa32c24458a22f5dc5e835264f689f4f5653210c69fd107363" dependencies = [ "crossbeam-utils", ] @@ -542,31 +542,27 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] [[package]] name = "crossbeam-channel" -version = "0.5.8" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" dependencies = [ - "cfg-if", "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" -dependencies = [ - "cfg-if", -] +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" [[package]] name = "crypto-common" @@ -647,9 +643,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "4.0.0" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770d968249b5d99410d61f5bf89057f3199a077a04d087092f58e7d10692baae" +checksum = "67b215c49b2b248c855fb73579eb1f4f26c38ffdc12973e20e07b91d78d5646e" dependencies = [ "concurrent-queue", "parking", @@ -662,7 +658,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "958e4d70b6d5e81971bebec42271ec641e7ff4e170a6fa605f2b8a8b65cb97d3" dependencies = [ - "event-listener 4.0.0", + "event-listener 4.0.3", "pin-project-lite", ] @@ -704,9 +700,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -719,9 +715,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -729,15 +725,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -746,9 +742,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -767,23 +763,22 @@ dependencies = [ [[package]] name = "futures-lite" -version = "2.0.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3831c2651acb5177cbd83943f3d9c8912c5ad03c76afcc0e9511ba568ec5ebb" +checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba" dependencies = [ "fastrand 2.0.1", "futures-core", "futures-io", - "memchr", "parking", "pin-project-lite", ] [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -792,15 +787,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -810,9 +805,9 @@ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -838,9 +833,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -867,9 +862,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -907,9 +902,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "home" @@ -933,9 +928,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -962,9 +957,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -977,7 +972,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -1067,13 +1062,13 @@ dependencies = [ [[package]] name = "is-terminal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" dependencies = [ "hermit-abi", - "rustix 0.38.25", - "windows-sys 0.48.0", + "rustix 0.38.30", + "windows-sys 0.52.0", ] [[package]] @@ -1087,15 +1082,15 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -1128,9 +1123,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "linked-hash-map" @@ -1146,9 +1141,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "log" @@ -1167,9 +1162,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "mime" @@ -1194,9 +1189,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "wasi", @@ -1215,7 +1210,7 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "cfg-if", "libc", ] @@ -1242,33 +1237,33 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl-src" -version = "300.1.6+3.1.4" +version = "300.2.1+3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439fac53e092cd7442a3660c85dde4643ab3b5bd39040912388dcdabf6b88085" +checksum = "3fe476c29791a5ca0d1273c697e96085bbabbbea2ef7afd5617e78a4b40332d3" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.96" +version = "0.9.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" +checksum = "c1665caf8ab2dc9aef43d1c0023bd904633a6a05cb30b0ad59bec2ae986e57a7" dependencies = [ "cc", "libc", @@ -1333,9 +1328,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae9cee2a55a544be8b89dc6848072af97a20f2422603c10865be2a42b580fff5" +checksum = "1f200d8d83c44a45b21764d1916299752ca035d15ecd46faca3e9a2a2bf6ad06" dependencies = [ "memchr", "thiserror", @@ -1344,9 +1339,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81d78524685f5ef2a3b3bd1cafbc9fcabb036253d9b1463e726a91cd16e2dfc2" +checksum = "bcd6ab1236bbdb3a49027e920e693192ebfe8913f6d60e294de57463a493cfde" dependencies = [ "pest", "pest_generator", @@ -1354,9 +1349,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68bd1206e71118b5356dae5ddc61c8b11e28b09ef6a31acbd15ea48a28e0c227" +checksum = "2a31940305ffc96863a735bef7c7994a00b325a7138fdbc5bda0f1a0476d3275" dependencies = [ "pest", "pest_meta", @@ -1367,9 +1362,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c747191d4ad9e4a4ab9c8798f1e82a39affe7ef9648390b7e5548d18e099de6" +checksum = "a7ff62f5259e53b78d1af898941cdcdccfae7385cf7d793a6e55de5d05bb4b7d" dependencies = [ "once_cell", "pest", @@ -1431,9 +1426,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "polling" @@ -1453,14 +1448,14 @@ dependencies = [ [[package]] name = "polling" -version = "3.3.1" +version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf63fa624ab313c11656b4cda960bfc46c410187ad493c41f6ba2d8c1e991c9e" +checksum = "545c980a3880efd47b2e262f6a4bb6daad6555cf3367aa9c4e52895f69537a41" dependencies = [ "cfg-if", "concurrent-queue", "pin-project-lite", - "rustix 0.38.25", + "rustix 0.38.30", "tracing", "windows-sys 0.52.0", ] @@ -1473,9 +1468,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", "syn", @@ -1483,9 +1478,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -1594,9 +1589,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -1712,15 +1707,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.25" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", - "linux-raw-sys 0.4.11", - "windows-sys 0.48.0", + "linux-raw-sys 0.4.13", + "windows-sys 0.52.0", ] [[package]] @@ -1731,9 +1726,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "sample-mqtt-connector" @@ -1779,18 +1774,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", @@ -1799,9 +1794,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.109" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0652c533506ad7a2e353cce269330d6afd8bdfb6d75e0ace5b35aacbd7b9e9" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -1901,9 +1896,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.41" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -1918,40 +1913,40 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand 2.0.1", "redox_syscall", - "rustix 0.38.25", - "windows-sys 0.48.0", + "rustix 0.38.30", + "windows-sys 0.52.0", ] [[package]] name = "termcolor" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", @@ -2054,7 +2049,7 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.5", + "base64 0.21.7", "bytes", "h2", "http", @@ -2150,9 +2145,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" @@ -2168,9 +2163,9 @@ checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -2228,9 +2223,9 @@ dependencies = [ [[package]] name = "value-bag" -version = "1.4.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72e1902dde2bd6441347de2b70b7f5d59bf157c6c62f0c44572607a1d55bbe" +checksum = "7cdbaf5e132e593e9fc1de6a15bbec912395b11fb9719e061cf64f804524c503" [[package]] name = "vcpkg" @@ -2267,9 +2262,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2277,9 +2272,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", @@ -2292,9 +2287,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -2304,9 +2299,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2314,9 +2309,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", @@ -2327,15 +2322,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", @@ -2350,7 +2345,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.25", + "rustix 0.38.30", ] [[package]] diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 5578f1d..bca13c7 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -234,7 +234,7 @@ where } #[cfg(test)] -mod pubsub_impl_tests { +mod config_utils_tests { use config::Value; use include_dir::{DirEntry, File}; From afc69a4ae1d84bd1391a77c37211e972f9f70ea5 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Wed, 17 Jan 2024 22:06:17 -0800 Subject: [PATCH 30/31] Update try_into_format func name --- common/src/config_utils.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index bca13c7..7b185c1 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -19,7 +19,7 @@ pub const FILE_SEPARATOR: &str = "."; /// /// # Arguments /// * `ext` - extension str to convert. -fn try_into_format(ext: &str) -> Result<FileFormat, Box<dyn std::error::Error + Send + Sync>> { +fn try_ext_into_file_format(ext: &str) -> Result<FileFormat, Box<dyn std::error::Error + Send + Sync>> { match ext { ext if FileFormat::Ini.file_extensions().contains(&ext) => Ok(FileFormat::Ini), ext if FileFormat::Json.file_extensions().contains(&ext) => Ok(FileFormat::Json), @@ -71,7 +71,7 @@ impl ConfigFileMetadata { } let parsed_ext = split_name.pop().unwrap(); - let ext = try_into_format(parsed_ext)?; + let ext = try_ext_into_file_format(parsed_ext)?; if split_name.join(FILE_SEPARATOR).is_empty() { return Err(Box::new(std::io::Error::new( @@ -244,51 +244,51 @@ mod config_utils_tests { fn try_into_format_success() { // Ini format let ini_ext = "ini"; - let ini_format = try_into_format(ini_ext).unwrap(); + let ini_format = try_ext_into_file_format(ini_ext).unwrap(); assert_eq!(ini_format, FileFormat::Ini); // Json format let json_ext = "json"; - let json_format = try_into_format(json_ext).unwrap(); + let json_format = try_ext_into_file_format(json_ext).unwrap(); assert_eq!(json_format, FileFormat::Json); // Json5 format let json5_ext = "json5"; - let json5_format = try_into_format(json5_ext).unwrap(); + let json5_format = try_ext_into_file_format(json5_ext).unwrap(); assert_eq!(json5_format, FileFormat::Json5); // Ron format let ron_ext = "ron"; - let ron_format = try_into_format(ron_ext).unwrap(); + let ron_format = try_ext_into_file_format(ron_ext).unwrap(); assert_eq!(ron_format, FileFormat::Ron); // Toml format let toml_ext = "toml"; - let toml_format = try_into_format(toml_ext).unwrap(); + let toml_format = try_ext_into_file_format(toml_ext).unwrap(); assert_eq!(toml_format, FileFormat::Toml); // Yaml format let yaml_ext = "yaml"; - let yaml_format = try_into_format(yaml_ext).unwrap(); + let yaml_format = try_ext_into_file_format(yaml_ext).unwrap(); assert_eq!(yaml_format, FileFormat::Yaml); let yaml_ext_2 = "yml"; - let yaml_format_2 = try_into_format(yaml_ext_2).unwrap(); + let yaml_format_2 = try_ext_into_file_format(yaml_ext_2).unwrap(); assert_eq!(yaml_format_2, FileFormat::Yaml); } #[test] fn try_into_format_invalid_err() { let ext_1 = "invalid"; - let result_1 = try_into_format(ext_1); + let result_1 = try_ext_into_file_format(ext_1); assert!(result_1.is_err()); let ext_2 = ""; - let result_2 = try_into_format(ext_2); + let result_2 = try_ext_into_file_format(ext_2); assert!(result_2.is_err()); let ext_3 = "123@"; - let result_3 = try_into_format(ext_3); + let result_3 = try_ext_into_file_format(ext_3); assert!(result_3.is_err()); } From 35ed665a1c4600a446d834d2da03fedc272a1339 Mon Sep 17 00:00:00 2001 From: Devin Kelley <105753233+devkelley@users.noreply.github.com> Date: Wed, 17 Jan 2024 22:07:09 -0800 Subject: [PATCH 31/31] Fix fmt --- common/src/config_utils.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 7b185c1..baadb7b 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -19,7 +19,9 @@ pub const FILE_SEPARATOR: &str = "."; /// /// # Arguments /// * `ext` - extension str to convert. -fn try_ext_into_file_format(ext: &str) -> Result<FileFormat, Box<dyn std::error::Error + Send + Sync>> { +fn try_ext_into_file_format( + ext: &str, +) -> Result<FileFormat, Box<dyn std::error::Error + Send + Sync>> { match ext { ext if FileFormat::Ini.file_extensions().contains(&ext) => Ok(FileFormat::Ini), ext if FileFormat::Json.file_extensions().contains(&ext) => Ok(FileFormat::Json),