From 1fc28a91ab10b806c7912e907692f3490a66d82a Mon Sep 17 00:00:00 2001 From: devkelley <105753233+devkelley@users.noreply.github.com> Date: Fri, 12 Jan 2024 17:08:49 -0800 Subject: [PATCH] Add command line configuration and simplified config hierarchy (#63) * config demo v1 * Add command line configuration * order config overrides * Remove curr dir config * Update docs and modify default files * fix doc errs * remove dockerfile cp as default config is now part of executable * Refactored proc macro, added detailed comments * fixed clippy warnings * fix unexpected delimiter * fixed doctest errs * second attempt * remove example as it is over complicated * add params to comments * remove unecessary clonse * add comment about required cmdline params * Add tests for config_source macro * fix clippy warnings * change to forward slash * Capitalized err messages * simplify collect generated code --- Cargo.lock | 142 ++++++- Cargo.toml | 6 + Dockerfile.amd64 | 3 - Dockerfile.arm64 | 3 - README.md | 2 +- common/Cargo.toml | 3 +- common/src/config_utils.rs | 50 ++- .../config => config}/constants.default.yaml | 2 - .../pub_sub_service_settings.default.yaml | 6 + .../template/pub_sub_service_settings.yaml | 0 docs/config-overrides.md | 28 +- docs/containers.md | 3 - proc-macros/Cargo.toml | 18 + proc-macros/src/config_source/generate.rs | 60 +++ proc-macros/src/config_source/mod.rs | 33 ++ proc-macros/src/config_source/parse.rs | 97 +++++ proc-macros/src/config_source/process.rs | 359 ++++++++++++++++++ proc-macros/src/lib.rs | 19 + pub-sub-service/Cargo.toml | 2 + pub-sub-service/README.md | 6 +- pub-sub-service/src/load_config.rs | 63 ++- pub-sub-service/src/main.rs | 16 +- 22 files changed, 865 insertions(+), 56 deletions(-) rename {.agemo/config => config}/constants.default.yaml (98%) rename {.agemo/config => config}/pub_sub_service_settings.default.yaml (90%) rename {.agemo/config => config}/template/pub_sub_service_settings.yaml (100%) create mode 100644 proc-macros/Cargo.toml create mode 100644 proc-macros/src/config_source/generate.rs create mode 100644 proc-macros/src/config_source/mod.rs create mode 100644 proc-macros/src/config_source/parse.rs create mode 100644 proc-macros/src/config_source/process.rs create mode 100644 proc-macros/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 6ec8c18..2de78fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,54 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d664a92ecae85fd0a7392615844904654d1d5f5514837f471ddef4a057aba1b6" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7079075b41f533b8c61d2a4d073c4676e1f8b249ff94a393b0595db304e0dd87" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "anyhow" version = "1.0.75" @@ -396,6 +444,46 @@ dependencies = [ "uuid", ] +[[package]] +name = "clap" +version = "4.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfaff671f6b22ca62406885ece523383b9b64022e341e53e009a62ebc47a45f2" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + [[package]] name = "cmake" version = "0.1.50" @@ -405,12 +493,19 @@ dependencies = [ "cc", ] +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "common" version = "0.1.0" dependencies = [ "config", "home", + "include_dir", "log", "serde", "serde_derive", @@ -911,6 +1006,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "include_dir" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18762faeff7122e89e0857b02f7ce6fcc0d101d5e9ad2ad7846cc01d61b7f19e" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b139284b5cf57ecfa712bcc66950bb635b31aff41c188e8a4cfc758eca374a3f" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indexmap" version = "1.9.3" @@ -1376,6 +1490,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proc-macros" +version = "0.1.0" +dependencies = [ + "config", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "prost" version = "0.12.3" @@ -1447,12 +1571,14 @@ version = "0.1.0" dependencies = [ "async-std", "async-trait", + "clap", "common", "config", "env_logger", "futures", "log", "paho-mqtt", + "proc-macros", "proto", "serde", "serde_derive", @@ -1747,6 +1873,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strum" version = "0.25.0" @@ -1768,9 +1900,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" dependencies = [ "proc-macro2", "quote", @@ -2065,6 +2197,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "uuid" version = "1.6.1" diff --git a/Cargo.toml b/Cargo.toml index bb59419..119b7c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ resolver = "2" members = [ "common", + "proc-macros", "pub-sub-service", "samples/chariott-publisher", "samples/chariott-subscriber", @@ -17,20 +18,25 @@ members = [ [workspace.dependencies] async-std = "1" async-trait = "0.1.76" +clap = { version = "4.4.11" } config = "0.13.3" ctrlc = { version = "3.4", features = ["termination"] } env_logger = "0.10" futures = "0.3" home = "0.5.9" +include_dir = "0.7.3" log = "^0.4" paho-mqtt = "0.12" +proc-macro2 = "1.0.70" prost = "0.12" prost-types = "0.12" +quote = "1.0.23" serde = "1.0.160" serde_derive = "1.0.163" serde_json = "^1.0" strum = "0.25" strum_macros = "0.25" +syn = { version = "2.0.40", features = ["extra-traits", "full"] } tokio = { version = "1.35.1", features = ["time"] } tonic = "0.10" tonic-build = "0.10" diff --git a/Dockerfile.amd64 b/Dockerfile.amd64 index 7700aa5..0265284 100644 --- a/Dockerfile.amd64 +++ b/Dockerfile.amd64 @@ -90,9 +90,6 @@ ENV AGEMO_HOME=/sdv/.agemo # Copy the executable from the "build" stage. COPY --from=build /sdv/service /sdv/ -# Copy default configs. -COPY --from=build /sdv/.agemo/config/ /sdv/.agemo/config/ - # Expose the port that the application listens on. EXPOSE 50051 diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index b28e72a..94e7e7f 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -93,9 +93,6 @@ ENV AGEMO_HOME=/sdv/.agemo # Copy the executable from the "build" stage. COPY --from=build /sdv/service /sdv/ -# Copy default configs. -COPY --from=build /sdv/.agemo/config/ /sdv/.agemo/config/ - # Expose the port that the application listens on. EXPOSE 50051 diff --git a/README.md b/README.md index 994f0ad..6b1ac36 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ cargo test ## Configuration Setup -The service configuration is defined in [.agemo/config](.agemo/config/). The default configuration +The service configuration is defined in [config](config/). The default configuration files will allow a user to run the service without any modification. Please read [config_overrides](./docs/config-overrides.md) for more information on how to modify the service's configuration. diff --git a/common/Cargo.toml b/common/Cargo.toml index a5a68a9..844eb52 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -11,7 +11,8 @@ license = "MIT" [dependencies] config = { workspace = true } home = { workspace = true } +include_dir = { workspace = true } log = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true } \ No newline at end of file diff --git a/common/src/config_utils.rs b/common/src/config_utils.rs index 4b15b54..3f7b47c 100644 --- a/common/src/config_utils.rs +++ b/common/src/config_utils.rs @@ -4,32 +4,44 @@ use std::{env, path::Path}; -use config::File; +use config::{File, FileFormat, Source}; use home::home_dir; +use include_dir::{include_dir, Dir}; use serde::Deserialize; pub const YAML_EXT: &str = "yaml"; const CONFIG_DIR: &str = "config"; +const DEFAULT: &str = "default"; const DOT_AGEMO_DIR: &str = ".agemo"; const AGEMO_HOME: &str = "AGEMO_HOME"; +const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../config"); + /// Read config from layered configuration files. -/// Searches for `{config_file_name}.default.{config_file_ext}` as the base configuration in `$AGEMO_HOME`, -/// then searches for overrides named `{config_file_name}.{config_file_ext}` in the current directory and `$AGEMO_HOME`. +/// Searches for `{config_file_name}.default.{config_file_ext}` as the base configuration, +/// then searches for overrides named `{config_file_name}.{config_file_ext}` in `$AGEMO_HOME`. /// If `$AGEMO_HOME` is not set, it defaults to `$HOME/.agemo`. /// /// # Arguments -/// - `config_file_name`: The config file name. This is used to construct the file names to search for. -/// - `config_file_ext`: The config file extension. This is used to construct the file names to search for. -pub fn read_from_files( +/// * `config_file_name` - The config file name. This is used to construct the file names to search for. +/// * `config_file_ext` - The config file extension. This is used to construct the file names to search for. +/// * `args` - Optional commandline arguments. Any values set will override values gathered from config files. +pub fn read_from_files( config_file_name: &str, config_file_ext: &str, + args: Option, ) -> Result> where T: for<'a> Deserialize<'a>, + A: Source + Send + Sync + 'static + Clone, { - let default_config_file = format!("{config_file_name}.default.{config_file_ext}"); + // Get default config. + let default_config_filename = format!("{config_file_name}.{DEFAULT}.{config_file_ext}"); + let default_config_file = DEFAULT_DIR.get_file(default_config_filename).unwrap(); + let default_config_contents_str = default_config_file.contents_utf8().unwrap(); + + // Get override_files let overrides_file = format!("{config_file_name}.{config_file_ext}"); let config_path = match env::var(AGEMO_HOME) { @@ -51,20 +63,22 @@ where } }; - // The path below resolves to {config_path}/{default_config_file}. - let default_config_file_path = config_path.join(default_config_file); - - // The path below resolves to {current_dir}/{overrides_file}. - let current_dir_config_file_path = env::current_dir()?.join(overrides_file.clone()); - // The path below resolves to {config_path}/{overrides_file} let overrides_config_file_path = config_path.join(overrides_file); - let config_store = config::Config::builder() - .add_source(File::from(default_config_file_path)) - .add_source(File::from(current_dir_config_file_path).required(false)) - .add_source(File::from(overrides_config_file_path).required(false)) - .build()?; + let mut config_sources = config::Config::builder() + .add_source(File::from_str( + default_config_contents_str, + FileFormat::Yaml, + )) + .add_source(File::from(overrides_config_file_path).required(false)); + + // Adds command line arguments if there are any. + if let Some(args) = args { + config_sources = config_sources.add_source(args); + } + + let config_store = config_sources.build()?; config_store.try_deserialize().map_err(|e| e.into()) } diff --git a/.agemo/config/constants.default.yaml b/config/constants.default.yaml similarity index 98% rename from .agemo/config/constants.default.yaml rename to config/constants.default.yaml index 969a653..08c5d33 100644 --- a/.agemo/config/constants.default.yaml +++ b/config/constants.default.yaml @@ -18,5 +18,3 @@ pub_sub_reference: "pubsub.v1.pubsub.proto" # Retry interval for connections. retry_interval_secs: 5 - -### diff --git a/.agemo/config/pub_sub_service_settings.default.yaml b/config/pub_sub_service_settings.default.yaml similarity index 90% rename from .agemo/config/pub_sub_service_settings.default.yaml rename to config/pub_sub_service_settings.default.yaml index c65099a..7db8a9a 100644 --- a/.agemo/config/pub_sub_service_settings.default.yaml +++ b/config/pub_sub_service_settings.default.yaml @@ -1,7 +1,11 @@ +### + # # Pub Sub Service Settings # +### Standalone Settings + # The IP address and port number that the Pub Sub Service listens on for requests. # Example: "0.0.0.0:50051" pub_sub_authority: "0.0.0.0:50051" @@ -9,3 +13,5 @@ pub_sub_authority: "0.0.0.0:50051" # The URI of the messaging service used to facilitate publish and subscribe functionality. # Example: "mqtt://0.0.0.0:1883" messaging_uri: "mqtt://0.0.0.0:1883" + +### diff --git a/.agemo/config/template/pub_sub_service_settings.yaml b/config/template/pub_sub_service_settings.yaml similarity index 100% rename from .agemo/config/template/pub_sub_service_settings.yaml rename to config/template/pub_sub_service_settings.yaml diff --git a/docs/config-overrides.md b/docs/config-overrides.md index dcf114d..cd520f0 100644 --- a/docs/config-overrides.md +++ b/docs/config-overrides.md @@ -2,26 +2,36 @@ The pub sub service supports configuration overrides that enable users to override the default settings and provide custom configuration. This is achieved with configuration layering. Default -configuration files are defined in `.agemo/config` at the root of the project, and this is often -suitable for basic scenarios or getting started quickly. The service relies on the environment -variable `$AGEMO_HOME` to find the default configuration files. This variable is set by default to -point to `{path_to_project_root}/.agemo` when running the service with `cargo run`. The default +configuration files are defined in `config` at the root of the project, and this is often +suitable for basic scenarios or getting started quickly. The service includes the default +configuration files at build time. The service relies on the environment variable `$AGEMO_HOME` to +find any override configuration files. This variable is set by default to point to +`{path_to_project_root}/.agemo` when running the service with `cargo run`. Template configuration +files to use to override can be found under [config/template](../config/template/). The default configuration files can be overridden at runtime using custom values. When loading configuration, the service will probe for and unify config in the following order, with values near the end of the list taking higher precedence: -- The default config -- A config file in the working directory of the executable (for example, the directory you were in -when you ran the `cargo run` command) - `$AGEMO_HOME/config/{config_name}.yaml`. If you have not set a `$AGEMO_HOME` directory or are not running the service with `cargo run`, this defaults to: - Unix: `$HOME/.agemo/config/{config_name}.yaml` - Windows: `%USERPROFILE%\.agemo\config\{config_name}.yaml` (note that Windows support is not guaranteed by Agemo) +- Command line arguments. Because the config is layered, the overrides can be partially defined and only specify the top-level configuration fields that should be overridden. Anything not specified in an override file will use the default value. -Samples handles configuration in the same way, except it utilizes the `.agemo-samples` directory -and the `$AGEMO_SAMPLES_HOME` env variable to point to that directory. +Samples handles configuration in the same way, except it utilizes the `$AGEMO_SAMPLES_HOME` env +variable to point to the `.agemo-samples` directory at the project root. + +## Command Line Arguments + +The service leverages [clap (command line argument parser)](https://github.com/clap-rs/clap) to +override individual configuration values through command line arguments when starting the service. +To see the list of possible parameters, run: + +```shell +cargo run -p pub-sub-service -- --help +``` diff --git a/docs/containers.md b/docs/containers.md index 8c5139e..50040d0 100644 --- a/docs/containers.md +++ b/docs/containers.md @@ -14,9 +14,6 @@ x86-64 architecture. - [Dockerfile.arm64](../Dockerfile.arm64) - Dockerfile used to build the `Pub Sub Service` for the aarch64 architecture. ->Note: The default configuration files are cloned from [.agemo/config](../.agemo/config/), defined -in the project's root. - #### Mosquitto MQTT Broker - [Dockerfile.mosquitto.amd64](../Dockerfile.mosquitto.amd64) - Dockerfile used to build the diff --git a/proc-macros/Cargo.toml b/proc-macros/Cargo.toml new file mode 100644 index 0000000..5b569a0 --- /dev/null +++ b/proc-macros/Cargo.toml @@ -0,0 +1,18 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. +# SPDX-License-Identifier: MIT + +[package] +name = "proc-macros" +version = "0.1.0" +edition = "2021" +license = "MIT" + +[lib] +proc-macro = true + +[dependencies] +config = { workspace = true } +proc-macro2 = { workspace = true } +quote = { workspace = true } +syn = { workspace = true } diff --git a/proc-macros/src/config_source/generate.rs b/proc-macros/src/config_source/generate.rs new file mode 100644 index 0000000..dfc34a7 --- /dev/null +++ b/proc-macros/src/config_source/generate.rs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +use proc_macro2::TokenStream; +use quote::quote; + +use super::process::StructDataOutput; + +/// Generate code for the ConfigSource derive macro. +/// +/// # Arguments +/// * `struct_data` - Data gathered from a Struct. +pub(crate) fn generate(struct_data: StructDataOutput) -> TokenStream { + // Define values for the code generation. + let struct_name = struct_data.struct_name; + let struct_entries = struct_data.struct_fields; + + // Define generics information for the code generation. + let (impl_generics, type_generics, where_clause) = struct_data.struct_generics.split_for_impl(); + + // Construct a list of entries from the fields of the Struct. + let entries = struct_entries.into_iter().map(|entry| { + let field_name = entry.name; + let field_name_str = entry.name_str; + + // Code snippet changes based on whether the entry is an optional field. + if entry.is_optional { + quote! { + (String::from(#field_name_str), (&self.#field_name).clone().map(|v| config::Value::from(v))), + } + } else { + quote! { + (String::from(#field_name_str), Some(config::Value::from((&self.#field_name).clone()))), + } + } + }); + + // Construct a code snippet that implements the `Source` Trait. + quote! { + #[automatically_derived] + impl #impl_generics config::Source #type_generics for #struct_name #where_clause { + fn clone_into_box(&self) -> Box { + Box::new((*self).clone()) + } + + fn collect(&self) -> Result, config::ConfigError> { + let entries: config::Map::> = config::Map::from([#(#entries)*]); + + // Filters out entries with value of None. + let valid_entries: config::Map:: = entries + .into_iter() + .filter_map(|(k, v)| v.map(|val| (k, val))) + .collect(); + + Ok(valid_entries) + } + } + } +} diff --git a/proc-macros/src/config_source/mod.rs b/proc-macros/src/config_source/mod.rs new file mode 100644 index 0000000..359201c --- /dev/null +++ b/proc-macros/src/config_source/mod.rs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +mod generate; +mod parse; +mod process; + +use proc_macro::TokenStream; +use syn::{parse_macro_input, DeriveInput}; + +use generate::generate; +use parse::parse_input; +use process::process; + +/// Implements the ConfigSource derive macro +/// +/// # Arguments: +/// +/// - `ts`: The token stream input +pub fn config_source(ts: TokenStream) -> TokenStream { + // Parse token stream into input. + let input: DeriveInput = parse_macro_input!(ts); + + // Parse input into Struct data. + let data = parse_input(input); + + // Process the Struct data. + let processed_data = process(data); + + // Generate the output code. + generate(processed_data).into() +} diff --git a/proc-macros/src/config_source/parse.rs b/proc-macros/src/config_source/parse.rs new file mode 100644 index 0000000..5d024c5 --- /dev/null +++ b/proc-macros/src/config_source/parse.rs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +use proc_macro2::Ident; +use syn::{punctuated::Punctuated, token::Comma, Field}; +use syn::{Data, DataStruct, DeriveInput, Fields, Generics}; + +/// Represents a Struct. +pub(crate) struct StructData { + /// The identifier of the Struct. + pub struct_name: Ident, + /// List of fields of the Struct. + pub struct_fields: Punctuated, + /// The generics associated with the Struct. + pub struct_generics: Generics, +} + +/// Parse input data for the ConfigSource derive macro. +/// Will panic if input is not gathered from a Struct. +/// +/// # Arguments +/// * `input` - Parsed derive macro input. +pub(crate) fn parse_input(input: DeriveInput) -> StructData { + let struct_name = input.ident; + let struct_generics = input.generics; + + // Processes input data into Struct Fields. Panics if data is not from a Struct. + let struct_fields = match input.data { + Data::Struct(DataStruct { + fields: Fields::Named(fields), + .. + }) => fields.named, + _ => panic!("This derive macro only works on structs with named fields"), + }; + + StructData { + struct_name, + struct_fields, + struct_generics, + } +} + +#[cfg(test)] +mod config_source_parse_tests { + use quote::quote; + use std::panic::catch_unwind; + + use super::*; + + #[test] + fn can_parse_struct() { + let struct_tok = quote! { + pub struct Foo { + pub bar: String, + pub baz: Option, + } + }; + + // Parses token stream into DeriveInput for test. + let derive_input = syn::parse2::(struct_tok).unwrap(); + + let output = parse_input(derive_input.clone()); + + assert_eq!(output.struct_name, derive_input.ident); + assert_eq!(output.struct_generics, derive_input.generics); + } + + #[test] + fn parse_panics_with_non_struct_type() { + let enum_tok = quote! { + pub enum Foo { + Bar(String), + Baz(Option), + } + }; + + // Parses token stream into DeriveInput for test. + let derive_input = syn::parse2::(enum_tok).unwrap(); + + let result = catch_unwind(|| parse_input(derive_input)); + assert!(result.is_err()); + } + + #[test] + fn parse_panics_with_non_named_fields() { + let unit_struct_tok = quote! { + pub struct Foo; + }; + + // Parses token stream into DeriveInput for test. + let derive_input = syn::parse2::(unit_struct_tok).unwrap(); + + let result = catch_unwind(|| parse_input(derive_input)); + assert!(result.is_err()); + } +} diff --git a/proc-macros/src/config_source/process.rs b/proc-macros/src/config_source/process.rs new file mode 100644 index 0000000..7fec5da --- /dev/null +++ b/proc-macros/src/config_source/process.rs @@ -0,0 +1,359 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +use proc_macro2::Ident; +use syn::{Generics, Path, Type}; + +use super::parse::StructData; + +/// Represents data gathered from a Struct. +pub(crate) struct StructDataOutput { + /// The identifier of the Struct. + pub struct_name: Ident, + /// A vector of fields for the Struct. + pub struct_fields: Vec, + /// The generics associated with the Struct. + pub struct_generics: Generics, +} + +/// Represents a single named field in a Struct. +pub(crate) struct FieldEntry { + /// The identifier of the field. + pub name: Ident, + /// The identifier of the field as a string. + pub name_str: String, + /// Whether the field is optional. + pub is_optional: bool, +} + +/// Process the data for the ConfigSource derive macro. +/// This method collects the relevent struct values for generation. +/// +/// # Arguments +/// * `data` - Parsed Struct data. +pub(crate) fn process(data: StructData) -> StructDataOutput { + // Process fields from Struct. + let struct_fields: Vec = data + .struct_fields + .into_iter() + .map(|field| { + let field_name = field.ident.unwrap(); + // Get the field name as a string. Will be used as a key in the code generation step. + let field_name_str = field_name.to_string(); + // Determine if field is optional. Relevant for the code generation step. + let is_optional = is_option(&field.ty); + + FieldEntry { + name: field_name, + name_str: field_name_str, + is_optional, + } + }) + .collect(); + + StructDataOutput { + struct_name: data.struct_name, + struct_fields, + struct_generics: data.struct_generics, + } +} + +/// Helper method to determine if a Type is of type `Option`. +/// +/// # Arguments +/// * `path` - Path to check. +fn path_is_option(path: &Path) -> bool { + path.leading_colon.is_none() && path.segments.len() == 1 && path.segments[0].ident == "Option" +} + +/// Determines if the provided Type is of type `Option`. +/// +/// # Arguments +/// * `ty` - Struct field type to check. +fn is_option(ty: &Type) -> bool { + matches!(ty, Type::Path(typepath) if typepath.qself.is_none() && path_is_option(&typepath.path)) +} + +#[cfg(test)] +mod config_source_process_tests { + use quote::format_ident; + use std::panic::catch_unwind; + use syn::{parse_quote, punctuated::Punctuated, token::Comma, Field, TypePath}; + + use crate::config_source::process::path_is_option; + + use super::*; + + #[test] + fn path_is_option_type() { + let option_string: TypePath = parse_quote!(Option); + assert!(path_is_option(&option_string.path)); + + let option_u64: TypePath = parse_quote!(Option); + assert!(path_is_option(&option_u64.path)); + + let option_bool: TypePath = parse_quote!(Option); + assert!(path_is_option(&option_bool.path)); + } + + #[test] + fn path_is_not_option_type() { + let string_type: TypePath = parse_quote!(String); + assert!(!path_is_option(&string_type.path)); + + let u64_type: TypePath = parse_quote!(u64); + assert!(!path_is_option(&u64_type.path)); + + let bool_type: TypePath = parse_quote!(bool); + assert!(!path_is_option(&bool_type.path)); + } + + #[test] + fn type_is_option() { + let option_string_type: Type = parse_quote!(Option); + assert!(is_option(&option_string_type)); + + let option_u64_type: Type = parse_quote!(Option); + assert!(is_option(&option_u64_type)); + + let option_bool_type: Type = parse_quote!(Option); + assert!(is_option(&option_bool_type)); + } + + #[test] + fn type_is_not_option() { + let string_type: Type = parse_quote!(String); + assert!(!is_option(&string_type)); + + let u64_type: Type = parse_quote!(u64); + assert!(!is_option(&u64_type)); + + let bool_type: Type = parse_quote!(bool); + assert!(!is_option(&bool_type)); + } + + #[test] + fn can_process_struct_data_with_optional_fields() { + let struct_name = format_ident!("Foo"); + let struct_generics = Generics::default(); + + let field_a: Field = parse_quote!(field_a: Option); + let field_b: Field = parse_quote!(field_b: Option); + let field_c: Field = parse_quote!(field_c: Option); + + // Create Punctuated list for input data. + let mut fields = Punctuated::::new(); + fields.push_value(field_a.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_b.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_c.clone()); + fields.push_punct(Comma::default()); + + let struct_data = StructData { + struct_name: struct_name.clone(), + struct_fields: fields, + struct_generics: struct_generics.clone(), + }; + + let output = process(struct_data); + + assert_eq!(output.struct_name, struct_name); + assert_eq!(output.struct_generics, struct_generics); + assert_eq!(output.struct_fields.len(), 3); + + // Check that each of the fields is present. + let mut field_iter = output.struct_fields.into_iter(); + let expected_field_a_name = field_a.ident.expect("Field_A ident should be present."); + let expected_field_b_name = field_b.ident.expect("Field_B ident should be present."); + let expected_field_c_name = field_c.ident.expect("Field_C ident should be present."); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_a_name) + && field.name_str.eq(&expected_field_a_name.to_string()) + && field.is_optional + }), + "Expected Field_A did not match processed Field_A" + ); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_b_name) + && field.name_str.eq(&expected_field_b_name.to_string()) + && field.is_optional + }), + "Expected Field_B did not match processed Field_B" + ); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_c_name) + && field.name_str.eq(&expected_field_c_name.to_string()) + && field.is_optional + }), + "Expected Field_C did not match processed Field_C" + ); + } + + #[test] + fn can_process_struct_data_with_non_optional_fields() { + let struct_name = format_ident!("Foo"); + let struct_generics = Generics::default(); + + let field_a: Field = parse_quote!(field_a: String); + let field_b: Field = parse_quote!(field_b: u64); + let field_c: Field = parse_quote!(field_c: bool); + + // Create Punctuated list for input data. + let mut fields = Punctuated::::new(); + fields.push_value(field_a.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_b.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_c.clone()); + fields.push_punct(Comma::default()); + + let struct_data = StructData { + struct_name: struct_name.clone(), + struct_fields: fields, + struct_generics: struct_generics.clone(), + }; + + let output = process(struct_data); + + assert_eq!(output.struct_name, struct_name); + assert_eq!(output.struct_generics, struct_generics); + assert_eq!(output.struct_fields.len(), 3); + + // Check that each of the fields is present. + let mut field_iter = output.struct_fields.into_iter(); + let expected_field_a_name = field_a.ident.expect("Field_A ident should be present."); + let expected_field_b_name = field_b.ident.expect("Field_B ident should be present."); + let expected_field_c_name = field_c.ident.expect("Field_C ident should be present."); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_a_name) + && field.name_str.eq(&expected_field_a_name.to_string()) + && !field.is_optional + }), + "Expected Field_A did not match processed Field_A" + ); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_b_name) + && field.name_str.eq(&expected_field_b_name.to_string()) + && !field.is_optional + }), + "Expected Field_B did not match processed Field_B" + ); + + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_c_name) + && field.name_str.eq(&expected_field_c_name.to_string()) + && !field.is_optional + }), + "Expected Field_C did not match processed Field_C" + ); + } + + #[test] + fn can_process_struct_data_with_mixed_fields() { + let struct_name = format_ident!("Foo"); + let struct_generics = Generics::default(); + + let field_a: Field = parse_quote!(field_a: String); + let field_b: Field = parse_quote!(field_b: Option); + let field_c: Field = parse_quote!(field_c: bool); + + // Create Punctuated list for input data. + let mut fields = Punctuated::::new(); + fields.push_value(field_a.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_b.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_c.clone()); + fields.push_punct(Comma::default()); + + let struct_data = StructData { + struct_name: struct_name.clone(), + struct_fields: fields, + struct_generics: struct_generics.clone(), + }; + + let output = process(struct_data); + + assert_eq!(output.struct_name, struct_name); + assert_eq!(output.struct_generics, struct_generics); + assert_eq!(output.struct_fields.len(), 3); + + // Check that each of the fields is present. + let mut field_iter = output.struct_fields.into_iter(); + let expected_field_a_name = field_a.ident.expect("Field_A ident should be present."); + let expected_field_b_name = field_b.ident.expect("Field_B ident should be present."); + let expected_field_c_name = field_c.ident.expect("Field_C ident should be present."); + + // Is a non optional field. + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_a_name) + && field.name_str.eq(&expected_field_a_name.to_string()) + && !field.is_optional + }), + "Expected Field_A did not match processed Field_A" + ); + + // Is an optional field. + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_b_name) + && field.name_str.eq(&expected_field_b_name.to_string()) + && field.is_optional + }), + "Expected Field_B did not match processed Field_B" + ); + + // Is a non optional field. + assert!( + field_iter.any(|field| { + field.name.eq(&expected_field_c_name) + && field.name_str.eq(&expected_field_c_name.to_string()) + && !field.is_optional + }), + "Expected Field_C did not match processed Field_C" + ); + } + + #[test] + fn panic_with_malformed_field_data() { + let struct_name = format_ident!("Foo"); + let struct_generics = Generics::default(); + + let field_a: Field = parse_quote!(field_a: String); + // Malformed Field entry with no name. + let field_b: Field = parse_quote!(Option); + let field_c: Field = parse_quote!(field_c: bool); + + // Create Punctuated list for input data. + let mut fields = Punctuated::::new(); + fields.push_value(field_a.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_b.clone()); + fields.push_punct(Comma::default()); + fields.push_value(field_c.clone()); + fields.push_punct(Comma::default()); + + let struct_data = StructData { + struct_name: struct_name.clone(), + struct_fields: fields, + struct_generics: struct_generics.clone(), + }; + + let result = catch_unwind(|| process(struct_data)); + assert!(result.is_err()); + } +} diff --git a/proc-macros/src/lib.rs b/proc-macros/src/lib.rs new file mode 100644 index 0000000..624bc18 --- /dev/null +++ b/proc-macros/src/lib.rs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// SPDX-License-Identifier: MIT + +mod config_source; + +use proc_macro::TokenStream; + +/// Derives `config::Source` Trait (from the config crate) for a Struct. +/// +/// Note: The Struct must have named fields and the Type for each field must be convertable into a +/// `config::Value` from the `config` crate. +/// +/// # Arguments +/// * `ts`: A token stream. +#[proc_macro_derive(ConfigSource)] +pub fn config_source(ts: TokenStream) -> TokenStream { + config_source::config_source(ts) +} diff --git a/pub-sub-service/Cargo.toml b/pub-sub-service/Cargo.toml index 8a3380f..88b8ddd 100644 --- a/pub-sub-service/Cargo.toml +++ b/pub-sub-service/Cargo.toml @@ -11,12 +11,14 @@ license = "MIT" [dependencies] async-std = { workspace = true } async-trait = { workspace = true } +clap = { workspace = true, features = [ "derive" ] } common = { path = "../common" } config = { workspace = true } env_logger = { workspace = true } futures = { workspace = true } log = { workspace = true } paho-mqtt = { workspace = true } +proc-macros = { path = "../proc-macros"} proto = { path = "../proto-build" } serde = { workspace = true } serde_derive = { workspace = true } diff --git a/pub-sub-service/README.md b/pub-sub-service/README.md index 7a469e1..250768e 100644 --- a/pub-sub-service/README.md +++ b/pub-sub-service/README.md @@ -63,11 +63,11 @@ service and directly communicate. ### Configure Pub Sub Service to use Chariott -1. Copy the `pub_sub_service_settings.yaml` template to [.agemo/config](../.agemo/config/) if the -file does not already exist. From the enlistment root, run: +1. Copy the `pub_sub_service_settings.yaml` template to `.agemo/config` if the file does not +already exist. From the enlistment root, run: ```shell - cp ./.agemo/config/template/pub_sub_service_settings.yaml ./.agemo/config/ + cp ./config/template/pub_sub_service_settings.yaml ./.agemo/config/ ``` 2. Uncomment and set the following values: diff --git a/pub-sub-service/src/load_config.rs b/pub-sub-service/src/load_config.rs index c1f1f80..b9ec89a 100644 --- a/pub-sub-service/src/load_config.rs +++ b/pub-sub-service/src/load_config.rs @@ -6,13 +6,46 @@ use std::env; +use clap::Parser; use common::config_utils; -use log::error; +use log::{debug, error}; +use proc_macros::ConfigSource; use serde_derive::{Deserialize, Serialize}; const CONFIG_FILE_NAME: &str = "pub_sub_service_settings"; const CONSTANTS_FILE_NAME: &str = "constants"; +/// Object containing commandline config options for the Pub Sub service. +/// Non-optional fields must be passed in via the commandline and will override any values from +/// configuration files. +#[derive(Clone, Debug, Parser, Serialize, Deserialize, ConfigSource)] +#[command(author, about, long_about = None)] +pub struct CmdConfigOptions { + /// The IP address and port number that the Pub Sub service listens on for requests. + /// Required if not set in configuration files. (eg. "0.0.0.0:50051"). + #[arg(short, long)] + pub pub_sub_authority: Option, + /// The URI of the messaging service used to facilitate publish and subscribe functionality. + /// Required if not set in configuration files. (eg. "mqtt://0.0.0.0:1883"). + #[arg(short, long)] + pub messaging_uri: Option, + /// The URI that the Chariott service listens on for requests. (eg. "http://0.0.0.0:50000"). + #[arg(short, long)] + pub chariott_uri: Option, + /// The namespace of the Pub Sub service. + #[arg(short = 's', long)] + pub namespace: Option, + /// The name of the Pub Sub service. + #[arg(short, long)] + pub name: Option, + /// The current version of the Pub Sub Service. + #[arg(short, long)] + pub version: Option, + /// The log level of the program. + #[arg(short, long, default_value = "info")] + pub log_level: String, +} + /// Object that contains constants used for establishing connection between services. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct CommunicationConstants { @@ -29,7 +62,7 @@ pub struct CommunicationConstants { } /// Object containing configuration settings to run the Pub Sub service. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Parser, Serialize, Deserialize)] pub struct Settings { /// The IP address and port number that the Pub Sub service listens on for requests. pub pub_sub_authority: String, @@ -49,19 +82,35 @@ pub struct Settings { /// /// # Arguments /// * `config_file_name` - Name of the config file to load settings from. -pub fn load_config(config_file_name: &str) -> Result> +/// * `args` - Optional commandline config arguments. +pub fn load_config( + config_file_name: &str, + args: Option, +) -> Result> where T: for<'de> serde::Deserialize<'de>, { - config_utils::read_from_files(config_file_name, config_utils::YAML_EXT) + config_utils::read_from_files(config_file_name, config_utils::YAML_EXT, args) } /// Load the settings. /// /// Will attempt to load the settings from the service configuration file. If the necessary config /// is set will run in Chariott enabled mode, otherwise the service will run in standalone mode. -pub fn load_settings() -> Result> { - let mut settings: Settings = load_config(CONFIG_FILE_NAME)?; +/// +/// # Arguments +/// * `args` - Commandline config arguments. +pub fn load_settings( + args: CmdConfigOptions, +) -> Result> { + let mut settings: Settings = load_config(CONFIG_FILE_NAME, Some(args)) + .map_err(|e| { + format!( + "Failed to load required configuration settings due to error: {e}. See --help for more details." + ) + })?; + + debug!("settings config: {:?}", settings); if settings.chariott_uri.is_some() { // Get version of the service for Chariott registration if not defined. @@ -96,5 +145,5 @@ pub fn load_constants() -> Result where T: for<'de> serde::Deserialize<'de>, { - load_config(CONSTANTS_FILE_NAME) + load_config(CONSTANTS_FILE_NAME, None) } diff --git a/pub-sub-service/src/main.rs b/pub-sub-service/src/main.rs index 1101dce..de1474b 100644 --- a/pub-sub-service/src/main.rs +++ b/pub-sub-service/src/main.rs @@ -13,8 +13,9 @@ // Tells cargo to warn if a doc comment is missing and should be provided. #![warn(missing_docs)] -use std::sync::mpsc; +use std::{str::FromStr, sync::mpsc}; +use clap::Parser; use env_logger::{Builder, Target}; use log::{error, info, warn, LevelFilter}; use pubsub_connector::PubSubConnector; @@ -25,7 +26,7 @@ use proto::pubsub::v1::pub_sub_server::PubSubServer; use crate::{ connectors::chariott_connector::{self, ServiceIdentifier}, - load_config::CommunicationConstants, + load_config::{CmdConfigOptions, CommunicationConstants}, pubsub_connector::MonitorMessage, }; @@ -37,14 +38,21 @@ pub mod topic_manager; #[tokio::main] async fn main() -> Result<(), Box> { + // Load command line arguments if any. + let parsed_args = CmdConfigOptions::parse(); + + // Get log level. Defaults to info. + let log_level = + LevelFilter::from_str(&parsed_args.log_level).expect("Could not parse log level"); + // Setup logging. Builder::new() - .filter(None, LevelFilter::Info) + .filter(None, log_level) .target(Target::Stdout) .init(); // Load settings in from config file. - let settings = load_config::load_settings()?; + let settings = load_config::load_settings(parsed_args)?; let communication_consts = load_config::load_constants::()?; // Initialize pub sub service