Add command line configuration and simplified config hierarchy (#63)

* config demo v1

* Add command line configuration

* order config overrides

* Remove curr dir config

* Update docs and modify default files

* fix doc errs

* remove dockerfile cp as default config is now part of executable

* Refactored proc macro, added detailed comments

* fixed clippy warnings

* fix unexpected delimiter

* fixed doctest errs

* second attempt

* remove example as it is over complicated

* add params to comments

* remove unecessary clonse

* add comment about required cmdline params

* Add tests for config_source macro

* fix clippy warnings

* change to forward slash

* Capitalized err messages

* simplify collect generated code

Cargo.toml

@@ -6,6 +6,7 @@
 resolver = "2"
 members = [
   "common",
+  "proc-macros",
   "pub-sub-service",
   "samples/chariott-publisher",
   "samples/chariott-subscriber",
@@ -17,20 +18,25 @@ members = [
 [workspace.dependencies]
 async-std = "1"
 async-trait = "0.1.76"
+clap = { version = "4.4.11" }
 config = "0.13.3"
 ctrlc = { version = "3.4", features = ["termination"] }
 env_logger = "0.10"
 futures = "0.3"
 home = "0.5.9"
+include_dir = "0.7.3"
 log = "^0.4"
 paho-mqtt = "0.12"
+proc-macro2 = "1.0.70"
 prost = "0.12"
 prost-types = "0.12"
+quote = "1.0.23"
 serde = "1.0.160"
 serde_derive = "1.0.163"
 serde_json = "^1.0"
 strum = "0.25"
 strum_macros = "0.25"
+syn = { version = "2.0.40", features = ["extra-traits", "full"] }
 tokio = { version = "1.35.1", features = ["time"] }
 tonic = "0.10"
 tonic-build = "0.10"

Dockerfile.amd64

@@ -90,9 +90,6 @@ ENV AGEMO_HOME=/sdv/.agemo
 # Copy the executable from the "build" stage.
 COPY --from=build /sdv/service /sdv/
 
-# Copy default configs.
-COPY --from=build /sdv/.agemo/config/ /sdv/.agemo/config/
-
 # Expose the port that the application listens on.
 EXPOSE 50051
 

Dockerfile.arm64

@@ -93,9 +93,6 @@ ENV AGEMO_HOME=/sdv/.agemo
 # Copy the executable from the "build" stage.
 COPY --from=build /sdv/service /sdv/
 
-# Copy default configs.
-COPY --from=build /sdv/.agemo/config/ /sdv/.agemo/config/
-
 # Expose the port that the application listens on.
 EXPOSE 50051
 

README.md

@@ -115,7 +115,7 @@ cargo test
 
 ## Configuration Setup
 
-The service configuration is defined in [.agemo/config](.agemo/config/). The default configuration
+The service configuration is defined in [config](config/). The default configuration
 files will allow a user to run the service without any modification. Please read
 [config_overrides](./docs/config-overrides.md) for more information on how to modify the service's
 configuration.

common/Cargo.toml

@@ -11,7 +11,8 @@ license = "MIT"
 [dependencies]
 config = { workspace = true }
 home = { workspace = true }
+include_dir = { workspace = true }
 log = { workspace = true }
 serde = { workspace = true }
 serde_derive = { workspace = true }
-tokio = { workspace = true }
+tokio = { workspace = true }

common/src/config_utils.rs

@@ -4,32 +4,44 @@
 
 use std::{env, path::Path};
 
-use config::File;
+use config::{File, FileFormat, Source};
 use home::home_dir;
+use include_dir::{include_dir, Dir};
 use serde::Deserialize;
 
 pub const YAML_EXT: &str = "yaml";
 
 const CONFIG_DIR: &str = "config";
+const DEFAULT: &str = "default";
 const DOT_AGEMO_DIR: &str = ".agemo";
 const AGEMO_HOME: &str = "AGEMO_HOME";
 
+const DEFAULT_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../config");
+
 /// Read config from layered configuration files.
-/// Searches for `{config_file_name}.default.{config_file_ext}` as the base configuration in `$AGEMO_HOME`,
-/// then searches for overrides named `{config_file_name}.{config_file_ext}` in the current directory and `$AGEMO_HOME`.
+/// Searches for `{config_file_name}.default.{config_file_ext}` as the base configuration,
+/// then searches for overrides named `{config_file_name}.{config_file_ext}` in `$AGEMO_HOME`.
 /// If `$AGEMO_HOME` is not set, it defaults to `$HOME/.agemo`.
 ///
 /// # Arguments
-/// - `config_file_name`: The config file name. This is used to construct the file names to search for.
-/// - `config_file_ext`: The config file extension. This is used to construct the file names to search for.
-pub fn read_from_files<T>(
+/// * `config_file_name` - The config file name. This is used to construct the file names to search for.
+/// * `config_file_ext` - The config file extension. This is used to construct the file names to search for.
+/// * `args` - Optional commandline arguments. Any values set will override values gathered from config files.
+pub fn read_from_files<T, A>(
     config_file_name: &str,
     config_file_ext: &str,
+    args: Option<A>,
 ) -> Result<T, Box<dyn std::error::Error + Send + Sync>>
 where
     T: for<'a> Deserialize<'a>,
+    A: Source + Send + Sync + 'static + Clone,
 {
-    let default_config_file = format!("{config_file_name}.default.{config_file_ext}");
+    // Get default config.
+    let default_config_filename = format!("{config_file_name}.{DEFAULT}.{config_file_ext}");
+    let default_config_file = DEFAULT_DIR.get_file(default_config_filename).unwrap();
+    let default_config_contents_str = default_config_file.contents_utf8().unwrap();
+
+    // Get override_files
     let overrides_file = format!("{config_file_name}.{config_file_ext}");
 
     let config_path = match env::var(AGEMO_HOME) {
@@ -51,20 +63,22 @@ where
         }
     };
 
-    // The path below resolves to {config_path}/{default_config_file}.
-    let default_config_file_path = config_path.join(default_config_file);
-
-    // The path below resolves to {current_dir}/{overrides_file}.
-    let current_dir_config_file_path = env::current_dir()?.join(overrides_file.clone());
-
     // The path below resolves to {config_path}/{overrides_file}
     let overrides_config_file_path = config_path.join(overrides_file);
 
-    let config_store = config::Config::builder()
-        .add_source(File::from(default_config_file_path))
-        .add_source(File::from(current_dir_config_file_path).required(false))
-        .add_source(File::from(overrides_config_file_path).required(false))
-        .build()?;
+    let mut config_sources = config::Config::builder()
+        .add_source(File::from_str(
+            default_config_contents_str,
+            FileFormat::Yaml,
+        ))
+        .add_source(File::from(overrides_config_file_path).required(false));
+
+    // Adds command line arguments if there are any.
+    if let Some(args) = args {
+        config_sources = config_sources.add_source(args);
+    }
+
+    let config_store = config_sources.build()?;
 
     config_store.try_deserialize().map_err(|e| e.into())
 }

.agemo/config/constants.default.yaml → config/constants.default.yaml

@@ -18,5 +18,3 @@ pub_sub_reference: "pubsub.v1.pubsub.proto"
 
 # Retry interval for connections.
 retry_interval_secs: 5
-
-###