From c7cf53d97a6d894ee895e6330290605617b69b78 Mon Sep 17 00:00:00 2001 From: James Adams Date: Wed, 7 Sep 2022 22:39:24 +0100 Subject: [PATCH] ncm-metaconfig: telegraf: Add support for config Telegraf config is currently TOML based. --- .../src/main/metaconfig/telegraf/main.tt | 7 + .../telegraf/nested_array_of_tables.tt | 7 + .../main/metaconfig/telegraf/pan/config.pan | 12 + .../main/metaconfig/telegraf/pan/schema.pan | 308 ++++++++++++++++++ .../src/main/metaconfig/telegraf/table.tt | 12 + .../telegraf/tests/profiles/config.pan | 77 +++++ .../telegraf/tests/regexps/config/base | 60 ++++ .../src/main/metaconfig/telegraf/value.tt | 13 + .../src/test/perl/service-telegraf.t | 8 + 9 files changed, 504 insertions(+) create mode 100644 ncm-metaconfig/src/main/metaconfig/telegraf/main.tt create mode 100644 ncm-metaconfig/src/main/metaconfig/telegraf/nested_array_of_tables.tt create mode 100644 ncm-metaconfig/src/main/metaconfig/telegraf/pan/config.pan create mode 100644 ncm-metaconfig/src/main/metaconfig/telegraf/pan/schema.pan create mode 100644 ncm-metaconfig/src/main/metaconfig/telegraf/table.tt create mode 100644 ncm-metaconfig/src/main/metaconfig/telegraf/tests/profiles/config.pan create mode 100644 ncm-metaconfig/src/main/metaconfig/telegraf/tests/regexps/config/base create mode 100644 ncm-metaconfig/src/main/metaconfig/telegraf/value.tt create mode 100644 ncm-metaconfig/src/test/perl/service-telegraf.t diff --git a/ncm-metaconfig/src/main/metaconfig/telegraf/main.tt b/ncm-metaconfig/src/main/metaconfig/telegraf/main.tt new file mode 100644 index 0000000000..f0d253b091 --- /dev/null +++ b/ncm-metaconfig/src/main/metaconfig/telegraf/main.tt @@ -0,0 +1,7 @@ +[global_tags] +[% INCLUDE 'metaconfig/telegraf/table.tt' table=global_tags FILTER indent -%] + +[agent] +[% INCLUDE 'metaconfig/telegraf/table.tt' table=agent FILTER indent %] +[%- INCLUDE 'metaconfig/telegraf/nested_array_of_tables.tt' name='inputs' nest=inputs -%] +[%- INCLUDE 'metaconfig/telegraf/nested_array_of_tables.tt' name='outputs' nest=outputs -%] diff --git a/ncm-metaconfig/src/main/metaconfig/telegraf/nested_array_of_tables.tt b/ncm-metaconfig/src/main/metaconfig/telegraf/nested_array_of_tables.tt new file mode 100644 index 0000000000..bf71fc6820 --- /dev/null +++ b/ncm-metaconfig/src/main/metaconfig/telegraf/nested_array_of_tables.tt @@ -0,0 +1,7 @@ +[%- FOREACH array IN nest -%] +[%- FOREACH table IN array.value -%] +[%- path = name _ '.' _ array.key %] +[% '[[' _ path _ ']]' %] +[% INCLUDE 'metaconfig/telegraf/table.tt' path=path table=table FILTER indent -%] +[%- END -%] +[%- END -%] diff --git a/ncm-metaconfig/src/main/metaconfig/telegraf/pan/config.pan b/ncm-metaconfig/src/main/metaconfig/telegraf/pan/config.pan new file mode 100644 index 0000000000..2e35404815 --- /dev/null +++ b/ncm-metaconfig/src/main/metaconfig/telegraf/pan/config.pan @@ -0,0 +1,12 @@ +unique template metaconfig/telegraf/config; + +include 'metaconfig/telegraf/schema'; + +bind "/software/components/metaconfig/services/{/etc/telegraf/telegraf.conf}/contents" = service_telegraf; + +prefix "/software/components/metaconfig/services/{/etc/telegraf/telegraf.conf}"; + +"mode" = 0644; +"owner" = "root"; +"group" = "root"; +"module" = "telegraf/main"; diff --git a/ncm-metaconfig/src/main/metaconfig/telegraf/pan/schema.pan b/ncm-metaconfig/src/main/metaconfig/telegraf/pan/schema.pan new file mode 100644 index 0000000000..03a6bfbfbb --- /dev/null +++ b/ncm-metaconfig/src/main/metaconfig/telegraf/pan/schema.pan @@ -0,0 +1,308 @@ +declaration template metaconfig/telegraf/schema; + +include 'pan/types'; + +type telegraf_time_interval_string = string with match(SELF, '\d+([num]?s|m|h|d)'); + +type telegraf_file_size_string = string with match(SELF, '\d+[KM]B'); + +type telegraf_global_tags = string_trimmed{}; + +type telegraf_agent = { + @{ Default data collection interval for all inputs } + 'interval' ? telegraf_time_interval_string + + @{ + Rounds collection interval to 'interval' + ie, if interval="10s" then always collect on :00, :10, :20, etc. + } + 'round_interval' ? boolean + + @{ + Telegraf will send metrics to outputs in batches of at most + metric_batch_size metrics. + This controls the size of writes that Telegraf sends to output plugins. + } + 'metric_batch_size' ? long(1..) + + @{ + Maximum number of unwritten metrics per output. Increasing this value + allows for longer periods of output downtime without dropping metrics at the + cost of higher maximum memory usage. + } + 'metric_buffer_limit' ? long(1..) + + @{ + Collection jitter is used to jitter the collection by a random amount. + Each plugin will sleep for a random time within jitter before collecting. + This can be used to avoid many plugins querying things like sysfs at the + same time, which can have a measurable effect on the system. + } + 'collection_jitter' ? telegraf_time_interval_string + + @{ + Default flushing interval for all outputs. Maximum flush_interval will be + flush_interval + flush_jitter + } + 'flush_interval' ? telegraf_time_interval_string + + @{ + Jitter the flush interval by a random amount. This is primarily to avoid + large write spikes for users running a large number of telegraf instances. + ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + } + 'flush_jitter' ? telegraf_time_interval_string + + @{ + By default or when set to "0s", precision will be set to the same + timestamp order as the collection interval, with the maximum being 1s. + ie, when interval = "10s", precision will be "1s" + when interval = "250ms", precision will be "1ms" + Precision will NOT be used for service inputs. It is up to each individual + service input to set the timestamp at the appropriate precision. + Valid time units are "ns", "us" (or "µs"), "ms", "s". + } + 'precision' ? telegraf_time_interval_string + + @{ Log at debug level. } + 'debug' ? boolean + + @{ Log only error level messages. } + 'quiet' ? boolean + + @{ + Log target controls the destination for logs and can be one of "file", + "stderr" or, on Windows, "eventlog". When set to "file", the output file + is determined by the "logfile" setting. + } + 'logtarget' ? choice('file', 'stderr') + + @{ + Name of the file to be logged to when using the "file" logtarget. + If set to the empty string then logs are written to stderr. + } + 'logfile' ? absolute_file_path + + @{ + The logfile will be rotated after the time interval specified. + When set to 0 no time based rotation is performed. + Logs are rotated only when written to, if there is no log activity rotation may be delayed. + } + 'logfile_rotation_interval' ? telegraf_time_interval_string + + @{ + The logfile will be rotated when it becomes larger than the specified size. + When set to 0 no size based rotation is performed. + } + 'logfile_rotation_max_size' ? telegraf_file_size_string + + @{ + Maximum number of rotated archives to keep, any older logs are deleted. + If set to -1, no archives are removed. + } + 'logfile_rotation_max_archives' ? long(-1..) + + @{ + Pick a timezone to use when logging or type 'local' for local time. + Example: America/Chicago + } + 'log_with_timezone' ? string_trimmed + + @{ + Override default hostname, if empty use os.Hostname() + } + 'hostname' ? type_hostname + + @{ + If set to true, do no set the "host" tag in the telegraf agent. + } + 'omit_hostname' ? boolean +}; + +# Common to all plugin types +type telegraf_plugin_common = { + @{ Name an instance of a plugin. } + 'alias' ? string_trimmed + + + ## Metric Filtering + + @{ + An array of glob pattern strings. + Only metrics whose measurement name matches a pattern in this list are emitted. + } + 'namepass' ? string_trimmed[] + + @{ + The inverse of namepass. If a match is found the metric is discarded. + This is tested on metrics after they have passed the namepass test. + } + 'namedrop' ? string_trimmed[] + + @{ + An array of glob pattern strings. + Only fields whose field key matches a pattern in this list are emitted. + } + 'fieldpass' ? string_trimmed[] + + @{ + The inverse of fieldpass. Fields with a field key matching one of the patterns will be discarded from the metric. + This is tested on metrics after they have passed the fieldpass test. + } + 'fielddrop' ? string_trimmed[] + + @{ + An array of glob pattern strings. Only tags with a tag key matching one of the patterns are emitted. + In contrast to tagpass, which will pass an entire metric based on its tag, + taginclude removes all non matching tags from the metric. + Any tag can be filtered including global tags and the agent host tag. + } + 'taginclude' ? string_trimmed[] + + @{ + The inverse of taginclude. Tags with a tag key matching one of the patterns will be discarded from the metric. + Any tag can be filtered including global tags and the agent host tag. + } + 'tagexclude' ? string_trimmed[] + + @{ + A table mapping tag keys to arrays of glob pattern strings. + Only metrics that contain a tag key in the table and a tag value matching one of its patterns is emitted. + } + 'tagpass' ? string_trimmed[]{} + + @{ + The inverse of tagpass. If a match is found the metric is discarded. + This is tested on metrics after they have passed the tagpass test. + } + 'tagdrop' ? string_trimmed[]{} +}; + +# Common to Input, Aggregator and Output plugins +type telegraf_iao_plugin_common = { + @{ Override the base name of the measurement. (Default is the name of the input). } + 'name_override' ? string_trimmed + + @{ Specifies a prefix to attach to the measurement name. } + 'name_prefix' ? string_trimmed + + @{ Specifies a suffix to attach to the measurement name. } + 'name_suffix' ? string_trimmed +}; + +type telegraf_plugin_input = extensible { + include telegraf_plugin_common + include telegraf_iao_plugin_common + + @{ + Overrides the interval setting of the agent for the plugin. + How often to gather this metric. Normal plugins use a single global interval, + but if one particular input should be run less or more often, you can configure that here. + } + 'interval' ? telegraf_time_interval_string + + @{ + Overrides the precision setting of the agent for the plugin. + Collected metrics are rounded to the precision specified as an interval. + When this value is set on a service input, multiple events occuring at the same timestamp + may be merged by the output database. + } + 'precision' ? telegraf_time_interval_string + + @{ + Overrides the collection_jitter setting of the agent for the plugin. + Collection jitter is used to jitter the collection by a random interval. + } + 'collection_jitter' ? telegraf_time_interval_string + + @{ + A map of tags to apply to a specific input's measurements. + } + 'tags' ? string_trimmed{} +}; + +type telegraf_plugin_output = extensible { + include telegraf_plugin_common + include telegraf_iao_plugin_common + + @{ + The maximum time between flushes. + Use this setting to override the agent flush_interval on a per plugin basis. + } + 'flush_interval' ? telegraf_time_interval_string + + @{ + The amount of time to jitter the flush interval. + Use this setting to override the agent flush_jitter on a per plugin basis. + } + 'flush_jitter' ? telegraf_time_interval_string + + @{ + The maximum number of metrics to send at once. + Use this setting to override the agent metric_batch_size on a per plugin basis. + } + 'metric_batch_size' ? long(1..) + + @{ + The maximum number of unsent metrics to buffer. + Use this setting to override the agent metric_buffer_limit on a per plugin basis. + } + 'metric_buffer_limit' ? long(1..) +}; + + +type telegraf_plugin_processor = extensible { + include telegraf_plugin_common + + @{ + The order in which the processor(s) are executed. + If this is not specified then processor execution order will be random. + } + 'order' ? long(1..) +}; + +type telegraf_plugin_aggregator = extensible { + include telegraf_plugin_common + include telegraf_iao_plugin_common + + @{ + The period on which to flush & clear each aggregator. + All metrics that are sent with timestamps outside of this period will be ignored by the aggregator. + } + 'period' ? telegraf_time_interval_string + + @{ + The delay before each aggregator is flushed. + This is to control how long for aggregators to wait before receiving metrics from input plugins, + in the case that aggregators are flushing and inputs are gathering on the same interval. + } + 'delay' ? telegraf_time_interval_string + + @{ + The duration when the metrics will still be aggregated by the plugin, + even though they're outside of the aggregation period. + This is needed in a situation when the agent is expected to receive + late metrics and it's acceptable to roll them up into next aggregation period. + } + 'grace' ? telegraf_time_interval_string + + @{ + If true, the original metric will be dropped by the aggregator and will not get sent to the output plugins. + } + 'drop_original' ? boolean + + @{ + A map of tags to apply to the measurement - behavior varies based on aggregator. + } + 'tags' ? string_trimmed{} +}; + + +type service_telegraf = { + 'global_tags' ? telegraf_global_tags + 'agent' ? telegraf_agent + 'inputs' ? telegraf_plugin_input[]{} + 'processors' ? telegraf_plugin_processor[]{} + 'aggregators' ? telegraf_plugin_aggregator[]{} + 'outputs' ? telegraf_plugin_output[]{} +}; diff --git a/ncm-metaconfig/src/main/metaconfig/telegraf/table.tt b/ncm-metaconfig/src/main/metaconfig/telegraf/table.tt new file mode 100644 index 0000000000..ed562559db --- /dev/null +++ b/ncm-metaconfig/src/main/metaconfig/telegraf/table.tt @@ -0,0 +1,12 @@ +[%- FOREACH pair IN table -%] +[%- IF not CCM.is_hash(pair.value) -%] +[% pair.key %] = [%- INCLUDE 'metaconfig/telegraf/value.tt' value=pair.value %] +[% END -%] +[%- END -%] +[%- FOREACH pair IN table -%] +[%- IF CCM.is_hash(pair.value) -%] +[% path = (path.empty ? '' : path _ '.') _ pair.key -%] +[% '[' _ path _ ']' %] +[% INCLUDE 'metaconfig/telegraf/table.tt' path=path table=pair.value FILTER indent -%] +[% END -%] +[%- END -%] diff --git a/ncm-metaconfig/src/main/metaconfig/telegraf/tests/profiles/config.pan b/ncm-metaconfig/src/main/metaconfig/telegraf/tests/profiles/config.pan new file mode 100644 index 0000000000..4fe8d090df --- /dev/null +++ b/ncm-metaconfig/src/main/metaconfig/telegraf/tests/profiles/config.pan @@ -0,0 +1,77 @@ +object template config; + +include 'metaconfig/telegraf/config'; + +prefix "/software/components/metaconfig/services/{/etc/telegraf/telegraf.conf}/contents"; + +'global_tags' = dict( + 'model', 'test_model', + 'personality', 'test_personality', +); + +'agent' = dict( + 'interval', '60s', + 'round_interval', true, + 'metric_buffer_limit', 5000, + 'collection_jitter', '0s', + 'flush_interval', '300s', + 'flush_jitter', '60s', + 'debug', false, + 'quiet', false, + 'omit_hostname', false, +); + +'inputs/cpu' = list( + dict( + 'percpu', false, + 'totalcpu', true, + 'fielddrop', list( + 'time_*', + ), + 'tagdrop', dict( + 'cpu', list( + 'cpu6', + 'cpu7', + ), + ), + ), +); + +'inputs/disk' = list( + dict( + 'mount_points', list( + '/', + '/pool', + ), + 'ignore_fs', list( + 'tmpfs', + 'devtmpfs', + ), + 'tagpass', dict( + 'fstype', list( + 'ext4', + 'xfs', + ), + 'path', list( + '/opt', + '/home*', + ), + ), + ), +); + +'outputs' = dict( + 'influxdb', list( + dict( + 'urls', list( + 'http://influxdb01.example.org:8086', + ), + 'database', 'testnodes', + 'precision', 's', + 'timeout', '5s', + 'username', 'write_testnodes', + 'password', 'test_password', + 'skip_database_creation', true, + ), + ), +); diff --git a/ncm-metaconfig/src/main/metaconfig/telegraf/tests/regexps/config/base b/ncm-metaconfig/src/main/metaconfig/telegraf/tests/regexps/config/base new file mode 100644 index 0000000000..c632d3b822 --- /dev/null +++ b/ncm-metaconfig/src/main/metaconfig/telegraf/tests/regexps/config/base @@ -0,0 +1,60 @@ +Base test for config +--- +/etc/telegraf/telegraf.conf +--- +^\[global_tags\]$ +^ model = "test_model"$ +^ personality = "test_personality"$ +^$ +^\[agent\]$ +^ collection_jitter = "0s"$ +^ debug = false$ +^ flush_interval = "300s"$ +^ flush_jitter = "60s"$ +^ interval = "60s"$ +^ metric_buffer_limit = 5000$ +^ omit_hostname = false$ +^ quiet = false$ +^ round_interval = true$ +^$ +^\[\[inputs\.cpu\]\]$ +^ fielddrop = \[$ +^ "time_\*",$ +^ \]$ +^ percpu = false$ +^ totalcpu = true$ +^ \[inputs\.cpu\.tagdrop\]$ +^ cpu = \[$ +^ "cpu6",$ +^ "cpu7",$ +^ \]$ +^$ +^\[\[inputs\.disk\]\]$ +^ ignore_fs = \[$ +^ "tmpfs",$ +^ "devtmpfs",$ +^ \]$ +^ mount_points = \[$ +^ "/",$ +^ "/pool",$ +^ \]$ +^ \[inputs\.disk\.tagpass\]$ +^ fstype = \[$ +^ "ext4",$ +^ "xfs",$ +^ \]$ +^ path = \[$ +^ "/opt",$ +^ "/home\*",$ +^ \]$ +^$ +^\[\[outputs\.influxdb\]\]$ +^ database = "testnodes"$ +^ password = "test_password"$ +^ precision = "s"$ +^ skip_database_creation = true$ +^ timeout = "5s"$ +^ urls = \[$ +^ "http://influxdb01\.example\.org:8086"\,$ +^ ]$ +^ username = "write_testnodes"$ diff --git a/ncm-metaconfig/src/main/metaconfig/telegraf/value.tt b/ncm-metaconfig/src/main/metaconfig/telegraf/value.tt new file mode 100644 index 0000000000..f6d29a379f --- /dev/null +++ b/ncm-metaconfig/src/main/metaconfig/telegraf/value.tt @@ -0,0 +1,13 @@ +[%- IF value.is_boolean -%] +[%- value ? 'true' : 'false' -%] +[%- ELSIF CCM.is_list(value) -%] +[ +[% FOREACH item IN value -%] +[% FILTER indent -%] +[% INCLUDE 'metaconfig/telegraf/value.tt' value=item -%], +[% END -%] +[% END -%] +] +[%- ELSE -%] +[%- value.is_string ? '"' _ value _ '"' : value -%] +[%- END -%] diff --git a/ncm-metaconfig/src/test/perl/service-telegraf.t b/ncm-metaconfig/src/test/perl/service-telegraf.t new file mode 100644 index 0000000000..33239ba5bd --- /dev/null +++ b/ncm-metaconfig/src/test/perl/service-telegraf.t @@ -0,0 +1,8 @@ +use Test::More; +use Test::Quattor::TextRender::Metaconfig; + +my $u = Test::Quattor::TextRender::Metaconfig->new( + service => 'telegraf', +)->test(); + +done_testing;