modified existing prom and alertmanager roles

.ansible-sign/sha256sum.txt

@@ -4,8 +4,8 @@ ca3f84e14f6aa778003094fd160710543fae3594d76dbc4335d2d0b77245e8ce  AnsibleSSP.cod
 e81d36021af399b6ecafeecde71a8634aacbf718a56ecc5878dec71180d8b704  ELKBuild.yml
 c8066ae4c79ca812f0abde5b42d4a0bf954a60c88b3eab2149d75d98b567c931  Elastic_restart.yml
 936cb966490bb452f91f2cb5b0821ef31c4bc8e62cf2c57a2c11640b60db7e18  MANIFEST.in
-2e796c430ae2e8f0d061770ae1f58b19aaa8aec077388a73051dd71a316c4218  Nodeexporter.yml
-00bd0602dc641aac3168f64d976d9e0d23b4c54d0030c98d7a4c880bde0f49c4  Promgraf.yml
+e23aacc90b0cced31153f1d5f62e018d3ee59fe03d1b7265cfdb9a10c242e875  Nodeexporter.yml
+86472c01b5463af6861466a9f297665d4cbfd9d0941ec3cb2e4a818ecb873969  Promgraf.yml
 798346886b7ec801a19ed365c33f092e2d5dfb564c034ae886bc51785c89b232  README.md
 88223aecf0ee1bfa38a215a481e1dd48054d043a7b52b9fd81243f9699d6142c  collections/requirements.yml
 bcc7a97fdb676c3f94875a674a87d25cd286cabba5820a5bb0a3eb228951b568  email.yml
@@ -23,18 +23,18 @@ d26a0f12ebb5835b5aa36074b24addf52fdca91939d97244eb682bb3bd05545e  roles/jenkins_
 3ce37f28b0b4cd83a86440fdfbc1148f6a61125dd4dcf900b953778a588ac12b  roles/jenkins_setup/tasks/main.yml
 fde421bd02e9d5d96effcbef92b4657f2ed9283855cebafcb3246c4829882382  roles/kafka/handlers/main.yml
 b6feda249d68b441cf3dedb363eee0aa78543b6566b276542ec93da0830c5890  roles/kafka/tasks/main.yml
-f74c4f7001a8f7e59756bd159e84514a64d9ff7ce2f43ee742f861af0e6ea307  roles/prometheus/defaults/main.yml
+7f721521e6af1761575abf516441d5d4752ca2d0d5908ee2cfbdb0aaa64a53ef  roles/prometheus/defaults/main.yml
 a8355ca3078663433c90e991a1a9622ae2d5b911696aef4a62701ed6a454690c  roles/prometheus/handlers/main.yml
 4116ff338c5ffed270d770729ff19e4f0a85365be1745f282fd35f50ca660fd6  roles/prometheus/tasks/main.yml
-652e2eb86794ca3d840668e2b69cb503dd1c1f13c71876ce862c4fc6e35f6a5d  roles/prometheus/tasks/setup-alertmanager.yml
+0851b3c911166154035af002069175750e67611180218be24015b2831e166c24  roles/prometheus/tasks/setup-alertmanager.yml
 ab401c6ab6918566be97cb084100c7c60148fc9c90fd856869737d410a6f0600  roles/prometheus/tasks/setup-facts.yml
 0f4c1f016548af369cddc1369030d7e8c827091f83d638249b8d36711e60be50  roles/prometheus/tasks/setup-nodeexporter.yml
-79b6447108a4061f0dd1a71444186ae3d2450b50e33c43b5875dc2163333f8ef  roles/prometheus/tasks/setup-prometheus.yml
-74e2466ae5d026dee56fb6a004033b3af71adc65044a5ee90d49da038dcc9acb  roles/prometheus/templates/alertmanager.service.j2
-9605093238d4b3e623a65c733625ac5135ce03bb58262df4fd79be813898e938  roles/prometheus/templates/alertmanager.yml.j2
+7ae95928dcecc94b606d1020bd6216cb1e8811f9aa694f15cf80c05b469d2831  roles/prometheus/tasks/setup-prometheus.yml
+f033f1b2030d560d5c1eacba15997573c0d730c4a486c1aa4bdafca823f818ed  roles/prometheus/templates/alertmanager.service.j2
+940cccbc5ae9ebd2ca89ad981961acab3429b43ff7f429896e2cf4a755725402  roles/prometheus/templates/alertmanager.yml.j2
 13425d7aa6d7877e7875d1b2ffd4ba2ebe499cce2c2f7ba48b1983241e326781  roles/prometheus/templates/nodeexporter.service.j2
-742e8f93db56cff779c6f1b61e496dabe9c956eff0d465ef2fdad225b3c0ba11  roles/prometheus/templates/prometheus.service.j2
-de6931951a93895849179f1d8f0d3b813af7644bdf82d545977ef53ba1989eb7  roles/prometheus/templates/prometheus.yml.j2
+5fb6b8d81e5f1a129a339f3c30c700d14477a3be7aa2f905b2af27e0aef2bc0a  roles/prometheus/templates/prometheus.service.j2
+f404c84a5f12e670c83485ab20e366f45c6a1ccd9efa779b4affa84b5e8c9fb8  roles/prometheus/templates/prometheus.yml.j2
 94c9adec62bc289d278eab57072ca157525763a09e5aebb22ab737c929650ff9  roles/prometheus/vars/main.yml
 8e1fb59a494984f0c85a460c2fcb9b5d857afa7609c597c5be46c8014d21d7fc  roles/requirements.yml
 16f3626f6c3a2f1511188992b1325bf99e6344a73483817802955ef79a374e51  roles/snort_build/defaults/main.yml

.ansible-sign/sha256sum.txt.sig

@@ -1,14 +1,14 @@
 -----BEGIN PGP SIGNATURE-----
 
-iQGzBAABCAAdFiEE/bJvyFHKKJdaZDOLTiIiIXrUDNEFAmVL+CYACgkQTiIiIXrU
-DNEduAwAkphpnyaq3LyhtpdfCHX7z4CAE1coSWeKuK+opmMoR6KWqgPwDyZ2XwaF
-wJ9cs2ewHmw+MNvxBkn2mMZnhxDENinELIrCL32mkJTuNvnvVuxPyS0vF0wUj8av
-C8YD25Vwgd3SzFBmuMyKfpYU9Opirqe8X52/LmeFHglQqyRPv3fBqaX1vDnz2hLh
-Ld7DbCvhjUWZzKKGV85zpvu2LulLSgdvXnb7EAS1oWSxKN0w9ClqnS6wX6Sxrt7T
-C4OlHh4UpXbiBaPvSkzWibCGGzxvegzTWwqtWSNqVZ3JL3zIsUvSIGkfyj/5Nlpp
-//JZq61ymAVp4FmSXkGxURfamYIkXp9SzuD3Vgc62CYsC1OOPIEUfX/jTV5bvEGP
-ra/knlBbG6suk520/nFtRtvZKq9TmL+ewSG5HtGgPP+3ya56gvehc8uGARso8qFc
-kgzC7QQuqiobB2HANO1EYjLWzNeR9+WxquCtovW+6NEUH0LjN76c7TuYJbf9yAAG
-mRRkjAI5
-=3SBT
+iQGzBAABCAAdFiEE/bJvyFHKKJdaZDOLTiIiIXrUDNEFAmVM36AACgkQTiIiIXrU
+DNGx/wv/bxBSaWAoaE4FXEwaYQZ7zKRWBrDUM/4QCEL/eB24YU1bnt8eRN1KOSZy
+x5rtlUxYZzWnCuNI9d7KDW4ghDwh9wyUl44ElvOFuSUBvhtD1QoW3tMrHybSLml2
+J+JIyVfbkFZL6p0IjC5cw1W+Fj2mrGscrEryXfTwVWpx7axLdDQC+/V5P1hiuqyt
+EuR54GsmGZcES/Lz7bVXH2QmKxgOsb++IotWKJoMZh5OOaUXwWFb/wxGhMnbWZvN
+OgzrDJsoEs8wp58GZycDunYpeTk3bx5QyDGxME74jVOrYQ+9viEK7tz7IjbYtqTC
+NxVJxHSKBYV416s0GxLpVAyIMX1hR7euhSB5Bw8S42yXPOKdbjt+CTJWRDtFujxb
+tDnSVm2rMIAmyUVZbMjS+E/pN4Spy51aKfUgcgkU5fXYf3HROmmGeX1mqNE2IkGN
+QczXaAAdktPlrBDdEOe0SuKcDTR6UMOhw63hQkF4R937q/CRIHbefmWuFGvi7K3V
+O1fNl1hO
+=OmbS
 -----END PGP SIGNATURE-----

Nodeexporter.yml

@@ -4,7 +4,7 @@
   vars:
     - prometheus_components:
         - node_exporter
-    - nodeexporter_version: 1.4.0
+    - nodeexporter_version: 1.6.1
 
   roles:
     - prometheus

Promgraf.yml

@@ -9,7 +9,7 @@
     - tower_scrape: 5m
     - prometheus_version: 2.47.2
     - alertmanager_version: 0.26.0
-    - nodeexporter_version: 1.4.0
+    - nodeexporter_version: 1.6.1
 
 # If issues arise with grafana yum install, then manually install
 # update /etc/yum.repos.d/grafana.repo to the info below

roles/prometheus/defaults/main.yml

@@ -12,6 +12,7 @@ prometheus_user: prometheus
 prometheus_path_install: /usr/local/bin/prometheus
 # Prometheus configuration path
 prometheus_path_config: /etc/prometheus
+alertmanager_path_config: /etc/alertmanager
 # Prometheus logs path
 prometheus_path_log: /var/log/prometheus
 # Prometheus PID path
@@ -23,6 +24,8 @@ prometheus_path_rules: "{{ prometheus_path_config }}/rules"
 # Prometheus file sd config path:
 prometheus_path_file_sd_config: "{{ prometheus_path_config }}/tgroups"
 
+_prometheus_binary_install_dir: '/usr/local/bin'
+
 # Prometheus components to install
 # Can be a mix of:
 #   - prometheus
@@ -90,6 +93,7 @@ alertmanager_smtp_auth_secret: null
 alertmanager_smtp_auth_identity: null
 # The default SMTP TLS requirement. default: false
 alertmanager_smtp_require_tls: false
+_alertmanager_binary_install_dir: '/usr/local/bin'
 
 # Slack API url. default empty
 alertmanager_slack_api_url: null

roles/prometheus/tasks/setup-alertmanager.yml

@@ -29,7 +29,8 @@
     - name: "Expand Alert Manager binaries"
       ansible.builtin.unarchive:
         src: "/tmp/{{ prometheus_am_archive }}"
-        dest: "{{ prometheus_path_install }}"
+        dest: "/tmp"
+        creates: "/tmp/alertmanager-{{ alertmanager_version }}.{{ prometheus_platform_suffix }}"
         owner: "{{ prometheus_user }}"
         group: "{{ prometheus_group }}"
         copy: false
@@ -39,6 +40,19 @@
         path: "/tmp/{{ prometheus_am_archive }}"
         state: absent
 
+    - name: Propagate official alertmanager and amtool binaries
+      ansible.builtin.copy:
+        src: "/tmp/alertmanager-{{ alertmanager_version }}.linux-{{ go_arch }}/{{ item }}"
+        dest: "{{ _alertmanager_binary_install_dir }}/{{ item }}"
+        mode: 0755
+        owner: root
+        group: root
+      with_items:
+        - alertmanager
+        - amtool
+      notify:
+        - "restart alertmanager service"
+
 - name: "Ensure files permissions"
   ansible.builtin.file:
     path: "{{ prometheus_path_install }}"
@@ -81,7 +95,7 @@
 - name: "Ensure Alert manager configuration"
   ansible.builtin.template:
     src: "alertmanager.yml.j2"
-    dest: "{{ prometheus_path_config }}/alertmanager.yml"
+    dest: "{{ alertmanager_path_config }}/alertmanager.yml"
     owner: "{{ prometheus_user }}"
     group: "{{ prometheus_group }}"
     mode: "u=rw,g=,o="

roles/prometheus/tasks/setup-prometheus.yml

@@ -29,7 +29,8 @@
     - name: "Expand Prometheus binaries"
       ansible.builtin.unarchive:
         src: "/tmp/{{ prometheus_prom_archive }}"
-        dest: "{{ prometheus_path_install }}"
+        dest: "/tmp"
+        creates: /tmp/prometheus-{{ prometheus_version }}.{{ prometheus_platform_suffix }}/prometheus"
         owner: "{{ prometheus_user }}"
         group: "{{ prometheus_group }}"
         copy: false
@@ -39,6 +40,32 @@
         path: "/tmp/{{ prometheus_prom_archive }}"
         state: absent
 
+    - name: Propagate official prometheus and promtool binaries
+      ansible.builtin.copy:
+        src: "/tmp/prometheus-{{ prometheus_version }}.linux-{{ go_arch }}/{{ item }}"
+        dest: "{{ _prometheus_binary_install_dir }}/{{ item }}"
+        mode: '0755'
+        owner: root
+        group: root
+      with_items:
+        - prometheus
+        - promtool
+      notify:
+        - restart prometheus service
+
+    - name: Propagate official console templates
+      ansible.builtin.copy:
+        src: "/tmp/prometheus-{{ prometheus_version }}.linux-{{ go_arch }}/{{ item }}/"
+        dest: "{{ prometheus_config_dir }}/{{ item }}/"
+        mode: '0644'
+        owner: root
+        group: root
+      with_items:
+        - console_libraries
+        - consoles
+      notify:
+        - restart prometheus service
+
 - name: "Ensure files permissions"
   ansible.builtin.file:
     path: "{{ prometheus_path_install }}"

roles/prometheus/templates/alertmanager.service.j2

@@ -7,8 +7,11 @@ After=network-online.target
 User=root
 Restart=on-failure
 RestartSec=10
-ExecStart={{ alertmanager_daemon_dir }}/alertmanager \
-              --config.file={{ prometheus_path_config }}/alertmanager.yml \
-              --web.external-url http://promgraf.shadowman.dev:9093
+ExecStart={{ _alertmanager_binary_install_dir }}/alertmanager \
+              --config.file={{ alertmanager_path_config }}/alertmanager.yml \
+              --web.config.file={{ alertmanager_path_config }}/web.yml \
+              --web.external-url http://{{ inventory_hostname }}:9093
+WorkingDirectory={{ alertmanager_path_config }}
+
 [Install]
 WantedBy=multi-user.target

roles/prometheus/templates/alertmanager.yml.j2

@@ -34,13 +34,12 @@ route:
   group_wait: 5s
   group_interval: 10s
   repeat_interval: 3h
-  receiver: 'snow'
+  receiver: 'EDA'
 
 receivers:
-  - name: 'snow'
+  - name: 'EDA'
     webhook_configs:
-    - url: "http://localhost:9877/webhook"
-#    - url: "http://eda.shadowman.dev:8000/endpoint"
+    - url: "http://eda.shadowman.dev:8000/endpoint"
       send_resolved: false
 
 

roles/prometheus/templates/prometheus.service.j2

@@ -8,8 +8,9 @@ After=network-online.target
 User=root
 Restart=on-failure
 RestartSec=10
-ExecStart={{ prometheus_daemon_dir }}/prometheus \
+ExecStart={{ prometheus_path_install }} \
               --config.file={{ prometheus_path_config }}/prometheus.yml \
+              --web.config.file={{ prometheus_path_config }}/web.yml \
 
 [Install]
 WantedBy=multi-user.target

roles/prometheus/templates/prometheus.yml.j2

@@ -12,21 +12,53 @@ global:
 
   # How frequently to evaluate rules.
   # [ evaluation_interval: <duration> | default = 1m ]
-  evaluation_interval:  1m
+  evaluation_interval:  {{ tower_scrape }}
 
 alerting:
   alertmanagers:
-  - static_configs:
+  - scheme: https
+    static_configs:
     - targets:
-       - promgraf.shadowman.dev:9093
+       - {{ inventory_hostname }}:9093
+
+rule_files:
+  - "/etc/prometheus/rules/*.yml"
 
 scrape_configs:
 
-  # Scrape job for prometheus himself
+  # Scrape job for prometheus
   - job_name: 'prometheus'
+    scheme: https
     static_configs:
     - targets: ['{{ inventory_hostname }}:9090']
 
+  # Scrape Config for Tower
+  - job_name: 'tower metrics'
+    metrics_path: /api/v2/metrics/
+    scrape_interval: {{ tower_scrape }}
+    scheme: https
+    bearer_token: "{{ bearer_token }}"
+    static_configs:
+    - targets: ['tower1.shadowman.dev:443']
+
+  # Scrape config for Tower Node Exporter
+  - job_name: 'tower1.shadowman.dev'
+    scrape_interval: {{ tower_scrape }}
+    static_configs:
+    - targets: ['tower1.shadowman.dev:9100']
+
+  # Scrape config for Tower Node Exporter
+  - job_name: 'tower2.shadowman.dev'
+    scrape_interval: {{ tower_scrape }}
+    static_configs:
+    - targets: ['tower2.shadowman.dev:9100']
+
+  # Scrape job for rhel8 node exporter
+  - job_name: 'rhel8.shadowman.dev'
+    scrape_interval: {{ tower_scrape }}
+    static_configs:
+    - targets: ['rhel8.shadowman.dev:9100']
+
 #  # The labels to add to any time series or alerts when communicating with
 #  # external systems (federation, remote storage, Alertmanager).
 #  external_labels: