diff --git a/.ansible-sign/sha256sum.txt b/.ansible-sign/sha256sum.txt index 29f11b9..adf0c6b 100644 --- a/.ansible-sign/sha256sum.txt +++ b/.ansible-sign/sha256sum.txt @@ -4,8 +4,8 @@ ca3f84e14f6aa778003094fd160710543fae3594d76dbc4335d2d0b77245e8ce AnsibleSSP.cod e81d36021af399b6ecafeecde71a8634aacbf718a56ecc5878dec71180d8b704 ELKBuild.yml c8066ae4c79ca812f0abde5b42d4a0bf954a60c88b3eab2149d75d98b567c931 Elastic_restart.yml 936cb966490bb452f91f2cb5b0821ef31c4bc8e62cf2c57a2c11640b60db7e18 MANIFEST.in -2e796c430ae2e8f0d061770ae1f58b19aaa8aec077388a73051dd71a316c4218 Nodeexporter.yml -00bd0602dc641aac3168f64d976d9e0d23b4c54d0030c98d7a4c880bde0f49c4 Promgraf.yml +e23aacc90b0cced31153f1d5f62e018d3ee59fe03d1b7265cfdb9a10c242e875 Nodeexporter.yml +86472c01b5463af6861466a9f297665d4cbfd9d0941ec3cb2e4a818ecb873969 Promgraf.yml 798346886b7ec801a19ed365c33f092e2d5dfb564c034ae886bc51785c89b232 README.md 88223aecf0ee1bfa38a215a481e1dd48054d043a7b52b9fd81243f9699d6142c collections/requirements.yml bcc7a97fdb676c3f94875a674a87d25cd286cabba5820a5bb0a3eb228951b568 email.yml @@ -23,18 +23,18 @@ d26a0f12ebb5835b5aa36074b24addf52fdca91939d97244eb682bb3bd05545e roles/jenkins_ 3ce37f28b0b4cd83a86440fdfbc1148f6a61125dd4dcf900b953778a588ac12b roles/jenkins_setup/tasks/main.yml fde421bd02e9d5d96effcbef92b4657f2ed9283855cebafcb3246c4829882382 roles/kafka/handlers/main.yml b6feda249d68b441cf3dedb363eee0aa78543b6566b276542ec93da0830c5890 roles/kafka/tasks/main.yml -f74c4f7001a8f7e59756bd159e84514a64d9ff7ce2f43ee742f861af0e6ea307 roles/prometheus/defaults/main.yml +7f721521e6af1761575abf516441d5d4752ca2d0d5908ee2cfbdb0aaa64a53ef roles/prometheus/defaults/main.yml a8355ca3078663433c90e991a1a9622ae2d5b911696aef4a62701ed6a454690c roles/prometheus/handlers/main.yml 4116ff338c5ffed270d770729ff19e4f0a85365be1745f282fd35f50ca660fd6 roles/prometheus/tasks/main.yml -652e2eb86794ca3d840668e2b69cb503dd1c1f13c71876ce862c4fc6e35f6a5d roles/prometheus/tasks/setup-alertmanager.yml +0851b3c911166154035af002069175750e67611180218be24015b2831e166c24 roles/prometheus/tasks/setup-alertmanager.yml ab401c6ab6918566be97cb084100c7c60148fc9c90fd856869737d410a6f0600 roles/prometheus/tasks/setup-facts.yml 0f4c1f016548af369cddc1369030d7e8c827091f83d638249b8d36711e60be50 roles/prometheus/tasks/setup-nodeexporter.yml -79b6447108a4061f0dd1a71444186ae3d2450b50e33c43b5875dc2163333f8ef roles/prometheus/tasks/setup-prometheus.yml -74e2466ae5d026dee56fb6a004033b3af71adc65044a5ee90d49da038dcc9acb roles/prometheus/templates/alertmanager.service.j2 -9605093238d4b3e623a65c733625ac5135ce03bb58262df4fd79be813898e938 roles/prometheus/templates/alertmanager.yml.j2 +7ae95928dcecc94b606d1020bd6216cb1e8811f9aa694f15cf80c05b469d2831 roles/prometheus/tasks/setup-prometheus.yml +f033f1b2030d560d5c1eacba15997573c0d730c4a486c1aa4bdafca823f818ed roles/prometheus/templates/alertmanager.service.j2 +940cccbc5ae9ebd2ca89ad981961acab3429b43ff7f429896e2cf4a755725402 roles/prometheus/templates/alertmanager.yml.j2 13425d7aa6d7877e7875d1b2ffd4ba2ebe499cce2c2f7ba48b1983241e326781 roles/prometheus/templates/nodeexporter.service.j2 -742e8f93db56cff779c6f1b61e496dabe9c956eff0d465ef2fdad225b3c0ba11 roles/prometheus/templates/prometheus.service.j2 -de6931951a93895849179f1d8f0d3b813af7644bdf82d545977ef53ba1989eb7 roles/prometheus/templates/prometheus.yml.j2 +5fb6b8d81e5f1a129a339f3c30c700d14477a3be7aa2f905b2af27e0aef2bc0a roles/prometheus/templates/prometheus.service.j2 +f404c84a5f12e670c83485ab20e366f45c6a1ccd9efa779b4affa84b5e8c9fb8 roles/prometheus/templates/prometheus.yml.j2 94c9adec62bc289d278eab57072ca157525763a09e5aebb22ab737c929650ff9 roles/prometheus/vars/main.yml 8e1fb59a494984f0c85a460c2fcb9b5d857afa7609c597c5be46c8014d21d7fc roles/requirements.yml 16f3626f6c3a2f1511188992b1325bf99e6344a73483817802955ef79a374e51 roles/snort_build/defaults/main.yml diff --git a/.ansible-sign/sha256sum.txt.sig b/.ansible-sign/sha256sum.txt.sig index 4c8caa2..f041c84 100644 --- a/.ansible-sign/sha256sum.txt.sig +++ b/.ansible-sign/sha256sum.txt.sig @@ -1,14 +1,14 @@ -----BEGIN PGP SIGNATURE----- -iQGzBAABCAAdFiEE/bJvyFHKKJdaZDOLTiIiIXrUDNEFAmVL+CYACgkQTiIiIXrU -DNEduAwAkphpnyaq3LyhtpdfCHX7z4CAE1coSWeKuK+opmMoR6KWqgPwDyZ2XwaF -wJ9cs2ewHmw+MNvxBkn2mMZnhxDENinELIrCL32mkJTuNvnvVuxPyS0vF0wUj8av -C8YD25Vwgd3SzFBmuMyKfpYU9Opirqe8X52/LmeFHglQqyRPv3fBqaX1vDnz2hLh -Ld7DbCvhjUWZzKKGV85zpvu2LulLSgdvXnb7EAS1oWSxKN0w9ClqnS6wX6Sxrt7T -C4OlHh4UpXbiBaPvSkzWibCGGzxvegzTWwqtWSNqVZ3JL3zIsUvSIGkfyj/5Nlpp -//JZq61ymAVp4FmSXkGxURfamYIkXp9SzuD3Vgc62CYsC1OOPIEUfX/jTV5bvEGP -ra/knlBbG6suk520/nFtRtvZKq9TmL+ewSG5HtGgPP+3ya56gvehc8uGARso8qFc -kgzC7QQuqiobB2HANO1EYjLWzNeR9+WxquCtovW+6NEUH0LjN76c7TuYJbf9yAAG -mRRkjAI5 -=3SBT +iQGzBAABCAAdFiEE/bJvyFHKKJdaZDOLTiIiIXrUDNEFAmVM36AACgkQTiIiIXrU +DNGx/wv/bxBSaWAoaE4FXEwaYQZ7zKRWBrDUM/4QCEL/eB24YU1bnt8eRN1KOSZy +x5rtlUxYZzWnCuNI9d7KDW4ghDwh9wyUl44ElvOFuSUBvhtD1QoW3tMrHybSLml2 +J+JIyVfbkFZL6p0IjC5cw1W+Fj2mrGscrEryXfTwVWpx7axLdDQC+/V5P1hiuqyt +EuR54GsmGZcES/Lz7bVXH2QmKxgOsb++IotWKJoMZh5OOaUXwWFb/wxGhMnbWZvN +OgzrDJsoEs8wp58GZycDunYpeTk3bx5QyDGxME74jVOrYQ+9viEK7tz7IjbYtqTC +NxVJxHSKBYV416s0GxLpVAyIMX1hR7euhSB5Bw8S42yXPOKdbjt+CTJWRDtFujxb +tDnSVm2rMIAmyUVZbMjS+E/pN4Spy51aKfUgcgkU5fXYf3HROmmGeX1mqNE2IkGN +QczXaAAdktPlrBDdEOe0SuKcDTR6UMOhw63hQkF4R937q/CRIHbefmWuFGvi7K3V +O1fNl1hO +=OmbS -----END PGP SIGNATURE----- diff --git a/Nodeexporter.yml b/Nodeexporter.yml index 38ec427..d58265b 100644 --- a/Nodeexporter.yml +++ b/Nodeexporter.yml @@ -4,7 +4,7 @@ vars: - prometheus_components: - node_exporter - - nodeexporter_version: 1.4.0 + - nodeexporter_version: 1.6.1 roles: - prometheus diff --git a/Promgraf.yml b/Promgraf.yml index 9645f3a..31b3420 100644 --- a/Promgraf.yml +++ b/Promgraf.yml @@ -9,7 +9,7 @@ - tower_scrape: 5m - prometheus_version: 2.47.2 - alertmanager_version: 0.26.0 - - nodeexporter_version: 1.4.0 + - nodeexporter_version: 1.6.1 # If issues arise with grafana yum install, then manually install # update /etc/yum.repos.d/grafana.repo to the info below diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml index 850803e..2dba57a 100644 --- a/roles/prometheus/defaults/main.yml +++ b/roles/prometheus/defaults/main.yml @@ -12,6 +12,7 @@ prometheus_user: prometheus prometheus_path_install: /usr/local/bin/prometheus # Prometheus configuration path prometheus_path_config: /etc/prometheus +alertmanager_path_config: /etc/alertmanager # Prometheus logs path prometheus_path_log: /var/log/prometheus # Prometheus PID path @@ -23,6 +24,8 @@ prometheus_path_rules: "{{ prometheus_path_config }}/rules" # Prometheus file sd config path: prometheus_path_file_sd_config: "{{ prometheus_path_config }}/tgroups" +_prometheus_binary_install_dir: '/usr/local/bin' + # Prometheus components to install # Can be a mix of: # - prometheus @@ -90,6 +93,7 @@ alertmanager_smtp_auth_secret: null alertmanager_smtp_auth_identity: null # The default SMTP TLS requirement. default: false alertmanager_smtp_require_tls: false +_alertmanager_binary_install_dir: '/usr/local/bin' # Slack API url. default empty alertmanager_slack_api_url: null diff --git a/roles/prometheus/tasks/setup-alertmanager.yml b/roles/prometheus/tasks/setup-alertmanager.yml index 1049f1a..6538abd 100644 --- a/roles/prometheus/tasks/setup-alertmanager.yml +++ b/roles/prometheus/tasks/setup-alertmanager.yml @@ -29,7 +29,8 @@ - name: "Expand Alert Manager binaries" ansible.builtin.unarchive: src: "/tmp/{{ prometheus_am_archive }}" - dest: "{{ prometheus_path_install }}" + dest: "/tmp" + creates: "/tmp/alertmanager-{{ alertmanager_version }}.{{ prometheus_platform_suffix }}" owner: "{{ prometheus_user }}" group: "{{ prometheus_group }}" copy: false @@ -39,6 +40,19 @@ path: "/tmp/{{ prometheus_am_archive }}" state: absent + - name: Propagate official alertmanager and amtool binaries + ansible.builtin.copy: + src: "/tmp/alertmanager-{{ alertmanager_version }}.linux-{{ go_arch }}/{{ item }}" + dest: "{{ _alertmanager_binary_install_dir }}/{{ item }}" + mode: 0755 + owner: root + group: root + with_items: + - alertmanager + - amtool + notify: + - "restart alertmanager service" + - name: "Ensure files permissions" ansible.builtin.file: path: "{{ prometheus_path_install }}" @@ -81,7 +95,7 @@ - name: "Ensure Alert manager configuration" ansible.builtin.template: src: "alertmanager.yml.j2" - dest: "{{ prometheus_path_config }}/alertmanager.yml" + dest: "{{ alertmanager_path_config }}/alertmanager.yml" owner: "{{ prometheus_user }}" group: "{{ prometheus_group }}" mode: "u=rw,g=,o=" diff --git a/roles/prometheus/tasks/setup-prometheus.yml b/roles/prometheus/tasks/setup-prometheus.yml index 63d8be2..221bc1c 100644 --- a/roles/prometheus/tasks/setup-prometheus.yml +++ b/roles/prometheus/tasks/setup-prometheus.yml @@ -29,7 +29,8 @@ - name: "Expand Prometheus binaries" ansible.builtin.unarchive: src: "/tmp/{{ prometheus_prom_archive }}" - dest: "{{ prometheus_path_install }}" + dest: "/tmp" + creates: /tmp/prometheus-{{ prometheus_version }}.{{ prometheus_platform_suffix }}/prometheus" owner: "{{ prometheus_user }}" group: "{{ prometheus_group }}" copy: false @@ -39,6 +40,32 @@ path: "/tmp/{{ prometheus_prom_archive }}" state: absent + - name: Propagate official prometheus and promtool binaries + ansible.builtin.copy: + src: "/tmp/prometheus-{{ prometheus_version }}.linux-{{ go_arch }}/{{ item }}" + dest: "{{ _prometheus_binary_install_dir }}/{{ item }}" + mode: '0755' + owner: root + group: root + with_items: + - prometheus + - promtool + notify: + - restart prometheus service + + - name: Propagate official console templates + ansible.builtin.copy: + src: "/tmp/prometheus-{{ prometheus_version }}.linux-{{ go_arch }}/{{ item }}/" + dest: "{{ prometheus_config_dir }}/{{ item }}/" + mode: '0644' + owner: root + group: root + with_items: + - console_libraries + - consoles + notify: + - restart prometheus service + - name: "Ensure files permissions" ansible.builtin.file: path: "{{ prometheus_path_install }}" diff --git a/roles/prometheus/templates/alertmanager.service.j2 b/roles/prometheus/templates/alertmanager.service.j2 index 0a1a9a0..7e9716a 100644 --- a/roles/prometheus/templates/alertmanager.service.j2 +++ b/roles/prometheus/templates/alertmanager.service.j2 @@ -7,8 +7,11 @@ After=network-online.target User=root Restart=on-failure RestartSec=10 -ExecStart={{ alertmanager_daemon_dir }}/alertmanager \ - --config.file={{ prometheus_path_config }}/alertmanager.yml \ - --web.external-url http://promgraf.shadowman.dev:9093 +ExecStart={{ _alertmanager_binary_install_dir }}/alertmanager \ + --config.file={{ alertmanager_path_config }}/alertmanager.yml \ + --web.config.file={{ alertmanager_path_config }}/web.yml \ + --web.external-url http://{{ inventory_hostname }}:9093 +WorkingDirectory={{ alertmanager_path_config }} + [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/roles/prometheus/templates/alertmanager.yml.j2 b/roles/prometheus/templates/alertmanager.yml.j2 index 03edcea..a62a9c5 100644 --- a/roles/prometheus/templates/alertmanager.yml.j2 +++ b/roles/prometheus/templates/alertmanager.yml.j2 @@ -34,13 +34,12 @@ route: group_wait: 5s group_interval: 10s repeat_interval: 3h - receiver: 'snow' + receiver: 'EDA' receivers: - - name: 'snow' + - name: 'EDA' webhook_configs: - - url: "http://localhost:9877/webhook" -# - url: "http://eda.shadowman.dev:8000/endpoint" + - url: "http://eda.shadowman.dev:8000/endpoint" send_resolved: false diff --git a/roles/prometheus/templates/prometheus.service.j2 b/roles/prometheus/templates/prometheus.service.j2 index 5471a20..7f4745b 100644 --- a/roles/prometheus/templates/prometheus.service.j2 +++ b/roles/prometheus/templates/prometheus.service.j2 @@ -8,8 +8,9 @@ After=network-online.target User=root Restart=on-failure RestartSec=10 -ExecStart={{ prometheus_daemon_dir }}/prometheus \ +ExecStart={{ prometheus_path_install }} \ --config.file={{ prometheus_path_config }}/prometheus.yml \ + --web.config.file={{ prometheus_path_config }}/web.yml \ [Install] WantedBy=multi-user.target \ No newline at end of file diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2 index 82f6518..f15a412 100644 --- a/roles/prometheus/templates/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus.yml.j2 @@ -12,21 +12,53 @@ global: # How frequently to evaluate rules. # [ evaluation_interval: | default = 1m ] - evaluation_interval: 1m + evaluation_interval: {{ tower_scrape }} alerting: alertmanagers: - - static_configs: + - scheme: https + static_configs: - targets: - - promgraf.shadowman.dev:9093 + - {{ inventory_hostname }}:9093 + +rule_files: + - "/etc/prometheus/rules/*.yml" scrape_configs: - # Scrape job for prometheus himself + # Scrape job for prometheus - job_name: 'prometheus' + scheme: https static_configs: - targets: ['{{ inventory_hostname }}:9090'] + # Scrape Config for Tower + - job_name: 'tower metrics' + metrics_path: /api/v2/metrics/ + scrape_interval: {{ tower_scrape }} + scheme: https + bearer_token: "{{ bearer_token }}" + static_configs: + - targets: ['tower1.shadowman.dev:443'] + + # Scrape config for Tower Node Exporter + - job_name: 'tower1.shadowman.dev' + scrape_interval: {{ tower_scrape }} + static_configs: + - targets: ['tower1.shadowman.dev:9100'] + + # Scrape config for Tower Node Exporter + - job_name: 'tower2.shadowman.dev' + scrape_interval: {{ tower_scrape }} + static_configs: + - targets: ['tower2.shadowman.dev:9100'] + + # Scrape job for rhel8 node exporter + - job_name: 'rhel8.shadowman.dev' + scrape_interval: {{ tower_scrape }} + static_configs: + - targets: ['rhel8.shadowman.dev:9100'] + # # The labels to add to any time series or alerts when communicating with # # external systems (federation, remote storage, Alertmanager). # external_labels: