diff --git a/Dockerfile b/Dockerfile index daa683ef..c173bd8e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,6 +26,7 @@ COPY dist/*.whl /tmp COPY config.yaml /work/config.yaml COPY lookups /work/lookups COPY mibs /work/mibs +COPY profiles /work/profiles RUN pip3.8 install $(ls /tmp/*.whl); rm -f /tmp/*.whl EXPOSE 5000 diff --git a/config.yaml b/config.yaml index 3bbf1806..c4e25225 100644 --- a/config.yaml +++ b/config.yaml @@ -18,10 +18,11 @@ snmp: dir: "/work/mibs/pysnmp" load_list: "lookups/mibs_list.csv" mibs_path: "/work/mibs" + profiles_path: "/work/profiles" mongo: oid: database: "mib_server" collection: "oids" mib: database: "files" - collection: "mib_files" + collection: "mib_files" \ No newline at end of file diff --git a/profiles/default.yaml b/profiles/default.yaml new file mode 100644 index 00000000..4fe621bc --- /dev/null +++ b/profiles/default.yaml @@ -0,0 +1,37 @@ +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +profiles: + basev1: + frequency: 10 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - '1.3.6.1.2.1.2.*' + basev1l2: + frequency: 20 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - ['IF-MIB','ifHCInOctets'] + - ['IF-MIB','ifHCOutOctets'] + - ['IF-MIB','ifInErrors'] + - ['IF-MIB','ifOutErrors'] + - ['IF-MIB','ifInDiscards'] + - ['IF-MIB','ifOutDiscards'] \ No newline at end of file diff --git a/splunk_connect_for_snmp_mib_server/mib_server.py b/splunk_connect_for_snmp_mib_server/mib_server.py index e89c8191..6f754a14 100644 --- a/splunk_connect_for_snmp_mib_server/mib_server.py +++ b/splunk_connect_for_snmp_mib_server/mib_server.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. # ######################################################################## - import logging from flask import Flask, request from flask_autoindex import AutoIndex +from splunk_connect_for_snmp_mib_server.profiles import merge_profiles from splunk_connect_for_snmp_mib_server.translator import Translator logger = logging.getLogger(__name__) @@ -39,6 +39,7 @@ def __init__(self, args, server_config): def build_flask_app(self): app = Flask(__name__) mibs_path = self._server_config["snmp"]["mibs"]["mibs_path"] + profiles_path = self._server_config["snmp"]["mibs"]["profiles_path"] files_index = AutoIndex(app, mibs_path, add_url_rules=False) @app.route("/") @@ -68,6 +69,10 @@ def translator(): result = self._translator.format_text_event(var_binds) return result + @app.route("/profiles", methods=["GET"]) + def get_profiles(): + return merge_profiles(profiles_path, "profiles") + return app def run_mib_server(self): diff --git a/splunk_connect_for_snmp_mib_server/profiles.py b/splunk_connect_for_snmp_mib_server/profiles.py new file mode 100644 index 00000000..c0c0b421 --- /dev/null +++ b/splunk_connect_for_snmp_mib_server/profiles.py @@ -0,0 +1,40 @@ +# ######################################################################## +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ######################################################################## +import logging +import os + +import yaml +from yaml.parser import ParserError + +logger = logging.getLogger(__name__) + + +def merge_profiles(directory, root_name): + result = {} + merged_profiles = {} + for root, directories, files in os.walk(directory, topdown=False): + for name in sorted(files): + with open(os.path.join(root, name), "r") as stream: + try: + data = yaml.safe_load(stream) + merged_profiles.update(data[root_name]) + except ParserError as pe: + logger.warning( + f"Error while parsing file {os.path.join(root, name)} : {pe}" + ) + + result[root_name] = merged_profiles + return result diff --git a/tests/local-config.yaml b/tests/local-config.yaml new file mode 100644 index 00000000..07a73768 --- /dev/null +++ b/tests/local-config.yaml @@ -0,0 +1,31 @@ +# +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +snmp: + mibs: + dir: "mibs/pysnmp" + load_list: "lookups/mibs_list.csv" + mibs_path: "mibs" + profiles_path: "profiles" +mongo: + oid: + database: "mib_server" + collection: "oids" + mib: + database: "files" + collection: "mib_files" + profile: + database: "profiles" + collection: "profiles" \ No newline at end of file diff --git a/tests/profiles/malformed_profile/default.yaml b/tests/profiles/malformed_profile/default.yaml new file mode 100644 index 00000000..68a91e25 --- /dev/null +++ b/tests/profiles/malformed_profile/default.yaml @@ -0,0 +1,37 @@ +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +profiles: + basev1: + frequency: 10 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - '1.3.6.1.2.1.2.*' + basev1l2: + frequency: 20 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - ['IF-MIB','ifHCInOctets'] + - ['IF-MIB','ifHCOutOctets'] + - ['IF-MIB','ifInErrors'] + - ['IF-MIB','ifOutErrors'] + - ['IF-MIB','ifInDiscards'] + - ['IF-MIB','ifOutDiscards'] \ No newline at end of file diff --git a/tests/profiles/malformed_profile/local.yaml b/tests/profiles/malformed_profile/local.yaml new file mode 100644 index 00000000..8c622e6c --- /dev/null +++ b/tests/profiles/malformed_profile/local.yaml @@ -0,0 +1,42 @@ +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +profiles: + basev2: + patterns: + - '*MY_DEFAULT_DEVICE_3*' + frequency: 60 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - '1.3.6.1.2.1.2.*' + basev2l2: + patterns: + - '*MY_DEFAULT_DEVICE_3*' + - '*MY_DEFAULT_DEVICE_NAME_3*' + frequency: 120 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - ['IF-MIB','ifHCInOctets'] + - ['IF-MIB','ifHCOutOctets'] + - ['IF-MIB','ifInErrors'] + - ['IF-MIB','ifOutErrors'] + - ['IF-MIB','ifInDiscards'] + - ['IF-MIB','ifOutDiscards'] \ No newline at end of file diff --git a/tests/profiles/one_profile/default.yaml b/tests/profiles/one_profile/default.yaml new file mode 100644 index 00000000..4fe621bc --- /dev/null +++ b/tests/profiles/one_profile/default.yaml @@ -0,0 +1,37 @@ +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +profiles: + basev1: + frequency: 10 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - '1.3.6.1.2.1.2.*' + basev1l2: + frequency: 20 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - ['IF-MIB','ifHCInOctets'] + - ['IF-MIB','ifHCOutOctets'] + - ['IF-MIB','ifInErrors'] + - ['IF-MIB','ifOutErrors'] + - ['IF-MIB','ifInDiscards'] + - ['IF-MIB','ifOutDiscards'] \ No newline at end of file diff --git a/tests/profiles/same_name_profiles/default.yaml b/tests/profiles/same_name_profiles/default.yaml new file mode 100644 index 00000000..4fe621bc --- /dev/null +++ b/tests/profiles/same_name_profiles/default.yaml @@ -0,0 +1,37 @@ +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +profiles: + basev1: + frequency: 10 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - '1.3.6.1.2.1.2.*' + basev1l2: + frequency: 20 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - ['IF-MIB','ifHCInOctets'] + - ['IF-MIB','ifHCOutOctets'] + - ['IF-MIB','ifInErrors'] + - ['IF-MIB','ifOutErrors'] + - ['IF-MIB','ifInDiscards'] + - ['IF-MIB','ifOutDiscards'] \ No newline at end of file diff --git a/tests/profiles/same_name_profiles/local.yaml b/tests/profiles/same_name_profiles/local.yaml new file mode 100644 index 00000000..85bdde7e --- /dev/null +++ b/tests/profiles/same_name_profiles/local.yaml @@ -0,0 +1,42 @@ +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +profiles: + basev1: + patterns: + - '*MY_DEFAULT_DEVICE_3*' + frequency: 60 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - '1.3.6.1.2.1.2.*' + basev1l2: + patterns: + - '*MY_DEFAULT_DEVICE_3*' + - '*MY_DEFAULT_DEVICE_NAME_3*' + frequency: 120 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - ['IF-MIB','ifHCInOctets'] + - ['IF-MIB','ifHCOutOctets'] + - ['IF-MIB','ifInErrors'] + - ['IF-MIB','ifOutErrors'] + - ['IF-MIB','ifInDiscards'] + - ['IF-MIB','ifOutDiscards'] \ No newline at end of file diff --git a/tests/profiles/two_profiles/default.yaml b/tests/profiles/two_profiles/default.yaml new file mode 100644 index 00000000..4fe621bc --- /dev/null +++ b/tests/profiles/two_profiles/default.yaml @@ -0,0 +1,37 @@ +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +profiles: + basev1: + frequency: 10 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - '1.3.6.1.2.1.2.*' + basev1l2: + frequency: 20 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - ['IF-MIB','ifHCInOctets'] + - ['IF-MIB','ifHCOutOctets'] + - ['IF-MIB','ifInErrors'] + - ['IF-MIB','ifOutErrors'] + - ['IF-MIB','ifInDiscards'] + - ['IF-MIB','ifOutDiscards'] \ No newline at end of file diff --git a/tests/profiles/two_profiles/local.yaml b/tests/profiles/two_profiles/local.yaml new file mode 100644 index 00000000..8c622e6c --- /dev/null +++ b/tests/profiles/two_profiles/local.yaml @@ -0,0 +1,42 @@ +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +profiles: + basev2: + patterns: + - '*MY_DEFAULT_DEVICE_3*' + frequency: 60 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - '1.3.6.1.2.1.2.*' + basev2l2: + patterns: + - '*MY_DEFAULT_DEVICE_3*' + - '*MY_DEFAULT_DEVICE_NAME_3*' + frequency: 120 + varBinds: + # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"] + - ['SNMPv2-MIB', 'sysDescr'] + - ['SNMPv2-MIB', 'sysUpTime',0] + - ['SNMPv2-MIB', 'sysName'] + - ['IF-MIB','ifHCInOctets'] + - ['IF-MIB','ifHCOutOctets'] + - ['IF-MIB','ifInErrors'] + - ['IF-MIB','ifOutErrors'] + - ['IF-MIB','ifInDiscards'] + - ['IF-MIB','ifOutDiscards'] \ No newline at end of file diff --git a/tests/test_profile_merge.py b/tests/test_profile_merge.py new file mode 100644 index 00000000..488d0610 --- /dev/null +++ b/tests/test_profile_merge.py @@ -0,0 +1,62 @@ +# ######################################################################## +# Copyright 2021 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ######################################################################## + +from unittest import TestCase + +from splunk_connect_for_snmp_mib_server.profiles import merge_profiles + + +class ProfileLoaderTest(TestCase): + def test_multiple_files_merging(self): + merged_profiles = merge_profiles("tests/profiles/two_profiles", "profiles")[ + "profiles" + ] + + assert len(merged_profiles.keys()) == 4 + assert "basev1" in merged_profiles.keys() + assert "basev2" in merged_profiles.keys() + assert "basev1l2" in merged_profiles.keys() + assert "basev2l2" in merged_profiles.keys() + + def test_one_file(self): + merged_profiles = merge_profiles("tests/profiles/one_profile", "profiles")[ + "profiles" + ] + + assert len(merged_profiles.keys()) == 2 + assert "basev1" in merged_profiles.keys() + assert "basev1l2" in merged_profiles.keys() + + def test_same_name_profiles_were_overwritten_in_alphabetical_order(self): + merged_profiles = merge_profiles( + "tests/profiles/same_name_profiles", "profiles" + )["profiles"] + + assert len(merged_profiles.keys()) == 2 + assert "basev1" in merged_profiles.keys() + assert "basev1l2" in merged_profiles.keys() + + assert merged_profiles["basev1"]["frequency"] == 60 + assert merged_profiles["basev1l2"]["frequency"] == 120 + + def test_malformed_file_is_ignored(self): + merged_profiles = merge_profiles( + "tests/profiles/malformed_profile", "profiles" + )["profiles"] + + assert len(merged_profiles.keys()) == 2 + assert "basev2" in merged_profiles.keys() + assert "basev2l2" in merged_profiles.keys() diff --git a/tests/test_translator.py b/tests/test_translator.py index 9ab34249..226da864 100644 --- a/tests/test_translator.py +++ b/tests/test_translator.py @@ -152,7 +152,10 @@ def test_format_trap_non_existing_oid(self): value_type = input_var_binds_list[i]["val_type"] oid = input_var_binds_list[i]["oid"] value = input_var_binds_list[i]["val"] - current = f'oid-type{i+1}="{oid_type}" value{i+1}-type="{value_type}" {oid}="{value}" value{i+1}="{value}"' + current = ( + f'oid-type{i + 1}="{oid_type}"' + f' value{i + 1}-type="{value_type}" {oid}="{value}" value{i + 1}="{value}"' + ) # these two additional spaces are not an error untranslated += f"{current} " if i < len(input_var_binds_list) - 1: