feat: create default profile definition in MIB server (#56)

* feat: create default profile definition in MIB server

* fix: fix failing build

* fix: fix failing build

* fix: fix failing build

* fix: fix failing build

* feat: PR comments

* fix: build fix

* fix: build fix

Dockerfile

@@ -26,6 +26,7 @@ COPY dist/*.whl /tmp
 COPY config.yaml /work/config.yaml
 COPY lookups /work/lookups
 COPY mibs /work/mibs
+COPY profiles /work/profiles
 RUN pip3.8 install $(ls /tmp/*.whl); rm -f /tmp/*.whl
 
 EXPOSE 5000

config.yaml

@@ -18,10 +18,11 @@ snmp:
     dir: "/work/mibs/pysnmp"
     load_list: "lookups/mibs_list.csv"
     mibs_path: "/work/mibs"
+    profiles_path: "/work/profiles"
 mongo:
   oid:
     database: "mib_server"
     collection: "oids"
   mib:
     database: "files"
-    collection: "mib_files"
+    collection: "mib_files"

profiles/default.yaml

@@ -0,0 +1,37 @@
+# Copyright 2021 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+profiles:
+  basev1:
+    frequency: 10
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - '1.3.6.1.2.1.2.*'
+  basev1l2:
+    frequency: 20
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - ['IF-MIB','ifHCInOctets']
+      - ['IF-MIB','ifHCOutOctets']
+      - ['IF-MIB','ifInErrors']
+      - ['IF-MIB','ifOutErrors']
+      - ['IF-MIB','ifInDiscards']
+      - ['IF-MIB','ifOutDiscards']

splunk_connect_for_snmp_mib_server/mib_server.py

@@ -13,12 +13,12 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #   ########################################################################
-
 import logging
 
 from flask import Flask, request
 from flask_autoindex import AutoIndex
 
+from splunk_connect_for_snmp_mib_server.profiles import merge_profiles
 from splunk_connect_for_snmp_mib_server.translator import Translator
 
 logger = logging.getLogger(__name__)
@@ -39,6 +39,7 @@ def __init__(self, args, server_config):
     def build_flask_app(self):
         app = Flask(__name__)
         mibs_path = self._server_config["snmp"]["mibs"]["mibs_path"]
+        profiles_path = self._server_config["snmp"]["mibs"]["profiles_path"]
         files_index = AutoIndex(app, mibs_path, add_url_rules=False)
 
         @app.route("/")
@@ -68,6 +69,10 @@ def translator():
                 result = self._translator.format_text_event(var_binds)
             return result
 
+        @app.route("/profiles", methods=["GET"])
+        def get_profiles():
+            return merge_profiles(profiles_path, "profiles")
+
         return app
 
     def run_mib_server(self):

splunk_connect_for_snmp_mib_server/profiles.py

@@ -0,0 +1,40 @@
+#   ########################################################################
+#   Copyright 2021 Splunk Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#   ########################################################################
+import logging
+import os
+
+import yaml
+from yaml.parser import ParserError
+
+logger = logging.getLogger(__name__)
+
+
+def merge_profiles(directory, root_name):
+    result = {}
+    merged_profiles = {}
+    for root, directories, files in os.walk(directory, topdown=False):
+        for name in sorted(files):
+            with open(os.path.join(root, name), "r") as stream:
+                try:
+                    data = yaml.safe_load(stream)
+                    merged_profiles.update(data[root_name])
+                except ParserError as pe:
+                    logger.warning(
+                        f"Error while parsing file {os.path.join(root, name)} : {pe}"
+                    )
+
+    result[root_name] = merged_profiles
+    return result

tests/local-config.yaml

@@ -0,0 +1,31 @@
+#
+# Copyright 2021 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+snmp:
+  mibs:
+    dir: "mibs/pysnmp"
+    load_list: "lookups/mibs_list.csv"
+    mibs_path: "mibs"
+    profiles_path: "profiles"
+mongo:
+  oid:
+    database: "mib_server"
+    collection: "oids"
+  mib:
+    database: "files"
+    collection: "mib_files"
+  profile:
+    database: "profiles"
+    collection: "profiles"

tests/profiles/malformed_profile/default.yaml

@@ -0,0 +1,37 @@
+# Copyright 2021 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+profiles:
+  basev1:
+    frequency: 10
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - '1.3.6.1.2.1.2.*'
+  basev1l2:
+      frequency: 20
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - ['IF-MIB','ifHCInOctets']
+      - ['IF-MIB','ifHCOutOctets']
+      - ['IF-MIB','ifInErrors']
+      - ['IF-MIB','ifOutErrors']
+      - ['IF-MIB','ifInDiscards']
+      - ['IF-MIB','ifOutDiscards']

tests/profiles/malformed_profile/local.yaml

@@ -0,0 +1,42 @@
+# Copyright 2021 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+profiles:
+  basev2:
+    patterns:
+      - '*MY_DEFAULT_DEVICE_3*'
+    frequency: 60
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - '1.3.6.1.2.1.2.*'
+  basev2l2:
+    patterns:
+      - '*MY_DEFAULT_DEVICE_3*'
+      - '*MY_DEFAULT_DEVICE_NAME_3*'
+    frequency: 120
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - ['IF-MIB','ifHCInOctets']
+      - ['IF-MIB','ifHCOutOctets']
+      - ['IF-MIB','ifInErrors']
+      - ['IF-MIB','ifOutErrors']
+      - ['IF-MIB','ifInDiscards']
+      - ['IF-MIB','ifOutDiscards']

tests/profiles/one_profile/default.yaml

@@ -0,0 +1,37 @@
+# Copyright 2021 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+profiles:
+  basev1:
+    frequency: 10
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - '1.3.6.1.2.1.2.*'
+  basev1l2:
+    frequency: 20
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - ['IF-MIB','ifHCInOctets']
+      - ['IF-MIB','ifHCOutOctets']
+      - ['IF-MIB','ifInErrors']
+      - ['IF-MIB','ifOutErrors']
+      - ['IF-MIB','ifInDiscards']
+      - ['IF-MIB','ifOutDiscards']

tests/profiles/same_name_profiles/default.yaml

@@ -0,0 +1,37 @@
+# Copyright 2021 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+profiles:
+  basev1:
+    frequency: 10
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - '1.3.6.1.2.1.2.*'
+  basev1l2:
+    frequency: 20
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - ['IF-MIB','ifHCInOctets']
+      - ['IF-MIB','ifHCOutOctets']
+      - ['IF-MIB','ifInErrors']
+      - ['IF-MIB','ifOutErrors']
+      - ['IF-MIB','ifInDiscards']
+      - ['IF-MIB','ifOutDiscards']

tests/profiles/same_name_profiles/local.yaml

@@ -0,0 +1,42 @@
+# Copyright 2021 Splunk Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+profiles:
+  basev1:
+    patterns:
+      - '*MY_DEFAULT_DEVICE_3*'
+    frequency: 60
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - '1.3.6.1.2.1.2.*'
+  basev1l2:
+    patterns:
+      - '*MY_DEFAULT_DEVICE_3*'
+      - '*MY_DEFAULT_DEVICE_NAME_3*'
+    frequency: 120
+    varBinds:
+      # Syntax: [ "MIB-Files", "MIB object name" "MIB index number"]
+      - ['SNMPv2-MIB', 'sysDescr']
+      - ['SNMPv2-MIB', 'sysUpTime',0]
+      - ['SNMPv2-MIB', 'sysName']
+      - ['IF-MIB','ifHCInOctets']
+      - ['IF-MIB','ifHCOutOctets']
+      - ['IF-MIB','ifInErrors']
+      - ['IF-MIB','ifOutErrors']
+      - ['IF-MIB','ifInDiscards']
+      - ['IF-MIB','ifOutDiscards']