Merge pull request #64 from fosslight/develop

Add yaml parsing code

src/fosslight_util/oss_item.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from fosslight_util.constant import LOGGER_NAME
+
+_logger = logging.getLogger(LOGGER_NAME)
+
+
+class OssItem:
+    name = "-"
+    version = ""
+    licenses = []
+    source = ""
+    files = []
+    copyright = ""
+    comment = ""
+    exclude = ""
+    homepage = ""
+    relative_path = ""
+
+    def __init__(self, value):
+        self.name = "-"
+        self.version = ""
+        self.licenses = []
+        self.source = ""
+        self.files = []
+        self.copyright = ""
+        self.comment = ""
+        self.exclude = ""
+        self.homepage = ""
+        self.relative_path = value
+
+    def __del__(self):
+        pass
+
+    def set_homepage(self, value):
+        self.homepage = value
+
+    def set_comment(self, value):
+        self.comment = value
+
+    def set_copyright(self, value):
+        if value != "":
+            if isinstance(value, list):
+                value = "\n".join(value)
+            value = value.strip()
+        self.copyright = value
+
+    def set_exclude(self, value):
+        if value:
+            self.exclude = "Exclude"
+        else:
+            self.exclude = ""
+
+    def set_name(self, value):
+        self.name = value
+
+    def set_version(self, value):
+        self.version = str(value)
+
+    def set_licenses(self, value):
+        if not isinstance(value, list):
+            value = value.split(",")
+        self.licenses.extend(value)
+        self.licenses = [item.strip() for item in self.licenses]
+        self.licenses = list(set(self.licenses))
+
+    def set_files(self, value):
+        if isinstance(value, list):
+            self.files.extend(value)
+        else:
+            self.files.append(value)
+        self.files = list(set(self.files))
+
+    def set_source(self, value):
+        self.source = value
+
+    def get_print_array(self):
+        items = []
+        if len(self.files) == 0:
+            self.files.append("")
+        if len(self.licenses) == 0:
+            self.licenses.append("")
+        for file in self.files:
+            lic = ",".join(self.licenses)
+            if self.relative_path != "" and not str(self.relative_path).endswith("/"):
+                self.relative_path += "/"
+            items.append([self.relative_path + file, self.name, self.version, lic, self.source, self.homepage,
+                          self.copyright, "", self.exclude, self.comment])
+        return items
+
+    def get_print_json(self):
+        json_item = {}
+        json_item["name"] = self.name
+
+        if self.version != "":
+            json_item["version"] = self.version
+        if self.source != "":
+            json_item["source"] = self.source
+        if self.copyright != "":
+            json_item["copyright"] = self.copyright
+        if self.exclude != "":
+            json_item["exclude"] = self.exclude
+        if self.comment != "":
+            json_item["comment"] = self.comment
+        if self.homepage != "":
+            json_item["homepage"] = self.homepage
+
+        if len(self.files) > 0:
+            json_item["file"] = self.files
+        if len(self.licenses) > 0:
+            json_item["license"] = self.licenses
+
+        return json_item
+
+
+def invalid(cmd):
+    _logger.info('[{}] is invalid'.format(cmd))

src/fosslight_util/parsing_yaml.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2021 LG Electronics Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+import yaml
+import logging
+import codecs
+import os
+import sys
+from .constant import LOGGER_NAME
+from .oss_item import OssItem, invalid
+
+_logger = logging.getLogger(LOGGER_NAME)
+
+
+def parsing_yml(yaml_file, base_path):
+    oss_list = []
+    license_list = []
+    idx = 1
+    try:
+        path_of_yml = os.path.normpath(os.path.dirname(yaml_file))
+        base_path = os.path.normpath(base_path)
+        relative_path = ""
+
+        if path_of_yml != base_path:
+            relative_path = os.path.relpath(path_of_yml, base_path)
+
+        doc = yaml.safe_load(codecs.open(yaml_file, "r", "utf-8"))
+        for root_element in doc:
+            oss_items = doc[root_element]
+            if oss_items:
+                for oss in oss_items:
+                    item = OssItem(relative_path)
+                    for key, value in oss.items():
+                        set_value_switch(item, key, value)
+                    items_to_print = item.get_print_array()
+                    for item_to_print in items_to_print:
+                        item_to_print.insert(0, str(idx))
+                    oss_list.extend(items_to_print)
+                    license_list.extend(item.licenses)
+                    idx += 1
+    except yaml.YAMLError as ex:
+        _logger.error('Parsing yaml :' + str(ex))
+    license_list = set(license_list)
+    return oss_list, license_list
+
+
+def find_all_oss_pkg_files(path_to_find, file_to_find):
+    oss_pkg_files = []
+
+    if not os.path.isdir(path_to_find):
+        _logger.error("Can't find a path :" + path_to_find)
+        sys.exit(os.EX_DATAERR)
+
+    try:
+        # oss_pkg_files.extend(glob.glob(path_to_find + '/**/'+file, recursive=True)) #glib is too slow
+        for p, d, f in os.walk(path_to_find):
+            for file in f:
+                file_name = file.lower()
+                if file_name in file_to_find or (
+                        (file_name.endswith("yml") or file_name.endswith("yaml"))
+                        and file_name.startswith("oss-pkg-info")):
+                    oss_pkg_files.append(os.path.join(p, file))
+    except Exception as ex:
+        _logger.debug("Error, find all oss-pkg-info files:" + str(ex))
+
+    return oss_pkg_files
+
+
+def set_value_switch(oss, key, value):
+    switcher = {
+        'name': oss.set_name,
+        'version': oss.set_version,
+        'source': oss.set_source,
+        'license': oss.set_licenses,
+        'file': oss.set_files,
+        'copyright': oss.set_copyright,
+        'exclude': oss.set_exclude,
+        'comment': oss.set_comment,
+        'homepage': oss.set_homepage
+    }
+    func = switcher.get(key, lambda key: invalid(key))
+    func(value)