[RFC] scripts: ci: check_compliance: Add python lint/format check

.github/workflows/compliance.yml

@@ -38,7 +38,7 @@ jobs:
       run: |
         pip3 install setuptools
         pip3 install wheel
-        pip3 install python-magic lxml junitparser gitlint pylint pykwalify yamllint clang-format unidiff sphinx-lint
+        pip3 install python-magic lxml junitparser gitlint pylint pykwalify yamllint clang-format unidiff sphinx-lint ruff
         pip3 install west
 
     - name: west setup

.gitignore

@@ -102,6 +102,7 @@ MaintainersFormat.txt
 ModulesMaintainers.txt
 Nits.txt
 Pylint.txt
+Ruff.txt
 SphinxLint.txt
 TextEncoding.txt
 YAMLLint.txt

.ruff.toml

@@ -0,0 +1,30 @@
+# Copyright (c) 2024 Basalte bv
+# SPDX-License-Identifier: Apache-2.0
+
+extend = ".ruff-excludes.toml"
+
+line-length = 100
+target-version = "py310"
+
+[lint]
+select = [
+    # zephyr-keep-sorted-start
+    "B",      # flake8-bugbear
+    "E",      # pycodestyle
+    "F",      # pyflakes
+    "I",      # isort
+    "SIM",    # flake8-simplify
+    "UP",     # pyupgrade
+    "W",      # pycodestyle warnings
+    # zephyr-keep-sorted-stop
+]
+
+ignore = [
+    # zephyr-keep-sorted-start
+    "UP027",  # deprecated pyupgrade rule
+    # zephyr-keep-sorted-stop
+]
+
+[format]
+quote-style = "preserve"
+line-ending = "lf"

scripts/ci/check_compliance.py

@@ -81,20 +81,25 @@ def get_files(filter=None, paths=None):
     return files
 
 class FmtdFailure(Failure):
-
-    def __init__(self, severity, title, file, line=None, col=None, desc=""):
+    def __init__(
+        self, severity, title, file, line=None, col=None, desc="", end_line=None, end_col=None
+    ):
         self.severity = severity
         self.title = title
         self.file = file
         self.line = line
         self.col = col
+        self.end_line = end_line
+        self.end_col = end_col
         self.desc = desc
         description = f':{desc}' if desc else ''
         msg_body = desc or title
 
         txt = f'\n{title}{description}\nFile:{file}' + \
               (f'\nLine:{line}' if line else '') + \
-              (f'\nColumn:{col}' if col else '')
+              (f'\nColumn:{col}' if col else '') + \
+              (f'\nEndLine:{end_line}' if end_line else '') + \
+              (f'\nEndColumn:{end_col}' if end_col else '')
         msg = f'{file}' + (f':{line}' if line else '') + f' {msg_body}'
         typ = severity.lower()
 
@@ -171,13 +176,15 @@ def failure(self, text, msg=None, type_="failure"):
         fail = Failure(msg or f'{type(self).name} issues', type_)
         self._result(fail, text)
 
-    def fmtd_failure(self, severity, title, file, line=None, col=None, desc=""):
+    def fmtd_failure(
+        self, severity, title, file, line=None, col=None, desc="", end_line=None, end_col=None
+    ):
         """
         Signals that the test failed, and store the information in a formatted
         standardized manner. Can be called many times within the same test to
         report multiple failures.
         """
-        fail = FmtdFailure(severity, title, file, line, col, desc)
+        fail = FmtdFailure(severity, title, file, line, col, desc, end_line, end_col)
         self._result(fail, fail.text)
         self.fmtd_failures.append(fail)
 
@@ -1611,6 +1618,54 @@ def run(self):
                 self.check_file(file, fp)
 
 
+class Ruff(ComplianceTest):
+    """
+    Ruff
+    """
+    name = "Ruff"
+    doc = "Check python files with ruff."
+    path_hint = "<git-top>"
+
+    def run(self):
+        for file in get_files(filter="d"):
+            if not file.endswith(".py"):
+                continue
+
+            try:
+                subprocess.run(
+                    f"ruff check --force-exclude --output-format=json {file}",
+                    check=True,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.STDOUT,
+                    shell=True,
+                    cwd=GIT_TOP,
+                )
+            except subprocess.CalledProcessError as ex:
+                output = ex.output.decode("utf-8")
+                messages = json.loads(output)
+                for m in messages:
+                    self.fmtd_failure(
+                        "error",
+                        f'Python lint error ({m.get("code")}) see {m.get("url")}',
+                        file,
+                        line=m.get("location", {}).get("row"),
+                        col=m.get("location", {}).get("column"),
+                        end_line=m.get("end_location", {}).get("row"),
+                        end_col=m.get("end_location", {}).get("column"),
+                        desc=m.get("message"),
+                    )
+            try:
+                subprocess.run(
+                    f"ruff format --force-exclude --diff {file}",
+                    check=True,
+                    shell=True,
+                    cwd=GIT_TOP,
+                )
+            except subprocess.CalledProcessError:
+                desc = f"Run 'ruff format {file}'"
+                self.fmtd_failure("error", "Python format error", file, desc=desc)
+
+
 class TextEncoding(ComplianceTest):
     """
     Check that any text file is encoded in ascii or utf-8.
@@ -1675,6 +1730,8 @@ def annotate(res):
     notice = f'::{res.severity} file={res.file}' + \
              (f',line={res.line}' if res.line else '') + \
              (f',col={res.col}' if res.col else '') + \
+             (f',endLine={res.end_line}' if res.end_line else '') + \
+             (f',endColumn={res.end_col}' if res.end_col else '') + \
              f',title={res.title}::{msg}'
     print(notice)
 

scripts/requirements-compliance.txt

@@ -10,3 +10,4 @@ pylint>=3
 unidiff
 yamllint
 sphinx-lint
+ruff

scripts/ruff/gen_format_exclude.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2024 Basalte bv
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+
+# A very simple script to convert ruff format output to toml
+# For example:
+# ruff format --check | ./scripts/ruff/gen_format_exclude.py >> .ruff-excludes.toml
+
+if __name__ == "__main__":
+    sys.stdout.write("[format]\n")
+    sys.stdout.write("exclude = [\n")
+    for line in sys.stdin:
+        if line.startswith("Would reformat: "):
+            sys.stdout.write(f'    "./{line[16:-1]}",\n')
+    sys.stdout.write("]\n")

scripts/ruff/gen_lint_exclude.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2024 Basalte bv
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import sys
+from pathlib import Path, PurePosixPath
+
+# A very simple script to convert ruff lint output from json to toml
+# For example:
+# ruff check --output-format=json | ./scripts/ruff/gen_lint_exclude.py >> .ruff-excludes.toml
+
+
+class RuffRule:
+    def __init__(self, code: str, url: str) -> None:
+        self.code = code
+        self.url = url
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, type(self)):
+            return NotImplemented
+        return self.code.__eq__(other.code)
+
+    def __hash__(self) -> int:
+        return self.code.__hash__()
+
+
+if __name__ == "__main__":
+    violations = json.load(sys.stdin)
+    sys.stdout.write("[lint.per-file-ignores]\n")
+
+    rules: dict[str, set[RuffRule]] = {}
+    for v in violations:
+        rules.setdefault(v["filename"], set()).add(RuffRule(v["code"], v["url"]))
+
+    for f, rs in rules.items():
+        path = PurePosixPath(f)
+        sys.stdout.write(f'"./{path.relative_to(Path.cwd())}" = [\n')
+        for r in sorted(rs, key=lambda x: x.code):
+            sys.stdout.write(f'    "{r.code}",\t# {r.url}\n'.expandtabs())
+        sys.stdout.write("]\n")