From 5241995d10ae89b8a44d8fbaa6f70929dd04233f Mon Sep 17 00:00:00 2001 From: Adrian Clay Lake Date: Fri, 6 Dec 2024 23:29:03 +0100 Subject: [PATCH] refactor: small improvements for push --- .gitignore | 2 + yaml_checker/README.md | 108 ++++++++++++++++++ yaml_checker/demo/basic.yaml | 13 --- yaml_checker/demo/factory.yaml | 9 -- yaml_checker/demo/slice.yaml | 28 +++-- yaml_checker/requirements.txt | 4 +- yaml_checker/sample.yaml | 8 -- yaml_checker/setup.py | 18 ++- yaml_checker/yaml_checker/__main__.py | 7 +- yaml_checker/yaml_checker/config/__init__.py | 4 +- yaml_checker/yaml_checker/config/base.py | 11 +- yaml_checker/yaml_checker/config/chisel.py | 60 +++++----- .../yaml_checker/config/oci_factory.py | 4 +- 13 files changed, 186 insertions(+), 90 deletions(-) delete mode 100644 yaml_checker/demo/basic.yaml delete mode 100644 yaml_checker/demo/factory.yaml delete mode 100644 yaml_checker/sample.yaml diff --git a/.gitignore b/.gitignore index 9b71fe7..47f0945 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .vscode __pycache__ *.rock +.venv/ +yaml_checker/yaml_checker.egg-info diff --git a/yaml_checker/README.md b/yaml_checker/README.md index aab047e..b84e1d4 100644 --- a/yaml_checker/README.md +++ b/yaml_checker/README.md @@ -1,3 +1,111 @@ # YAML Checker +An internal CLI util for formatting and validating YAML files. This project +relies on Pydantic and Ruamel libraries. +**Installation** +```bash +pip install -e yaml_checker +``` + +**Usage** +``` +usage: yaml_checker [-h] [-v] [-w] [--config CONFIG] [files ...] + +positional arguments: + files Additional files to process (optional). + +options: + -h, --help show this help message and exit + -v, --verbose Enable verbose output. + -w, --write Write yaml output to disk. + --config CONFIG CheckYAML subclass to load +``` + +**Example** + +```bash +# Lets cat a demonstration file for comparison. +$ cat yaml_checker/demo/slice.yaml +# yaml_checker --config=Chisel demo/slice.yaml + +package: grep + +essential: + - grep_copyright + +# hello: world + +slices: + bins: + essential: + - libpcre2-8-0_libs # tests + + # another test + - libc6_libs + contents: + /usr/bin/grep: + + deprecated: + # These are shell scripts requiring a symlink from /usr/bin/dash to + # /usr/bin/sh. + # See: https://manpages.ubuntu.com/manpages/noble/en/man1/grep.1.html + essential: + - dash_bins + - grep_bins + contents: + # we ned this leading comment + /usr/bin/rgrep: # this should be last + + /usr/bin/fgrep: + + # careful with this path ... + /usr/bin/egrep: # it is my favorite + copyright: + contents: + /usr/share/doc/grep/copyright: +# Note: Missing new line at EOF + +# Now we can run the yaml_checker to format the same file. +# Note how comments are preserved during sorting of lists and +# dict type objects. If you want to test the validator, +# uncomment the hello field. +$ yaml_checker --config=Chisel yaml_checker/demo/slice.yaml +# yaml_checker --config=Chisel demo/slice.yaml + +package: grep + +essential: + - grep_copyright + +# hello: world + +slices: + bins: + essential: + - libc6_libs + - libpcre2-8-0_libs # tests + + # another test + contents: + /usr/bin/grep: + + deprecated: + # These are shell scripts requiring a symlink from /usr/bin/dash to + # /usr/bin/sh. + # See: https://manpages.ubuntu.com/manpages/noble/en/man1/grep.1.html + essential: + - dash_bins + - grep_bins + contents: + # we ned this leading comment + + # careful with this path ... + /usr/bin/egrep: # it is my favorite + /usr/bin/fgrep: + /usr/bin/rgrep: # this should be last + copyright: + contents: + /usr/share/doc/grep/copyright: + +``` diff --git a/yaml_checker/demo/basic.yaml b/yaml_checker/demo/basic.yaml deleted file mode 100644 index 1d40b77..0000000 --- a/yaml_checker/demo/basic.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# python3 -m yaml_checker demo/basic.yaml - -hello: - - world: - - 1 - - 2 # inline comment - - 3 - - # a comment heading a field - foo: - bar - baz: "This is a really long string in this document. By the end I hope to reach over 150 characters in total. But I need a few more words to reach that goal. Done" \ No newline at end of file diff --git a/yaml_checker/demo/factory.yaml b/yaml_checker/demo/factory.yaml deleted file mode 100644 index 6ac228f..0000000 --- a/yaml_checker/demo/factory.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# python3 -m yaml_checker --config=OCIFactory demo/factory.yaml - -hello: - world: "42" - foo: - - "bar" - - "baz" - - "etc" - tricky_string: "it's a trap!" \ No newline at end of file diff --git a/yaml_checker/demo/slice.yaml b/yaml_checker/demo/slice.yaml index 51a12a8..db26dce 100644 --- a/yaml_checker/demo/slice.yaml +++ b/yaml_checker/demo/slice.yaml @@ -1,33 +1,37 @@ -# python3 -m yaml_checker --config=Chisel demo/factory.yaml +# yaml_checker --config=Chisel demo/slice.yaml package: grep essential: - - grep_copyright + - grep_copyright # hello: world slices: bins: essential: - - libpcre2-8-0_libs # tests - - libc6_libs + - libpcre2-8-0_libs # tests + + # another test + - libc6_libs contents: - /usr/bin/grep: + /usr/bin/grep: deprecated: # These are shell scripts requiring a symlink from /usr/bin/dash to # /usr/bin/sh. # See: https://manpages.ubuntu.com/manpages/noble/en/man1/grep.1.html - essential: ["dash_bins", "grep_bins"] + essential: + - dash_bins + - grep_bins contents: - /usr/bin/rgrep: + # we ned this leading comment + /usr/bin/rgrep: # this should be last - # tests /usr/bin/fgrep: - - /usr/bin/egrep: # tests1 + # careful with this path ... + /usr/bin/egrep: # it is my favorite copyright: - contents: - /usr/share/doc/grep/copyright: \ No newline at end of file + contents: + /usr/share/doc/grep/copyright: \ No newline at end of file diff --git a/yaml_checker/requirements.txt b/yaml_checker/requirements.txt index 7e6282c..ece35be 100644 --- a/yaml_checker/requirements.txt +++ b/yaml_checker/requirements.txt @@ -1,2 +1,2 @@ -ruyaml -pytest \ No newline at end of file +pydantic==2.8.2 +ruamel.yaml==0.18.6 \ No newline at end of file diff --git a/yaml_checker/sample.yaml b/yaml_checker/sample.yaml deleted file mode 100644 index 71ac108..0000000 --- a/yaml_checker/sample.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# comment - -test: - test2: "1" # another comment - test3: [a, b, c] - - # more comments - test4: [1, 2, 3] \ No newline at end of file diff --git a/yaml_checker/setup.py b/yaml_checker/setup.py index 0edaf93..a140857 100644 --- a/yaml_checker/setup.py +++ b/yaml_checker/setup.py @@ -1,17 +1,23 @@ from pathlib import Path -from setuptools import setup, find_packages +from setuptools import find_packages, setup -def read(filename): + +def read_text(filename): filepath = Path(__file__).parent / filename - file = open(filepath, "r") - return file.read() + return filepath.read_text() setup( name="yaml_checker", version="0.1.0", - long_description=read("README.md"), + long_description=read_text("README.md"), packages=find_packages(), - install_requires=read("requirements.txt"), + install_requires=read_text("requirements.txt"), + entry_points={ + "console_scripts": [ + "yaml_checker=yaml_checker.__main__:main", + "clayaml=yaml_checker.__main__:main", + ], + }, ) diff --git a/yaml_checker/yaml_checker/__main__.py b/yaml_checker/yaml_checker/__main__.py index 338e851..53e1618 100644 --- a/yaml_checker/yaml_checker/__main__.py +++ b/yaml_checker/yaml_checker/__main__.py @@ -1,9 +1,10 @@ -from .config.base import YAMLCheckConfigBase -from pathlib import Path +import argparse import logging +from pathlib import Path -import argparse +from .config.base import YAMLCheckConfigBase +# TODO: display all available configs in help parser = argparse.ArgumentParser() parser.add_argument( diff --git a/yaml_checker/yaml_checker/config/__init__.py b/yaml_checker/yaml_checker/config/__init__.py index 23a1fca..3724559 100644 --- a/yaml_checker/yaml_checker/config/__init__.py +++ b/yaml_checker/yaml_checker/config/__init__.py @@ -1,10 +1,10 @@ -from pathlib import Path from importlib import import_module +from pathlib import Path submodule_root = Path(__file__).parent package_name = __name__ - +# import all submodules so our configs registry is populated for submodule in submodule_root.glob("*.py"): submodule_name = submodule.stem diff --git a/yaml_checker/yaml_checker/config/base.py b/yaml_checker/yaml_checker/config/base.py index 6763946..b4dc75a 100644 --- a/yaml_checker/yaml_checker/config/base.py +++ b/yaml_checker/yaml_checker/config/base.py @@ -1,11 +1,12 @@ import fnmatch -from io import StringIO import logging +from io import StringIO from pathlib import Path -from pydantic import BaseModel -from ruyaml import YAML from typing import Any +from pydantic import BaseModel +from ruamel.yaml import YAML + class YAMLCheckConfigReg(type): def __init__(cls, *args, **kwargs): @@ -27,7 +28,7 @@ class Config: extra = "allow" class Config: - """ruyaml.YAML configuration set before loading.""" + """ruamel.yaml configuration set before loading.""" preserve_quotes = True width = 80 @@ -49,7 +50,7 @@ def __init__(self): if hasattr(self.yaml, attr): setattr(self.yaml, attr, attr_val) else: - raise AttributeError(f"Invalid ruyaml.YAML attribute: {attr}") + raise AttributeError(f"Invalid ruamel.yaml attribute: {attr}") def load(self, yaml_str: str): """Load YAML data from string""" diff --git a/yaml_checker/yaml_checker/config/chisel.py b/yaml_checker/yaml_checker/config/chisel.py index 07a4991..0397861 100644 --- a/yaml_checker/yaml_checker/config/chisel.py +++ b/yaml_checker/yaml_checker/config/chisel.py @@ -1,14 +1,14 @@ -from ruyaml.scalarstring import PlainScalarString -from ruyaml.comments import CommentedMap +from typing import Any, Dict, List from pydantic import BaseModel, Field, RootModel -from typing import List, Dict, Any - +from ruamel.yaml.comments import CommentedMap +from ruamel.yaml.scalarstring import PlainScalarString from .base import YAMLCheckConfigBase class Slices(RootModel): + # TODO: expand slices model to validate individual slices root: Dict[str, Any] @@ -28,40 +28,40 @@ class Chisel(YAMLCheckConfigBase): } def sort_content(self, path, data): - # print(path, type(data)) - # print(dir(data)) - # print() - # return CommentedMap(sorted(data)) + """Sort dict and list objects.""" + + def prep_comment_content(value): + # remove whitespace and leading pound sign + value = value.strip() + value = value.strip("#") + return value if isinstance(data, dict): - # data.ordereddict() - print(path, type(data), str(data)) - print(data.ca.items) - # print(dir(data)) sorted_dict = CommentedMap() - for key, value in data.items(): - # print(key, "before", data.get_comment_inline(key)) - + for key in sorted(data.keys()): sorted_dict[key] = data[key] - return sorted_dict + if key in data.ca.items: + _, key_comments, eol_comment, _ = data.ca.items[key] - # sorted_items = sorted( - # data.items(), - # key=lambda item: item[0], # Sort by key - # ) + # Migrate comments to new sorted dictionary. This works for most + # but not all cases + if key_comments is not None: + if not isinstance(key_comments, list): + key_comments = [key_comments] - # sorted_settings = CommentedMap() - # for key, value in sorted_items: - # # Attach comments manually - # if isinstance(value, dict) and isinstance(value, CommentedMap): - # sorted_settings[key] = value - # else: - # sorted_settings[key] = value + for key_comment in key_comments: + content = prep_comment_content(key_comment.value) + sorted_dict.yaml_set_comment_before_after_key( + key, before=content, indent=key_comment.column + ) - # # print(dir(data)) + if eol_comment is not None: + # These should be sorted ok, no need for warning + content = prep_comment_content(eol_comment.value) + sorted_dict.yaml_add_eol_comment(content, key) - # return sorted_settings + return sorted_dict elif isinstance(data, list): data.sort() @@ -70,9 +70,11 @@ def sort_content(self, path, data): return data def no_quotes(self, path, data): + """Remove quotes form strings""" if isinstance(data, str): return PlainScalarString(data) return data + # validate documents with basic SDF model Model = SDF diff --git a/yaml_checker/yaml_checker/config/oci_factory.py b/yaml_checker/yaml_checker/config/oci_factory.py index fedcf68..f60270d 100644 --- a/yaml_checker/yaml_checker/config/oci_factory.py +++ b/yaml_checker/yaml_checker/config/oci_factory.py @@ -1,5 +1,7 @@ import logging -from ruyaml.scalarstring import SingleQuotedScalarString, DoubleQuotedScalarString + +from ruamel.yaml.scalarstring import (DoubleQuotedScalarString, + SingleQuotedScalarString) from .base import YAMLCheckConfigBase