Generate data classes

Tweak code gen settings

Use frozen model

Parsing works

Delete dead code

All other tests to continue even when one of the sibling jobs fail

Use older syntax for Py3.9

Cleaning up type errors

WIP:  De-uglifying

Fix tests to minimize false positives

Simplify element extraction

With data digestion

Fixup test results

Improve enums

Parse refactoring done

Lint fixes

More fixes

Cleanup and auto-scale the values

Note that we are taking perfomance hit by doing some extra
pre-processing for the convinience to the user

Shuffle things for clarity

Small fixes

.github/workflows/python.yml

@@ -12,6 +12,7 @@ jobs:
  strategy:
  matrix:
  python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+ continue-on-error: true
  steps:
  - uses: actions/checkout@v4
  - name: Set up Python ${{ matrix.python-version }}

.pre-commit-config.yaml

@@ -32,7 +32,9 @@ repos:
  hooks:
  - id: ruff
  args: [--fix, --exit-non-zero-on-fix, --show-fixes]
+ exclude: src/greenbutton_objects/data
  - id: ruff-format
+ exclude: src/greenbutton_objects/data
 
  - repo: https://github.com/lsst-ts/pre-commit-xmllint
  rev: v1.0.0

.xsdata.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Config xmlns="http://pypi.org/project/xsdata" version="24.7">
+ <Output maxLineLength="110" subscriptableTypes="false" unionType="false">
+ <Package>greenbutton_objects.data</Package>
+ <Format repr="true" eq="true" order="true" unsafeHash="true" frozen="false" slots="false" kwOnly="false">dataclasses</Format>
+ <Structure>filenames</Structure>
+ <DocstringStyle>reStructuredText</DocstringStyle>
+ <RelativeImports>false</RelativeImports>
+ <CompoundFields defaultName="choice" useSubstitutionGroups="false" forceDefaultName="false" maxNameParts="3">false</CompoundFields>
+ <WrapperFields>false</WrapperFields>
+ <PostponedAnnotations>false</PostponedAnnotations>
+ <UnnestClasses>false</UnnestClasses>
+ <IgnorePatterns>false</IgnorePatterns>
+ <IncludeHeader>true</IncludeHeader>
+ </Output>
+ <Conventions>
+ <ClassName case="pascalCase" safePrefix="type"/>
+ <FieldName case="snakeCase" safePrefix="value"/>
+ <ConstantName case="screamingSnakeCase" safePrefix="value"/>
+ <ModuleName case="snakeCase" safePrefix="mod"/>
+ <PackageName case="snakeCase" safePrefix="pkg"/>
+ </Conventions>
+ <Substitutions>
+ <Substitution type="package" search="http://www.w3.org/2001/XMLSchema" replace="xs"/>
+ <Substitution type="package" search="http://www.w3.org/XML/1998/namespace" replace="xml"/>
+ <Substitution type="package" search="http://www.w3.org/2001/XMLSchema-instance" replace="xsi"/>
+ <Substitution type="package" search="http://www.w3.org/1998/Math/MathML" replace="mathml3"/>
+ <Substitution type="package" search="http://www.w3.org/1999/xlink" replace="xlink"/>
+ <Substitution type="package" search="http://www.w3.org/1999/xhtml" replace="xhtml"/>
+ <Substitution type="package" search="http://schemas.xmlsoap.org/wsdl/soap/" replace="soap"/>
+ <Substitution type="package" search="http://schemas.xmlsoap.org/wsdl/soap12/" replace="soap12"/>
+ <Substitution type="package" search="http://schemas.xmlsoap.org/soap/envelope/" replace="soapenv"/>
+ <Substitution type="class" search="(.*)Class$" replace="\1Type"/>
+ </Substitutions>
+ <Extensions/>
+</Config>

Makefile

@@ -1,3 +1,6 @@
+CONFIG := $(abspath .xsdata.xml)
+
+
 black:
  black --check .
 
@@ -10,7 +13,11 @@ pydocstyle:
 lint: black isort pydocstyle
 
 mypy:
- mypy .
+ mypy src
+
+gen_classes:
+ cd src && xsdata generate -c $(CONFIG) -p greenbutton_objects.data.espi https://www.naesb.org/espi.xsd
+ cd src && xsdata generate -c $(CONFIG) -p greenbutton_objects.data.atom https://greenbuttondata.org/xsd/3_3/atom.xsd
 
 gen_examples:
  python .\tests\test_rules_engine\generate_example_data.py

pyproject.toml

@@ -24,7 +24,10 @@ classifiers = [
  "Programming Language :: Python",
  "Programming Language :: Python :: 3"
 ]
-keywords = ["feed", "reader", "tutorial"]
+dependencies = [
+ "xsdata"
+]
+keywords = ["feed", "reader"]
 requires-python = ">=3.9"
 
 [project.optional-dependencies]
@@ -37,6 +40,7 @@ dev = [
  "pytest==8.3.2",
  "pytest-cov==5.0.0",
  "pytest_sugar==1.0.0",
+ "xsdata[cli]==24.7",
  "ruff==0.5.5"
 ]
 

src/greenbutton_objects/atom/href_tree.py

@@ -0,0 +1,97 @@
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+from greenbutton_objects.data.atom import ContentType, EntryType, Feed
+
+
+@dataclass
+class HRefTreeNode:
+ uri: str
+ parent: Optional[str] = None
+ contentType: type = type(None)
+ content: List[ContentType] = field(default_factory=list)
+ children: List[str] = field(default_factory=list)
+ related: List[str] = field(default_factory=list)
+ title: str = ""
+
+
+class HRefForest:
+ def __init__(self) -> None:
+ self.forest: Dict[str, HRefTreeNode] = {}
+
+ def __ensure_container(self, uri: str) -> None:
+ if uri not in self.forest:
+ self.forest[uri] = HRefTreeNode(uri)
+
+ def __link_parents(self) -> "HRefForest":
+ for node in self.forest.values():
+ if node.parent:
+ parent_node = self.forest.get(node.parent)
+ if parent_node:
+ parent_node.children.append(node.uri)
+ return self
+
+ def __ensure_containers(self) -> "HRefForest":
+ for key in list(self.forest.keys()):
+ node = self.forest[key]
+ if node.parent:
+ self.__ensure_container(node.parent)
+ for related_uri in node.related:
+ self.__ensure_container(related_uri)
+ return self
+
+ def __add_nodes(self, feed: Feed) -> "HRefForest":
+ def entry_content_type(entry: EntryType) -> type:
+ if entry.content and entry.content[0].content:
+ content_type = type(entry.content[0].content[0])
+ else:
+ content_type = type(None)
+ return content_type
+
+ for entry in feed.entry:
+ related = []
+ parent = None
+ uri = ""
+
+ content_type = entry_content_type(entry)
+
+ for link in entry.link:
+ # Skip links without URIs
+ if not link.href:
+ continue
+ if link.rel == "self":
+ uri = link.href
+ elif link.rel == "related":
+ related.append(link.href)
+ elif link.rel == "up":
+ parent = link.href
+
+ title = self.get_entry_title(entry)
+
+ self.forest[uri] = HRefTreeNode(
+ uri=uri,
+ title=title,
+ parent=parent,
+ related=related,
+ contentType=content_type,
+ content=entry.content,
+ )
+
+ return self
+
+ @staticmethod
+ def get_entry_title(entry: EntryType) -> str:
+ if entry.title:
+ title_parts = []
+ for text in entry.title:
+ if len(text.content) > 0:
+ title_parts.append(text.content[0])
+ return "".join(title_parts) # type: ignore
+ else:
+ return ""
+
+ def build(self, feed: Feed) -> "HRefForest":
+ return self.__add_nodes(feed).__ensure_containers().__link_parents()
+
+ def root_nodes(self) -> List[str]:
+ return [node.uri for node in self.forest.values() if node.parent is None]

src/greenbutton_objects/atom/object_tree.py

@@ -0,0 +1,104 @@
+from dataclasses import dataclass, field
+from itertools import chain
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Union
+
+from greenbutton_objects.atom.href_tree import HRefForest, HRefTreeNode
+from greenbutton_objects.util import get_first
+
+
+@dataclass
+class EntryNode:
+ title: str
+ uri: str
+ content: Sequence[object]
+
+ parent: Optional["EntryNode"] = None
+ content_type: type = type(None)
+ children_type: type = type(None)
+ children: List["EntryNode"] = field(default_factory=list)
+ # TODO: Should related be a list or dict keyed by type?
+ related: List["EntryNode"] = field(default_factory=list)
+
+ def infer_children_type(self) -> None:
+ # We are making a big assumption here that all children are of the same type.
+ # This is to make things simpler for the consumer who's using the library
+ if self.children and self.children[0].content:
+ content_ = self.children[0].content[0]
+ if content_.content: # type: ignore
+ self.children_type = content_.content[0].__class__ # type: ignore
+ else:
+ self.children_type = type(None)
+
+ def first_content(self) -> Any:
+ first_node = get_first(self.content)
+ first_node_content = first_node.content # type: ignore
+ return get_first(first_node_content)
+
+ def get_related_of_type(self, elements_type: type) -> Iterable["EntryNode"]:
+ containers = [obj for obj in self.related if obj.children_type is elements_type]
+ if containers:
+ elements: Iterable[EntryNode] = chain.from_iterable(
+ container.children for container in containers
+ )
+ else:
+ elements = [obj for obj in self.related if obj.content_type is elements_type]
+ return elements
+
+ def safe_get_content(self, content_type: type) -> Union[Any, None]:
+ obj = get_first(self.get_related_of_type(content_type))
+ return obj.first_content() if obj else None
+
+
+class ObjectForest:
+ def __init__(
+ self,
+ ) -> None:
+ self.__roots: List[EntryNode] = []
+
+ def build(self, href_forest: HRefForest) -> "ObjectForest":
+ node_cache: Dict[str, EntryNode] = {}
+
+ def add_node(href_node: HRefTreeNode) -> EntryNode:
+ entry_node = EntryNode(
+ title=href_node.title,
+ uri=href_node.uri,
+ content=href_node.content,
+ content_type=href_node.contentType,
+ )
+ node_cache[href_node.uri] = entry_node
+ return entry_node
+
+ def build_tree(node_uri: str) -> EntryNode:
+ if node_uri not in node_cache:
+ href_node = href_forest.forest[node_uri]
+ entry_node = add_node(href_node)
+
+ # Children
+ entry_node.children = [build_tree(child) for child in href_node.children]
+ for child in entry_node.children:
+ child.parent = entry_node
+ entry_node.infer_children_type()
+
+ # Relatives
+ entry_node.related = [build_tree(child) for child in href_node.related]
+
+ return node_cache[node_uri]
+
+ self.__roots = [build_tree(uri) for uri in href_forest.root_nodes()]
+
+ return self
+
+ @staticmethod
+ def get_elements_by_type(elements_type: type, source: list[EntryNode]) -> Iterable[EntryNode]:
+ containers = [obj for obj in source if obj.children_type is elements_type]
+ if containers:
+ elements: Iterable[EntryNode] = chain.from_iterable(
+ container.children for container in containers
+ )
+ else:
+ elements = [obj for obj in source if obj.content_type is elements_type]
+ return elements
+
+ @property
+ def roots(self) -> List[EntryNode]:
+ return self.__roots

src/greenbutton_objects/data/__init__.py

@@ -0,0 +1 @@
+# nothing here

src/greenbutton_objects/data/atom/__init__.py

@@ -0,0 +1,47 @@
+"""This file was generated by xsdata, v24.7, on 2024-08-08 14:41:00
+
+Generator: DataclassGenerator
+See: https://xsdata.readthedocs.io/
+"""
+
+from greenbutton_objects.data.atom.atom import (
+ CategoryType,
+ ContentType,
+ DateTimeType,
+ Entry,
+ EntryType,
+ Feed,
+ FeedType,
+ GeneratorType,
+ IconType,
+ IdType,
+ LinkType,
+ LogoType,
+ PersonType,
+ SourceType,
+ TextType,
+ TextTypeType,
+ UriType,
+)
+from greenbutton_objects.data.atom.xml import LangValue
+
+__all__ = [
+ "CategoryType",
+ "ContentType",
+ "DateTimeType",
+ "Entry",
+ "EntryType",
+ "Feed",
+ "FeedType",
+ "GeneratorType",
+ "IconType",
+ "IdType",
+ "LinkType",
+ "LogoType",
+ "PersonType",
+ "SourceType",
+ "TextType",
+ "TextTypeType",
+ "UriType",
+ "LangValue",
+]