Merge pull request #6 from linkml/multfile-process

Adding MultiFileTransformer

Makefile

@@ -44,6 +44,9 @@ all: gen-project gendoc
 %.yaml: gen-project
 deploy: all mkd-gh-deploy
 
+src/linkml_transformer/datamodel/transformer_model.py: src/linkml_transformer/datamodel/transformer_model.yaml
+	$(RUN) gen-pydantic $< > $@.tmp && mv $@.tmp $@
+
 # generates all project files
 gen-project: $(PYMODEL)
 	$(RUN) gen-project -d $(DEST) $(SOURCE_SCHEMA_PATH) && mv $(DEST)/*.py $(PYMODEL)

pyproject.toml

@@ -7,12 +7,12 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.8"
-linkml-runtime = "^1.4.5"
+linkml-runtime = ">=1.5.2"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.3.1"
 pytest-cov = "^4.0.0"
-linkml = "^1.3.7"
+linkml = ">=1.5.0"
 mkdocs-mermaid2-plugin = "^0.6.0"
 
 [tool.poetry.scripts]

src/linkml_transformer/cli/cli.py

@@ -11,9 +11,7 @@
 import yaml
 from linkml_runtime import SchemaView
 from linkml_runtime.dumpers import yaml_dumper
-from linkml_runtime.loaders import yaml_loader
 
-from linkml_transformer.datamodel.transformer_model import TransformationSpecification
 from linkml_transformer.schema_mapper.schema_mapper import SchemaMapper
 from linkml_transformer.transformer.object_transformer import ObjectTransformer
 
@@ -76,9 +74,7 @@ def map_data(
     logging.info(f"Transforming {input} conforming to {schema} using {transformer_specification}")
     tr = ObjectTransformer()
     tr.source_schemaview = SchemaView(schema)
-    tr.specification = yaml_loader.load(
-        transformer_specification, target_class=TransformationSpecification
-    )
+    tr.load_transformer_specification(transformer_specification)
     with open(input) as file:
         input_obj = yaml.safe_load(file)
     tr.index(input_obj, source_type)
@@ -111,12 +107,11 @@ def derive_schema(schema, transformer_specification, output, output_format, **kw
         linkml-tr derive-schema -T transform/personinfo-to-agent.transform.yaml source/personinfo.yaml
     """
     logging.info(f"Transforming {schema} using {transformer_specification}")
-    tr = SchemaMapper()
-    tr.source_schemaview = SchemaView(schema)
-    specification = yaml_loader.load(
-        transformer_specification, target_class=TransformationSpecification
-    )
-    target_schema = tr.derive_schema(specification)
+    tr = ObjectTransformer()
+    tr.load_transformer_specification(transformer_specification)
+    mapper = SchemaMapper(transformer=tr)
+    mapper.source_schemaview = SchemaView(schema)
+    target_schema = mapper.derive_schema()
     if output:
         outfile = open(output, "w", encoding="utf-8")
     else:

src/linkml_transformer/compiler/__init__.py

@@ -0,0 +1 @@
+"""Compiles schemas into other transformation frameworks."""

src/linkml_transformer/compiler/awk_compiler.py

@@ -1,6 +1,4 @@
-from linkml_runtime.utils.yamlutils import YAMLRoot
-
-from linkml_transformer.compiler.compiler import Compiler
+from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler
 from linkml_transformer.datamodel.transformer_model import TransformationSpecification
 
 
@@ -11,5 +9,5 @@ class AWKCompiler(Compiler):
     Note: this is only expected to work for flat schemas.
     """
 
-    def compile(self, specification: TransformationSpecification) -> YAMLRoot:
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
         raise NotImplementedError

src/linkml_transformer/compiler/compiler.py

@@ -1,16 +1,21 @@
-from abc import ABC, abstractmethod
+from abc import ABC
 from dataclasses import dataclass
+from typing import Iterator
 
 from linkml_runtime import SchemaView
-from linkml_runtime.utils.yamlutils import YAMLRoot
 
 from linkml_transformer.datamodel.transformer_model import TransformationSpecification
 
 
+@dataclass
+class CompiledSpecification:
+    serialization: str
+
+
 @dataclass
 class Compiler(ABC):
     """
-    Base class for all compiler.
+    Base class for all compilers.
 
     A compiler will compile a transformation specification into
     an alternative representation.
@@ -21,12 +26,17 @@ class Compiler(ABC):
     source_schemaview: SchemaView = None
     """A view over the schema describing the source."""
 
-    @abstractmethod
-    def compile(self, specification: TransformationSpecification) -> YAMLRoot:
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
         """
         Transform source object into an instance of the target class.
 
         :param specification:
         :return:
         """
+        s = ""
+        for chunk in self._compile_iterator(specification):
+            s += chunk
+        return CompiledSpecification(serialization=s)
+
+    def _compile_iterator(self, specification: TransformationSpecification) -> Iterator[str]:
         raise NotImplementedError

src/linkml_transformer/compiler/python_compiler.py

@@ -0,0 +1,73 @@
+from copy import deepcopy
+from typing import Iterator
+
+from jinja2 import Template
+
+from linkml_transformer.compiler.compiler import Compiler
+from linkml_transformer.datamodel.transformer_model import (
+    ClassDerivation,
+    TransformationSpecification,
+)
+from linkml_transformer.transformer.inference import induce_missing_values
+
+CD_TEMPLATE = """
+{% macro gen_slot_derivation_value(sd, var) -%}
+{%- if sd.range -%}
+derive_{{ sd.range }}({{ var }})
+{%- else -%}
+{{ var }}
+{%- endif -%}
+{%- endmacro %}
+{% macro gen_slot_derivation(sd, force_singlevalued=False) -%}
+{%- if not force_singlevalued and sd.populated_from and induced_slots[sd.populated_from].multivalued -%}
+ [ {{ gen_slot_derivation_value(sd, "x") }} for x in {{ gen_slot_derivation(sd, force_singlevalued=True) }} ]
+{%- else -%}
+ {%- if sd.populated_from -%}
+  source_object.{{ sd.populated_from }}
+ {%- elif sd.expr -%}
+  {{ sd.expr }}
+ {%- else -%}
+  None
+ {%- endif -%}
+{%- endif -%}
+{%- endmacro %}
+def derive_{{ cd.name }}(
+        source_object: {{ source_module }}.{{ cd.populated_from }}
+    ) -> {{ target_module }}.{{ cd.name }}:
+    return {{ cd.populated_from }}(
+       {%- for sd in cd.slot_derivations.values() %}
+       {{ sd.name }}={{ gen_slot_derivation(sd) }},
+       {%- endfor %}
+    )
+
+"""
+
+
+class PythonCompiler(Compiler):
+    """
+    Compiles a Transformation Specification to Python code.
+    """
+
+    def _compile_iterator(self, specification: TransformationSpecification) -> Iterator[str]:
+        specification = deepcopy(specification)
+        induce_missing_values(specification, self.source_schemaview)
+        for cd in specification.class_derivations.values():
+            yield from self._yield_compile_class_derivation(cd)
+
+    def _yield_compile_class_derivation(self, cd: ClassDerivation) -> Iterator[str]:
+        sv = self.source_schemaview
+        if cd.populated_from:
+            populated_from = cd.populated_from
+        else:
+            populated_from = cd.name
+        if populated_from not in sv.all_classes():
+            return
+        induced_slots = {s.name: s for s in sv.class_induced_slots(populated_from)}
+        t = Template(CD_TEMPLATE)
+        yield t.render(
+            cd=cd,
+            source_module="src",
+            target_module="tgt",
+            induced_slots=induced_slots,
+            schemaview=sv,
+        )

src/linkml_transformer/compiler/r2rml_compiler.py

@@ -1,6 +1,4 @@
-from linkml_runtime.utils.yamlutils import YAMLRoot
-
-from linkml_transformer.compiler.compiler import Compiler
+from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler
 from linkml_transformer.datamodel.transformer_model import TransformationSpecification
 
 
@@ -10,5 +8,5 @@ class R2RMLCompiler(Compiler):
 
     """
 
-    def compile(self, specification: TransformationSpecification) -> YAMLRoot:
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
         raise NotImplementedError

src/linkml_transformer/compiler/sparql_compiler.py

@@ -1,6 +1,4 @@
-from linkml_runtime.utils.yamlutils import YAMLRoot
-
-from linkml_transformer.compiler.compiler import Compiler
+from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler
 from linkml_transformer.datamodel.transformer_model import TransformationSpecification
 
 
@@ -10,5 +8,5 @@ class SPARQLCompiler(Compiler):
 
     """
 
-    def compile(self, specification: TransformationSpecification) -> YAMLRoot:
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
         raise NotImplementedError

src/linkml_transformer/compiler/sql_compiler.py

@@ -1,6 +1,4 @@
-from linkml_runtime.utils.yamlutils import YAMLRoot
-
-from linkml_transformer.compiler.compiler import Compiler
+from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler
 from linkml_transformer.datamodel.transformer_model import TransformationSpecification
 
 
@@ -10,5 +8,5 @@ class SQLCompiler(Compiler):
 
     """
 
-    def compile(self, specification: TransformationSpecification) -> YAMLRoot:
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
         raise NotImplementedError

src/linkml_transformer/compiler/sssom_compiler.py

@@ -1,6 +1,4 @@
-from linkml_runtime.utils.yamlutils import YAMLRoot
-
-from linkml_transformer.compiler.compiler import Compiler
+from linkml_transformer.compiler.compiler import CompiledSpecification, Compiler
 from linkml_transformer.datamodel.transformer_model import TransformationSpecification
 
 
@@ -11,5 +9,5 @@ class SSSOMCompiler(Compiler):
     Note: SSSOM has less expressivity so this is expected to be highly lossy
     """
 
-    def compile(self, specification: TransformationSpecification) -> YAMLRoot:
+    def compile(self, specification: TransformationSpecification) -> CompiledSpecification:
         raise NotImplementedError