Add workflow script for GitHub Actions

.github/workflows/main.yml

@@ -0,0 +1,194 @@
+---
+name: pysqlsync database tests
+
+on:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  postgresql:
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:15-alpine
+        env:
+          POSTGRES_PORT: 5432
+          POSTGRES_USER: levente.hunyadi
+          POSTGRES_PASSWORD: "<YourStrong@Passw0rd>"
+          POSTGRES_DB: levente.hunyadi
+        ports:
+          - 5432:5432
+        # set health checks to wait until PostgreSQL has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --health-start-period 10s
+
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip --disable-pip-version-check install -r requirements.txt
+
+      - name: Build package
+        run: |
+          ./check.sh
+          python -m build
+
+      - name: Run integration tests
+        run: |
+          TEST_INTEGRATION=1 TEST_POSTGRESQL=1 python -m unittest discover
+
+  oracle:
+    runs-on: ubuntu-latest
+
+    services:
+      oracle:
+        image: container-registry.oracle.com/database/free:latest
+        env:
+          ORACLE_PWD: "<YourStrong@Passw0rd>"
+        ports:
+          - 1521:1521
+        # set health checks to wait until Oracle has started
+        options: >-
+          --health-interval 20s
+          --health-timeout 10s
+          --health-retries 10
+          --health-start-period 30s
+
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip --disable-pip-version-check install -r requirements.txt
+
+      - name: Build package
+        run: |
+          ./check.sh
+          python -m build
+
+      - name: Run integration tests
+        run: |
+          TEST_INTEGRATION=1 TEST_ORACLE=1 python -m unittest discover
+
+  mysql:
+    runs-on: ubuntu-latest
+
+    services:
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: "<YourStrong@Passw0rd>"
+          MYSQL_DATABASE: levente_hunyadi
+        ports:
+          - 3306:3306
+        # set health checks to wait until MySQL has started
+        options: >-
+          --health-cmd "mysqladmin ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --health-start-period 10s
+
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip --disable-pip-version-check install -r requirements.txt
+
+      - name: Build package
+        run: |
+          ./check.sh
+          python -m build
+
+      - name: Run integration tests
+        run: |
+          TEST_INTEGRATION=1 TEST_MYSQL=1 python -m unittest discover
+
+  mssql:
+    runs-on: ubuntu-latest
+
+    services:
+      mssql:
+        image: mcr.microsoft.com/mssql/server:2022-latest
+        env:
+          ACCEPT_EULA: Y
+          MSSQL_SA_PASSWORD: "<YourStrong@Passw0rd>"
+          SQLCMDPASSWORD: "<YourStrong@Passw0rd>"
+        ports:
+          - 1433:1433
+        # set health checks to wait until Microsoft SQL Server has started
+        options: >-
+          --health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -Q 'SELECT 1' -b"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --health-start-period 10s
+
+    strategy:
+      matrix:
+        python-version: ["3.12"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Microsoft ODBC
+        run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip --disable-pip-version-check install -r requirements.txt
+
+      - name: Build package
+        run: |
+          ./check.sh
+          python -m build
+
+      - name: Run integration tests
+        run: |
+          TEST_INTEGRATION=1 TEST_MSSQL=1 python -m unittest discover

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 Levente Hunyadi
+Copyright (c) 2023-2024 Levente Hunyadi
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

pysqlsync/__init__.py

@@ -9,7 +9,7 @@
 
 __version__ = "0.3.0"
 __author__ = "Levente Hunyadi"
-__copyright__ = "Copyright 2023, Levente Hunyadi"
+__copyright__ = "Copyright 2023-2024, Levente Hunyadi"
 __license__ = "MIT"
 __maintainer__ = "Levente Hunyadi"
 __status__ = "Production"

pysqlsync/base.py

@@ -919,6 +919,10 @@ async def _get_transformer(
         if table.is_relation(column):
             relation = generator.state.get_referenced_table(table.name, column.name)
             if relation.is_lookup_table():
+                LOGGER.debug(
+                    f"found lookup table column {column.name} in table {table.name}"
+                )
+
                 enum_dict: dict[str, int]
 
                 if field_type is str:

pysqlsync/formation/py_to_sql.py

@@ -541,21 +541,21 @@ def dataclass_to_table(self, cls: type[DataclassInstance]) -> Table:
         if self.options.foreign_constraints:
             constraints.extend(self.dataclass_to_constraints(cls))
 
-            # relationships for enumeration types
-            if self.options.enum_mode is EnumMode.RELATION:
-                for enum_field in dataclass_enum_fields(cls):
-                    constraints.append(
-                        ForeignConstraint(
-                            LocalId(f"fk_{cls.__name__}_{enum_field.name}"),
-                            (LocalId(enum_field.name),),
-                            ConstraintReference(
-                                self.create_qualified_id(
-                                    enum_field.type.__module__, enum_field.type.__name__
-                                ),
-                                (LocalId("id"),),
+        # relationships for enumeration types ignore foreign constraints option and always create a foreign key
+        if self.options.enum_mode is EnumMode.RELATION:
+            for enum_field in dataclass_enum_fields(cls):
+                constraints.append(
+                    ForeignConstraint(
+                        LocalId(f"fk_{cls.__name__}_{enum_field.name}"),
+                        (LocalId(enum_field.name),),
+                        ConstraintReference(
+                            self.create_qualified_id(
+                                enum_field.type.__module__, enum_field.type.__name__
                             ),
+                            (LocalId("id"),),
                         ),
-                    )
+                    ),
+                )
 
         if self.options.enum_mode is EnumMode.CHECK:
             for enum_field in dataclass_enum_fields(cls):

pysqlsync/formation/sql_to_py.py

@@ -4,6 +4,7 @@
 import keyword
 import sys
 import types
+import typing
 from dataclasses import dataclass
 from io import StringIO
 from typing import Annotated, Any, Optional, Union
@@ -122,7 +123,7 @@ def qual_to_module(self, id: SupportsQualifiedId) -> str:
 
     def column_to_field(
         self, table: Table, column: Column
-    ) -> tuple[str, TypeLike, dataclasses.Field]:
+    ) -> tuple[str, type, dataclasses.Field]:
         """
         Generates a dataclass field corresponding to a table column.
 
@@ -154,11 +155,10 @@ def column_to_field(
                 union_types = tuple(self.qual_to_module(r.table) for r in c.references)
                 field_type = Union[union_types]
 
-        return (
-            field_name,
-            field_type,
-            dataclasses.field(default=default),
-        )
+        # use cast to ensure compatibility with signature of `make_dataclass`
+        data_type = typing.cast(type, field_type)
+
+        return (field_name, data_type, dataclasses.field(default=default))
 
     def table_to_dataclass(self, table: Table) -> type[DataclassInstance]:
         """
@@ -183,7 +183,7 @@ def table_to_dataclass(self, table: Table) -> type[DataclassInstance]:
             typ = dataclasses.make_dataclass(class_name, fields, module=module.__name__)
         else:
             typ = dataclasses.make_dataclass(
-                class_name, fields, namespace={"__module__": module.__name__}  # type: ignore
+                class_name, fields, namespace={"__module__": module.__name__}
             )
         with StringIO() as out:
             for field in dataclasses.fields(typ):

requirements.txt

@@ -1,11 +1,28 @@
 # core
-json_strong_typing >= 0.2.9
+json_strong_typing >= 0.3.2
+typing_extensions >= 4.8; python_version<"3.12"
+
+# development tools
+build >= 1.0
+mypy >= 1.8
+flake8 >= 7.0
 
 # data export/import
-tsv2py
+tsv2py >= 0.5.2
+
+# PostgreSQL
+asyncpg >= 0.29
+asyncpg-stubs >= 0.29
 
-# database drivers
-asyncpg >= 0.28
+# Oracle
+oracledb >= 1.4
+
+# MySQL
 aiomysql >= 0.2
-aiotrino >= 0.2
+PyMySQL[rsa]
+
+# Microsoft SQL Server
 pyodbc >= 5.0
+
+# Trino
+aiotrino >= 0.2

setup.cfg

@@ -29,7 +29,7 @@ include_package_data = True
 packages = find:
 python_requires = >=3.9
 install_requires =
-    json_strong_typing >= 0.3.1
+    json_strong_typing >= 0.3.2
     typing_extensions >= 4.8; python_version<"3.12"
 
 [options.extras_require]

tests/params.py

@@ -48,7 +48,7 @@ def parameters(self) -> ConnectionParameters:
             host="localhost",
             port=5432,
             username="levente.hunyadi",
-            password=None,
+            password="<YourStrong@Passw0rd>",
             database="levente.hunyadi",
         )
 
@@ -84,7 +84,7 @@ def parameters(self) -> ConnectionParameters:
             host="localhost",
             port=3306,
             username="root",
-            password=None,  # "<YourStrong@Passw0rd>",
+            password="<YourStrong@Passw0rd>",
             database="levente_hunyadi",
         )
 

tests/test_synchronize.py

@@ -131,11 +131,22 @@ async def get_rows(self, conn: BaseContext, table: Table) -> int:
         return count
 
     async def test_insert_update_delete_rows(self) -> None:
-        async with self.engine.create_connection(self.parameters, self.options) as conn:
+        await self.insert_update_delete_rows(self.options)
+
+    async def test_insert_update_delete_rows_relation(self) -> None:
+        options = GeneratorOptions(
+            namespaces={tables: "sample", event: "event", school: "school", user: None},
+            foreign_constraints=False,
+            enum_mode=EnumMode.RELATION,
+        )
+        await self.insert_update_delete_rows(options)
+
+    async def insert_update_delete_rows(self, options: GeneratorOptions) -> None:
+        async with self.engine.create_connection(self.parameters, options) as conn:
             explorer = self.engine.create_explorer(conn)
             await explorer.synchronize(module=tables)
 
-        async with self.engine.create_connection(self.parameters, self.options) as conn:
+        async with self.engine.create_connection(self.parameters, options) as conn:
             explorer = self.engine.create_explorer(conn)
             await explorer.synchronize(module=tables)
             entity_types = get_entity_types([tables])