diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..c7447c2 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,194 @@ +--- +name: pysqlsync database tests + +on: + workflow_dispatch: + +defaults: + run: + shell: bash + +jobs: + postgresql: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:15-alpine + env: + POSTGRES_PORT: 5432 + POSTGRES_USER: levente.hunyadi + POSTGRES_PASSWORD: "" + POSTGRES_DB: levente.hunyadi + ports: + - 5432:5432 + # set health checks to wait until PostgreSQL has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + --health-start-period 10s + + strategy: + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip --disable-pip-version-check install -r requirements.txt + + - name: Build package + run: | + ./check.sh + python -m build + + - name: Run integration tests + run: | + TEST_INTEGRATION=1 TEST_POSTGRESQL=1 python -m unittest discover + + oracle: + runs-on: ubuntu-latest + + services: + oracle: + image: container-registry.oracle.com/database/free:latest + env: + ORACLE_PWD: "" + ports: + - 1521:1521 + # set health checks to wait until Oracle has started + options: >- + --health-interval 20s + --health-timeout 10s + --health-retries 10 + --health-start-period 30s + + strategy: + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip --disable-pip-version-check install -r requirements.txt + + - name: Build package + run: | + ./check.sh + python -m build + + - name: Run integration tests + run: | + TEST_INTEGRATION=1 TEST_ORACLE=1 python -m unittest discover + + mysql: + runs-on: ubuntu-latest + + services: + mysql: + image: mysql:8.0 + env: + MYSQL_ROOT_PASSWORD: "" + MYSQL_DATABASE: levente_hunyadi + ports: + - 3306:3306 + # set health checks to wait until MySQL has started + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + --health-start-period 10s + + strategy: + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip --disable-pip-version-check install -r requirements.txt + + - name: Build package + run: | + ./check.sh + python -m build + + - name: Run integration tests + run: | + TEST_INTEGRATION=1 TEST_MYSQL=1 python -m unittest discover + + mssql: + runs-on: ubuntu-latest + + services: + mssql: + image: mcr.microsoft.com/mssql/server:2022-latest + env: + ACCEPT_EULA: Y + MSSQL_SA_PASSWORD: "" + SQLCMDPASSWORD: "" + ports: + - 1433:1433 + # set health checks to wait until Microsoft SQL Server has started + options: >- + --health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -Q 'SELECT 1' -b" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + --health-start-period 10s + + strategy: + matrix: + python-version: ["3.12"] + + steps: + - uses: actions/checkout@v3 + + - name: Install Microsoft ODBC + run: sudo ACCEPT_EULA=Y apt-get install msodbcsql18 -y + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip --disable-pip-version-check install -r requirements.txt + + - name: Build package + run: | + ./check.sh + python -m build + + - name: Run integration tests + run: | + TEST_INTEGRATION=1 TEST_MSSQL=1 python -m unittest discover diff --git a/LICENSE b/LICENSE index 51af19b..5d2d004 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2023 Levente Hunyadi +Copyright (c) 2023-2024 Levente Hunyadi Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pysqlsync/__init__.py b/pysqlsync/__init__.py index cf2b651..ee63321 100644 --- a/pysqlsync/__init__.py +++ b/pysqlsync/__init__.py @@ -9,7 +9,7 @@ __version__ = "0.3.0" __author__ = "Levente Hunyadi" -__copyright__ = "Copyright 2023, Levente Hunyadi" +__copyright__ = "Copyright 2023-2024, Levente Hunyadi" __license__ = "MIT" __maintainer__ = "Levente Hunyadi" __status__ = "Production" diff --git a/pysqlsync/base.py b/pysqlsync/base.py index 66f5a17..14c9fc5 100644 --- a/pysqlsync/base.py +++ b/pysqlsync/base.py @@ -919,6 +919,10 @@ async def _get_transformer( if table.is_relation(column): relation = generator.state.get_referenced_table(table.name, column.name) if relation.is_lookup_table(): + LOGGER.debug( + f"found lookup table column {column.name} in table {table.name}" + ) + enum_dict: dict[str, int] if field_type is str: diff --git a/pysqlsync/formation/py_to_sql.py b/pysqlsync/formation/py_to_sql.py index c5b1de1..26e6d15 100644 --- a/pysqlsync/formation/py_to_sql.py +++ b/pysqlsync/formation/py_to_sql.py @@ -541,21 +541,21 @@ def dataclass_to_table(self, cls: type[DataclassInstance]) -> Table: if self.options.foreign_constraints: constraints.extend(self.dataclass_to_constraints(cls)) - # relationships for enumeration types - if self.options.enum_mode is EnumMode.RELATION: - for enum_field in dataclass_enum_fields(cls): - constraints.append( - ForeignConstraint( - LocalId(f"fk_{cls.__name__}_{enum_field.name}"), - (LocalId(enum_field.name),), - ConstraintReference( - self.create_qualified_id( - enum_field.type.__module__, enum_field.type.__name__ - ), - (LocalId("id"),), + # relationships for enumeration types ignore foreign constraints option and always create a foreign key + if self.options.enum_mode is EnumMode.RELATION: + for enum_field in dataclass_enum_fields(cls): + constraints.append( + ForeignConstraint( + LocalId(f"fk_{cls.__name__}_{enum_field.name}"), + (LocalId(enum_field.name),), + ConstraintReference( + self.create_qualified_id( + enum_field.type.__module__, enum_field.type.__name__ ), + (LocalId("id"),), ), - ) + ), + ) if self.options.enum_mode is EnumMode.CHECK: for enum_field in dataclass_enum_fields(cls): diff --git a/pysqlsync/formation/sql_to_py.py b/pysqlsync/formation/sql_to_py.py index 8ed918b..a924f5a 100644 --- a/pysqlsync/formation/sql_to_py.py +++ b/pysqlsync/formation/sql_to_py.py @@ -4,6 +4,7 @@ import keyword import sys import types +import typing from dataclasses import dataclass from io import StringIO from typing import Annotated, Any, Optional, Union @@ -122,7 +123,7 @@ def qual_to_module(self, id: SupportsQualifiedId) -> str: def column_to_field( self, table: Table, column: Column - ) -> tuple[str, TypeLike, dataclasses.Field]: + ) -> tuple[str, type, dataclasses.Field]: """ Generates a dataclass field corresponding to a table column. @@ -154,11 +155,10 @@ def column_to_field( union_types = tuple(self.qual_to_module(r.table) for r in c.references) field_type = Union[union_types] - return ( - field_name, - field_type, - dataclasses.field(default=default), - ) + # use cast to ensure compatibility with signature of `make_dataclass` + data_type = typing.cast(type, field_type) + + return (field_name, data_type, dataclasses.field(default=default)) def table_to_dataclass(self, table: Table) -> type[DataclassInstance]: """ @@ -183,7 +183,7 @@ def table_to_dataclass(self, table: Table) -> type[DataclassInstance]: typ = dataclasses.make_dataclass(class_name, fields, module=module.__name__) else: typ = dataclasses.make_dataclass( - class_name, fields, namespace={"__module__": module.__name__} # type: ignore + class_name, fields, namespace={"__module__": module.__name__} ) with StringIO() as out: for field in dataclasses.fields(typ): diff --git a/requirements.txt b/requirements.txt index 1dbf396..a1b14a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,28 @@ # core -json_strong_typing >= 0.2.9 +json_strong_typing >= 0.3.2 +typing_extensions >= 4.8; python_version<"3.12" + +# development tools +build >= 1.0 +mypy >= 1.8 +flake8 >= 7.0 # data export/import -tsv2py +tsv2py >= 0.5.2 + +# PostgreSQL +asyncpg >= 0.29 +asyncpg-stubs >= 0.29 -# database drivers -asyncpg >= 0.28 +# Oracle +oracledb >= 1.4 + +# MySQL aiomysql >= 0.2 -aiotrino >= 0.2 +PyMySQL[rsa] + +# Microsoft SQL Server pyodbc >= 5.0 + +# Trino +aiotrino >= 0.2 diff --git a/setup.cfg b/setup.cfg index 3919a2d..f708d92 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,7 +29,7 @@ include_package_data = True packages = find: python_requires = >=3.9 install_requires = - json_strong_typing >= 0.3.1 + json_strong_typing >= 0.3.2 typing_extensions >= 4.8; python_version<"3.12" [options.extras_require] diff --git a/tests/params.py b/tests/params.py index 011efe4..50e3210 100644 --- a/tests/params.py +++ b/tests/params.py @@ -48,7 +48,7 @@ def parameters(self) -> ConnectionParameters: host="localhost", port=5432, username="levente.hunyadi", - password=None, + password="", database="levente.hunyadi", ) @@ -84,7 +84,7 @@ def parameters(self) -> ConnectionParameters: host="localhost", port=3306, username="root", - password=None, # "", + password="", database="levente_hunyadi", ) diff --git a/tests/test_synchronize.py b/tests/test_synchronize.py index 17a5378..5f4c3d2 100644 --- a/tests/test_synchronize.py +++ b/tests/test_synchronize.py @@ -131,11 +131,22 @@ async def get_rows(self, conn: BaseContext, table: Table) -> int: return count async def test_insert_update_delete_rows(self) -> None: - async with self.engine.create_connection(self.parameters, self.options) as conn: + await self.insert_update_delete_rows(self.options) + + async def test_insert_update_delete_rows_relation(self) -> None: + options = GeneratorOptions( + namespaces={tables: "sample", event: "event", school: "school", user: None}, + foreign_constraints=False, + enum_mode=EnumMode.RELATION, + ) + await self.insert_update_delete_rows(options) + + async def insert_update_delete_rows(self, options: GeneratorOptions) -> None: + async with self.engine.create_connection(self.parameters, options) as conn: explorer = self.engine.create_explorer(conn) await explorer.synchronize(module=tables) - async with self.engine.create_connection(self.parameters, self.options) as conn: + async with self.engine.create_connection(self.parameters, options) as conn: explorer = self.engine.create_explorer(conn) await explorer.synchronize(module=tables) entity_types = get_entity_types([tables])