Fixes #149 and restructures db tests. Bumped version

odmf/__init__.py

@@ -1,2 +1,2 @@
-__version__ = '2024.2.8'
+__version__ = '2024.3.20'
 prefix = '.'

odmf/dataimport/base.py

@@ -477,7 +477,7 @@ def config_getlist(section, option):
 
             if project:
                 rows = session.query(db.Project)\
-                    .filter(db.Project.id == project).count()
+                    .filter(db.Project.id == int(project)).count()
                 if rows != 1:
                     raise ValueError('Error in import description: \'%s\' is no'
                                      ' valid project identifier' % project)

odmf/dataimport/pandas_import.py

@@ -31,7 +31,7 @@ def __init__(self, session, idescr: ImportDescription, col: ImportColumn,
                  id: int, user: db.Person, site: db.Site, inst: db.Datasource,
                  valuetypes: typing.Dict[int, db.ValueType], raw: db.Quality,
                  start: datetime.datetime, end: datetime.datetime,
-                 filename: typing.Optional[str] = None
+                 filename: typing.Optional[str] = None, project: db.Project = None
                  ):
 
         self.column = col
@@ -62,7 +62,7 @@ def __init__(self, session, idescr: ImportDescription, col: ImportColumn,
                 access=col.access if col.access is not None else 1,
                 # Get timezone from descriptor or, if not present from global conf
                 timezone=idescr.timezone or conf.datetime_default_timezone,
-                project=idescr.project)
+                project=project)
             session.add(self.dataset)
 
         self.id = self.dataset.id
@@ -104,6 +104,7 @@ def columndatasets_from_description(
         inst = session.get(db.Datasource, idescr.instrument)
         user = session.get(db.Person, user)
         site = session.get(db.Site, siteid)
+        project = session.get(db.Project, idescr.project)
         # Get "raw" as data quality, to use as a default value
         raw = session.get(db.Quality, 0)
         # Get all the relevant valuetypes (vt) from db as a dict for fast look up
@@ -123,7 +124,7 @@ def columndatasets_from_description(
                     ColumnDataset(
                         session, idescr, col, None,
                         user, site, inst, valuetypes, raw,
-                        start, end, filepath.name
+                        start, end, filepath.name, project
                     )
                 )
             elif not col.ds_column:
@@ -132,7 +133,7 @@ def columndatasets_from_description(
                         session, idescr, col,
                         newid + len(newdatasets),
                         user, site, inst, valuetypes, raw,
-                        start, end, filepath.name
+                        start, end, filepath.name, project
                     )
                 )
 

tests/test_db/db_fixtures.py

@@ -0,0 +1,107 @@
+import datetime
+
+import pytest
+from PIL import Image
+import random
+import string
+from . import temp_in_database, db, session
+
+
+def randomstring(n):
+    return ''.join(random.sample(string.ascii_lowercase, n))
+
+@pytest.fixture()
+def site1_in_db(db, session):
+    with temp_in_database(
+            db.Site(
+                id=1, lat=50.5, lon=8.5, height=100,
+                name='site_1', comment='This is a comment', icon='an_icon.png'
+            ),
+            session) as site:
+        yield site
+
+
+@pytest.fixture()
+def datasource1_in_db(db, session):
+    with temp_in_database(
+            db.Datasource(
+                id=1, name='instr_1', comment='This is a comment',
+                sourcetype='sourcetype', manuallink='path/to/manual'
+            ),
+            session) as site:
+        yield site
+
+
+@pytest.fixture()
+def installation(db, session, site1_in_db, datasource1_in_db):
+    with temp_in_database(
+            db.Installation(
+                site1_in_db, datasource1_in_db,
+                id=1, installdate=datetime.datetime(2021, 2, 15),
+                comment='A test installation'
+            ),
+            session) as installation:
+        yield installation
+
+
+@pytest.fixture()
+def person(db, session):
+    with temp_in_database(
+        db.Person(
+            username=randomstring(10), email='This is an email', firstname='first',
+            surname='last', telephone='this is a phone number', comment='this is a comment',
+            can_supervise= False, mobile='this is a mobile number', car_available=0, access_level=2
+        ),
+        session) as person:
+        yield person
+
+
+@pytest.fixture()
+def log(db, session, person, site1_in_db):
+    with temp_in_database(
+        db.Log(
+            id=1, time=datetime.datetime(2022, 2, 17),
+            user=person, site=site1_in_db, message='this is a message'
+        ),
+        session) as log:
+        yield log
+
+@pytest.fixture()
+def job(db, session, person):
+    with temp_in_database(
+        db.Job(
+            id=1, name='this is a name', description='this is a description',
+            due=datetime.datetime(2023, 5, 20), author=person,
+            responsible=person, done=True, repeat=3,
+            link='/path/to/link', type='this is a type',
+            donedate=datetime.datetime(2023, 2,20)
+        ),
+        session) as job:
+        yield job
+
+
+@pytest.fixture()
+def project(db, session, person):
+    with temp_in_database(
+        db.Project(
+            id=1, person_responsible=person, name='this is a name', comment='this is a comment'
+        ),
+        session) as project:
+        yield project
+
+
+@pytest.fixture()
+def image(tmp_path, db, session, site1_in_db, person):
+
+    img = Image.new(mode='RGB', size=(400, 300), color=(30, 226, 76))
+    img.save(tmp_path / 'test_image.png')
+
+    with temp_in_database(
+        db.Image(
+            time=datetime.datetime(2021, 7, 10), format='png',
+            site=site1_in_db, by=person,
+            imagefile=str(tmp_path / 'test_image.png')
+        ),
+        session) as image:
+        yield image
+

tests/test_db/test_dbobjects.py

@@ -1,24 +1,8 @@
 import datetime
 
 import pytest
-from PIL import Image
-import random
-import string
-from . import temp_in_database, db, session, conf
-
-
-def randomstring(n):
-    return ''.join(random.sample(string.ascii_lowercase, n))
-
-@pytest.fixture()
-def site1_in_db(db, session):
-    with temp_in_database(
-            db.Site(
-                id=1, lat=50.5, lon=8.5, height=100,
-                name='site_1', comment='This is a comment', icon='an_icon.png'
-            ),
-            session) as site:
-        yield site
+from . import  db, session, conf
+from .db_fixtures import site1_in_db, datasource1_in_db, log, job, project, person, installation, image
 
 
 class TestSite:
@@ -65,16 +49,6 @@ def test_site_load(self, db, session, site1_in_db):
         assert 'id' in d
 
 
-@pytest.fixture()
-def datasource1_in_db(db, session):
-    with temp_in_database(
-            db.Datasource(
-                id=1, name='instr_1', comment='This is a comment',
-                sourcetype='sourcetype', manuallink='path/to/manual'
-            ),
-            session) as site:
-        yield site
-
 
 class TestDatasource:
     def test_create_datasource(self, datasource1_in_db):
@@ -93,16 +67,6 @@ def test_comparison(self, db, session, datasource1_in_db):
         with pytest.raises(TypeError):
             _ = datasource1_in_db < 2
 
-@pytest.fixture()
-def installation(db, session, site1_in_db, datasource1_in_db):
-    with temp_in_database(
-            db.Installation(
-                site1_in_db, datasource1_in_db,
-                id=1, installdate=datetime.datetime(2021, 2, 15),
-                comment='A test installation'
-            ),
-            session) as installation:
-        yield installation
 
 
 class TestInstallation:
@@ -139,17 +103,6 @@ def test_no_instrument(self, installation):
         installation.session().rollback()
 
 
-@pytest.fixture()
-def person(db, session):
-    with temp_in_database(
-        db.Person(
-            username=randomstring(10), email='This is an email', firstname='first',
-            surname='last', telephone='this is a phone number', comment='this is a comment',
-            can_supervise= False, mobile='this is a mobile number', car_available=0, access_level=2
-        ),
-        session) as person:
-        yield person
-
 class TestPerson:
     def test_person(self, person):
         assert person
@@ -160,16 +113,6 @@ def test_person(self, person):
         assert 'username' in d
 
 
-@pytest.fixture()
-def log(db, session, person, site1_in_db):
-    with temp_in_database(
-        db.Log(
-            id=1, time=datetime.datetime(2022, 2, 17),
-            user=person, site=site1_in_db, message='this is a message'
-        ),
-        session) as log:
-        yield log
-
 class TestLog:
     def test_log(self, log):
         assert log
@@ -187,18 +130,6 @@ def test_log_load(self, log, session, db):
         assert not log < log
 
 
-@pytest.fixture()
-def job(db, session, person):
-    with temp_in_database(
-        db.Job(
-            id=1, name='this is a name', description='this is a description',
-            due=datetime.datetime(2023, 5, 20), author=person,
-            responsible=person, done=True, repeat=3,
-            link='/path/to/link', type='this is a type',
-            donedate=datetime.datetime(2023, 2,20)
-        ),
-        session) as job:
-        yield job
 
 class TestJob:
     def test_job(self, job):
@@ -219,15 +150,6 @@ def test_due_time(self, job):
         assert job.due - datetime.timedelta(days=1) > datetime.datetime(2023, 3, 10)
 
 
-@pytest.fixture()
-def project(db, session, person):
-    with temp_in_database(
-        db.Project(
-            id=1, person_responsible=person, name='this is a name', comment='this is a comment'
-        ),
-        session) as project:
-        yield project
-
 
 class TestProject:
     def test_project(self, project):
@@ -258,23 +180,6 @@ def test_person_add_project(self, project, person):
         assert projects[0][1] == 1 , f'Project member has unexpected access level of {projects[0][1]}!=1'
 
 
-
-@pytest.fixture()
-def image(tmp_path, db, session, site1_in_db, person):
-
-    img = Image.new(mode='RGB', size=(400, 300), color=(30, 226, 76))
-    img.save(tmp_path / 'test_image.png')
-
-    with temp_in_database(
-        db.Image(
-            time=datetime.datetime(2021, 7, 10), format='png',
-            site=site1_in_db, by=person,
-            imagefile=str(tmp_path / 'test_image.png')
-        ),
-        session) as image:
-        yield image
-
-
 class TestImage:
     def test_image(self, image):
         assert image

tests/test_db/test_di_base.py

@@ -2,19 +2,22 @@
 import configparser
 
 from .. import conf
-from ..test_db import db
-
+from . import db, session
+from .db_fixtures import project, person
 # Create a config file for the Odyssey Logger
 @pytest.fixture()
 def di_conf_file(tmp_path):
 
     config = configparser.ConfigParser(interpolation=None)
-    config['Tipping Bucket (rain intensity)'] = {'instrument': '5',
-                                                 'skiplines': '9',
-                                                 'delimiter': ',',
-                                                 'decimalpoint': '.',
-                                                 'dateformat': '%d/%m/%Y %H:%M:%S',
-                                                 'datecolumns': '1, 2'}
+    config['Tipping Bucket (rain intensity)'] = {
+        'instrument': '5',
+        'skiplines': '9',
+        'delimiter': ',',
+        'decimalpoint': '.',
+        'dateformat': '%d/%m/%Y %H:%M:%S',
+        'datecolumns': '1, 2',
+        'project': '1'
+    }
 
     config['rain tips'] = {'column': '4',
                            'name': 'rain tips',
@@ -30,7 +33,7 @@ def di_conf_file(tmp_path):
     return config_path
 
 
-def test_from_file(di_conf_file, db):
+def test_from_file(di_conf_file, db, session, project):
     from odmf.dataimport import base
     pattern = '*.conf'
     descr = base.ImportDescription.from_file(path=di_conf_file, pattern=pattern)

tests/test_db/test_di_pandas_import.py

@@ -1,8 +1,7 @@
 import pytest
-import pandas as pd
-from .. import conf
-from ..test_db import db
+from ..test_db import db, session, conf
 from .test_di_base import di_conf_file
+from .db_fixtures import project, person
 
 @pytest.fixture
 def csv_file_for_import(tmp_path, di_conf_file):
@@ -17,13 +16,13 @@ def csv_file_for_import(tmp_path, di_conf_file):
     return sample_file
 
 
-def test_load_dataframe(csv_file_for_import, db):
+def test_load_dataframe(csv_file_for_import, db, project, person):
     from odmf.dataimport import pandas_import as pi
     idescr = pi.ImportDescription.from_file(csv_file_for_import)
     df = pi.load_dataframe(idescr=idescr, filepath=csv_file_for_import)
     assert not df.empty
 
-def test_load_dataframe_column_problem(csv_file_for_import, db):
+def test_load_dataframe_column_problem(csv_file_for_import, db, project, person):
     """
     This test is for issue #103, to see if it is working with total_columns
     https://github.com/jlu-ilr-hydro/odmf/issues/103