diff --git a/odmf/__init__.py b/odmf/__init__.py index 354af6fb..c26fc5ba 100644 --- a/odmf/__init__.py +++ b/odmf/__init__.py @@ -1,2 +1,2 @@ -__version__ = '2024.2.8' +__version__ = '2024.3.20' prefix = '.' diff --git a/odmf/dataimport/base.py b/odmf/dataimport/base.py index c6bba6a4..78b5f7fe 100644 --- a/odmf/dataimport/base.py +++ b/odmf/dataimport/base.py @@ -477,7 +477,7 @@ def config_getlist(section, option): if project: rows = session.query(db.Project)\ - .filter(db.Project.id == project).count() + .filter(db.Project.id == int(project)).count() if rows != 1: raise ValueError('Error in import description: \'%s\' is no' ' valid project identifier' % project) diff --git a/odmf/dataimport/pandas_import.py b/odmf/dataimport/pandas_import.py index b93cec91..80d9e4bc 100644 --- a/odmf/dataimport/pandas_import.py +++ b/odmf/dataimport/pandas_import.py @@ -31,7 +31,7 @@ def __init__(self, session, idescr: ImportDescription, col: ImportColumn, id: int, user: db.Person, site: db.Site, inst: db.Datasource, valuetypes: typing.Dict[int, db.ValueType], raw: db.Quality, start: datetime.datetime, end: datetime.datetime, - filename: typing.Optional[str] = None + filename: typing.Optional[str] = None, project: db.Project = None ): self.column = col @@ -62,7 +62,7 @@ def __init__(self, session, idescr: ImportDescription, col: ImportColumn, access=col.access if col.access is not None else 1, # Get timezone from descriptor or, if not present from global conf timezone=idescr.timezone or conf.datetime_default_timezone, - project=idescr.project) + project=project) session.add(self.dataset) self.id = self.dataset.id @@ -104,6 +104,7 @@ def columndatasets_from_description( inst = session.get(db.Datasource, idescr.instrument) user = session.get(db.Person, user) site = session.get(db.Site, siteid) + project = session.get(db.Project, idescr.project) # Get "raw" as data quality, to use as a default value raw = session.get(db.Quality, 0) # Get all the relevant valuetypes (vt) from db as a dict for fast look up @@ -123,7 +124,7 @@ def columndatasets_from_description( ColumnDataset( session, idescr, col, None, user, site, inst, valuetypes, raw, - start, end, filepath.name + start, end, filepath.name, project ) ) elif not col.ds_column: @@ -132,7 +133,7 @@ def columndatasets_from_description( session, idescr, col, newid + len(newdatasets), user, site, inst, valuetypes, raw, - start, end, filepath.name + start, end, filepath.name, project ) ) diff --git a/tests/test_db/db_fixtures.py b/tests/test_db/db_fixtures.py new file mode 100644 index 00000000..9ee9783b --- /dev/null +++ b/tests/test_db/db_fixtures.py @@ -0,0 +1,107 @@ +import datetime + +import pytest +from PIL import Image +import random +import string +from . import temp_in_database, db, session + + +def randomstring(n): + return ''.join(random.sample(string.ascii_lowercase, n)) + +@pytest.fixture() +def site1_in_db(db, session): + with temp_in_database( + db.Site( + id=1, lat=50.5, lon=8.5, height=100, + name='site_1', comment='This is a comment', icon='an_icon.png' + ), + session) as site: + yield site + + +@pytest.fixture() +def datasource1_in_db(db, session): + with temp_in_database( + db.Datasource( + id=1, name='instr_1', comment='This is a comment', + sourcetype='sourcetype', manuallink='path/to/manual' + ), + session) as site: + yield site + + +@pytest.fixture() +def installation(db, session, site1_in_db, datasource1_in_db): + with temp_in_database( + db.Installation( + site1_in_db, datasource1_in_db, + id=1, installdate=datetime.datetime(2021, 2, 15), + comment='A test installation' + ), + session) as installation: + yield installation + + +@pytest.fixture() +def person(db, session): + with temp_in_database( + db.Person( + username=randomstring(10), email='This is an email', firstname='first', + surname='last', telephone='this is a phone number', comment='this is a comment', + can_supervise= False, mobile='this is a mobile number', car_available=0, access_level=2 + ), + session) as person: + yield person + + +@pytest.fixture() +def log(db, session, person, site1_in_db): + with temp_in_database( + db.Log( + id=1, time=datetime.datetime(2022, 2, 17), + user=person, site=site1_in_db, message='this is a message' + ), + session) as log: + yield log + +@pytest.fixture() +def job(db, session, person): + with temp_in_database( + db.Job( + id=1, name='this is a name', description='this is a description', + due=datetime.datetime(2023, 5, 20), author=person, + responsible=person, done=True, repeat=3, + link='/path/to/link', type='this is a type', + donedate=datetime.datetime(2023, 2,20) + ), + session) as job: + yield job + + +@pytest.fixture() +def project(db, session, person): + with temp_in_database( + db.Project( + id=1, person_responsible=person, name='this is a name', comment='this is a comment' + ), + session) as project: + yield project + + +@pytest.fixture() +def image(tmp_path, db, session, site1_in_db, person): + + img = Image.new(mode='RGB', size=(400, 300), color=(30, 226, 76)) + img.save(tmp_path / 'test_image.png') + + with temp_in_database( + db.Image( + time=datetime.datetime(2021, 7, 10), format='png', + site=site1_in_db, by=person, + imagefile=str(tmp_path / 'test_image.png') + ), + session) as image: + yield image + diff --git a/tests/test_db/test_dbobjects.py b/tests/test_db/test_dbobjects.py index f7d7a275..e1f052e1 100644 --- a/tests/test_db/test_dbobjects.py +++ b/tests/test_db/test_dbobjects.py @@ -1,24 +1,8 @@ import datetime import pytest -from PIL import Image -import random -import string -from . import temp_in_database, db, session, conf - - -def randomstring(n): - return ''.join(random.sample(string.ascii_lowercase, n)) - -@pytest.fixture() -def site1_in_db(db, session): - with temp_in_database( - db.Site( - id=1, lat=50.5, lon=8.5, height=100, - name='site_1', comment='This is a comment', icon='an_icon.png' - ), - session) as site: - yield site +from . import db, session, conf +from .db_fixtures import site1_in_db, datasource1_in_db, log, job, project, person, installation, image class TestSite: @@ -65,16 +49,6 @@ def test_site_load(self, db, session, site1_in_db): assert 'id' in d -@pytest.fixture() -def datasource1_in_db(db, session): - with temp_in_database( - db.Datasource( - id=1, name='instr_1', comment='This is a comment', - sourcetype='sourcetype', manuallink='path/to/manual' - ), - session) as site: - yield site - class TestDatasource: def test_create_datasource(self, datasource1_in_db): @@ -93,16 +67,6 @@ def test_comparison(self, db, session, datasource1_in_db): with pytest.raises(TypeError): _ = datasource1_in_db < 2 -@pytest.fixture() -def installation(db, session, site1_in_db, datasource1_in_db): - with temp_in_database( - db.Installation( - site1_in_db, datasource1_in_db, - id=1, installdate=datetime.datetime(2021, 2, 15), - comment='A test installation' - ), - session) as installation: - yield installation class TestInstallation: @@ -139,17 +103,6 @@ def test_no_instrument(self, installation): installation.session().rollback() -@pytest.fixture() -def person(db, session): - with temp_in_database( - db.Person( - username=randomstring(10), email='This is an email', firstname='first', - surname='last', telephone='this is a phone number', comment='this is a comment', - can_supervise= False, mobile='this is a mobile number', car_available=0, access_level=2 - ), - session) as person: - yield person - class TestPerson: def test_person(self, person): assert person @@ -160,16 +113,6 @@ def test_person(self, person): assert 'username' in d -@pytest.fixture() -def log(db, session, person, site1_in_db): - with temp_in_database( - db.Log( - id=1, time=datetime.datetime(2022, 2, 17), - user=person, site=site1_in_db, message='this is a message' - ), - session) as log: - yield log - class TestLog: def test_log(self, log): assert log @@ -187,18 +130,6 @@ def test_log_load(self, log, session, db): assert not log < log -@pytest.fixture() -def job(db, session, person): - with temp_in_database( - db.Job( - id=1, name='this is a name', description='this is a description', - due=datetime.datetime(2023, 5, 20), author=person, - responsible=person, done=True, repeat=3, - link='/path/to/link', type='this is a type', - donedate=datetime.datetime(2023, 2,20) - ), - session) as job: - yield job class TestJob: def test_job(self, job): @@ -219,15 +150,6 @@ def test_due_time(self, job): assert job.due - datetime.timedelta(days=1) > datetime.datetime(2023, 3, 10) -@pytest.fixture() -def project(db, session, person): - with temp_in_database( - db.Project( - id=1, person_responsible=person, name='this is a name', comment='this is a comment' - ), - session) as project: - yield project - class TestProject: def test_project(self, project): @@ -258,23 +180,6 @@ def test_person_add_project(self, project, person): assert projects[0][1] == 1 , f'Project member has unexpected access level of {projects[0][1]}!=1' - -@pytest.fixture() -def image(tmp_path, db, session, site1_in_db, person): - - img = Image.new(mode='RGB', size=(400, 300), color=(30, 226, 76)) - img.save(tmp_path / 'test_image.png') - - with temp_in_database( - db.Image( - time=datetime.datetime(2021, 7, 10), format='png', - site=site1_in_db, by=person, - imagefile=str(tmp_path / 'test_image.png') - ), - session) as image: - yield image - - class TestImage: def test_image(self, image): assert image diff --git a/tests/test_db/test_di_base.py b/tests/test_db/test_di_base.py index d4ce12cd..df99e4a0 100644 --- a/tests/test_db/test_di_base.py +++ b/tests/test_db/test_di_base.py @@ -2,19 +2,22 @@ import configparser from .. import conf -from ..test_db import db - +from . import db, session +from .db_fixtures import project, person # Create a config file for the Odyssey Logger @pytest.fixture() def di_conf_file(tmp_path): config = configparser.ConfigParser(interpolation=None) - config['Tipping Bucket (rain intensity)'] = {'instrument': '5', - 'skiplines': '9', - 'delimiter': ',', - 'decimalpoint': '.', - 'dateformat': '%d/%m/%Y %H:%M:%S', - 'datecolumns': '1, 2'} + config['Tipping Bucket (rain intensity)'] = { + 'instrument': '5', + 'skiplines': '9', + 'delimiter': ',', + 'decimalpoint': '.', + 'dateformat': '%d/%m/%Y %H:%M:%S', + 'datecolumns': '1, 2', + 'project': '1' + } config['rain tips'] = {'column': '4', 'name': 'rain tips', @@ -30,7 +33,7 @@ def di_conf_file(tmp_path): return config_path -def test_from_file(di_conf_file, db): +def test_from_file(di_conf_file, db, session, project): from odmf.dataimport import base pattern = '*.conf' descr = base.ImportDescription.from_file(path=di_conf_file, pattern=pattern) diff --git a/tests/test_db/test_di_pandas_import.py b/tests/test_db/test_di_pandas_import.py index 79fc9741..9d9b1dd3 100644 --- a/tests/test_db/test_di_pandas_import.py +++ b/tests/test_db/test_di_pandas_import.py @@ -1,8 +1,7 @@ import pytest -import pandas as pd -from .. import conf -from ..test_db import db +from ..test_db import db, session, conf from .test_di_base import di_conf_file +from .db_fixtures import project, person @pytest.fixture def csv_file_for_import(tmp_path, di_conf_file): @@ -17,13 +16,13 @@ def csv_file_for_import(tmp_path, di_conf_file): return sample_file -def test_load_dataframe(csv_file_for_import, db): +def test_load_dataframe(csv_file_for_import, db, project, person): from odmf.dataimport import pandas_import as pi idescr = pi.ImportDescription.from_file(csv_file_for_import) df = pi.load_dataframe(idescr=idescr, filepath=csv_file_for_import) assert not df.empty -def test_load_dataframe_column_problem(csv_file_for_import, db): +def test_load_dataframe_column_problem(csv_file_for_import, db, project, person): """ This test is for issue #103, to see if it is working with total_columns https://github.com/jlu-ilr-hydro/odmf/issues/103