Merge branch 'refs/heads/master' into develop

# Conflicts: # odmf/__init__.py
jlu-ilr-hydro · Jul 15, 2024 · 1cc5168 · 1cc5168
2 parents 2eea41f + 706389a
commit 1cc5168
Show file tree

Hide file tree

Showing 26 changed files with 442 additions and 228 deletions.
diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml
@@ -6,7 +6,6 @@ name: ODMF - Observatory Data Management Framework
 on:
   push:
     branches: [ master ]
-    paths: [ .github/workflows ]
   pull_request:
     branches: [ master ]
 
@@ -17,7 +16,9 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.11" ]
+        python-version: 
+        - "3.11"
+        - "3.12"
 
     steps:
       - uses: actions/checkout@v4

diff --git a/README.md b/README.md
@@ -26,7 +26,7 @@ While the docker installation in preferred, odmf can be installed in a python vi
 
 Install virtual environment in directory `venv` and upgrade infrastructure, and activate it
 
-    $ python3.9 -m venv venv
+    $ python -m venv venv
     $ source venv/bin/activate
     $ python -m pip install --upgrade pip wheel setuptools
 
@@ -39,13 +39,3 @@ Check command line tool
     $ odmf --help
 
 
-    $ pip install -e odmf-src
-    $ odmf --help
-    $ 
-
-Change the config with the instructions from [conf.py](https://jlu-ilr-hydro.github.io/odmf/source/conf.py) wiki page.
-When the configuration is edited to meet your requirements, start the server and browse to https://localhost:8080
-
-    $ python3 start.py
-
-[Visit the institute's homepage](http://www.uni-giessen.de/faculties/f09/institutes/ilr/hydro?set_language=en)
diff --git a/odmf/dataimport/base.py b/odmf/dataimport/base.py
@@ -477,7 +477,7 @@ def config_getlist(section, option):
 
             if project:
                 rows = session.query(db.Project)\
-                    .filter(db.Project.id == project).count()
+                    .filter(db.Project.id == int(project)).count()
                 if rows != 1:
                     raise ValueError('Error in import description: \'%s\' is no'
                                      ' valid project identifier' % project)

diff --git a/odmf/dataimport/pandas_import.py b/odmf/dataimport/pandas_import.py
@@ -31,7 +31,7 @@ def __init__(self, session, idescr: ImportDescription, col: ImportColumn,
                  id: int, user: db.Person, site: db.Site, inst: db.Datasource,
                  valuetypes: typing.Dict[int, db.ValueType], raw: db.Quality,
                  start: datetime.datetime, end: datetime.datetime,
-                 filename: typing.Optional[str] = None
+                 filename: typing.Optional[str] = None, project: db.Project = None
                  ):
 
         self.column = col
@@ -62,7 +62,7 @@ def __init__(self, session, idescr: ImportDescription, col: ImportColumn,
                 access=col.access if col.access is not None else 1,
                 # Get timezone from descriptor or, if not present from global conf
                 timezone=idescr.timezone or conf.datetime_default_timezone,
-                project=idescr.project)
+                project=project)
             session.add(self.dataset)
 
         self.id = self.dataset.id
@@ -104,6 +104,7 @@ def columndatasets_from_description(
         inst = session.get(db.Datasource, idescr.instrument)
         user = session.get(db.Person, user)
         site = session.get(db.Site, siteid)
+        project = session.get(db.Project, idescr.project)
         # Get "raw" as data quality, to use as a default value
         raw = session.get(db.Quality, 0)
         # Get all the relevant valuetypes (vt) from db as a dict for fast look up
@@ -123,7 +124,7 @@ def columndatasets_from_description(
                     ColumnDataset(
                         session, idescr, col, None,
                         user, site, inst, valuetypes, raw,
-                        start, end, filepath.name
+                        start, end, filepath.name, project
                     )
                 )
             elif not col.ds_column:
@@ -132,7 +133,7 @@ def columndatasets_from_description(
                         session, idescr, col,
                         newid + len(newdatasets),
                         user, site, inst, valuetypes, raw,
-                        start, end, filepath.name
+                        start, end, filepath.name, project
                     )
                 )
 

diff --git a/odmf/db/person.py b/odmf/db/person.py
@@ -26,7 +26,7 @@ class Person(Base):
     mobile = sql.Column(sql.String)
     car_available = sql.Column(sql.Integer, default=0)
     password = sql.Column(sql.VARCHAR)
-    access_level = sql.Column(sql.INTEGER)
+    access_level = sql.Column(sql.INTEGER, nullable=False, default=0)
     active = sql.Column(sql.Boolean, default=True, nullable=False)
 
     def __str__(self):

diff --git a/odmf/db/project.py b/odmf/db/project.py
@@ -40,6 +40,7 @@ def members(self, access_level=0, with_responsible=True):
         if not self.session(): # For a new project no session and no member exists!
             return None
         from ..webpage.auth import Level
+        access_level = Level(access_level)
         for pm in (
                 self.members_query.filter(ProjectMember.access_level>=access_level)
                         .order_by(ProjectMember.access_level.desc(), ProjectMember._member)

diff --git a/odmf/plot/plot.py b/odmf/plot/plot.py
@@ -105,7 +105,11 @@ def load(self, start=None, end=None):
             series = group.asseries(session, self.name)
 
         if self.subplot.plot.aggregate:
-            sampler = series.resample(self.subplot.plot.aggregate)
+            if self.subplot.plot.aggregate == 'decade':
+                from ..tools.exportdatasets import DecadeMonthStart
+                sampler = series.resample(DecadeMonthStart())
+            else:
+                sampler = series.resample(self.subplot.plot.aggregate)
             series = sampler.aggregate(self.aggregatefunction or 'mean')
 
         # There were problems with arrays from length 0

diff --git a/odmf/static/media/help/export/index.md b/odmf/static/media/help/export/index.md
@@ -0,0 +1,58 @@
+# Export data from the structured database
+
+## Export observations
+
+### Export plotted data
+
+The most convenient way to export observation records is to construct a plot of the data
+and export the underlying data. A click on "export" opens the export Window, where you can choose the data
+format, the output timesteps and evtl. interpolation methods
+
+#### Data formats
+- Excel file
+- Textfile with comma seperated variables (CSV)
+- Textfile with tab seperated variables (TSV)
+- JSON Notation (best for web applications)
+- Apache parquet format (best for data science applications in eg. Python and R)
+
+### Time steps
+
+The next option chooses the scheme, which time steps should be exported. If you have regular time step
+data, eg. from a logger or you have created aggregated timesteps, the selection hardly matters. If
+the data you want to export has very irregular timesteps or different times, use the options below
+to get data at the time steps you are really interested.
+The options are:
+
+- union: All timesteps from all datasets are exported as an own row. Records with the same time are
+  put on the same line. If a plot line has no datapoint at the same time as another, a blank field is
+  exported. To join nearly identical time steps, you can use a tolerance (see below) 
+- intersection: Export only time steps where all plot lines have records. To get more exported lines,
+  tolerances are used again
+- Regular time steps: Define a time grid, eg. every day (`D`) or every three hours (`3h`) for export.
+  records with another time are only exported if they fit the output time tolerance
+- Timesteps from plot line: Use the time steps of a certain line and export data from other lines if their
+  data points are within the tolerance
+
+### Regular grid and tolerance
+
+If regular grid is selected you can specify the grid using frequncy codes. ODMF uses a
+[library which defines many codes](https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases). The most important are:
+- `s` Second
+- `min` Minute
+- `h` Hour
+- `D` Day
+- `B` Business day
+- `W` Week
+- `M` Month
+- `Y` Year
+
+These codes can be scaled, you can export biweekly with `2W` or every 3 hours as `3h`. A special code,
+that cannot be scaled is `decade`, here you get the data reported for three parts of a month, 
+day 1-10, day 11-20 and day 21-28/29/30/31.
+
+## Export metadata
+
+Exporting metadata is currently under development. You can export **all** sites at the site list, but 
+the export does not respect the filters.
+
+Exporting datasets, logs and pictures is not developed yet.
diff --git a/odmf/static/media/js/dataset-edit.js b/odmf/static/media/js/dataset-edit.js
@@ -170,7 +170,7 @@ $(function() {
                     $('#error').html(data);
                     $('#error-row').removeClass('d-none');
                 } else {
-                    window.location.href = odmf_ref('/dataset/?success=ds' + btn.data('dsid') + ' deleted');
+                    window.location.href = odmf_ref('/dataset/?message=ds' + btn.data('dsid') + ' deleted');
                 }
             });
     }

diff --git a/odmf/static/templates/dataset-edit.html b/odmf/static/templates/dataset-edit.html
@@ -230,11 +230,7 @@ <h4 class="col-lg-4">Sorry, ${user()}</h4>
                             title="Create a transformed dataset for this">
                         $~f(x)~$ transform
                     </button>
-                    <button py:if="access(Level.admin)" class="btn btn-outline-danger mr-2"
-                            title="remove dataset permanently" data-toggle="tooltip" id="removeds"
-                            data-dsid="${ds_act.id}" data-dsname="${str(ds_act)}" data-dssize="${ds_act.size()}">
-                        <i class="fas fa-trash mr-2"/> delete dataset
-                    </button>
+
                 </div>
                 <div id="edit" class="tab-pane w-100 mr-4 ${['', 'active'][not ds_act.is_defined()]}" role="tabpanel" py:if="access(Level.editor)">
                     <form action="save" method="post">
@@ -555,7 +551,11 @@ <h4 class="card-title">Some help for the Python and latex expressions</h4>
                                 <button type="submit" id="save" name="save" class="btn btn-success mr-2" >
                                     <i class="fas fa-check mr-2"/> save changes
                                 </button>
-
+                                <button py:if="access(Level.admin)" class="btn btn-danger mr-2"
+                                        title="remove dataset permanently" data-toggle="tooltip" id="removeds"
+                                        data-dsid="${ds_act.id}" data-dsname="${str(ds_act)}" data-dssize="${ds_act.size()}">
+                                    <i class="fas fa-trash mr-2"/> delete dataset
+                                </button>
                             </div>
                         </div>
                     </form>

diff --git a/odmf/static/templates/download.html b/odmf/static/templates/download.html
@@ -25,9 +25,10 @@
                 });
                 $('.copy-button').on('click', function() {
                     let name = $(this).data('name');
-                    let newfilename = window.prompt("New filename", 'copy_of_' + name)
+                    let action = $(this).data('action')
+                    let newfilename = window.prompt("New filename", action + '_of_' + name)
                     if (newfilename) {
-                        $.post(odmf_ref('/download/copyfile/'), {dir: '${curdir}',filename: name, newfilename: newfilename},
+                        $.post(odmf_ref('/download/copyfile/'), {dir: '${curdir}',filename: name, newfilename: newfilename, action:action},
                             function(success_url) {
                                 window.location = success_url;
                             }
@@ -155,59 +156,93 @@ <h5 class="card-title">
                             <span py:for="bc in curdir.breadcrumbs()">/<a href="${bc.href}" py:content="bc.basename" /></span>
                             <span class="badge bg-warning ml-8 float-right" py:content="modes[curdir].name"/>
                         </h5>
-                        <ul class="nav flex-column card-text mb-3" id="files">
-
-                            <li class="nav-item flexbox mt-1 pt-1 border-top" py:if="not curdir.isroot()" >
+                        <div class="list-group flex-column card-text mb-3" id="files">
+                            <a class="list-group-item list-group-action flexbox mt-1 pt-1 border-top"
+                               py:if="not curdir.isroot()"
+                               href="${curdir.parent().href}"
+                            >
                                 <span >
-                                    <a href="${curdir.parent().href}" class="nav-link">
-                                        <i class="fas fa-folder-open" />
-                                        <span >..</span>
-                                    </a>
+                                    <i class="fas fa-folder-open" />
+                                    <span >..</span>
                                 </span>
-                            </li>
-                            <li py:for="d in directories" class="nav-item flexbox mt-1 pt-1 border-top">
-                                <span class="nav-link">
-                                    <a href="${d.href}" class="${'' if modes[d] else 'disabled'}">
+                            </a>
+                            <div py:for="d in directories"
+                               py:if="modes[d]"
+                               class="list-group-item list-group-item-action"
+                               href="${d.href}"
+                            >
+                                <nav class="nav flexbox">
+                                    <a href="${d.href}" class="nav-link active">
                                         <i class="fas fa-folder fa-lg" />
                                         <span py:content="d.basename" />
                                     </a>
-                                    <span class="badge bg-warning text-bg-warning" py:content="modes[d].name"/>
-                                </span>
-                                <span>
-                                    <button class="btn btn-danger remove-button" data-name="${d.basename}"
-                                            py:if="modes[d]>=Mode.admin and d.isempty()"
-                                            title="remove directory" data-toggle="tooltip">
-                                        <i class="fas fa-trash-can" />
-                                    </button>
-                                </span>
-                            </li>
-                            <li py:for="f in files" class="nav-item flexbox mt-1 pt-1 border-top ${'hiddenfile' if f.ishidden() else ''}" >
-                                <span >
-                                    <a href="${f.href}" class="nav-link">
+                                    <div class="dropdown">
+                                        <span class="badge bg-warning text-bg-warning" py:content="modes[d].name"/>
+                                        <button class="btn btn-sm btn-light dropdown-toggle"
+                                                type="button" data-toggle="dropdown"
+                                                aria-expanded="false" aria-haspopup="true"
+                                                py:if="modes[curdir]>=Mode.write">
+
+                                        </button>
+
+                                        <div class="dropdown-menu" >
+                                            <a class="dropdown-item copy-button" data-name="${d.basename}" data-action="rename"
+                                               py:if="modes[d]>=Mode.admin">
+                                                <i class="fas fa-pen-to-square" /> rename
+                                            </a>
+                                            <a class="dropdown-item remove-button" data-name="${d.basename}"
+                                               py:if="modes[d]>=Mode.admin and d.isempty()">
+                                                <i class="fas fa-trash-can" /> remove empty directory
+                                            </a>
+                                        </div>
+                                    </div>
+                                </nav>
+                            </div>
+                            <div py:for="f in files" class="list-group-item  border-top ${class_if(f.ishidden(), 'hiddenfile')}">
+                                <nav class="nav flexbox">
+                                    <a href="${f.href}" class="nav-link active">
                                         <i class="fas fa-${handler[f].icon} fa-lg" />
                                         <span py:content="f.basename" class="mr-auto"/>
                                         <span class="ml-4 badge badge-secondary" py:content="f.formatsize()" />
                                     </a>
-                                </span>
-                                <span>
 
-                                    <button py:if="f.basename in import_history" class="btn btn-success" aria-expanded="false" aria-controls="import-history-${f.basename}"
-                                            data-toggle="popover" title="Already imported" data-content="${markdown(import_history[f.basename])}" data-html="true"
-                                    >
-                                        <i class="fas fa-check"/>
-                                    </button>
-                                    <button class="btn btn-secondary copy-button" data-name="${f.basename}" py:if="modes[curdir]>=Mode.write"
-                                            title="copy file" data-toggle="tooltip">
-                                        <i class="fas fa-copy" />
-                                    </button>
-                                    <button class="btn btn-danger remove-button" data-name="${f.basename}" py:if="modes[curdir]>=Mode.admin"
-                                            title="remove file" data-toggle="tooltip">
-                                        <i class="fas fa-trash-can" />
-                                    </button>
-                                </span>
+                                    <div class="nav-item dropdown">
+                                        <a py:if="f.basename in import_history"
+                                             class="text-success" href="#"
+                                             aria-expanded="false" aria-controls="import-history-${f.basename}"
+                                             data-toggle="popover" title="Already imported" data-content="${markdown(import_history[f.basename])}" data-html="true"
+                                        >
+                                            <i class="fas fa-check"/>
+                                        </a>
+                                        <button class="btn btn-sm btn-light dropdown-toggle"
+                                                type="button" data-toggle="dropdown"
+                                                aria-expanded="false" aria-haspopup="true"
+                                                py:if="modes[curdir]>=Mode.write">
 
-                            </li>
-                        </ul>
+                                        </button>
+
+                                        <div class="dropdown-menu" >
+                                            <a class="dropdown-item copy-button" data-name="${f.basename}" py:if="modes[curdir]>=Mode.write" data-action="copy"
+                                            >
+                                                <i class="fas fa-copy" /> copy
+                                            </a>
+                                            <a class="dropdown-item copy-button" data-name="${f.basename}" py:if="modes[curdir]>=Mode.write" data-action="rename">
+                                                <i class="fas fa-file-pen"/> rename
+                                            </a>
+                                            <a class="dropdown-item remove-button"
+                                               data-name="${f.basename}" py:if="modes[curdir]>=Mode.admin"
+                                            >
+                                                <i class="fas fa-trash-can" /> delete
+                                            </a>
+
+                                        </div>
+
+                                    </div>
+
+                                </nav>
+                            </div>
+
+                        </div>
                         <div class="nav">
                             <button class="btn btn-primary dropdown-toggle mr-3 nav-item" type="button"
                                     id="addFileDropdownButton" data-toggle="dropdown" py:attrs="prop(disabled=modes[curdir]==Mode.read)"