diff --git a/odmf/__init__.py b/odmf/__init__.py index a6625246..8df826d9 100644 --- a/odmf/__init__.py +++ b/odmf/__init__.py @@ -1,2 +1,2 @@ -__version__ = '2024.8.28' +__version__ = '2024.16.12dev' prefix = '.' diff --git a/odmf/plot/draw_plotly.py b/odmf/plot/draw_plotly.py index f96373ec..eef1c27b 100644 --- a/odmf/plot/draw_plotly.py +++ b/odmf/plot/draw_plotly.py @@ -3,12 +3,11 @@ Each backend needs to implement the function to_image(plot, format, dpi) and to_html(plot). """ - +import pandas as pd from plotly.subplots import make_subplots import plotly.graph_objects as go -import io +from datetime import datetime from . import Plot, Line -from ..config import conf def _draw_line(line: Line, start, end) -> go.Scatter: @@ -17,7 +16,13 @@ def _draw_line(line: Line, start, end) -> go.Scatter: mode = '' linestyle = None marker = None - if line.linestyle: + if line.linestyle == 'bar': + marker = { + 'color': line.color, + 'line': {'color': '#000000', 'width': 1} + } + return go.Bar(x=data.index, y=data, marker=marker, name=line.name) + elif line.linestyle: mode = 'lines' dash_dict = {'-': 'solid', ':': 'dot', '.': 'dot', '--': 'dash', '-.': 'dashdot'} linestyle = {'color': line.color, 'dash': dash_dict[line.linestyle], 'width': line.linewidth} @@ -31,7 +36,7 @@ def _draw_line(line: Line, start, end) -> go.Scatter: marker = {'color': line.color, 'symbol': symbol} - return go.Scatter(x=data.index, y=data, mode=mode, line=linestyle, marker=marker, name=line.name) + return go.Scatter(x=data.index, y=data, mode=mode, line=linestyle, marker=marker, name=line.name, connectgaps=False) def _make_figure(plot: Plot) -> go.Figure: @@ -54,6 +59,24 @@ def _make_figure(plot: Plot) -> go.Figure: rows=rows, cols=cols ) + for i, sp in enumerate(plot.subplots): + if sp.logsite in [l.siteid for l in sp.lines]: + # Traverse logs and draw them + logtime: datetime + for logtime, logtype, logtext in sp.get_logs(): + x = logtime.timestamp() * 1000 + row, col = 1 + i // plot.columns, 1 + i % plot.columns + fig.add_vline( + x, row=row, col=col, + opacity=0.5, + line={ + 'color': '#FF0000', + 'width': 4, + }, + annotation={ + 'text': f'{logtype}: {logtext}', + } + ) fig.update_yaxes() fig.update_layout(width=plot.size[0], height=plot.size[1], template='none') diff --git a/odmf/plot/plot.py b/odmf/plot/plot.py index 65b69818..98191172 100644 --- a/odmf/plot/plot.py +++ b/odmf/plot/plot.py @@ -188,7 +188,7 @@ def get_logs(self): logs = session.query(db.Log).filter_by(_site=self.logsite).filter( db.Log.time >= start).filter(db.Log.time <= end) return [ - (log.time, log.type, str(log)) + (log.time, log.type, log.message) for log in logs ] @@ -220,7 +220,7 @@ class Plot: Represents a full plot (matplotlib figure) """ - def __init__(self, height=None, width=None, columns=None, start=None, end=None, **kwargs): + def __init__(self, height=None, width=None, columns=None, start=None, end=None, name=None, path=None, aggregate=None, description=None, **kwargs): """ @param size: A tuple (width,height), the size of the plot in inches (with 100dpi) @param columns: number of subplot columns @@ -240,11 +240,13 @@ def __init__(self, height=None, width=None, columns=None, start=None, end=None, self.start = start or -90 self.end = end or -90 self.size = (width or 640, height or 480) + self.name = name or '' + self.path = path or '' + self.aggregate = aggregate or '' + self.description = description or '' + self.columns = columns or 1 self.subplots = [] - self.name = kwargs.pop('name', '') - self.aggregate = kwargs.pop('aggregate', '') - self.description = kwargs.pop('description', '') self.subplots = [ Subplot(self, **spargs) for i, spargs in enumerate(kwargs.pop('subplots', [])) @@ -266,7 +268,6 @@ def get_time_span(self): def lines(self): - return [line for sp in self.subplots for line in sp.lines] def fontsize(self, em): @@ -279,9 +280,11 @@ def __jdict__(self): """ Creates a dictionary with all properties of the plot, the subplots and their lines """ - return dict(width=self.size[0], height=self.size[1], columns=self.columns, - start=self.start, end=self.end, - subplots=asdict(self.subplots), - aggregate=self.aggregate, - description=self.description) + return dict( + width=self.size[0], height=self.size[1], columns=self.columns, + start=self.start, end=self.end, aggregate=self.aggregate, + name=self.name, path=self.path, + description=self.description, + subplots=asdict(self.subplots), + ) diff --git a/odmf/static/media/help/import/lab.md b/odmf/static/media/help/import/lab.md index a9398d34..a8cca5a7 100644 --- a/odmf/static/media/help/import/lab.md +++ b/odmf/static/media/help/import/lab.md @@ -11,10 +11,14 @@ Analysis carried out in a lab usually result in Excel files with a somewhat enco The samples in one batch can origin from multiple sites. Such files can be described with `.labimport` file. This import is ment for rather small datafiles, like a couple of hundred rows, as it is rather slow. + + The `.labimport` file consists of two main parts: -1. The decription of the **file format** eg. how to interprete the file as a table -2. the decription of each column. For each value in the table a fitting dataset **must** exist. Creating - a `.labimport` file is difficult and should be mentored by a person with programming experience. +1. The description of the **file format** eg. how to interprete the file as a table +2. the description of each column. For each value in the table a fitting dataset **must** exist. + If you create new datasets, the start and end must be set to a timerange inside the time range covered by the data. + +Creating a `.labimport` file is difficult and should be mentored by a person with programming experience. ## File format @@ -88,7 +92,7 @@ A column indicating the level (eg. depth) of a sampling. If the dataset is given This is a very complex type and can be used to derive metadata like site, time, depth from a sample name (this is called parsing). If additional columns for this information exists already, use the type `samplename`, then no parsing takes place and the -name is only copied. This is not easy - if you are no programmer, you might need help from someone with programming kno +name is only copied. This is not easy - if you are no programmer, you might need help from someone with programming knowledge The sample column contains a regex-pattern to describe the content of the sample name. [Regular expression (regex)](https://en.wikipedia.org/wiki/Regular_expression) is @@ -113,7 +117,7 @@ This pattern matches sample names like F1_4.12.2023_12:54_60 and can mean a samp at 12:54 in a depth of 60 cm. To derive the site, date and level, the parts must be translated. The date can be translated by the group number -(in the example 2) and the date format, in the example %d.%m.%Y_%H:%M using Pythons notation of date formats +(in the example 2) and the date format, in the example %d.%m.%Y_%H:%M using [Pythons notation of date formats](https://docs.python.org/3/library/datetime.html#format-codes) (which is used by a number of other programming languages). The site needs also the right group (in the example 1), and you can add a map to translate site names into site id's of the database. If your data uses already the official site id's, that map can be left out. @@ -143,6 +147,10 @@ The table may look like: | F1_6.5.2023_11:15_60 |2.5785|0.9456 | Site: F1 (#137), time: May 6th, 2023 in 60cm depth | B1_7.5.2023_12:45 |2.5785|0.9456 | Site: B1 (#123), time: May 7th, 2023 no level +Note that the date/time is encoded in the sample with a 4-digit year (6.5.2023_11:15 => Myy 6th, 2023, 11:15) and uses +as a date format `%d.%m.%Y_%H:%M`. The upper case `%Y` is the 4-digit year. [More about date formats...](https://docs.python.org/3/library/datetime.html#format-codes) + + ## `.labimport` file ``` driver: read_excel # pandas function to read the table. See: https://pandas.pydata.org/docs/reference/io.html @@ -164,7 +172,7 @@ columns: # Description of each column, use the column name as ob B3: 203 time: group: 2 - format: "%d.%m.%y_%H:%M" + format: "%d.%m.%Y_%H:%M" level: group: 3 factor: -0.01 @@ -196,6 +204,8 @@ twice, for higher accuracy. The duplicated measurements should be aggregated as | | 19_030321_10:42 | 0.1213 | 25.5579| n.a.| | 17.2184| 27.4383| 0.6143 | | 134_030321_11:28 | 0.1544 | 5.9271| n.a.| | 18.0146| 11.4721| n.a. +Note that the date/time is encoded in the sample with a 2-digit year (030321_10:30 => March 3rd, 2021, 10:30) and uses +as a date format `%d%m%y_%H:%M`. The lower case `%y` is the 2-digit year. [More about date formats...](https://docs.python.org/3/library/datetime.html#format-codes) ## `.labimport` file diff --git a/odmf/static/media/js/plot.js b/odmf/static/media/js/plot.js index 4727bebd..6c58ab35 100644 --- a/odmf/static/media/js/plot.js +++ b/odmf/static/media/js/plot.js @@ -10,19 +10,15 @@ function seterror(jqhxr ,textStatus, errorThrown) { } function gettime(startOrEnd) { - let timespan = $('#timeselect').val() + let timespan = $('#prop-timeselect').val() if (timespan < 0) { return timespan * 1; } - let res = $('#'+ startOrEnd + 'date').val(); + let res = $('#prop-'+ startOrEnd + 'date').val(); if (res) { - res += ' ' + ($('#'+ startOrEnd + 'time').val() || '00:00:00'); + res += ' ' + ($('#prop-'+ startOrEnd + 'time').val() || '00:00:00'); } else { - let today = new Date(); - if (startOrEnd == 'start') { - today.setFullYear(today.getFullYear() - 1) - } - res = today.toISOString(); + return -90 } return res; } @@ -72,7 +68,7 @@ function set_content_tree_handlers() { } }) - + $('.moveline').on('click', event =>{ let btn = $(event.currentTarget) let sp = plot.subplots[btn.data('subplot')] @@ -86,20 +82,24 @@ function set_content_tree_handlers() { } }) - $('.sp-logsite-button').on('click', event =>{ + $('.sp-logsite-list').on('change', event =>{ + let div = $(event.currentTarget) + let subplot=div.data('subplot'); + let site = div.val() * 1; + window.plot.subplots[subplot].logsite = site || null; + window.plot.apply() + }); + $('.sp-tools').on('show.bs.dropdown', event =>{ let subplot=$(event.currentTarget).data('subplot'); - let html = `
no logs
\n` + - window.plot.get_sites(subplot).map( - value => `
#${value}
` - ).reduce((acc, v)=>acc + '\n' + v) - $(`.sp-logsite-list[data-subplot=${subplot}]`).html(html) - $(`.sp-logsite-item[data-subplot=${subplot}]`).on('click', event =>{ - let div = $(event.currentTarget) - let subplot=div.data('subplot'); - let site = div.data('site'); - window.plot.subplots[subplot].logsite = site || null; - window.plot.apply() - }); + let act_site = window.plot.subplots[subplot].logsite + let html = `\n` + let sites = window.plot.get_sites(subplot).map(value => ``) + if (sites.length) html += sites.reduce((acc, v)=>acc + '\n' + v) + + let sp_select = $(`.sp-logsite-list[data-subplot=${subplot}]`) + sp_select.html(html) + sp_select.val(act_site) + }); $('.sp-remove-button').on('click', event=>{ @@ -118,6 +118,7 @@ class Plot { let saved_plot = JSON.parse(sessionStorage.getItem('plot')) if (saved_plot && !$.isEmptyObject(saved_plot)) { this.name = saved_plot.name || '' + this.path = saved_plot.path || '' this.start = saved_plot.start || gettime('start') this.end = saved_plot.end || gettime('end') this.columns = saved_plot.columns || 1 @@ -128,6 +129,7 @@ class Plot { } else { this.name = 'Unnamed plot' + this.path = '' this.start = gettime('start') this.end = gettime('end') this.columns = 1 @@ -144,7 +146,7 @@ class Plot { this[key] = obj[key] } } - this.apply() + this.render() return this } render(do_apply=true) { @@ -183,6 +185,8 @@ class Plot { return this.apply() } + /** Updates all plot related widgets on the site using the current plot object + **/ apply(width, height) { if (width) { this.width = width @@ -194,8 +198,32 @@ class Plot { } let txt_plot = JSON.stringify(this, null, 4); $('#plot-name').html(this.name) + let href = odmf_ref('/download/' + this.path) + $('#plot-path').html(this.path + '/' + this.name + '.plot') + + + if (plot.start < 0) { + $('#prop-timeselect').val(plot.start) + let today = new Date(); + $('#prop-enddate').val(today.toISOString().split(/[T,\s]/)[0]) + today.setFullYear(today.getFullYear() - 1) + $('#prop-startdate').val(today.toISOString().split(/[T,\s]/)[0]) + + + } else if (plot.start) { + $('#timeselect').val('') + $('#startdate').val(plot.start.split(/[T,\s]/)[0]) + } + if (plot.end && !(plot.end <0)) { + $('#enddate').val(plot.end.split(/[T,\s]/)[0]) + } + $('#prop-columns').val(plot.columns).attr('max', Math.max(1, plot.subplots.length)) + $('#prop-aggregate').val(plot.aggregate || '') + $('#prop-description').val(plot.description) + $('#content-tree .subplot').remove(); let autoreload = $('#autoreload_switch').prop('checked'); + this.subplots.forEach((subplot, index) => { let txt = $('#subplot-template').html() .replace(/§position§/g, index) @@ -212,9 +240,10 @@ class Plot { nl.before(line_html); }) $('#ct-new-subplot').before(obj); + }) sessionStorage.setItem('plot', txt_plot); - $('#property-summary').html(plot.toString()) + $('#property-summary').html($('#prop-timeselect :selected').text() + ' / ' + $('#prop-aggregate :selected').text()) $('#json-row pre').html(txt_plot); set_content_tree_handlers(); if (autoreload) { @@ -262,9 +291,11 @@ class Plot { toString() { if (this.start < 0) { - return 'last ' + (this.start*-1) + ' days, aggregate: ' + this.aggregate + let txt = 'last ' + (this.start*-1) + ' days' + if (this.aggregate) txt += ' agg: ' + this.aggregate + return txt } - return this.start.toString().slice(0,10) + ' - ' + this.end.toString().slice(0,10) + ', aggregate: ' + this.aggregate + return this.start.toString().slice(0,10) + ' - ' + this.end.toString().slice(0,10) + ', agg: ' + this.aggregate } } @@ -378,22 +409,22 @@ function set_line_dialog_handlers() { let dlg =$('#newline-dialog') let sp = button.data('subplot') let ln = button.data('lineno') - dlg.data('subplot', sp); - dlg.data('lineno', ln); - dlg.data('replace', button.data('replace')); - let plot = window.plot; - let line = {linestyle: '-'} - if (!(sp >= 0)) { - $('#error').html('#' + button.id + ' has no subplot').parent().parent().fadeIn() + dlg.data('subplot', sp); + dlg.data('lineno', ln); + dlg.data('replace', button.data('replace')); + let plot = window.plot; + let line = {linestyle: '-'} + if (!(sp >= 0)) { + $('#error').html('#' + button.id + ' has no subplot').parent().parent().fadeIn() } - else if (ln >= 0) { - line = plot.subplots[sp].lines[ln] + else if (ln >= 0) { + line = plot.subplots[sp].lines[ln] } lineDialogPopSelect(line.valuetype, line.site, ()=>{line_to_dialog(line)}) - }); + }); - $('#newline-dialog .dataset-select').on('change', () => { + $('#newline-dialog .dataset-select').on('change', () => { let dlg =$('#newline-dialog') let line = line_from_dialog() let valuetype = parseInt($('#nl-value').val()) @@ -408,7 +439,7 @@ function set_line_dialog_handlers() { }); - $('#newline-dialog .nl-style').on('change', () => { + $('#newline-dialog .nl-style').on('change', () => { let valuetype = parseInt($('#nl-value').val()) let site = parseInt($('#nl-site').val()) let linestyle = $('#nl-linestyle').val() @@ -421,9 +452,9 @@ function set_line_dialog_handlers() { }) - $('#nl-OK').on('click', () => { - let dlg =$('#newline-dialog') - let plot = window.plot + $('#nl-OK').on('click', () => { + let dlg =$('#newline-dialog') + let plot = window.plot let line = line_from_dialog() let sp_no = dlg.data('subplot') let line_no = dlg.data('lineno') @@ -478,27 +509,29 @@ $(() => { $('#addsubplot').prop('disabled', false); $('#btn-clf').on('click', function() { - let plot = window.plot - plot.subplots = []; - plot.aggregate = null - plot.columns = 1 - plot.apply() + if (confirm('Are you sure to delete the plot?')) { + let plot = window.plot + plot.subplots = []; + plot.aggregate = null + plot.columns = 1 + plot.apply() + } }); $('#addsubplot').on('click', e => { window.plot.addsubplot() }) - // Fluid layout doesn't seem to support 100% height; manually set it - $(window).resize(() => { - let plotElement = $('#plot'); - let po = plotElement.offset(); - po.totalHeight = $(window).height(); + // Fluid layout doesn't seem to support 100% height; manually set it + $(window).resize(() => { + let plotElement = $('#plot'); + let po = plotElement.offset(); + po.totalHeight = $(window).height(); po.em1 = parseFloat(getComputedStyle(plotElement[0]).fontSize); - let plotHeight = po.totalHeight - po.top - 2 * po.em1; - plotElement.height(plotHeight); + let plotHeight = po.totalHeight - po.top - 2 * po.em1; + plotElement.height(plotHeight); window.plot.apply(plotElement.width(), plotHeight) - }); + }); - $(window).resize(); + $(window).resize(); set_line_dialog_handlers() @@ -506,32 +539,74 @@ $(() => { window.plot.render() }); + $('.do-apply').on('change', () => { + window.plot.apply() + }) + $('#plot-save').on('click', () => { + $.post(odmf_ref('/plot/filedialog/save/'), { + plot: JSON.stringify(window.plot, null, 4), + path: window.plot.path + '/' + window.plot.name + '.plot' + }) + .done(() => {window.plot.apply()}) + .fail(seterror) + }) + - $('#fig-export button').on('click', event => { + $('.figure-export').on('click', event => { let fmt=$(event.currentTarget).data('format'); if (fmt) { download_on_post('image', {format: fmt, plot: JSON.stringify(window.plot, null, 4)}) } }) - $('#property-dialog').on('show.bs.modal', event => { - $('#property-dialog-content').load('property/') - }) $('#file-dialog').on('show.bs.modal', event => { - $('#file-dialog-content').load('filedialog/') + let path= encodeURIComponent(window.plot.path) + $('#file-dialog-content').load('filedialog/?path=' + path) }) $('#export-dialog').on('show.bs.modal', makeExportDialog) - + $('#autoreload_switch').on('change', event=> { + if ($(event.currentTarget).prop('checked')) { + window.plot.render() + } + }) $('.killplotfn').on('click', function() { var fn = $(this).html(); if (confirm('Do you really want to delete your plot "' + fn + '" from the server')) $.post('deleteplotfile',{filename:fn},seterror); }); - $('#autoreload_div').on('click', event => { - alert('hä?') - window.plot.render() + + $('#manualTimeControl').toggleClass('d-none', plot.start < 0) + + $('.prop-time').on('change', event => { + let start = $('#prop-timeselect').val() + $('#manualTimeControl').toggleClass('d-none', !(start==='manual')) + + plot.start = gettime('start') + plot.end = gettime('end') + plot.apply() + + }) + $('#prop-aggregate').on('change', () => { + plot.aggregate = $('#prop-aggregate').val() + plot.apply() + }) + $('#prop-columns').on('change', () => { + plot.columns = parseInt($('#prop-columns').val()) + plot.apply() + }) + $('#prop-description').on('change', () => { + plot.description = $('#prop-description').val() + }) + + $('#prop-OK').on('click', event => { + let plot = window.plot + plot.apply() + }); + + + }); diff --git a/odmf/static/templates/log.html b/odmf/static/templates/log.html index a60a8055..a24451c0 100644 --- a/odmf/static/templates/log.html +++ b/odmf/static/templates/log.html @@ -176,7 +176,7 @@

-