diff --git a/HISTORY.rst b/HISTORY.rst index 41966c4..df32625 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,13 @@ History ======= +------ +v0.7.1 +------ + +- improved cache handling +- improved user feedback for filtering & differential expression + ------ v0.7.0 ------ diff --git a/README.rst b/README.rst index b699507..2dd146c 100644 --- a/README.rst +++ b/README.rst @@ -25,7 +25,7 @@ SCelVis: Easy Single-Cell Visualization | -.. image:: scelvis/assets/movie.gif +.. image:: tutorial/scelvis_movie.gif :height: 400px :align: center @@ -216,6 +216,7 @@ You can use the following environment variables to configure the server. - ``SCELVIS_CACHE_DIR`` -- directory to use for the cache (default is to create a temporary directory) - ``SCELVIS_CACHE_REDIS_URL`` -- enable caching with REDIS and provide connection URL - ``SCELVIS_CACHE_DEFAULT_TIMEOUT`` -- cache lifetime coverage +- ``SCELVIS_CACHE_PRELOAD_DATA`` -- will preload all data at startup - ``SCELVIS_UPLOAD_DIR`` -- the directory to store uploaded data sets in (default is to create a temporary directory) - ``SCELVIS_UPLOAD_DISABLED`` -- set to "0" to disable upload feature - ``SCELVIS_CONVERSION_DISABLED`` -- set to "0" to disable the conversion feature diff --git a/scelvis/app.py b/scelvis/app.py index 0ec6e18..5d725e5 100644 --- a/scelvis/app.py +++ b/scelvis/app.py @@ -51,6 +51,14 @@ # Setup the cache. cache.setup_cache(app) +# preload data if required +if settings.CACHE_PRELOAD_DATA: + from . import store + + logger.info("preloading data ...") + store.load_all_metadata() + logger.info("preloading done.") + # Set app title app.title = "SCelVis v%s" % __version__ @@ -58,8 +66,6 @@ app.css.config.serve_locally = True app.scripts.config.serve_locally = True -# TODO: Better use the approach from this URL: -# - https://community.plot.ly/t/dynamic-controls-and-dynamic-output-components/5519 app.config.suppress_callback_exceptions = True # Setup the application's main layout. @@ -75,13 +81,12 @@ callbacks.register_select_cell_plot_type(app) callbacks.register_update_cell_scatter_plot_params(app) callbacks.register_toggle_select_cells_controls(app) -callbacks.register_update_select_cells_choices(app) +callbacks.register_update_select_cells_selected(app) callbacks.register_activate_select_cells_buttons(app) callbacks.register_update_cell_violin_plot_params(app) callbacks.register_update_cell_bar_chart_params(app) callbacks.register_toggle_filter_cells_controls(app, "meta") -callbacks.register_update_filter_cells_controls(app, "meta") - +callbacks.register_update_filter_cells_controls(app) # Cellbacks for the "genes" tab pane. callbacks.register_select_gene_plot_type(app) @@ -90,7 +95,6 @@ callbacks.register_select_gene_violin_plot(app) callbacks.register_select_gene_dot_plot(app) callbacks.register_toggle_filter_cells_controls(app, "expression") -callbacks.register_update_filter_cells_controls(app, "expression") # callbacks for the filter cells div (on "cells" tab pane, but required for both) callbacks.register_update_filter_cells_filters(app) @@ -116,9 +120,6 @@ def find(name, path): if name in files: return os.path.join(root, name) - # TODO: error handling, error handling, error handling - # TODO: prettify the form - # TODO: protect against "zip bomb" if request.method == "POST": if "file" not in request.files or not request.files["file"].filename: flash("No file uploaded!") @@ -185,7 +186,6 @@ def find(name, path): out_file, mimetype="application/binary", as_attachment=True ) else: - # TODO: prettify HTML form return """ Convert File diff --git a/scelvis/assets/cells.png b/scelvis/assets/cells.png deleted file mode 100644 index 068ea18..0000000 Binary files a/scelvis/assets/cells.png and /dev/null differ diff --git a/scelvis/assets/genes.png b/scelvis/assets/genes.png deleted file mode 100644 index ef54022..0000000 Binary files a/scelvis/assets/genes.png and /dev/null differ diff --git a/scelvis/assets/movie.gif b/scelvis/assets/movie.gif deleted file mode 100644 index aec5a2a..0000000 Binary files a/scelvis/assets/movie.gif and /dev/null differ diff --git a/scelvis/callbacks.py b/scelvis/callbacks.py index 6ff8aad..fa8ee3b 100644 --- a/scelvis/callbacks.py +++ b/scelvis/callbacks.py @@ -135,13 +135,14 @@ def register_update_cell_scatter_plot_params(app): Input("meta_scatter_select_x", "value"), Input("meta_scatter_select_y", "value"), Input("meta_scatter_select_color", "value"), - Input("filter_cells_filters", "children"), + Input("meta_filter_cells_update", "n_clicks"), ], + [State("filter_cells_filters", "children"), State("select_cells_selected", "children")], ) - def get_meta_plot_scatter(pathname, xc, yc, col, filters_json): + def get_meta_plot_scatter(pathname, xc, yc, col, n_clicks, filters_json, select_json): _, kwargs = get_route(pathname) data = store.load_data(kwargs.get("dataset")) - return ui.cells.render_plot_scatter(data, xc, yc, col, filters_json) + return ui.cells.render_plot_scatter(data, xc, yc, col, filters_json, select_json) def register_toggle_select_cells_controls(app): @@ -156,34 +157,34 @@ def toggle_select_cells_controls(n, is_open): return is_open -def register_update_select_cells_choices(app): +def register_update_select_cells_selected(app): @app.callback( - Output("select_cells_choices", "children"), + Output("select_cells_selected", "children"), [ Input("meta_scatter_plot", "selectedData"), Input("select_cells_group_A", "n_clicks"), Input("select_cells_group_B", "n_clicks"), Input("select_cells_reset", "n_clicks"), ], - [State("select_cells_choices", "children")], + [State("select_cells_selected", "children")], ) - def update_select_cells_choices( - selectedData, n_clicks_A, n_clicks_B, n_clicks_reset, choices_json + def update_select_cells_selected( + selectedData, n_clicks_A, n_clicks_B, n_clicks_reset, selected_json ): ctx = dash.callback_context - if choices_json is not None: - choices = json.loads(choices_json) + if selected_json is not None: + selected = json.loads(selected_json) else: - choices = {} + selected = {} if ctx.triggered and "group_A" in ctx.triggered[0]["prop_id"] and selectedData is not None: - choices["group_A"] = [p["text"] for p in selectedData["points"]] + selected["group_A"] = [p["text"] for p in selectedData["points"]] elif ( ctx.triggered and "group_B" in ctx.triggered[0]["prop_id"] and selectedData is not None ): - choices["group_B"] = [p["text"] for p in selectedData["points"]] + selected["group_B"] = [p["text"] for p in selectedData["points"]] elif ctx.triggered and "reset" in ctx.triggered[0]["prop_id"]: - choices = {} - return json.dumps(choices) + selected = {} + return json.dumps(selected) def register_activate_select_cells_buttons(app): @@ -194,17 +195,17 @@ def register_activate_select_cells_buttons(app): Output("select_cells_reset", "disabled"), Output("select_cells_run", "disabled"), ], - [Input("meta_scatter_plot", "selectedData"), Input("select_cells_choices", "children")], + [Input("meta_scatter_plot", "selectedData"), Input("select_cells_selected", "children")], ) - def activate_select_cells_buttons(selectedData, choices_json): - if choices_json is not None: - choices = json.loads(choices_json) + def activate_select_cells_buttons(selectedData, selected_json): + if selected_json is not None: + selected = json.loads(selected_json) else: - choices = {} - disabled_A = (selectedData is None) | ("group_A" in choices) - disabled_B = (selectedData is None) | ("group_B" in choices) - disabled_reset = ("group_A" not in choices) & ("group_B" not in choices) - disabled_run = ("group_A" not in choices) | ("group_B" not in choices) + selected = {} + disabled_A = (selectedData is None) | ("group_A" in selected) + disabled_B = (selectedData is None) | ("group_B" in selected) + disabled_reset = ("group_A" not in selected) & ("group_B" not in selected) + disabled_run = ("group_A" not in selected) | ("group_B" not in selected) return (disabled_A, disabled_B, disabled_reset, disabled_run) @@ -215,15 +216,18 @@ def register_run_differential_expression(app): Output("select_cells_results", "children"), Output("select_cells_results_download", "href"), Output("select_cells_parameters_download", "href"), + Output("select_cells_status", "children"), Output("select_cells_get_results", "style"), + Output("meta_scatter_select_color", "options"), ], [ Input("url", "pathname"), Input("select_cells_run", "n_clicks"), - Input("select_cells_choices", "children"), + Input("select_cells_selected", "children"), ], + [State("meta_scatter_select_color", "options")], ) - def run_differential_expression(pathname, n_clicks, select_json): + def run_differential_expression(pathname, n_clicks, select_json, options): ctx = dash.callback_context if ctx.triggered and "run" in ctx.triggered[0]["prop_id"] and n_clicks: _, kwargs = get_route(pathname) @@ -231,11 +235,11 @@ def run_differential_expression(pathname, n_clicks, select_json): res = ui.cells.run_differential_expression(data, select_json) return res else: - return "", "", "", {"display": "none"} + return "", "", "", "", {"display": "none"}, options @app.callback( [Output("main-tabs", "active_tab"), Output("expression_toggle_gene_list", "value")], - [Input("select_cells_view", "n_clicks"), Input("select_cells_reset", "n_clicks")], + [Input("select_cells_view_table", "n_clicks"), Input("select_cells_reset", "n_clicks")], [State("expression_toggle_gene_list", "value"), State("main-tabs", "active_tab")], ) def switch_view(n1, n2, selected, at): @@ -249,6 +253,13 @@ def switch_view(n1, n2, selected, at): selected.remove("diffexp") return at, selected + @app.callback( + Output("meta_scatter_select_color", "value"), + [Input("select_cells_view_groups", "n_clicks")], + ) + def switch_color(n): + return "DE_group" + def register_update_cell_violin_plot_params(app): """Register handlers on violin plot.""" @@ -264,13 +275,14 @@ def register_update_cell_violin_plot_params(app): Input("meta_violin_select_vars", "value"), Input("meta_violin_select_group", "value"), Input("meta_violin_select_split", "value"), - Input("filter_cells_filters", "children"), + Input("meta_filter_cells_update", "n_clicks"), ], + [State("filter_cells_filters", "children")], ) - def get_meta_plot_violin(pathname, variables, group, split, choices_json): + def get_meta_plot_violin(pathname, variables, group, split, n, filters_json): _, kwargs = get_route(pathname) data = store.load_data(kwargs.get("dataset")) - return ui.cells.render_plot_violin(data, variables, group, split, choices_json) + return ui.cells.render_plot_violin(data, variables, group, split, filters_json) def register_update_cell_bar_chart_params(app): @@ -297,13 +309,14 @@ def toggle_meta_bar_options(split): Input("meta_bar_select_group", "value"), Input("meta_bar_select_split", "value"), Input("meta_bar_options", "value"), - Input("filter_cells_filters", "children"), + Input("meta_filter_cells_update", "n_clicks"), ], + [State("filter_cells_filters", "children")], ) - def get_meta_plot_bars(pathname, group, split, options, choices_json): + def get_meta_plot_bars(pathname, group, split, options, n, filters_json): _, kwargs = get_route(pathname) data = store.load_data(kwargs.get("dataset")) - return ui.cells.render_plot_bars(data, group, split, options, choices_json) + return ui.cells.render_plot_bars(data, group, split, options, filters_json) def register_select_gene_plot_type(app): @@ -407,13 +420,14 @@ def register_select_gene_scatter_plot(app): Input("expression_scatter_select_x", "value"), Input("expression_scatter_select_y", "value"), Input("expression_select_genes", "value"), - Input("filter_cells_filters", "children"), + Input("expression_filter_cells_update", "n_clicks"), ], + [State("filter_cells_filters", "children")], ) - def get_expression_plot_scatter(pathname, xc, yc, genelist, choices_json): + def get_expression_plot_scatter(pathname, xc, yc, genelist, n, filters_json): _, kwargs = get_route(pathname) data = store.load_data(kwargs.get("dataset")) - return ui.genes.render_plot_scatter(data, xc, yc, genelist, choices_json) + return ui.genes.render_plot_scatter(data, xc, yc, genelist, filters_json) def register_select_gene_violin_plot(app): @@ -430,13 +444,14 @@ def register_select_gene_violin_plot(app): Input("expression_select_genes", "value"), Input("expression_violin_select_group", "value"), Input("expression_violin_select_split", "value"), - Input("filter_cells_filters", "children"), + Input("expression_filter_cells_update", "n_clicks"), ], + [State("filter_cells_filters", "children")], ) - def get_expression_plot_violin(pathname, genelist, group, split, choices_json): + def get_expression_plot_violin(pathname, genelist, group, split, n, filters_json): _, kwargs = get_route(pathname) data = store.load_data(kwargs.get("dataset")) - return ui.genes.render_plot_violin(data, pathname, genelist, group, split, choices_json) + return ui.genes.render_plot_violin(data, pathname, genelist, group, split, filters_json) def register_select_gene_dot_plot(app): @@ -453,13 +468,14 @@ def register_select_gene_dot_plot(app): Input("expression_select_genes", "value"), Input("expression_dot_select_group", "value"), Input("expression_dot_select_split", "value"), - Input("filter_cells_filters", "children"), + Input("expression_filter_cells_update", "n_clicks"), ], + [State("filter_cells_filters", "children")], ) - def get_expression_plot_dot(pathname, genelist, group, split, choices_json): + def get_expression_plot_dot(pathname, genelist, group, split, n, filters_json): _, kwargs = get_route(pathname) data = store.load_data(kwargs.get("dataset")) - return ui.genes.render_plot_dot(data, pathname, genelist, group, split, choices_json) + return ui.genes.render_plot_dot(data, pathname, genelist, group, split, filters_json) def register_toggle_filter_cells_controls(app, token): @@ -474,37 +490,86 @@ def toggle_filter_cells_controls(n, is_open): return is_open -def register_update_filter_cells_controls(app, token): +def register_update_filter_cells_controls(app): @app.callback( [ - Output("%s_filter_cells_ncells_div" % token, "style"), - Output("%s_filter_cells_ncells" % token, "marks"), - Output("%s_filter_cells_ncells" % token, "min"), - Output("%s_filter_cells_ncells" % token, "max"), - Output("%s_filter_cells_ncells" % token, "value"), - Output("%s_filter_cells_ncells" % token, "step"), - Output("%s_filter_cells_choice_div" % token, "style"), - Output("%s_filter_cells_choice" % token, "options"), - Output("%s_filter_cells_choice" % token, "value"), - Output("%s_filter_cells_range_div" % token, "style"), - Output("%s_filter_cells_range" % token, "marks"), - Output("%s_filter_cells_range" % token, "min"), - Output("%s_filter_cells_range" % token, "max"), - Output("%s_filter_cells_range" % token, "value"), - Output("%s_filter_cells_range" % token, "step"), + Output("meta_filter_cells_ncells_div", "style"), + Output("meta_filter_cells_ncells", "marks"), + Output("meta_filter_cells_ncells", "min"), + Output("meta_filter_cells_ncells", "max"), + Output("meta_filter_cells_ncells", "value"), + Output("meta_filter_cells_ncells", "step"), + Output("meta_filter_cells_choice_div", "style"), + Output("meta_filter_cells_choice", "options"), + Output("meta_filter_cells_choice", "value"), + Output("meta_filter_cells_range_div", "style"), + Output("meta_filter_cells_range", "marks"), + Output("meta_filter_cells_range", "min"), + Output("meta_filter_cells_range", "max"), + Output("meta_filter_cells_range", "value"), + Output("meta_filter_cells_range", "step"), + Output("expression_filter_cells_ncells_div", "style"), + Output("expression_filter_cells_ncells", "marks"), + Output("expression_filter_cells_ncells", "min"), + Output("expression_filter_cells_ncells", "max"), + Output("expression_filter_cells_ncells", "value"), + Output("expression_filter_cells_ncells", "step"), + Output("expression_filter_cells_choice_div", "style"), + Output("expression_filter_cells_choice", "options"), + Output("expression_filter_cells_choice", "value"), + Output("expression_filter_cells_range_div", "style"), + Output("expression_filter_cells_range", "marks"), + Output("expression_filter_cells_range", "min"), + Output("expression_filter_cells_range", "max"), + Output("expression_filter_cells_range", "value"), + Output("expression_filter_cells_range", "step"), + ], + [ + Input("url", "pathname"), + Input("meta_filter_cells_attribute", "value"), + Input("meta_filter_cells_reset", "n_clicks"), + Input("expression_filter_cells_attribute", "value"), + Input("expression_filter_cells_reset", "n_clicks"), ], - [Input("url", "pathname"), Input("%s_filter_cells_attribute" % token, "value")], [State("filter_cells_filters", "children")], ) - def update_filter_cells_controls(pathname, attribute, filters_json): + def update_filter_cells_controls( + pathname, meta_attribute, reset_meta, expression_attribute, reset_expression, filters_json + ): _, kwargs = get_route(pathname) data = store.load_data(kwargs.get("dataset")) - hidden_slider = ({"display": "none"}, {0: "0", 1: "1"}, 0, 1, 1, 0) + hidden_slider = ({"display": "none"}, {}, None, None, None, None) hidden_checklist = ({"display": "none"}, [], None) - hidden_rangeslider = ({"display": "none"}, {0: "0", 1: "1"}, 0, 1, [0, 1], 0) + hidden_rangeslider = ({"display": "none"}, {}, None, None, None, None) + ctx = dash.callback_context + + if ctx.triggered and "reset" in ctx.triggered[0]["prop_id"]: + return ( + hidden_slider + + hidden_checklist + + hidden_rangeslider + + hidden_slider + + hidden_checklist + + hidden_rangeslider + ) + + if ctx.triggered and "meta" in ctx.triggered[0]["prop_id"]: + token = "meta" + attribute = meta_attribute + elif ctx.triggered and "expression" in ctx.triggered[0]["prop_id"]: + token = "expression" + attribute = expression_attribute if attribute is None or attribute == "None": - return hidden_slider + hidden_checklist + hidden_rangeslider + return ( + hidden_slider + + hidden_checklist + + hidden_rangeslider + + hidden_slider + + hidden_checklist + + hidden_rangeslider + ) + filters = json.loads(filters_json) if attribute == "ncells": ncells_tot = data.ad.obs.shape[0] @@ -512,7 +577,7 @@ def update_filter_cells_controls(pathname, attribute, filters_json): ncells_selected = filters[attribute] else: ncells_selected = ncells_tot - return ( + ret = ( ( {"display": "block"}, dict( @@ -531,12 +596,14 @@ def update_filter_cells_controls(pathname, attribute, filters_json): values = data.ad.obs_vector(attribute) if not pd.api.types.is_numeric_dtype(values): categories = list(data.ad.obs[attribute].cat.categories) - return ( + ret = ( hidden_slider + ( {"display": "block"}, [{"label": v, "value": v} for v in categories], - filters[attribute] if attribute in filters else categories, + filters[attribute] + if attribute in filters and filters[attribute] is not None + else categories, ) + hidden_rangeslider ) @@ -549,7 +616,7 @@ def update_filter_cells_controls(pathname, attribute, filters_json): else: val_min = range_min val_max = range_max - return ( + ret = ( hidden_slider + hidden_checklist + ( @@ -566,14 +633,18 @@ def update_filter_cells_controls(pathname, attribute, filters_json): (range_max - range_min) / 1000, ) ) + if token == "meta": + return ret + hidden_slider + hidden_checklist + hidden_rangeslider + else: + return hidden_slider + hidden_checklist + hidden_rangeslider + ret def register_update_filter_cells_filters(app): @app.callback( [ Output("filter_cells_filters", "children"), - Output("meta_filter_cells_div", "title"), - Output("expression_filter_cells_div", "title"), + Output("meta_filter_cells_status", "children"), + Output("expression_filter_cells_status", "children"), ], [ Input("url", "pathname"), @@ -611,14 +682,12 @@ def update_filter_cells_filters( ctx = dash.callback_context filters = json.loads(filters_json) - active_filters = set() # if reset button was hit, remove entries in filters_json attributes = list(filters.keys()) - status = "active filters: " if ctx.triggered and "reset" in ctx.triggered[0]["prop_id"]: for attribute in attributes: del filters[attribute] - return (json.dumps(filters), status, status) + return (json.dumps(filters), "", "") # else update filters_json depending on inputs for ncells_value, cat_value, range_value, attribute in [ @@ -631,23 +700,36 @@ def update_filter_cells_filters( ), ]: if attribute is not None and attribute != "None": - if attribute == "ncells": + if attribute == "ncells" and ncells_value is not None: filters[attribute] = ncells_value - ncells_tot = data.ad.obs.shape[0] - if ncells_value < ncells_tot: - active_filters.add(attribute) else: values = data.ad.obs_vector(attribute) - if not pd.api.types.is_numeric_dtype(values): + if not pd.api.types.is_numeric_dtype(values) and cat_value is not None: filters[attribute] = cat_value - if cat_value is not None and set(cat_value) != set(values): - active_filters.add(attribute) - else: + elif range_value is not None: filters[attribute] = range_value - if range_value[0] > values.min() or range_value[1] < values.max(): - active_filters.add(attribute) - status += ", ".join(active_filters) + active_filters = set() + for attribute, filter_values in filters.items(): + if filter_values is None: + continue + if attribute == "ncells": + ncells_tot = data.ad.obs.shape[0] + if filter_values < ncells_tot: + active_filters.add(attribute) + else: + values = data.ad.obs_vector(attribute) + if not pd.api.types.is_numeric_dtype(values): + if set(filter_values) != set(values): + active_filters.add(attribute) + else: + if filter_values[0] > values.min() or filter_values[1] < values.max(): + active_filters.add(attribute) + + if len(active_filters) > 0: + status = "active filters: " + ", ".join(active_filters) + else: + status = "" return (json.dumps(filters), status, status) @@ -668,6 +750,8 @@ def activate_filter_cells_reset(pathname, filters_json): filters = {} disabled = True for attribute, selected in filters.items(): + if selected is None: + continue if attribute == "ncells": ncells_tot = data.ad.obs.shape[0] if selected < ncells_tot: @@ -691,9 +775,6 @@ def activate_filter_cells_reset(pathname, filters_json): def register_file_upload(app): """Register callbacks for the file upload""" - logger.info("-> file_upload") - - # TODO: move main handler into "upload" module? @app.callback( Output("url", "pathname"), [Input("file-upload", "contents")], diff --git a/scelvis/convert.py b/scelvis/convert.py index db9a436..731dee6 100644 --- a/scelvis/convert.py +++ b/scelvis/convert.py @@ -172,8 +172,6 @@ def _load_expression(self, clustering, tsne, diffexp): ad = sc.read_10x_h5(expression_file) ad.var_names_make_unique() logger.info("Combining meta data") - # TODO: do we need to make variable names unique here or can we suppress the warning - # TODO: with ``with_log_level(anndata.utils.logger, logging.WARN)``? ad.obs["cluster"] = clustering ad.obs["n_counts"] = ad.X.sum(1).A1 ad.obs["n_genes"] = (ad.X > 0).sum(1).A1 @@ -216,8 +214,6 @@ def _split_samples(self, ad): def _normalize_filter_dge(ad): logger.info("Normalizing and filtering DGE") - # TODO: do we need to make variable names unique here or can we suppress the warning - # TODO: with ``with_log_level(anndata.utils.logger, logging.WARN)``? sc.pp.filter_cells(ad, min_genes=100) sc.pp.filter_genes(ad, min_cells=5) sc.pp.normalize_per_cell(ad, counts_per_cell_after=1.0e4) @@ -226,7 +222,6 @@ def _normalize_filter_dge(ad): def _write_output(self, ad): logger.info("Saving anndata object to %s", self.args.out_file) - # TODO: explicitely set column types to get rid of warning? ad.write(self.args.out_file) @@ -305,8 +300,6 @@ def _load_expression(self, coords, annotation, markers): obs=pd.concat([coords.loc[DGE.columns], annotation.loc[DGE.columns]], axis=1), var=pd.DataFrame([], index=DGE.index), ) - # TODO: do we need to make variable names unique here or can we suppress the warning - # TODO: with ``with_log_level(anndata.utils.logger, logging.WARN)``? for col in markers.columns: ad.uns["marker_" + col] = markers[col] @@ -314,7 +307,6 @@ def _load_expression(self, coords, annotation, markers): def _write_output(self, ad): logger.info("Saving anndata object to %s", self.args.out_file) - # TODO: explicitely set column types to get rid of warning? ad.write(self.args.out_file) @@ -381,7 +373,6 @@ def _load_loom(self, markers): def _write_output(self, ad): logger.info("Saving anndata object to %s", self.args.out_file) - # TODO: explicitely set column types to get rid of warning? ad.write(self.args.out_file) @@ -411,7 +402,6 @@ class Config: def run(args, _parser=None): """Main entry point after argument parsing.""" - # TODO: detect pipeline output format_ = args.format if format_ == "auto": if os.path.exists(os.path.join(args.indir, "coords.tsv")): diff --git a/scelvis/settings.py b/scelvis/settings.py index 9916f24..bdf4049 100644 --- a/scelvis/settings.py +++ b/scelvis/settings.py @@ -36,6 +36,8 @@ CACHE_DIR = None #: For "redis" cache: the URL to use for connecting to the cache. CACHE_REDIS_URL = None +#: whether or not to preload data on startup +CACHE_PRELOAD_DATA = False #: The height of the plot. PLOT_HEIGHT = 500 diff --git a/scelvis/static/home.md b/scelvis/static/home.md index 95ca851..751307b 100644 --- a/scelvis/static/home.md +++ b/scelvis/static/home.md @@ -5,5 +5,5 @@ SCelVis is a web app for the visualization and interactive exploration of single **Start by selecting the dataset you want to work with from the `Go To` menu on the top right**: choose an existing dataset, or upload a new one. -![movie](assets/movie.gif) +![go to https://github.com/bihealth/scelvis for a tutorial](https://github.com/bihealth/scelvis/tutorial/scelvis_movie.gif) diff --git a/scelvis/ui/cells.py b/scelvis/ui/cells.py index bc3a8f0..6eee599 100644 --- a/scelvis/ui/cells.py +++ b/scelvis/ui/cells.py @@ -20,6 +20,7 @@ def render_controls_scatter(data): """Render "top left" controls for the scatter plot for the given ``data``.""" + return [ html.Div( children=[ @@ -179,20 +180,34 @@ def render_select_cells_controls(data): ] ), html.P(), + html.P(children="", id="select_cells_status", style={"fontSize": 14}), dbc.Row( [ + "view", + dbc.Button( + "groups", + id="select_cells_view_groups", + color="link", + style={ + "padding-left": 2, + "padding-right": 2, + "padding-top": 0, + "padding-bottom": 3, + }, + ), + "or", dbc.Button( - "view table", - id="select_cells_view", + "table", + id="select_cells_view_table", color="link", style={ - "padding-left": 0, + "padding-left": 2, "padding-right": 0, "padding-top": 0, - "padding-bottom": 2, + "padding-bottom": 3, }, ), - " or get ", + "; download ", html.A( children=[html.I(className="fas fa-cloud-download-alt pr-1"), "results"], download="results.csv", @@ -212,7 +227,6 @@ def render_select_cells_controls(data): id="select_cells_get_results", style={"display": "none"}, ), - html.Div(id="select_cells_choices", style={"display": "none"}), html.Div(id="select_cells_results", style={"display": "none"}), ] ) @@ -256,14 +270,17 @@ def render(data): """Render the "Cell Annotation" content.""" return dbc.Row( children=[ + # hidden div to store selection of cells + html.Div(id="select_cells_selected", style={"display": "none"}), dbc.Col(children=render_controls(data), className="col-3"), # Placeholder for the plot. dbc.Col(children=[dcc.Loading(id="meta_plot", type="circle")], className="col-9"), + # dbc.Col(children=[html.Div(id="meta_plot")], className="col-9"), ] ) -def render_plot_scatter(data, xc, yc, col, filters_json): +def render_plot_scatter(data, xc, yc, col, filters_json, select_json): """Render the scatter plot figure.""" if xc is None or yc is None or col is None: @@ -271,10 +288,38 @@ def render_plot_scatter(data, xc, yc, col, filters_json): ad_here = common.apply_filter_cells_filters(data, filters_json) - if col in data.categorical_meta: - # select color palette - colvals = ad_here.obs[col].unique() - cm = colors.get_cm(colvals) + if col in data.categorical_meta or col == "DE_group": + + if col == "DE_group": + + if select_json is None: + return {}, "", True + + # color by DE groups selected from "differential expression" + selected = json.loads(select_json) + + if "group_A" not in selected or "group_B" not in selected: + return {}, "", True + + # make sure groups are disjoint + group_A = list(set(selected["group_A"]) - set(selected["group_B"])) + group_B = list(set(selected["group_B"]) - set(selected["group_A"])) + + if len(group_A) == 0 or len(group_B) == 0: + return {}, "", True + + ad_here.obs["DE_group"] = "none" + ad_here.obs.loc[group_A, "DE_group"] = "A" + ad_here.obs.loc[group_B, "DE_group"] = "B" + + colvals = ["A", "B", "none"] + cm = ["#1f77b4", "#ff7f0e", "#cccccc"] + + else: + + # select color palette + colvals = ad_here.obs[col].unique() + cm = colors.get_cm(colvals) # plot scatter for each category separately traces = [] @@ -498,13 +543,24 @@ def run_differential_expression(data, select_json): selected = json.loads(select_json) - ad_here.obs["group"] = np.nan - ad_here.obs.loc[selected["group_A"], "group"] = "A" - ad_here.obs.loc[selected["group_B"], "group"] = "B" + # make sure these groups are disjoint + group_A = list(set(selected["group_A"]) - set(selected["group_B"])) + group_B = list(set(selected["group_B"]) - set(selected["group_A"])) - ad_here = ad_here[~ad_here.obs["group"].isnull(), :] + options = [{"label": c, "value": c} for c in data.categorical_meta + data.numerical_meta] - res = sc.tl.rank_genes_groups(ad_here, "group", copy=True).uns["rank_genes_groups"] + if len(group_A) == 0 or len(group_B) == 0: + return "{}", "", "", "no valid groups selected!", {"display": "none"}, options + + options += [{"label": "DE_group", "value": "DE_group"}] + + ad_here.obs["DE_group"] = np.nan + ad_here.obs.loc[group_A, "DE_group"] = "A" + ad_here.obs.loc[group_B, "DE_group"] = "B" + + ad_here = ad_here[~ad_here.obs["DE_group"].isnull(), :] + + res = sc.tl.rank_genes_groups(ad_here, "DE_group", copy=True).uns["rank_genes_groups"] res_df = pd.DataFrame( { @@ -519,6 +575,8 @@ def run_differential_expression(data, select_json): res_df = res_df.reset_index().drop("n", axis=1) res_df = res_df[res_df["adjp"] < 0.05] + status_string = "%d DE genes at 5%% FDR" % res_df.shape[0] + results_string = "data:text/csv;charset=utf-8," + urllib.parse.quote( res_df.to_csv(index=True, header=True, encoding="utf-8") ) @@ -531,4 +589,11 @@ def run_differential_expression(data, select_json): pd.Series(params).to_csv(index=True, header=False, encoding="utf-8") ) - return res_df.to_json(), results_string, params_string, {"display": "block"} + return ( + res_df.to_json(), + results_string, + params_string, + status_string, + {"display": "block"}, + options, + ) diff --git a/scelvis/ui/common.py b/scelvis/ui/common.py index c2f3665..a0334c7 100644 --- a/scelvis/ui/common.py +++ b/scelvis/ui/common.py @@ -51,7 +51,6 @@ def render_filter_cells_collapse(data, token): ), ], id="%s_filter_cells_div" % token, - title="active filters: ", ) @@ -115,12 +114,24 @@ def render_filter_cells_controls(data, token): ], style={"display": "none"}, ), - dbc.Button( - "reset filters", - id="%s_filter_cells_reset" % token, - color="link", - className="text-left", - size="sm", + html.P(children="", id="%s_filter_cells_status" % token, style={"fontSize": 14}), + dbc.Row( + [ + dbc.Button( + "update plot", + id="%s_filter_cells_update" % token, + color="primary", + className="mr-1", + size="sm", + ), + dbc.Button( + "reset filters", + id="%s_filter_cells_reset" % token, + className="mr-1", + color="secondary", + size="sm", + ), + ] ), ] @@ -140,6 +151,8 @@ def apply_filter_cells_filters(data, filters_json): take = np.ones(ncells_tot, dtype=bool) filters = json.loads(filters_json) for col, selected in filters.items(): + if selected is None: + continue if col == "ncells": take = take & (np.random.random(ncells_tot) < selected / ncells_tot) elif col in data.categorical_meta: diff --git a/scelvis/ui/genes.py b/scelvis/ui/genes.py index a19d9bd..9bd49a3 100644 --- a/scelvis/ui/genes.py +++ b/scelvis/ui/genes.py @@ -166,6 +166,7 @@ def render(data): # Placeholder for the plot. dbc.Col( children=[dcc.Loading(id="expression_plot", type="circle")], + # children=[html.Div(id="expression_plot")], className="col-9", ), ] diff --git a/scelvis/webserver.py b/scelvis/webserver.py index cfb9ab4..e56598c 100644 --- a/scelvis/webserver.py +++ b/scelvis/webserver.py @@ -91,6 +91,7 @@ def run(args, parser): settings.CACHE_REDIS_URL = args.cache_redis_url elif args.cache_dir: settings.CACHE_DIR = args.cache_dir + settings.CACHE_PRELOAD_DATA = args.cache_preload_data settings.UPLOAD_ENABLED = not args.upload_disabled settings.UPLOAD_DIR = args.upload_dir settings.CONVERSION_ENABLED = not args.conversion_disabled @@ -131,10 +132,15 @@ def setup_argparse(parser): ) parser.add_argument( "--cache-default-timeout", - default=os.environ.get("SCELVIS_CACHE_DEFAULT_TIMEOUT", 600), + default=os.environ.get("SCELVIS_CACHE_DEFAULT_TIMEOUT", 7 * 24 * 60 * 60), help="Default timeout for cache", ) - + parser.add_argument( + "--cache-preload-data", + default=os.environ.get("SCELVIS_CACHE_PRELOAD_DATA", "0") not in ("", "0", "N", "n"), + action="store_true", + help="whether to preload data at startup", + ) parser.add_argument( "--upload-dir", default=os.environ.get("SCELVIS_UPLOAD_DIR"), diff --git a/tests/test_convert.py b/tests/test_convert.py index 0eb7d02..c7707bb 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -65,11 +65,9 @@ def test_run_converter_directly_with_about_md(config_with_about_md): config = config_with_about_md convert.CellRangerConverter(config).run() assert os.path.exists(config.out_file) - # TODO: test title/short title/readme is set def test_run_main_with_about_md(config_with_about_md): config = config_with_about_md assert convert.run(config) is None assert os.path.exists(config.out_file) - # TODO: test title/short title/readme is set diff --git a/tests/test_dash.py b/tests/test_dash.py index c6da79e..7fb00eb 100644 --- a/tests/test_dash.py +++ b/tests/test_dash.py @@ -48,7 +48,6 @@ def test_render_upload(dash_duo, scelvis_settings): item_upload.click() # Make sure that the site navigates to the "upload data" page. dash_duo.wait_for_text_to_equal("#page-brand", "Upload Data") - # TODO: actually test the upload page def test_render_cell_annotation(dash_duo, scelvis_settings): @@ -87,6 +86,9 @@ def test_render_cell_annotation(dash_duo, scelvis_settings): options = checklist.find_elements_by_css_selector("*") options[0].click() + item = dash_duo.wait_for_element_by_css_selector("#meta_filter_cells_update") + item.click() + button = dash_duo.wait_for_element_by_css_selector("#meta_filter_cells_reset") button.click() diff --git a/tutorial/scelvis_movie.gif b/tutorial/scelvis_movie.gif new file mode 100644 index 0000000..4c83fe4 Binary files /dev/null and b/tutorial/scelvis_movie.gif differ