From bc1627a82484503fa505782ef7579328bf6719e7 Mon Sep 17 00:00:00 2001 From: Benedikt Obermayer Date: Tue, 28 Jan 2020 15:45:04 +0100 Subject: [PATCH] fix iRODS authentication issues --- HISTORY.rst | 7 ++++ docs_manual/tutorial_input_formats.rst | 2 +- scelvis/app.py | 4 +- scelvis/data.py | 11 ++++- scelvis/settings.py | 9 ++++ scelvis/ui/main.py | 8 ++-- scelvis/webserver.py | 58 ++++++++++++++++++++++---- 7 files changed, 82 insertions(+), 17 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index b0ad707..a767c67 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,13 @@ History ======= +------ +v0.8.2 +------ + +- fix iRODS authentication issues +- update docs for .h5ad input + ------ v0.8.1 ------ diff --git a/docs_manual/tutorial_input_formats.rst b/docs_manual/tutorial_input_formats.rst index 60950d8..0f12f23 100644 --- a/docs_manual/tutorial_input_formats.rst +++ b/docs_manual/tutorial_input_formats.rst @@ -9,7 +9,7 @@ HDF5 Input For HDF5 input (no conversion necessary), you can do your analysis with `scanpy `__ to create an anndata object ``ad``. SCelVis will use embedding coordinates from ``ad.obsm``, cell annotation from ``ad.obs`` and expression data directly from ``ad.X`` (this should contain normalized and log-transformed expression values for all genes). If present, information about the dataset will be extracted from strings stored in ``ad.uns['about_title']``, ``ad.uns['about_short_title']`` and ``ad.uns['about_readme']`` (assumed to be Markdown). -Information about marker genes will be taken either from the ``rank_genes_groups`` slot in ``ad.uns`` or from entries starting with ``marker_`` in ``ad.uns``: entries called ``marker_gene`` (required!), ``marker_cluster``, ``marker_padj``, ``marker_LFC`` will create a table with the columns ``gene``, ``cluster``, ``padj``, and ``LFC``. +Information about marker genes will be taken from entries starting with ``marker_`` in ``ad.uns``: entries called ``marker_gene`` (required!), ``marker_cluster``, ``marker_padj``, ``marker_LFC`` will create a table with the columns ``gene``, ``cluster``, ``padj``, and ``LFC``. ``SCelVis`` will also extract marker information from ``ad.uns['rank_genes_groups']``. However, certain datatypes in ``ad.uns`` together with version mismatches of ``scanpy``, ``h5py`` and ``anndata`` can lead to ``.h5ad`` files that are not readable by ``SCelVis`` (see `issue #832 `__). To be on the safe side, it's recommended to delete unneccessary slots in ``ad.uns`` (e.g., ``del ad.uns['rank_genes_groups']``). Also, ``ad.obsm['X_pca']`` and ``ad.varm['PCs']`` are likely dispensable. If you prepared your data with ``Seurat`` (v2), you can use ``Convert(from = sobj, to = "anndata", filename = "data.h5ad")`` to get an HDF5 file. diff --git a/scelvis/app.py b/scelvis/app.py index d0d2898..93e8155 100644 --- a/scelvis/app.py +++ b/scelvis/app.py @@ -13,7 +13,7 @@ import dash import flask import uuid -import scanpy as sc +import anndata from flask import request, helpers from logzero import logger from werkzeug.utils import secure_filename @@ -143,7 +143,7 @@ def upload_route(): logger.info("Writing to %s", filepath) file.save(filepath) try: - sc.read(filepath) + anndata.read_h5ad(filepath) return """

upload successful!

diff --git a/scelvis/data.py b/scelvis/data.py index a6aab11..34cb87b 100644 --- a/scelvis/data.py +++ b/scelvis/data.py @@ -92,7 +92,7 @@ def create_irods_session(url): if "ssl" in url.scheme: ssl_settings = { "ssl_context": ssl.create_default_context( - purpose=ssl.Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None + purpose=ssl.Purpose.CLIENT_AUTH, cafile=None, capath=None, cadata=None ) } else: @@ -109,8 +109,15 @@ def create_irods_session(url): port=(url.port or 1247), user=url.username, password=url.password or "", - irods_authentication_scheme=irods_authentication_scheme, zone=(url.path or "/").split("/")[1], + irods_authentication_scheme=irods_authentication_scheme, + irods_client_server_negotiation=settings.IRODS_CLIENT_SERVER_NEGOTIATION, + irods_client_server_policy=settings.IRODS_CLIENT_SERVER_POLICY, + irods_ssl_verify_server=settings.IRODS_SSL_VERIFY_SERVER, + irods_encryption_algorithm=settings.IRODS_ENCRYPTION_ALGORITHM, + irods_encryption_key_size=settings.IRODS_ENCRYPTION_KEY_SIZE, + irods_encryption_num_hash_rounds=settings.IRODS_ENCRYPTION_NUM_HASH_ROUNDS, + irods_encryption_salt_size=settings.IRODS_ENCRYPTION_SALT_SIZE, **ssl_settings ) query = parse_qs(url.query) diff --git a/scelvis/settings.py b/scelvis/settings.py index 61ab775..99fe5e3 100644 --- a/scelvis/settings.py +++ b/scelvis/settings.py @@ -50,3 +50,12 @@ #: Whether or not to enable file upload for conversion, default is ``True``. CONVERSION_ENABLED = False + +#: various default iRODS settings +IRODS_CLIENT_SERVER_NEGOTIATION = "request_server_negotiation" +IRODS_CLIENT_SERVER_POLICY = "CS_NEG_REQUIRE" +IRODS_SSL_VERIFY_SERVER = "none" +IRODS_ENCRYPTION_ALGORITHM = "AES-256-CBC" +IRODS_ENCRYPTION_KEY_SIZE = 32 +IRODS_ENCRYPTION_NUM_HASH_ROUNDS = 16 +IRODS_ENCRYPTION_SALT_SIZE = 8 diff --git a/scelvis/ui/main.py b/scelvis/ui/main.py index 5f8f66a..6f3face 100644 --- a/scelvis/ui/main.py +++ b/scelvis/ui/main.py @@ -53,12 +53,14 @@ def render_convert(): def render_upload(): return html.Div( children=[ - "upload .h5ad file here (you can convert your pipeline output to such a file using ", + "upload .h5ad file here (should be smaller than %.0f MB)" + % (int(settings.MAX_UPLOAD_DATA_SIZE) / 1.0e6), + html.P(), + "you can convert your pipeline output to such a file using ", html.A( children=[html.I(className="fas fa-redo mr-1"), "Convert Data"], - href="%s/dash/convert/" % settings.PUBLIC_URL_PREFIX, + href="/dash/convert", ), - ")", html.P(), html.Iframe(src="/upload/", width="400", height="200"), ] diff --git a/scelvis/webserver.py b/scelvis/webserver.py index 416ca7b..0bd7e21 100644 --- a/scelvis/webserver.py +++ b/scelvis/webserver.py @@ -121,6 +121,13 @@ def setup_argparse(parser): action="append", help="Path to data source(s)", ) + + parser.add_argument( + "--public-url-prefix", + default=os.environ.get("SCELVIS_URL_PREFIX", ""), + help="The prefix that this app will be served under (e.g., if behind a reverse proxy.)", + ) + parser.add_argument( "--cache-dir", default=os.environ.get("SCELVIS_CACHE_DIR"), @@ -143,11 +150,18 @@ def setup_argparse(parser): action="store_true", help="whether to preload data at startup", ) + parser.add_argument( "--upload-dir", default=os.environ.get("SCELVIS_UPLOAD_DIR"), help="Directory for visualization uploads, default is to create temporary directory", ) + parser.add_argument( + "--max-upload-data-size", + default=os.environ.get("SCELVIS_MAX_UPLOAD_DATA_SIZE", "1000000000"), + type=int, + help="Maximal size for data upload in bytes", + ) parser.add_argument( "--disable-upload", default=os.environ.get("SCELVIS_UPLOAD_DISABLED", "0") not in ("", "0", "N", "n"), @@ -156,12 +170,6 @@ def setup_argparse(parser): help="Whether or not to disable visualization uploads", ) - parser.add_argument( - "--public-url-prefix", - default=os.environ.get("SCELVIS_URL_PREFIX", ""), - help="The prefix that this app will be served under (e.g., if behind a reverse proxy.)", - ) - parser.add_argument( "--disable-conversion", default=os.environ.get("SCELVIS_CONVERSION_DISABLED", "0") not in ("", "0", "N", "n"), @@ -171,8 +179,40 @@ def setup_argparse(parser): ) parser.add_argument( - "--max-upload-data-size", - default=os.environ.get("SCELVIS_MAX_UPLOAD_DATA_SIZE", "1000000000"), + "--irods-client-server-negotiation", + default=os.environ.get("IRODS_CLIENT_SERVER_NEGOTIATION", None), + help="IRODS setting", + ) + parser.add_argument( + "--irods-client-server-policy", + default=os.environ.get("IRODS_CLIENT_SERVER_POLICY", None), + help="IRODS setting", + ) + parser.add_argument( + "--irods-ssl-verify-server", + default=os.environ.get("IRODS_SSL_VERIFY_SERVER", None), + help="IRODS setting", + ) + parser.add_argument( + "--irods-encryption-algorithm", + default=os.environ.get("IRODS_ENCRYPTION_ALGORITHM", None), + help="IRODS setting", + ) + parser.add_argument( + "--irods-encryption-key-size", + default=os.environ.get("IRODS_ENCRYPTION_KEY_SIZE", None), type=int, - help="Maximal size for data upload in bytes", + help="IRODS setting", + ) + parser.add_argument( + "--irods-encryption-num-hash-rounds", + default=os.environ.get("IRODS_ENCRYPTION_NUM_HASH_ROUNDS", None), + type=int, + help="IRODS setting", + ) + parser.add_argument( + "--irods-encryption-salt-size", + default=os.environ.get("IRODS_ENCRYPTION_SALT_SIZE", None), + type=int, + help="IRODS setting", )