diff --git a/.travis.yml b/.travis.yml index c72172289..2bebca400 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,71 +1,100 @@ -dist: xenial -sudo: required language: cpp +stages: + - name: basic + if: branch != master + - name: full linux + if: branch = master + - name: singularity build push + if: branch = master OR tag IS present + +env: + global: + - DISPLAY=":99.0" + services: - xvfb -matrix: +jobs: + allow_failures: + - stage: singularity build push include: - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-6 - env: COMPILER=gcc GCC=6 - - os: linux - if: branch = master + - stage: basic + env: PYTHON_VERSION="3.7.5" COMPILER="gcc" GCCv="7" + - stage: full linux + os: linux + env: PYTHON_VERSION="3.7.5" COMPILER="gcc" GCCv="7" + - stage: full linux + os: linux + env: PYTHON_VERSION="3.7.5" COMPILER="gcc" GCCv="6" + - stage: full linux + os: linux + env: PYTHON_VERSION="3.6.9" COMPILER="gcc" GCCv="7" + - stage: full linux + os: linux + env: PYTHON_VERSION="3.6.9" COMPILER="gcc" GCCv="6" + - stage: singularity build push + env: PYTHON_VERSION="3.7.5" + language: go + go: "1.13" + git: { submodules: false, depth: 1 } addons: apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-7 - env: COMPILER=gcc GCC=7 - -env: - global: - - DISPLAY=":99.0" - - MINCONDA_VERSION="latest" - - MINCONDA_LINUX="Linux-x86_64" + packages: [ flawfinder, squashfs-tools, uuid-dev, libuuid1, libffi-dev, libssl-dev, libssl1.0.0, libarchive-dev, libgpgme11-dev, libseccomp-dev ] + homebrew: { packages: [ squashfs ], update: true } + before_install: skip + install: + - SINGULARITY_BASE="${GOPATH}/src/github.com/sylabs/singularity" + - export PATH="${GOPATH}/bin:${PATH}" + - mkdir -p "${GOPATH}/src/github.com/sylabs" + - cd "${GOPATH}/src/github.com/sylabs" + - git clone -b v3.5.0 https://github.com/sylabs/singularity + - cd singularity + - ./mconfig -v -p /usr/local + - make -j `nproc 2>/dev/null || echo 1` -C ./builddir all + - sudo make -C ./builddir install + before_script: + # token used for push commnad + - echo -e "$SYLABS_TOKEN" > token + - singularity remote login --tokenfile ./token || exit 1 + # key used for image signing + # for travis ci pgp key block you must replace newline with '\\n', and replace spaces with '\ ' + - echo -e "$SINGULARITY_KEY" > skey + - head -n 4 ./skey + - echo $SINGULARITY_KEY_PW | singularity key import ./skey || exit 1 + script: + - cd $TRAVIS_BUILD_DIR + - export SINGULARITYENV_PYTHON_VERSION=$PYTHON_VERSION + - export SINGULARITYENV_GIT_COMMIT_HASH=$TRAVIS_COMMIT + - sudo singularity build ./extra-foam.sif extra-foam.def + after_script: + - echo $SINGULARITY_KEY_PW | singularity sign ./extra-foam.sif + - singularity push ./extra-foam.sif library://robert.rosca/default/extra-foam:$TRAVIS_BRANCH before_install: - - | - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - if [[ "$COMPILER" == "gcc" ]]; then - export CXX=g++-$GCC CC=gcc-$GCC; - fi - fi - -install: - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - MINCONDA_OS=$MINCONDA_LINUX; - fi - - wget "http://repo.continuum.io/miniconda/Miniconda3-$MINCONDA_VERSION-$MINCONDA_OS.sh" -O miniconda.sh + - export GXX="g++-$GCCv" GCC="gcc-$GCCv" + - sudo -E apt-add-repository -y "ppa:ubuntu-toolchain-r/test" + - sudo apt-get -q update + - sudo -E apt-get -yq --no-install-suggests --no-install-recommends $(travis_apt_get_options) install libxkbcommon-x11-0 $GXX + - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/$GXX 0 + - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/$GCC 0 + - g++ --version + - gcc --version + - wget "http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" +install: + - conda install -y python=$PYTHON_VERSION + - echo $PYTHON_VERSION - conda install -y cmake -c conda-forge - - which python - - python --version - - which pip - - pip --version - # QT_DEBUG_PLUGINS - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - sudo apt install -y libxkbcommon-x11-0; - pip install -e .[test]; - fi - + - pip install -e .[test] before_script: - /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1400x900x24 - sleep 3 - script: # test parallel version - python setup.py build_ext --with-tests - python setup.py test -v - - python setup.py benchmark -v # test series version - export FOAM_WITH_TBB=0 @@ -74,4 +103,3 @@ script: - export XTENSOR_WITH_XSIMD=0 - python setup.py build_ext --with-tests - python setup.py test -v - - python setup.py benchmark -v diff --git a/README.md b/README.md index e8b6ebb54..67edfda89 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ EXtra-foam [![Build Status](https://travis-ci.org/European-XFEL/EXtra-foam.svg?branch=master)](https://travis-ci.org/European-XFEL/EXtra-foam) -*EXtra-foam* (previously known as *[karaboFAI](https://in.xfel.eu/readthedocs/docs/karabofai/en/latest/)*) is an application that provides -super fast on-line (real-time) and off-line data analysis and visualization for experiments at European XFEL that using 2D -detectors, namely AGIPD, DSSC, LPD, FastCCD, JungFrau, etc., together with other 1D detectors (e.g. -XGM, digitizer, etc.) and various control data. +*EXtra-foam* (previously known as *[karaboFAI](https://in.xfel.eu/readthedocs/docs/karabofai/en/latest/)*) is a +framework that provides super fast on-line (real-time) and off-line data analysis and visualization for +experiments at European XFEL that using 2D detectors (e.g. AGIPD, DSSC, LPD, ePix100, FastCCD, JungFrau, +etc.), together with other 1D detectors (e.g. XGM, digitizer, etc.) and various control data. [Documentation](https://extra-foam.readthedocs.io/en/latest/) diff --git a/benchmarks/benchmark_demo.ipynb b/benchmarks/benchmark_demo.ipynb new file mode 100644 index 000000000..f24ac95b7 --- /dev/null +++ b/benchmarks/benchmark_demo.ipynb @@ -0,0 +1,381 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Benchmark of EXtra-foam\n", + "\n", + "Author: Jun Zhu, March 03, 2020" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from extra_foam.algorithms import mask_image_data, nanmean_image_data, nansum\n", + "import random\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "72" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import multiprocessing as mp\n", + "\n", + "mp.cpu_count()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Assume we want to process a train of detector images." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "images = np.random.randn(128, 1200, 1200).astype(np.float32)\n", + "images[:, ::2, ::2] = np.nan" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Benchmark nanmean" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.21 s ± 2.58 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "%timeit np.nanmean(images, axis=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "13.5 ms ± 759 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" + ] + } + ], + "source": [ + "%timeit nanmean_image_data(images)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A common use case is to apply `nanmean` to a list of selected images." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# select 120 images out of 128 ones\n", + "selected = random.sample(range(images.shape[0]), 120)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.43 s ± 1.66 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "# numpy is slower than operating on all the images since it copies the data when 'selected' is a list.\n", + "%timeit np.nanmean(images[selected], axis=0)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "12.1 ms ± 117 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" + ] + } + ], + "source": [ + "%timeit nanmean_image_data(images, kept=selected)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Benchmark masking" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "threshold mask" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "353 ms ± 216 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "imgs = images.copy()\n", + "%timeit imgs[(imgs > 1) | (imgs < -1)] = np.nan" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "19.6 ms ± 533 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" + ] + } + ], + "source": [ + "imgs = images.copy()\n", + "%timeit mask_image_data(imgs, threshold_mask=(-1, 1))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "image mask" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "image_mask = np.ones((1200, 1200), dtype=bool)\n", + "image_mask[::3, ::3] = np.nan" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "723 ms ± 424 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "imgs = images.copy()\n", + "%timeit imgs[:, image_mask] = np.nan" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "27.5 ms ± 71.5 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" + ] + } + ], + "source": [ + "imgs = images.copy()\n", + "%timeit mask_image_data(imgs, image_mask=image_mask)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "threshold mask and image mask" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "637 ms ± 595 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)\n" + ] + } + ], + "source": [ + "imgs = images.copy()\n", + "%timeit imgs[(image_mask) | (imgs > 1) | (imgs < -1)] = np.nan" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "27.5 ms ± 37.6 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n" + ] + } + ], + "source": [ + "imgs = images.copy()\n", + "%timeit mask_image_data(imgs, image_mask=image_mask, threshold_mask=(-1, 1))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Benchmark nan-statistics" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Non-paralleled funtion can be also much faster than `numpy`." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "images = np.random.randn(1200, 1200).astype(np.float32)\n", + "images[::2, ::2] = np.nan" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "4.64 ms ± 4.84 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n" + ] + } + ], + "source": [ + "%timeit np.nansum(images)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.48 ms ± 60.1 ns per loop (mean ± std. dev. of 7 runs, 1000 loops each)\n" + ] + } + ], + "source": [ + "%timeit nansum(images)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/benchmark_geometry.py b/benchmarks/benchmark_geometry.py index e52a7241a..24df0ce13 100644 --- a/benchmarks/benchmark_geometry.py +++ b/benchmarks/benchmark_geometry.py @@ -18,7 +18,7 @@ _geom_path = osp.join(osp.dirname(osp.abspath(__file__)), "../extra_foam/geometries") -def _benchmark_1m_imp(geom_fast_cls, geom_cls, geom_file, quad_positions): +def _benchmark_1m_imp(geom_fast_cls, geom_cls, geom_file, quad_positions=None): for from_dtype, from_str in _data_sources: n_pulses = 64 @@ -37,7 +37,10 @@ def _benchmark_1m_imp(geom_fast_cls, geom_cls, geom_file, quad_positions): # assemble with geometry and quad position - geom = geom_fast_cls.from_h5_file_and_quad_positions(geom_file, quad_positions) + if quad_positions is not None: + geom = geom_fast_cls.from_h5_file_and_quad_positions(geom_file, quad_positions) + else: + geom = geom_fast_cls.from_crystfel_geom(geom_file) out = np.full((n_pulses, *geom.assembledShape()), np.nan, dtype=np.float32) t0 = time.perf_counter() geom.position_all_modules(modules, out) @@ -45,8 +48,10 @@ def _benchmark_1m_imp(geom_fast_cls, geom_cls, geom_file, quad_positions): # assemble with geometry and quad position in EXtra-geom - geom = geom_cls.from_h5_file_and_quad_positions( - geom_file, quad_positions) + if quad_positions is not None: + geom = geom_cls.from_h5_file_and_quad_positions(geom_file, quad_positions) + else: + geom = geom_cls.from_crystfel_geom(geom_file) out = geom.output_array_for_position_fast((n_pulses,)) t0 = time.perf_counter() geom.position_all_modules(modules, out=out) @@ -87,6 +92,15 @@ def benchmark_lpd_1m(): _benchmark_1m_imp(LPD_1MGeometryFast, LPD_1MGeometry, geom_file, quad_positions) +def benchmark_agipd_1m(): + from extra_foam.geometries import AGIPD_1MGeometryFast + from extra_geom import AGIPD_1MGeometry + + geom_file = osp.join(_geom_path, "agipd_mar18_v11.geom") + + _benchmark_1m_imp(AGIPD_1MGeometryFast, AGIPD_1MGeometry, geom_file) + + if __name__ == "__main__": print("*" * 80) print("Benchmark geometry") @@ -95,3 +109,5 @@ def benchmark_lpd_1m(): benchmark_dssc_1m() benchmark_lpd_1m() + + benchmark_agipd_1m() diff --git a/docs/changelog.rst b/docs/changelog.rst index d63c042db..f561a794f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,31 @@ CHANGELOG ========= +0.8.1 (16 March 2020) +------------------------ + +- **Improvement** + + - Automatically reset empty image mask with inconsistent shape. #104 + +- **New Feature** + + - Implement AGIPD 1M geometry in C++. #102 + - Add ROI1_DIV_ROI2 as an option for ROI FOM. #103 + - Implement normalization for ROI FOM. #96 + - Implement ROI FOM master-slave scan. #93 + - Add branch-based CI and Singularity image deployment. #92 + - Add support for ePix100 detector. #90 + - Implement save and load metadata. #87 + +0.8.0.1 (3 March 2020) +------------------------ + +- **Bug Fix** + + - Fix display bug in ImageTool #85 + + 0.8.0 (2 March 2020) ------------------------ diff --git a/docs/image_tool.rst b/docs/image_tool.rst index 860db210d..81680cf8c 100644 --- a/docs/image_tool.rst +++ b/docs/image_tool.rst @@ -78,6 +78,15 @@ ROI FOM setup +----------------------------+--------------------------------------------------------------------+ | ``FOM`` | ROI FOM type, e.g. *SUM*, *MEAN*, *MEDIAN*, *MIN*, *MAX*. | +----------------------------+--------------------------------------------------------------------+ +| ``Norm`` | Normalizer of ROI FOM. *Only applicable for train-resolved and | +| | pump-probe analysis*. | ++----------------------------+--------------------------------------------------------------------+ +| ``Master-slave`` | Check to activate the *master-slave* model. This model is used | +| | exclusively in correlation plots (see :ref:`statistics analysis`). | +| | When it is activated, FOMs of ROI1 (master) and ROI2 (slave) will | +| | be plotted in the same correlation plot. For other statistics | +| | analysis like binning and histogram, only ROI1 FOM will be used. | ++----------------------------+--------------------------------------------------------------------+ ROI histogram setup """"""""""""""""""" @@ -118,7 +127,8 @@ Define the 1D projection of ROI (region of interest) analysis setup. +----------------------------+--------------------------------------------------------------------+ | ``Direction`` | Direction of 1D projection (x or y). | +----------------------------+--------------------------------------------------------------------+ -| ``Norm`` | Normalizer of the 1D-projection VFOM. | +| ``Norm`` | Normalizer of the 1D-projection VFOM. *Only applicable for | +| | train-resolved and pump-probe analysis*. | +----------------------------+--------------------------------------------------------------------+ | ``AUC range`` | AUC (area under a curve) integration range. | +----------------------------+--------------------------------------------------------------------+ @@ -239,7 +249,8 @@ aforementioned coordinate system, respectively. +----------------------------+--------------------------------------------------------------------+ | ``Integ range (1/A)`` | Azimuthal integration range. | +----------------------------+--------------------------------------------------------------------+ -| ``Norm`` | Normalizer of the azimuthal integration result. | +| ``Norm`` | Normalizer of the scattering curve. *Only applicable for | +| | train-resolved and pump-probe analysis*. | +----------------------------+--------------------------------------------------------------------+ | ``AUC range (1/A)`` | AUC (area under curve) range. | +----------------------------+--------------------------------------------------------------------+ diff --git a/docs/images/configurator.png b/docs/images/configurator.png new file mode 100644 index 000000000..64367d041 Binary files /dev/null and b/docs/images/configurator.png differ diff --git a/docs/introduction.rst b/docs/introduction.rst index a16b12818..bc0532937 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -4,7 +4,7 @@ INTRODUCTION **EXtra-foam** ( **F**\ ast **O**\ nline **A**\ nalysis **M**\ onitor) is a tool that provides real-time and off-line data analysis (**azimuthal integration**, **ROI**, **correlation**, **binning**, etc.) and visualization for experiments using **2D area detectors** (*AGIPD*, -*DSSC* , *FastCCD*, *JungFrau*, *LPD*, etc.) at European XFEL. +*LPD*, *DSSC* , *FastCCD*, *JungFrau*, *ePix100*, etc.) at European XFEL. +------------+-------------------------+-------------------------+ @@ -22,6 +22,8 @@ real-time and off-line data analysis (**azimuthal integration**, **ROI**, **corr +------------+------------+------------+------------+------------+ | FastCCD | True | True | True | True | +------------+------------+------------+------------+------------+ +| ePix100 | True | True | True | True | ++------------+------------+------------+------------+------------+ Why use **EXtra-foam** diff --git a/docs/main_gui.rst b/docs/main_gui.rst index 575e72837..ec07f5b7d 100644 --- a/docs/main_gui.rst +++ b/docs/main_gui.rst @@ -117,3 +117,51 @@ Statistics analysis ------------------- See :ref:`statistics analysis` + + +Configurator +------------ + +.. image:: images/configurator.png + :width: 640 + +*Configurator* is a new feature introduced in version 0.8.1, it allows users to save and load +different analysis setups (a snapshot in the Redis database) conveniently. To apply a setup, +simply **double-click** the name of the snapshot listed in the table. Please distinguish it +from :ref:`config file`, which is mainly used for data source management. Due to the historical +reason, some setups in the :ref:`config file` can also be saved and loaded via the configurator, +like ``photon energy``, ``sample distance``, etc. :ref:`config file` defines the default setups +which will be overwritten when a setup snapshot is applied. The default setups can be recovered by +clicking the ``Reset to default`` button. + ++----------------------------+--------------------------------------------------------------------+ +| Input | Description | ++============================+====================================================================+ +| ``Take snapshot`` | Take a snapshot of the current setup and store in ``Last saved``. | ++----------------------------+--------------------------------------------------------------------+ +| ``Reset to default`` | Reset the current setup to default. ``Last saved`` will not be | +| | affected. | ++----------------------------+--------------------------------------------------------------------+ +| ``Save setups in file`` | Save all the snapshots listed in the table to file. All the | +| | snapshots in the setup file will be lost. | ++----------------------------+--------------------------------------------------------------------+ +| ``Load setups from file`` | Load all the snapshots from file. In case of name conflict, the | +| | listed snapshot in the table will be overwritten. | ++----------------------------+--------------------------------------------------------------------+ + +When right-clicking the name of a snapshot, a menu will show up: + ++----------------------------+--------------------------------------------------------------------+ +| Input | Description | ++============================+====================================================================+ +| ``Copy snapshot`` | Make a copy of the snapshot. | ++----------------------------+--------------------------------------------------------------------+ +| ``Delete snapshot`` | Delete the snapshot. | ++----------------------------+--------------------------------------------------------------------+ +| ``Rename snapshot`` | Rename the snapshot. | ++----------------------------+--------------------------------------------------------------------+ + +.. warning:: + + *Configurator* is still in the testing phase and we are collecting feedbacks from users. + It should be noted that there is no backup recovery mechanism for now. diff --git a/docs/start.rst b/docs/start.rst index 903722990..7e460d43f 100644 --- a/docs/start.rst +++ b/docs/start.rst @@ -18,7 +18,7 @@ features than the **stable** version. .. code-block:: bash - module load exfel EXtra-foam + module load exfel EXtra-foam/beta extra-foam DETECTOR TOPIC More info on command line arguments can be obtained as @@ -78,7 +78,7 @@ To start the **stable** version on online or `Maxwell` clusters: .. code-block:: bash - module load exfel EXtra-foam/beta + module load exfel EXtra-foam extra-foam DETECTOR TOPIC diff --git a/docs/statistics.rst b/docs/statistics.rst index 57731ca9f..305180b1b 100644 --- a/docs/statistics.rst +++ b/docs/statistics.rst @@ -123,3 +123,6 @@ Setup the visualization of correlations of a given FOM with various slow data. .. image:: images/correlation_window.png :width: 800 + +One can also plot FOMs of ROI1 and ROI2 together when the *master-slave* mode is activated in +:ref:`ROI FOM setup`. diff --git a/extra-foam.def b/extra-foam.def new file mode 100644 index 000000000..26c5b7be1 --- /dev/null +++ b/extra-foam.def @@ -0,0 +1,87 @@ +Bootstrap: library +From: centos:7.6.1810 + +%help + EXtra-foam (previously known as karaboFAI) is an application that provides + super fast on-line (real-time) and off-line data analysis and visualization + for experiments at European XFEL that using 2D detectors, namely AGIPD, DSSC, + LPD, FastCCD, JungFrau, etc., together with other 1D detectors (e.g. XGM, + digitizer, etc.) and various control data. + + GitHub: https://github.com/European-XFEL/EXtra-foam + Docs: https://extra-foam.readthedocs.io/en/latest/ + +%environment + __conda_setup="$('/usr/local/bin/conda' 'shell.bash' 'hook' 2> /dev/null)" + if [ $? -eq 0 ]; then + eval "$__conda_setup" + else + if [ -f "/usr/local/etc/profile.d/conda.sh" ]; then + . "/usr/local/etc/profile.d/conda.sh" + else + export PATH="/usr/local/bin:$PATH" + fi + fi + unset __conda_setup + + conda activate base + +%runscript + extra-foam "$@" + +%post + export logpath=/.singularity.d/logs/ + mkdir -p $logpath + + # Set default python version if missing + PYTHON_VERSION="${PYTHON_VERSION:-3.7.5}" + echo "Building for python version: $PYTHON_VERSION" + + # Basic yum dependencies + ## Install and update + yum update -y | tee $logpath/00-yum-update.log + yum install -y epel-release | tee $logpath/01-yum-epel-release.log + yum groupinstall -y 'Development Tools' | tee $logpath/02-yum-dev-tools.log + yum install -y \ + nano \ + curl \ + wget \ + tar \ + bzip2 \ + git \ + e4fsprogs \ + xeyes \ + mesa-libGL \ + qt5-qtbase \ + libxkbcommon-x11 \ + | tee $logpath/03-yum-install.log + + ## Log yum packages + yum list installed | tee $logpath/04-yum-list-installed.log + + # Install and setup miniconda + ## Download and install minconda + curl -sSL http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh + chmod +x /tmp/miniconda.sh + /tmp/miniconda.sh -bfp /usr/local/ + rm -f /tmp/miniconda.sh + source /usr/local/bin/activate + + ## Install python dependencies + conda install -y python=$PYTHON_VERSION + conda install -y -c anaconda cmake numpy + conda install -y -c omgarcia gcc-6 + + ## Download EXtra-foam source + mkdir -p /usr/local/src/ + cd /usr/local/src/ + git clone https://github.com/European-XFEL/EXtra-foam + cd EXtra-foam + git checkout $GIT_COMMIT_HASH + git submodule update --init | tee -a $logpath/05-extra-foam-submodules.log + git log -n 1 | tee -a $logpath/06-extra-foam-clone.log + + pip install . + + ## Export environment to concretise + conda env export -n base -f /.singularity.d/logs/07-conda-env-export.log diff --git a/extra_foam/__init__.py b/extra_foam/__init__.py index 8e3b2f3a9..9456dc322 100644 --- a/extra_foam/__init__.py +++ b/extra_foam/__init__.py @@ -37,7 +37,7 @@ import os -__version__ = "0.8.0" +__version__ = "0.8.1" # root path for storing config and log files ROOT_PATH = os.path.join(os.path.expanduser("~"), ".EXtra-foam") diff --git a/extra_foam/algorithms/__init__.py b/extra_foam/algorithms/__init__.py index 7017e843d..e000938c2 100644 --- a/extra_foam/algorithms/__init__.py +++ b/extra_foam/algorithms/__init__.py @@ -8,7 +8,7 @@ All rights reserved. """ from .statistics_py import ( - hist_with_stats, nanhist_with_stats, compute_statistics, find_actual_range, + hist_with_stats, nanhist_with_stats, compute_statistics, nanmean, nansum ) diff --git a/extra_foam/algorithms/statistics_py.py b/extra_foam/algorithms/statistics_py.py index 43977ef93..7c9996082 100644 --- a/extra_foam/algorithms/statistics_py.py +++ b/extra_foam/algorithms/statistics_py.py @@ -13,19 +13,20 @@ from .statistics import nanmean, nansum -def find_actual_range(arr, range): - """Find the actual range for an array of data. +def _get_outer_edges(arr, range): + """Determine the outer bin edges to use. - This is a helper function to find the non-infinite range (lb, ub) as - input for some other functions. + From both the data and the range argument. :param numpy.ndarray arr: data. - :param tuple range: desired range. + :param tuple range: desired range (min, max). - :return tuple: actual range. - - Note: the input data is assume to be non-free. + :return tuple: outer edges (min, max). + Note: the input array is assumed to be nan-free but could contain +-inf. + The returned outer edges could be inf or -inf if both the min/max + value of array and the corresponding boundary of the range argument + are inf or -inf. """ v_min, v_max = range assert v_min < v_max @@ -77,6 +78,8 @@ def nanhist_with_stats(roi, bin_range=(-np.inf, np.inf), n_bins=10): :param numpy.ndarray roi: image ROI. :param tuple bin_range: (lb, ub) of histogram. :param int n_bins: number of bins of histogram. + + :raise ValueError: if finite outer edges cannot be found. """ # Note: Since the nan functions in numpy is typically 5-8 slower # than the non-nan counterpart, it is always faster to remove nan @@ -88,8 +91,8 @@ def nanhist_with_stats(roi, bin_range=(-np.inf, np.inf), n_bins=10): mask_image_data(filtered, threshold_mask=bin_range, keep_nan=True) filtered = filtered[~np.isnan(filtered)] - actual_range = find_actual_range(filtered, bin_range) - hist, bin_edges = np.histogram(filtered, range=actual_range, bins=n_bins) + outer_edges = _get_outer_edges(filtered, bin_range) + hist, bin_edges = np.histogram(filtered, range=outer_edges, bins=n_bins) bin_centers = (bin_edges[1:] + bin_edges[:-1]) / 2.0 mean, median, std = compute_statistics(filtered) @@ -102,8 +105,10 @@ def hist_with_stats(data, bin_range=(-np.inf, np.inf), n_bins=10): :param numpy.ndarray data: input data. :param tuple bin_range: (lb, ub) of histogram. :param int n_bins: number of bins of histogram. + + :raise ValueError: if finite outer edges cannot be found. """ - v_min, v_max = find_actual_range(data, bin_range) + v_min, v_max = _get_outer_edges(data, bin_range) filtered = data[(data >= v_min) & (data <= v_max)] hist, bin_edges = np.histogram( diff --git a/extra_foam/algorithms/tests/test_statistics.py b/extra_foam/algorithms/tests/test_statistics.py index 5fb4a562d..49191e98a 100644 --- a/extra_foam/algorithms/tests/test_statistics.py +++ b/extra_foam/algorithms/tests/test_statistics.py @@ -4,8 +4,8 @@ import numpy as np -from extra_foam.algorithms import ( - hist_with_stats, nanhist_with_stats, compute_statistics, find_actual_range, +from extra_foam.algorithms.statistics_py import ( + hist_with_stats, nanhist_with_stats, compute_statistics, _get_outer_edges, nanmean, nansum ) @@ -58,6 +58,11 @@ def testNanhistWithStats(self): assert np.median(arr_gt) == median assert np.std(arr_gt) == pytest.approx(std) + # case 5 (finite outer edges cannot be found) + roi = np.array([[-np.inf, np.nan, 3], [np.nan, 5, 6]], dtype=np.float32) + with pytest.raises(ValueError): + nanhist_with_stats(roi, (-np.inf, np.inf), 4) + def testHistWithStats(self): data = np.array([0, 1, 2, 3, 6, 0], dtype=np.float32) # 1D hist, bin_centers, mean, median, std = hist_with_stats(data, (1, 3), 4) @@ -79,40 +84,54 @@ def testHistWithStats(self): # case 3 (elements have the same value) roi = np.array([[1, 1, 1], [1, 1, 1]], dtype=np.float32) # 2D - hist, bin_centers, mean, median, std = nanhist_with_stats(roi, (0, 3), 4) + hist, bin_centers, mean, median, std = hist_with_stats(roi, (0, 3), 4) np.testing.assert_array_equal([0, 6, 0, 0], hist) np.testing.assert_array_equal([0.375, 1.125, 1.875, 2.625], bin_centers) assert 1 == mean assert 1 == median assert 0 == std + # case 4 (finite outer edges cannot be found) + roi = np.array([[np.inf, 2, 3], [4, 5, 6]], dtype=np.float32) + with pytest.raises(ValueError): + hist_with_stats(roi, (-np.inf, np.inf), 4) + def testFindActualRange(self): arr = np.array([1, 2, 3, 4]) - assert (-1.5, 2.5) == find_actual_range(arr, (-1.5, 2.5)) + assert (-1.5, 2.5) == _get_outer_edges(arr, (-1.5, 2.5)) arr = np.array([1, 2, 3, 4]) - assert (1, 4) == find_actual_range(arr, (-math.inf, math.inf)) + assert (1, 4) == _get_outer_edges(arr, (-math.inf, math.inf)) arr = np.array([1, 1, 1, 1]) - assert (0.5, 1.5) == find_actual_range(arr, (-math.inf, math.inf)) + assert (0.5, 1.5) == _get_outer_edges(arr, (-math.inf, math.inf)) arr = np.array([1, 2, 3, 4]) - assert (3, 4) == find_actual_range(arr, (3, math.inf)) + assert (3, 4) == _get_outer_edges(arr, (3, math.inf)) arr = np.array([1, 2, 3, 4]) - assert (4, 5) == find_actual_range(arr, (4, math.inf)) + assert (4, 5) == _get_outer_edges(arr, (4, math.inf)) arr = np.array([1, 2, 3, 4]) - assert (5, 6) == find_actual_range(arr, (5, math.inf)) + assert (5, 6) == _get_outer_edges(arr, (5, math.inf)) arr = np.array([1, 2, 3, 4]) - assert (0, 1) == find_actual_range(arr, (-math.inf, 1)) + assert (0, 1) == _get_outer_edges(arr, (-math.inf, 1)) arr = np.array([1, 2, 3, 4]) - assert (-1, 0) == find_actual_range(arr, (-math.inf, 0)) + assert (-1, 0) == _get_outer_edges(arr, (-math.inf, 0)) arr = np.array([1, 2, 3, 4]) - assert (-1, 0) == find_actual_range(arr, (-math.inf, 0)) + assert (-1, 0) == _get_outer_edges(arr, (-math.inf, 0)) + + arr = np.array([1, 2, 3, -np.inf]) + assert (-np.inf, 0) == _get_outer_edges(arr, (-math.inf, 0)) + + arr = np.array([1, 2, 3, np.inf]) + assert (0, np.inf) == _get_outer_edges(arr, (0, math.inf)) + + arr = np.array([1, -np.inf, 3, np.inf]) + assert (-np.inf, np.inf) == _get_outer_edges(arr, (-math.inf, math.inf)) def testComputeStatistics(self): with np.warnings.catch_warnings(): diff --git a/extra_foam/config.py b/extra_foam/config.py index c88eda7c9..28bfbd659 100644 --- a/extra_foam/config.py +++ b/extra_foam/config.py @@ -41,6 +41,7 @@ class RoiCombo(IntEnum): ROI2 = 2 ROI1_SUB_ROI2 = 3 ROI1_ADD_ROI2 = 4 + ROI1_DIV_ROI2 = 5 ROI3 = 11 ROI4 = 12 ROI3_SUB_ROI4 = 13 @@ -274,8 +275,8 @@ class _Config(dict): "GUI_BACKGROUND_COLOR": (225, 225, 225, 255), # colors of for ROI bounding boxes 1 to 4 "GUI_ROI_COLORS": ('b', 'r', 'g', 'o'), - # colors for correlation plots 1 to 4 - "GUI_CORRELATION_COLORS": ('b', 'o', 'g', 'r'), + # colors (master, slave) for correlation plots 1 and 2 + "GUI_CORRELATION_COLORS": (('b', 'r'), ('g', 'p')), # color of the image mask bounding box while drawing "GUI_MASK_BOUNDING_BOX_COLOR": 'b', # ------------------------------------------------------------- @@ -341,6 +342,13 @@ class _Config(dict): NUMBER_OF_MODULES=1, MODULE_SHAPE=(1934, 960), PIXEL_SIZE=0.030e-3), + "ePix100": _AreaDetectorConfig( + REDIS_PORT=6384, + PULSE_RESOLVED=False, + REQUIRE_GEOMETRY=False, + NUMBER_OF_MODULES=1, + MODULE_SHAPE=(708, 768), + PIXEL_SIZE=0.110e-3), "BaslerCamera": _AreaDetectorConfig( REDIS_PORT=6389, PULSE_RESOLVED=False, @@ -516,6 +524,11 @@ def config_file(self): return _Config.config_file(topic) raise ValueError("TOPIC is not specified!") + @property + def setup_file(self): + detector = self._data['DETECTOR'] + return osp.join(ROOT_PATH, f".{detector.lower()}.setup.yaml") + @property def control_sources(self): return self._data.control_sources @@ -546,6 +559,9 @@ def parse_detector_name(detector): if detector == 'JUNGFRAUPR': return 'JungFrauPR' + if detector == 'EPIX100': + return 'ePix100' + return detector.upper() diff --git a/extra_foam/configs/mid.config.yaml b/extra_foam/configs/mid.config.yaml index c3c58cdd1..327738d9e 100644 --- a/extra_foam/configs/mid.config.yaml +++ b/extra_foam/configs/mid.config.yaml @@ -8,19 +8,22 @@ SOURCE: - image.data MID_DET_AGIPD1M-1/DET/*CH0:xtdf: - image.data + ePix100: + PIPELINE: + # the property name for the raw data is 'data.image.data' + MID_EXP_EPIX-1/DET/RECEIVER:output: + - data.image + MID_EXP_EPIX-2/DET/RECEIVER:output: + - data.image + MID_EXP_EPIX-1/DET/RECEIVER:daqOutput: + - data.image.pixels + MID_EXP_EPIX-2/DET/RECEIVER:daqOutput: + - data.image.pixels DETECTOR: AGIPD: + # AGIPD uses the CFEL geometry format GEOMETRY_FILE: agipd_mar18_v11.geom - QUAD_POSITIONS: - x1: -526 - y1: 630 - x2: -549 - y2: -4 - x3: 522 - y3: -157 - x4: 543 - y4: 477 BRIDGE_ADDR: 10.253.0.51 BRIDGE_PORT: 45012 LOCAL_ADDR: 127.0.0.1 diff --git a/extra_foam/configs/spb.config.yaml b/extra_foam/configs/spb.config.yaml index c526a48da..43eb3884d 100644 --- a/extra_foam/configs/spb.config.yaml +++ b/extra_foam/configs/spb.config.yaml @@ -11,16 +11,8 @@ SOURCE: DETECTOR: AGIPD: + # AGIPD uses the CFEL geometry format GEOMETRY_FILE: agipd_mar18_v11.geom - QUAD_POSITIONS: - x1: -526 - y1: 630 - x2: -549 - y2: -4 - x3: 522 - y3: -157 - x4: 543 - y4: 477 BRIDGE_ADDR: 10.253.0.51 BRIDGE_PORT: 45012 LOCAL_ADDR: 127.0.0.1 diff --git a/extra_foam/database/base_proxy.py b/extra_foam/database/base_proxy.py index b0e820f4a..472c19d82 100644 --- a/extra_foam/database/base_proxy.py +++ b/extra_foam/database/base_proxy.py @@ -8,10 +8,10 @@ All rights reserved. """ from .db_utils import redis_except_handler -from ..ipc import RedisConnection +from ..ipc import ProcessLogger, RedisConnection -class _AbstractProxy: +class _AbstractProxy(ProcessLogger): """_AbstractProxy. Base class for communicate with Redis server. diff --git a/extra_foam/database/metadata.py b/extra_foam/database/metadata.py index be36d343a..c165dc570 100644 --- a/extra_foam/database/metadata.py +++ b/extra_foam/database/metadata.py @@ -7,21 +7,33 @@ Copyright (C) European X-Ray Free-Electron Laser Facility GmbH. All rights reserved. """ +from datetime import datetime +import os.path as osp +from collections import OrderedDict + +import redis +import yaml +from yaml.scanner import ScannerError +from yaml.parser import ParserError + from .base_proxy import _AbstractProxy from .db_utils import redis_except_handler -from ..config import AnalysisType +from ..config import AnalysisType, config class MetaMetadata(type): def __new__(mcs, name, bases, class_dict): proc_list = [] + proc_keys = [] for k, v in class_dict.items(): if isinstance(v, str): s = v.split(":") if len(s) == 3 and s[1] == 'proc': proc_list.append(s[2]) + proc_keys.append(v) class_dict['processors'] = proc_list + class_dict['processor_keys'] = proc_keys cls = type.__new__(mcs, name, bases, class_dict) return cls @@ -35,6 +47,7 @@ class Metadata(metaclass=MetaMetadata): ANALYSIS_TYPE = "meta:analysis_type" # The key of processors' metadata must end with '_PROC' + META_PROC = "meta:proc:meta" GLOBAL_PROC = "meta:proc:global" IMAGE_PROC = "meta:proc:image" GEOMETRY_PROC = "meta:proc:geometry" @@ -142,3 +155,153 @@ def remove_data_source(self, item): return self._db.pipeline().execute_command( 'DEL', key).execute_command( 'PUBLISH', Metadata.DATA_SOURCE, key).execute() + + @redis_except_handler + def take_snapshot(self, name): + """Take a snapshot of the current metadata. + + :param str name: name of the snapshot. + """ + # return (unix time in seconds, microseconds) + timestamp = self._db.execute_command("TIME") + datetime_str = datetime.fromtimestamp(timestamp[0]).strftime( + "%m/%d/%Y, %H:%M:%S") + + cfg = self._read_configuration() + cfg[Metadata.META_PROC]["timestamp"] = datetime_str + cfg[Metadata.META_PROC]["description"] = "" + self._write_configuration(cfg, None, name) + return name, datetime_str, "" + + @redis_except_handler + def load_snapshot(self, name): + """Load metadata snapshot by name. + + :param str name: name of the snapshot. + """ + self._write_configuration(self._read_configuration(name), name, None) + + @redis_except_handler + def copy_snapshot(self, old, new): + """Copy metadata snapshot. + + :param str old: name of the old snapshot. + :param str new: name of the new snapshot. + """ + self._write_configuration(self._read_configuration(old), old, new) + + @redis_except_handler + def remove_snapshot(self, name): + """Remove metadata snapshot by name. + + :param str name: name of the snapshot + """ + pipe = self.pipeline() + for k in Metadata.processor_keys: + pipe.execute_command("DEL", f"{k}:{name}") + pipe.execute() + + @redis_except_handler + def rename_snapshot(self, old, new): + """Rename a metadata snapshot. + + :param str old: old configuration name. + :param str new: new configuration name. + """ + for k in Metadata.processor_keys: + try: + self._db.execute_command("RENAME", f"{k}:{old}", f"{k}:{new}") + except redis.ResponseError: + pass + + def _read_configuration(self, name=None): + """Read a configuration from Redis. + + :param str name: configuration name. + """ + cfg = dict() + for k in Metadata.processor_keys: + if name is not None: + k = f"{k}:{name}" + cfg[k] = self.hget_all(k) + return cfg + + def _write_configuration(self, cfg, old, new): + """Write a configuration into Redis. + + :param dict cfg: configuration. + :param str old: old configuration name. + :param str new: new configuration name. + """ + invalid_keys = [] + for k, v in cfg.items(): + if old is not None: + k_root = k.rsplit(':', maxsplit=1)[0] + else: + k_root = k + + if new is not None: + k_new = f"{k_root}:{new}" + else: + k_new = k_root + + if k_root in Metadata.processor_keys: + if v: + self._db.hmset(k_new, v) + else: + self._db.execute_command("DEL", k_new) + else: + invalid_keys.append(k) + + if invalid_keys: + self.warning( + f"Invalid keys when writing configuration: {invalid_keys}") + + def dump_configurations(self, lst): + """Dump all GUI configurations into file. + + :param list lst: a list of (name, description) of configurations in the + Configurator widget. + """ + filepath = config.setup_file + with open(filepath, 'w') as fp: + configurations = OrderedDict() + for name, description in lst: + configurations[name] = self._read_configuration(name) + meta_key = f"{Metadata.META_PROC}:{name}" + configurations[name][meta_key]["description"] = description + + if configurations: + try: + yaml.dump(configurations, fp, Dumper=yaml.Dumper) + except (ScannerError, ParserError) as e: + self.error(f"Invalid setup file: {filepath}\n{repr(e)}") + + def load_configurations(self): + """Load all GUI configurations from file.""" + filepath = config.setup_file + lst = [] + if osp.isfile(filepath): + with open(filepath, 'r') as fp: + try: + configurations = yaml.load(fp, Loader=yaml.Loader) + for name, cfg in configurations.items(): + meta_key = f"{Metadata.META_PROC}:{name}" + try: + timestamp = cfg[meta_key]["timestamp"] + except KeyError: + self.error(f"Invalid configuration: {name}! " + f"timestamp is missing") + continue + + try: + description = cfg[meta_key]["description"] + except KeyError: + description = "" + + lst.append((name, timestamp, description)) + self._write_configuration(cfg, name, name) + + except (ScannerError, ParserError) as e: + self.error(f"Invalid setup file: {filepath}\n{repr(e)}") + return lst diff --git a/extra_foam/database/tests/test_metadata.py b/extra_foam/database/tests/test_metadata.py index c70b58100..ecdc4f6b3 100644 --- a/extra_foam/database/tests/test_metadata.py +++ b/extra_foam/database/tests/test_metadata.py @@ -1,12 +1,23 @@ import unittest +from unittest.mock import MagicMock, patch +import os +import tempfile -from extra_foam.config import AnalysisType +from extra_foam.config import AnalysisType, config from extra_foam.database.metadata import Metadata, MetaMetadata from extra_foam.database import MetaProxy, MonProxy from extra_foam.processes import wait_until_redis_shutdown from extra_foam.services import start_redis_server +from extra_foam.gui.misc_widgets.configurator import Configurator +LAST_SAVED = Configurator.LAST_SAVED +DEFAULT = Configurator.DEFAULT +_tmp_cfg_dir = tempfile.mkdtemp() + + +@patch("extra_foam.config.ROOT_PATH", _tmp_cfg_dir) +@patch.dict(config._data, {"DETECTOR": "DSSC"}) class TestDataProxy(unittest.TestCase): @classmethod def setUpClass(cls): @@ -15,6 +26,8 @@ def setUpClass(cls): cls._meta = MetaProxy() cls._mon = MonProxy() + cls._meta.error = MagicMock() + @classmethod def tearDownClass(cls): wait_until_redis_shutdown() @@ -23,6 +36,8 @@ def tearDownClass(cls): cls._meta.reset() cls._mon.reset() + os.rmdir(_tmp_cfg_dir) + def testAnalysisType(self): type1 = AnalysisType.AZIMUTHAL_INTEG type2 = AnalysisType.PUMP_PROBE @@ -61,3 +76,100 @@ class Dummy(metaclass=MetaMetadata): GEOMETRY_PROC = "meta:proc:geometry" self.assertListEqual(['global', 'image', 'geometry'], Dummy.processors) + self.assertListEqual([Dummy.GLOBAL_PROC, + Dummy.IMAGE_PROC, + Dummy.GEOMETRY_PROC], Dummy.processor_keys) + + def testSnapshotOperation(self): + data = { + Metadata.IMAGE_PROC: {"aaa": '1', "bbb": "(-1, 1)", "ccc": "sea"}, + Metadata.GLOBAL_PROC: {"a1": '10', "b1": 'True'} + } + + def _write_to_redis(name=None): + for k, v in data.items(): + kk = k if name is None else f"{k}:{name}" + self._meta.hmset(kk, v) + + def _assert_data(meta, name=None): + for k, v in data.items(): + kk = k if name is None else f"{k}:{name}" + self.assertDictEqual(v, meta.hget_all(kk)) + + def _assert_empty_data(meta, name=None): + for k in data: + kk = k if name is None else f"{k}:{name}" + self.assertDictEqual({}, meta.hget_all(kk)) + + # test take snapshot + self._meta.take_snapshot(DEFAULT) + _assert_empty_data(self._meta, DEFAULT) + _write_to_redis() + + _assert_empty_data(self._meta, DEFAULT) + name, timestamp, description = self._meta.take_snapshot(LAST_SAVED) + self.assertEqual(LAST_SAVED, name) + self.assertEqual("", description) + _assert_data(self._meta, LAST_SAVED) + + # test copy snapshot + self._meta.copy_snapshot(LAST_SAVED, "abc") + _assert_data(self._meta, LAST_SAVED) + _assert_data(self._meta, "abc") + + # test load snapshot + self._meta.load_snapshot(DEFAULT) + _assert_empty_data(self._meta) + self._meta.load_snapshot("abc") + _assert_data(self._meta, "abc") + + # test rename snapshot + self._meta.rename_snapshot("abc", "efg") + _assert_empty_data(self._meta, "abc") + _assert_data(self._meta, "efg") + + # test remove snapshot + self._meta.remove_snapshot("efg") + _assert_empty_data(self._meta, "efg") + + def testDumpLoadConfigurations(self): + filepath = config.setup_file + + image_proc_data = {"aaa": '1', "bbb": "(-1, 1)", "ccc": "sea"} + self._meta.hmset(f"{Metadata.IMAGE_PROC}:Last saved", image_proc_data) + self._meta.hmset(f"{Metadata.META_PROC}:Last saved", { + "timestamp": "2020-03-03 03:03:03", + "description": "" + }) + + self._meta.hmset(f"{Metadata.IMAGE_PROC}:abc", image_proc_data) + # no 'description' + self._meta.hmset(f"{Metadata.META_PROC}:abc", { + "timestamp": "2020-03-04 03:03:03", + }) + + # no 'timestamp' + self._meta.hmset(f"{Metadata.IMAGE_PROC}:efg", image_proc_data) + + try: + self._meta.dump_configurations([("Last saved", ''), ("abc", "abc setup"), ("efg", "efg setup")]) + + self._meta.remove_snapshot("Last saved") + self._meta.remove_snapshot("abc") + self._meta.remove_snapshot("efg") + self.assertDictEqual({}, self._meta.hget_all(f"{Metadata.IMAGE_PROC}:Last saved")) + + lst = self._meta.load_configurations() + + self._meta.error.assert_called_once() + self.assertListEqual([('Last saved', '2020-03-03 03:03:03', ''), + ('abc', '2020-03-04 03:03:03', 'abc setup')], lst) + + self.assertDictEqual(image_proc_data, self._meta.hget_all(f"{Metadata.IMAGE_PROC}:Last saved")) + self.assertDictEqual(image_proc_data, self._meta.hget_all(f"{Metadata.IMAGE_PROC}:abc")) + + finally: + os.remove(filepath) + self._meta.remove_snapshot("Last saved") + self._meta.remove_snapshot("abc") + self._meta.remove_snapshot("efg") diff --git a/extra_foam/geometries/__init__.py b/extra_foam/geometries/__init__.py index d589e2289..33749a5b7 100644 --- a/extra_foam/geometries/__init__.py +++ b/extra_foam/geometries/__init__.py @@ -12,6 +12,7 @@ import numpy as np import h5py +from ..algorithms.geometry import AGIPD_1MGeometry as _AGIPD_1MGeometry from ..algorithms.geometry import LPD_1MGeometry as _LPD_1MGeometry from ..algorithms.geometry import DSSC_1MGeometry as _DSSC_1MGeometry @@ -105,3 +106,25 @@ def from_h5_file_and_quad_positions(cls, filepath, positions): modules.append(tiles) return cls(modules) + + +class AGIPD_1MGeometryFast(_AGIPD_1MGeometry, _1MGeometryPyMixin): + """AGIPD_1MGeometryFast. + + Extend the functionality of AGIPD_1MGeometry implementation in C++. + """ + @classmethod + def from_crystfel_geom(cls, filename): + from cfelpyutils.crystfel_utils import load_crystfel_geometry + from extra_geom.detectors import GeometryFragment + + geom_dict = load_crystfel_geometry(filename) + modules = [] + for p in range(cls.n_modules): + tiles = [] + modules.append(tiles) + for a in range(cls.n_tiles_per_module): + d = geom_dict['panels']['p{}a{}'.format(p, a)] + tiles.append(GeometryFragment.from_panel_dict(d).corner_pos) + + return cls(modules) diff --git a/extra_foam/geometries/tests/test_1M_geometry.py b/extra_foam/geometries/tests/test_1M_geometry.py index e652a4dfb..41ceffdcb 100644 --- a/extra_foam/geometries/tests/test_1M_geometry.py +++ b/extra_foam/geometries/tests/test_1M_geometry.py @@ -5,7 +5,9 @@ import numpy as np from extra_foam.pipeline.processors.image_assembler import StackView -from extra_foam.geometries import DSSC_1MGeometryFast, LPD_1MGeometryFast +from extra_foam.geometries import ( + DSSC_1MGeometryFast, LPD_1MGeometryFast, AGIPD_1MGeometryFast +) import extra_geom as eg from extra_foam.config import config @@ -60,7 +62,7 @@ def testAssemblingFile(self, dtype): # FIXME for i in range(2): assert abs(out_fast.shape[i] - out_gt.shape[i]) <= 1 - # np.testing.assert_equal(out_fast, out) + # np.testing.assert_equal(out_fast, out_gt) class TestDSSC_1MGeometryFast(_Test1MGeometryMixin): @@ -120,3 +122,17 @@ def setup_class(cls): cls.n_pulses = 2 cls.n_modules = LPD_1MGeometryFast.n_modules cls.module_shape = LPD_1MGeometryFast.module_shape + + +class TestAGIPD_1MGeometryFast(_Test1MGeometryMixin): + @classmethod + def setup_class(cls): + geom_file = osp.join(_geom_path, "agipd_mar18_v11.geom") + + cls.geom_stack = AGIPD_1MGeometryFast() + cls.geom_fast = AGIPD_1MGeometryFast.from_crystfel_geom(geom_file) + cls.geom = eg.AGIPD_1MGeometry.from_crystfel_geom(geom_file) + + cls.n_pulses = 2 + cls.n_modules = AGIPD_1MGeometryFast.n_modules + cls.module_shape = AGIPD_1MGeometryFast.module_shape diff --git a/extra_foam/gui/ctrl_widgets/analysis_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/analysis_ctrl_widget.py index e1cdbd231..eaa9f2f7e 100644 --- a/extra_foam/gui/ctrl_widgets/analysis_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/analysis_ctrl_widget.py @@ -14,6 +14,7 @@ from .base_ctrl_widgets import _AbstractGroupBoxCtrlWidget from .smart_widgets import SmartLineEdit from ...config import config +from ...database import Metadata as mt class AnalysisCtrlWidget(_AbstractGroupBoxCtrlWidget): @@ -80,3 +81,11 @@ def updateMetaData(self): w.returnPressed.emit() self._ma_window_le.returnPressed.emit() return True + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.GLOBAL_PROC) + self._ma_window_le.setText(cfg["ma_window"]) + if self._pulse_resolved: + self._poi_index_les[0].setText(cfg["poi1_index"]) + self._poi_index_les[1].setText(cfg["poi2_index"]) diff --git a/extra_foam/gui/ctrl_widgets/azimuthal_integ_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/azimuthal_integ_ctrl_widget.py index 8baeea555..64171b6d0 100644 --- a/extra_foam/gui/ctrl_widgets/azimuthal_integ_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/azimuthal_integ_ctrl_widget.py @@ -7,15 +7,18 @@ Copyright (C) European X-Ray Free-Electron Laser Facility GmbH. All rights reserved. """ +from collections import OrderedDict + from PyQt5.QtCore import Qt from PyQt5.QtGui import QDoubleValidator, QIntValidator from PyQt5.QtWidgets import QComboBox, QGridLayout, QLabel from .base_ctrl_widgets import _AbstractCtrlWidget from .smart_widgets import SmartBoundaryLineEdit, SmartLineEdit +from ..gui_helpers import invert_dict from ...algorithms import compute_q -from ...config import config, list_azimuthal_integ_methods - +from ...config import config, list_azimuthal_integ_methods, Normalizer +from ...database import Metadata as mt _DEFAULT_AZIMUTHAL_INTEG_POINTS = 512 @@ -31,6 +34,15 @@ def _estimate_q_range(): class AzimuthalIntegCtrlWidget(_AbstractCtrlWidget): """Widget for setting up azimuthal integration parameters.""" + _available_norms = OrderedDict({ + "": Normalizer.UNDEFINED, + "AUC": Normalizer.AUC, + "XGM": Normalizer.XGM, + "DIGITIZER": Normalizer.DIGITIZER, + "ROI": Normalizer.ROI, + }) + _available_norms_inv = invert_dict(_available_norms) + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -152,8 +164,7 @@ def initConnections(self): mediator.onAiIntegMethodChange) self._norm_cb.currentTextChanged.connect( - lambda x: mediator.onCurveNormalizerChange( - self._available_norms[x])) + lambda x: mediator.onAiNormChange(self._available_norms[x])) self._integ_range_le.value_changed_sgn.connect( mediator.onAiIntegRangeChange) @@ -162,7 +173,7 @@ def initConnections(self): lambda x: mediator.onAiIntegPointsChange(int(x))) self._auc_range_le.value_changed_sgn.connect( - mediator.onAiAucChangeChange) + mediator.onAiAucRangeChange) self._fom_integ_range_le.value_changed_sgn.connect( mediator.onAiFomIntegRangeChange) @@ -193,3 +204,22 @@ def updateMetaData(self): self._fom_integ_range_le.returnPressed.emit() return True + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.GLOBAL_PROC) + self._photon_energy_le.setText(cfg["photon_energy"]) + self._sample_dist_le.setText(cfg["sample_distance"]) + + cfg = self._meta.hget_all(mt.AZIMUTHAL_INTEG_PROC) + self._px_le.setText(cfg['pixel_size_x']) + self._py_le.setText(cfg['pixel_size_y']) + self._cx_le.setText(cfg['integ_center_x']) + self._cy_le.setText(cfg['integ_center_y']) + self._integ_method_cb.setCurrentText(cfg['integ_method']) + self._integ_range_le.setText(cfg['integ_range'][1:-1]) + self._integ_pts_le.setText(cfg['integ_points']) + self._norm_cb.setCurrentText( + self._available_norms_inv[int(cfg['normalizer'])]) + self._auc_range_le.setText(cfg['auc_range'][1:-1]) + self._fom_integ_range_le.setText(cfg['fom_integ_range'][1:-1]) diff --git a/extra_foam/gui/ctrl_widgets/base_ctrl_widgets.py b/extra_foam/gui/ctrl_widgets/base_ctrl_widgets.py index 7780cec48..408ec99f1 100644 --- a/extra_foam/gui/ctrl_widgets/base_ctrl_widgets.py +++ b/extra_foam/gui/ctrl_widgets/base_ctrl_widgets.py @@ -8,12 +8,11 @@ All rights reserved. """ import abc -from collections import OrderedDict from PyQt5.QtWidgets import QFrame, QGroupBox from ..mediator import Mediator -from ...config import Normalizer +from ...database import MetaProxy class _AbstractCtrlWidgetMixin: @@ -36,6 +35,12 @@ def updateMetaData(self): """ raise NotImplementedError + @abc.abstractmethod + def loadMetaData(self): + """Load metadata from Redis and set this control widget.""" + # raise NotImplementedError + pass + @abc.abstractmethod def onStart(self): raise NotImplementedError @@ -47,14 +52,6 @@ def onStop(self): class _AbstractCtrlWidget(QFrame, _AbstractCtrlWidgetMixin): - _available_norms = OrderedDict({ - "": Normalizer.UNDEFINED, - "AUC": Normalizer.AUC, - "XGM": Normalizer.XGM, - "DIGITIZER": Normalizer.DIGITIZER, - "ROI": Normalizer.ROI, - }) - def __init__(self, *, pulse_resolved=True, parent=None): """Initialization. @@ -64,6 +61,7 @@ def __init__(self, *, pulse_resolved=True, parent=None): super().__init__(parent=parent) self._mediator = Mediator() + self._meta = MetaProxy() # widgets whose values are not allowed to change after the "run" # button is clicked @@ -103,6 +101,7 @@ def __init__(self, title, *, pulse_resolved=True, parent=None): self.setStyleSheet(self.GROUP_BOX_STYLE_SHEET) self._mediator = Mediator() + self._meta = MetaProxy() # widgets whose values are not allowed to change after the "run" # button is clicked diff --git a/extra_foam/gui/ctrl_widgets/bin_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/bin_ctrl_widget.py index 2c7b73d25..b3b88df65 100644 --- a/extra_foam/gui/ctrl_widgets/bin_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/bin_ctrl_widget.py @@ -18,7 +18,9 @@ from .base_ctrl_widgets import _AbstractGroupBoxCtrlWidget from .smart_widgets import SmartBoundaryLineEdit, SmartLineEdit +from ..gui_helpers import invert_dict from ...config import AnalysisType, BinMode, config +from ...database import Metadata as mt _N_PARAMS = 2 @@ -37,14 +39,18 @@ class BinCtrlWidget(_AbstractGroupBoxCtrlWidget): "ROI proj": AnalysisType.ROI_PROJ, "azimuthal integ": AnalysisType.AZIMUTHAL_INTEG, }) + _analysis_types_inv = invert_dict(_analysis_types) _bin_modes = OrderedDict({ "average": BinMode. AVERAGE, "accumulcate": BinMode.ACCUMULATE, }) + _bin_modes_inv = invert_dict(_bin_modes) _user_defined_key = config["SOURCE_USER_DEFINED_CATEGORY"] + _UNDEFINED_CATEGORY = '' + def __init__(self, *args, **kwargs): super().__init__("Binning setup", *args, **kwargs) @@ -119,7 +125,7 @@ def initParamTable(self): # loop over bin parameters for i_row in range(n_row): category_cb = QComboBox() - category_cb.addItem('') # default is empty + category_cb.addItem(self._UNDEFINED_CATEGORY) for k, v in self._src_metadata.items(): if v: category_cb.addItem(k) @@ -253,3 +259,47 @@ def updateMetaData(self): self.onBinParamChangeCb(i_row) return True + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.BIN_PROC) + self._analysis_type_cb.setCurrentText( + self._analysis_types_inv[int(cfg["analysis_type"])]) + + self._mode_cb.setCurrentText(self._bin_modes_inv[int(cfg["mode"])]) + + for i in range(_N_PARAMS): + src = cfg[f'source{i+1}'] + if not src: + self._table.cellWidget(i, 0).setCurrentText( + self._UNDEFINED_CATEGORY) + self.onCategoryChange(i, self._UNDEFINED_CATEGORY) + else: + device_id, ppt = src.split(' ') + bin_range = cfg[f'bin_range{i+1}'][1:-1] + n_bins = cfg[f'n_bins{i+1}'] + ctg = self._find_category(device_id, ppt) + + self._table.cellWidget(i, 0).setCurrentText(ctg) + self.onCategoryChange(i, ctg) + if ctg == self._user_defined_key: + self._table.cellWidget(i, 1).setText(device_id) + self._table.cellWidget(i, 2).setText(ppt) + else: + self._table.cellWidget(i, 1).setCurrentText(device_id) + self._table.cellWidget(i, 2).setCurrentText(ppt) + self._table.cellWidget(i, 3).setText(bin_range) + self._table.cellWidget(i, 4).setText(n_bins) + + def _find_category(self, device_id, ppt): + for ctg in self._src_instrument: + ctg_srcs = self._src_instrument[ctg] + if device_id in ctg_srcs and ppt in ctg_srcs[device_id]: + return ctg + + for ctg in self._src_metadata: + ctg_srcs = self._src_metadata[ctg] + if device_id in ctg_srcs and ppt in ctg_srcs[device_id]: + return ctg + + return self._user_defined_key diff --git a/extra_foam/gui/ctrl_widgets/calibration_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/calibration_ctrl_widget.py index 150e99798..6b69d31b9 100644 --- a/extra_foam/gui/ctrl_widgets/calibration_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/calibration_ctrl_widget.py @@ -16,7 +16,8 @@ from .base_ctrl_widgets import _AbstractCtrlWidget from .smart_widgets import SmartSliceLineEdit -from ..gui_helpers import create_icon_button +from ..gui_helpers import create_icon_button, parse_slice_inv +from ...database import Metadata as mt from ...ipc import CalConstantsPub @@ -128,6 +129,20 @@ def updateMetaData(self): self._offset_slicer_le.returnPressed.emit() return True + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.IMAGE_PROC) + + self._correct_gain_cb.setChecked(cfg["correct_gain"] == 'True') + self._correct_offset_cb.setChecked(cfg["correct_offset"] == 'True') + self._dark_as_offset_cb.setChecked(cfg["dark_as_offset"] == 'True') + + if self._pulse_resolved: + self._gain_slicer_le.setText( + parse_slice_inv(cfg["gain_slicer"])) + self._offset_slicer_le.setText( + parse_slice_inv(cfg["offset_slicer"])) + @pyqtSlot() def _loadGainConst(self): filepath = QFileDialog.getOpenFileName( diff --git a/extra_foam/gui/ctrl_widgets/correlation_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/correlation_ctrl_widget.py index 1bda5da2f..c39543d4b 100644 --- a/extra_foam/gui/ctrl_widgets/correlation_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/correlation_ctrl_widget.py @@ -14,11 +14,14 @@ from PyQt5.QtGui import QDoubleValidator from PyQt5.QtWidgets import ( QComboBox, QGridLayout, QHeaderView, QLabel, QPushButton, QTableWidget, + QTableWidgetItem ) from .base_ctrl_widgets import _AbstractGroupBoxCtrlWidget from .smart_widgets import SmartLineEdit +from ..gui_helpers import invert_dict from ...config import AnalysisType, config +from ...database import Metadata as mt _N_PARAMS = 2 # maximum number of correlated parameters _DEFAULT_RESOLUTION = 0.0 @@ -34,9 +37,12 @@ class CorrelationCtrlWidget(_AbstractGroupBoxCtrlWidget): "ROI proj": AnalysisType.ROI_PROJ, "azimuthal integ": AnalysisType.AZIMUTHAL_INTEG, }) + _analysis_types_inv = invert_dict(_analysis_types) _user_defined_key = config["SOURCE_USER_DEFINED_CATEGORY"] + _UNDEFINED_CATEGORY = '' + def __init__(self, *args, **kwargs): super().__init__("Correlation setup", *args, **kwargs) @@ -95,7 +101,7 @@ def initParamTable(self): for i_row in range(_N_PARAMS): category_cb = QComboBox() - category_cb.addItem('') # default is empty + category_cb.addItem(self._UNDEFINED_CATEGORY) for k, v in self._src_metadata.items(): if v: category_cb.addItem(k) @@ -212,3 +218,43 @@ def updateMetaData(self): else: self.onCorrelationParamChangeCb(i_row) return True + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.CORRELATION_PROC) + self._analysis_type_cb.setCurrentText( + self._analysis_types_inv[int(cfg["analysis_type"])]) + + for i in range(_N_PARAMS): + src = cfg[f'source{i+1}'] + if not src: + self._table.cellWidget(i, 0).setCurrentText( + self._UNDEFINED_CATEGORY) + self.onCategoryChange(i, self._UNDEFINED_CATEGORY) + else: + device_id, ppt = src.split(' ') + resolution = cfg[f'resolution{i+1}'] + ctg = self._find_category(device_id, ppt) + + self._table.cellWidget(i, 0).setCurrentText(ctg) + self.onCategoryChange(i, ctg) + if ctg == self._user_defined_key: + self._table.cellWidget(i, 1).setText(device_id) + self._table.cellWidget(i, 2).setText(ppt) + else: + self._table.cellWidget(i, 1).setCurrentText(device_id) + self._table.cellWidget(i, 2).setCurrentText(ppt) + self._table.cellWidget(i, 3).setText(resolution) + + def _find_category(self, device_id, ppt): + for ctg in self._src_instrument: + ctg_srcs = self._src_instrument[ctg] + if device_id in ctg_srcs and ppt in ctg_srcs[device_id]: + return ctg + + for ctg in self._src_metadata: + ctg_srcs = self._src_metadata[ctg] + if device_id in ctg_srcs and ppt in ctg_srcs[device_id]: + return ctg + + return self._user_defined_key diff --git a/extra_foam/gui/ctrl_widgets/filter_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/filter_ctrl_widget.py index b8e1cf190..0a5441d2b 100644 --- a/extra_foam/gui/ctrl_widgets/filter_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/filter_ctrl_widget.py @@ -14,7 +14,9 @@ from .base_ctrl_widgets import _AbstractGroupBoxCtrlWidget from .smart_widgets import SmartBoundaryLineEdit +from ..gui_helpers import invert_dict from ...config import AnalysisType +from ...database import Metadata as mt class FomFilterCtrlWidget(_AbstractGroupBoxCtrlWidget): @@ -24,6 +26,7 @@ class FomFilterCtrlWidget(_AbstractGroupBoxCtrlWidget): "": AnalysisType.UNDEFINED, "ROI FOM": AnalysisType.ROI_FOM, }) + _analysis_types_inv = invert_dict(_analysis_types) def __init__(self, *args, **kwargs): super().__init__("FOM filter setup", *args, **kwargs) @@ -77,3 +80,12 @@ def updateMetaData(self): self._pulse_resolved_cb.toggled.emit( self._pulse_resolved_cb.isChecked()) return True + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.FOM_FILTER_PROC) + self._analysis_type_cb.setCurrentText( + self._analysis_types_inv[int(cfg["analysis_type"])]) + self._fom_range_le.setText(cfg["fom_range"][1:-1]) + if self._pulse_resolved: + self._pulse_resolved_cb.setChecked(cfg["pulse_resolved"] == 'True') diff --git a/extra_foam/gui/ctrl_widgets/geometry_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/geometry_ctrl_widget.py index 13cc7e028..8af316ba0 100644 --- a/extra_foam/gui/ctrl_widgets/geometry_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/geometry_ctrl_widget.py @@ -7,20 +7,30 @@ Copyright (C) European X-Ray Free-Electron Laser Facility GmbH. All rights reserved. """ -import os.path as osp from collections import OrderedDict +import json + from PyQt5.QtCore import Qt +from PyQt5.QtGui import QDoubleValidator from PyQt5.QtWidgets import ( QCheckBox, QComboBox, QFileDialog, QGridLayout, QHeaderView, QHBoxLayout, - QLabel, QLineEdit, QPushButton, QTableWidget, QTableWidgetItem, - QVBoxLayout + QLabel, QPushButton, QTableWidget, QVBoxLayout ) from .base_ctrl_widgets import _AbstractCtrlWidget -from ..gui_helpers import parse_table_widget +from .smart_widgets import SmartLineEdit, SmartStringLineEdit +from ..gui_helpers import invert_dict from ...config import config, GeomAssembler -from ...logger import logger +from ...database import Metadata as mt + + +def _parse_table_widget(widget): + ret = [] + for i in range(widget.columnCount()): + ret.append([float(widget.cellWidget(j, i).text()) + for j in range(widget.rowCount())]) + return ret class GeometryCtrlWidget(_AbstractCtrlWidget): @@ -30,6 +40,7 @@ class GeometryCtrlWidget(_AbstractCtrlWidget): "EXtra-foam": GeomAssembler.OWN, "EXtra-geom": GeomAssembler.EXTRA_GEOM, }) + _assemblers_inv = invert_dict(_assemblers) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -41,16 +52,14 @@ def __init__(self, *args, **kwargs): self._stack_only_cb = QCheckBox("Stack only") self._stack_only_cb.setChecked(False) - if config["DETECTOR"] == "AGIPD": - # FIXME: native AGIPD geometry is not implemented yet - self._assembler_cb.removeItem(0) - self._stack_only_cb.setEnabled(False) - self._quad_positions_tb = QTableWidget() + if config["DETECTOR"] == "AGIPD": + self._quad_positions_tb.setEnabled(False) + self._assembler_cb.setCurrentText("EXtra-geom") - self._geom_file_le = QLineEdit(config["GEOMETRY_FILE"]) + self._geom_file_le = SmartStringLineEdit(config["GEOMETRY_FILE"]) self._geom_file_open_btn = QPushButton("Load geometry file") - self._geom_file_open_btn.clicked.connect(self.loadGeometryFile) + self._geom_file_open_btn.clicked.connect(self._loadGeometryFile) self._non_reconfigurable_widgets = [ self @@ -96,41 +105,53 @@ def initConnections(self): lambda x: mediator.onGeomAssemblerChange( self._assemblers[x])) + self._geom_file_le.value_changed_sgn.connect( + mediator.onGeomFileChange) + def initQuadTable(self): n_row = 2 n_col = 4 - widget = self._quad_positions_tb - widget.setRowCount(n_row) - widget.setColumnCount(n_col) - try: - for i in range(n_row): - for j in range(n_col): - widget.setItem(i, j, QTableWidgetItem( - str(config["QUAD_POSITIONS"][j][i]))) - except IndexError: - pass - - widget.move(0, 0) - widget.setVerticalHeaderLabels(['x', 'y']) - widget.setHorizontalHeaderLabels(['1', '2', '3', '4']) - - header = widget.horizontalHeader() + + table = self._quad_positions_tb + table.setRowCount(n_row) + table.setColumnCount(n_col) + + for i in range(n_row): + for j in range(n_col): + if config["DETECTOR"] in ["LPD", "DSSC"]: + widget = SmartLineEdit(str(config["QUAD_POSITIONS"][j][i])) + else: + widget = SmartLineEdit('0') + widget.setValidator(QDoubleValidator(-999, 999, 6)) + widget.value_changed_sgn.connect(self._updateQuadPositions) + table.setCellWidget(i, j, widget) + + table.move(0, 0) + table.setVerticalHeaderLabels(['x', 'y']) + table.setHorizontalHeaderLabels(['1', '2', '3', '4']) + + header = table.horizontalHeader() + for i in range(n_col): header.setSectionResizeMode(i, QHeaderView.Stretch) - header = widget.verticalHeader() + header = table.verticalHeader() for i in range(n_row): header.setSectionResizeMode(i, QHeaderView.Stretch) - header_height = widget.horizontalHeader().height() - widget.setMinimumHeight(header_height * (n_row + 1.5)) - widget.setMaximumHeight(header_height * (n_row + 2.0)) + header_height = table.horizontalHeader().height() + table.setMinimumHeight(header_height * (n_row + 1.5)) + table.setMaximumHeight(header_height * (n_row + 2.0)) - def loadGeometryFile(self): + def _loadGeometryFile(self): filename = QFileDialog.getOpenFileName()[0] if filename: self._geom_file_le.setText(filename) + def _updateQuadPositions(self): + self._mediator.onGeomQuadPositionsChange( + _parse_table_widget(self._quad_positions_tb)) + def updateMetaData(self): """Override""" if not config['REQUIRE_GEOMETRY']: @@ -141,18 +162,28 @@ def updateMetaData(self): self._assembler_cb.currentTextChanged.emit( self._assembler_cb.currentText()) - geom_file = self._geom_file_le.text() - if not osp.isfile(geom_file): - logger.error(f": {geom_file} is not a valid file") - return False - self._mediator.onGeomFilenameChange(geom_file) + self._geom_file_le.returnPressed.emit() - try: - quad_positions = parse_table_widget(self._quad_positions_tb) - except ValueError as e: - logger.error(": " + repr(e)) - return False - - self._mediator.onGeomQuadPositionsChange(quad_positions) + self._updateQuadPositions() return True + + def loadMetaData(self): + """Override.""" + if not config['REQUIRE_GEOMETRY']: + return + + cfg = self._meta.hget_all(mt.GEOMETRY_PROC) + + self._assembler_cb.setCurrentText( + self._assemblers_inv[int(cfg["assembler"])]) + self._stack_only_cb.setChecked(cfg["stack_only"] == 'True') + self._geom_file_le.setText(cfg["geometry_file"]) + + quad_positions = json.loads(cfg["quad_positions"], encoding='utf8') + table = self._quad_positions_tb + n_rows = table.rowCount() + n_cols = table.columnCount() + for j in range(n_cols): + for i in range(n_rows): + table.cellWidget(i, j).setText(str(quad_positions[j][i])) diff --git a/extra_foam/gui/ctrl_widgets/histogram_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/histogram_ctrl_widget.py index 6119bb783..244c0ce25 100644 --- a/extra_foam/gui/ctrl_widgets/histogram_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/histogram_ctrl_widget.py @@ -17,7 +17,9 @@ from .base_ctrl_widgets import _AbstractGroupBoxCtrlWidget from .smart_widgets import SmartBoundaryLineEdit, SmartLineEdit +from ..gui_helpers import invert_dict from ...config import AnalysisType +from ...database import Metadata as mt class HistogramCtrlWidget(_AbstractGroupBoxCtrlWidget): @@ -27,6 +29,7 @@ class HistogramCtrlWidget(_AbstractGroupBoxCtrlWidget): "": AnalysisType.UNDEFINED, "ROI FOM": AnalysisType.ROI_FOM, }) + _analysis_types_inv = invert_dict(_analysis_types) def __init__(self, *args, **kwargs): super().__init__("Histogram setup", *args, **kwargs) @@ -95,3 +98,13 @@ def updateMetaData(self): self._pulse_resolved_cb.toggled.emit( self._pulse_resolved_cb.isChecked()) return True + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.HISTOGRAM_PROC) + self._analysis_type_cb.setCurrentText( + self._analysis_types_inv[int(cfg["analysis_type"])]) + self._bin_range_le.setText(cfg["bin_range"][1:-1]) + self._n_bins_le.setText(cfg['n_bins']) + if self._pulse_resolved: + self._pulse_resolved_cb.setChecked(cfg["pulse_resolved"] == 'True') diff --git a/extra_foam/gui/ctrl_widgets/image_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/image_ctrl_widget.py index b0ea28ad6..f40470f1a 100644 --- a/extra_foam/gui/ctrl_widgets/image_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/image_ctrl_widget.py @@ -17,6 +17,7 @@ from ..ctrl_widgets.smart_widgets import ( SmartLineEdit, SmartBoundaryLineEdit ) +from ...database import Metadata as mt class ImageCtrlWidget(_AbstractCtrlWidget): @@ -82,3 +83,8 @@ def updateMetaData(self): """Override.""" self.threshold_mask_le.returnPressed.emit() return True + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.IMAGE_PROC) + self.threshold_mask_le.setText(cfg["threshold_mask"][1:-1]) diff --git a/extra_foam/gui/ctrl_widgets/pump_probe_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/pump_probe_ctrl_widget.py index 2e0f74e76..705e221c7 100644 --- a/extra_foam/gui/ctrl_widgets/pump_probe_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/pump_probe_ctrl_widget.py @@ -8,6 +8,7 @@ All rights reserved. """ from collections import OrderedDict +import copy from PyQt5.QtCore import Qt from PyQt5.QtWidgets import ( @@ -15,19 +16,21 @@ ) from .base_ctrl_widgets import _AbstractGroupBoxCtrlWidget -from .smart_widgets import SmartIdLineEdit +from .smart_widgets import SmartSliceLineEdit +from ..gui_helpers import invert_dict, parse_slice_inv from ...config import PumpProbeMode, AnalysisType +from ...database import Metadata as mt class PumpProbeCtrlWidget(_AbstractGroupBoxCtrlWidget): """Widget for setting up pump-probe analysis parameters.""" - _available_modes = OrderedDict({ + __available_modes = OrderedDict({ "": PumpProbeMode.UNDEFINED, "reference as off": PumpProbeMode.REFERENCE_AS_OFF, - "same train": PumpProbeMode.SAME_TRAIN, "even/odd train": PumpProbeMode.EVEN_TRAIN_ON, - "odd/even train": PumpProbeMode.ODD_TRAIN_ON + "odd/even train": PumpProbeMode.ODD_TRAIN_ON, + "same train": PumpProbeMode.SAME_TRAIN, }) _analysis_types = OrderedDict({ @@ -36,23 +39,24 @@ class PumpProbeCtrlWidget(_AbstractGroupBoxCtrlWidget): "ROI proj": AnalysisType.ROI_PROJ, "azimuthal integ": AnalysisType.AZIMUTHAL_INTEG, }) + _analysis_types_inv = invert_dict(_analysis_types) def __init__(self, *args, **kwargs): super().__init__("Pump-probe setup", *args, **kwargs) self._mode_cb = QComboBox() - self._on_pulse_le = SmartIdLineEdit(":") - self._off_pulse_le = SmartIdLineEdit(":") + self._on_pulse_le = SmartSliceLineEdit(":") + self._off_pulse_le = SmartSliceLineEdit(":") - all_keys = list(self._available_modes.keys()) - if self._pulse_resolved: - self._mode_cb.addItems(all_keys) - else: - all_keys.remove("same train") - self._mode_cb.addItems(all_keys) + self._available_modes = copy.copy(self.__available_modes) + self._available_modes_inv = invert_dict(self._available_modes) + if not self._pulse_resolved: + del self._available_modes["same train"] + del self._available_modes_inv[PumpProbeMode.SAME_TRAIN] self._on_pulse_le.setEnabled(False) self._off_pulse_le.setEnabled(False) + self._mode_cb.addItems(list(self._available_modes.keys())) self._analysis_type_cb = QComboBox() self._analysis_type_cb.addItems(list(self._analysis_types.keys())) @@ -101,32 +105,46 @@ def initConnections(self): self._mode_cb.currentTextChanged.connect( lambda x: mediator.onPpModeChange(self._available_modes[x])) - self._mode_cb.currentTextChanged.connect( lambda x: self.onPpModeChange(self._available_modes[x])) self._on_pulse_le.value_changed_sgn.connect( - mediator.onPpOnPulseIdsChange) - + mediator.onPpOnPulseSlicerChange) self._off_pulse_le.value_changed_sgn.connect( - mediator.onPpOffPulseIdsChange) + mediator.onPpOffPulseSlicerChange) def updateMetaData(self): """Override""" - self._abs_difference_cb.toggled.emit( - self._abs_difference_cb.isChecked()) - self._analysis_type_cb.currentTextChanged.emit( self._analysis_type_cb.currentText()) self._mode_cb.currentTextChanged.emit(self._mode_cb.currentText()) self._on_pulse_le.returnPressed.emit() - self._off_pulse_le.returnPressed.emit() + self._abs_difference_cb.toggled.emit( + self._abs_difference_cb.isChecked()) + return True + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.PUMP_PROBE_PROC) + self._analysis_type_cb.setCurrentText( + self._analysis_types_inv[int(cfg["analysis_type"])]) + + self._mode_cb.setCurrentText( + self._available_modes_inv[int(cfg["mode"])]) + + self._abs_difference_cb.setChecked(cfg["abs_difference"] == 'True') + + if self._pulse_resolved: + self._on_pulse_le.setText( + parse_slice_inv(cfg["on_pulse_slicer"])) + self._off_pulse_le.setText( + parse_slice_inv(cfg["off_pulse_slicer"])) + def onPpModeChange(self, pp_mode): if not self._pulse_resolved: return diff --git a/extra_foam/gui/ctrl_widgets/roi_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/roi_ctrl_widget.py index 7015ba768..88fb2f9a1 100644 --- a/extra_foam/gui/ctrl_widgets/roi_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/roi_ctrl_widget.py @@ -17,8 +17,8 @@ from ..plot_widgets.plot_items import RectROI from ..ctrl_widgets import _AbstractCtrlWidget, SmartLineEdit from ..misc_widgets import FColor +from ...database import Metadata as mt from ...config import config -from ...pipeline.data_model import RectRoiGeom class _SingleRoiCtrlWidget(QWidget): @@ -26,14 +26,13 @@ class _SingleRoiCtrlWidget(QWidget): Widget which controls a single ROI. """ - # (idx, x, y, w, h) where idx starts from 1 + # TODO: locked currently is always 0 + # (idx, activated, locked, x, y, w, h) where idx starts from 1 roi_geometry_change_sgn = pyqtSignal(object) _pos_validator = QIntValidator(-10000, 10000) _size_validator = QIntValidator(1, 10000) - INVALID_GEOM = RectRoiGeom.INVALID - def __init__(self, roi: RectROI, *, parent=None): super().__init__(parent=parent) self._roi = roi @@ -102,14 +101,14 @@ def onToggleRoiActivation(self, state): if state == Qt.Checked: self._roi.show() self.enableAllEdit() - x, y = [int(v) for v in self._roi.pos()] - w, h = [int(v) for v in self._roi.size()] else: self._roi.hide() self.disableAllEdit() - x, y, w, h = self.INVALID_GEOM - self.roi_geometry_change_sgn.emit((self._roi.index, x, y, w, h)) + x, y = [int(v) for v in self._roi.pos()] + w, h = [int(v) for v in self._roi.size()] + self.roi_geometry_change_sgn.emit( + (self._roi.index, state == Qt.Checked, 0, x, y, w, h)) @pyqtSlot(object) def onRoiPositionEdited(self, value): @@ -129,10 +128,9 @@ def onRoiPositionEdited(self, value): # otherwise triggers infinite recursion self._roi.stateChanged(finish=False) - if not self._activate_cb.isChecked(): - x, y, w, h = self.INVALID_GEOM - - self.roi_geometry_change_sgn.emit((self._roi.index, x, y, w, h)) + state = self._activate_cb.isChecked() + self.roi_geometry_change_sgn.emit( + (self._roi.index, state, 0, x, y, w, h)) @pyqtSlot(object) def onRoiSizeEdited(self, value): @@ -151,10 +149,8 @@ def onRoiSizeEdited(self, value): # otherwise triggers infinite recursion self._roi.stateChanged(finish=False) - if not self._activate_cb.isChecked(): - x, y, w, h = self.INVALID_GEOM - - self.roi_geometry_change_sgn.emit((self._roi.index, x, y, w, h)) + self.roi_geometry_change_sgn.emit( + (self._roi.index, self._activate_cb.isChecked(), 0, x, y, w, h)) @pyqtSlot(object) def onRoiGeometryChangeFinished(self, roi): @@ -164,15 +160,24 @@ def onRoiGeometryChangeFinished(self, roi): self.updateParameters(x, y, w, h) # inform widgets outside this window - if not self._activate_cb.isChecked(): - x, y, w, h = self.INVALID_GEOM - - self.roi_geometry_change_sgn.emit((roi.index, x, y, w, h)) + self.roi_geometry_change_sgn.emit( + (roi.index, self._activate_cb.isChecked(), 0, x, y, w, h)) def notifyRoiParams(self): # fill the QLineEdit(s) and Redis self._roi.sigRegionChangeFinished.emit(self._roi) + def reloadRoiParams(self, cfg): + state, _, x, y, w, h = [v.strip() for v in cfg.split(',')] + + self.roi_geometry_change_sgn.disconnect() + self._px_le.setText(x) + self._py_le.setText(y) + self._width_le.setText(w) + self._height_le.setText(h) + self.roi_geometry_change_sgn.connect(Mediator().onRoiGeometryChange) + self._activate_cb.setChecked(bool(int(state))) + def updateParameters(self, x, y, w, h): self.roi_geometry_change_sgn.disconnect() self._px_le.setText(str(x)) @@ -230,6 +235,12 @@ def initConnections(self): def updateMetaData(self): """Override.""" - for i, widget in enumerate(self._roi_ctrls, 1): + for _, widget in enumerate(self._roi_ctrls, 1): widget.notifyRoiParams() return True + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.ROI_PROC) + for i, widget in enumerate(self._roi_ctrls, 1): + widget.reloadRoiParams(cfg[f"geom{i}"][1:-1]) diff --git a/extra_foam/gui/ctrl_widgets/roi_fom_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/roi_fom_ctrl_widget.py index 72d6cbcd6..f7ece7178 100644 --- a/extra_foam/gui/ctrl_widgets/roi_fom_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/roi_fom_ctrl_widget.py @@ -10,24 +10,33 @@ from collections import OrderedDict from PyQt5.QtCore import Qt -from PyQt5.QtWidgets import QComboBox, QGridLayout, QLabel +from PyQt5.QtWidgets import QCheckBox, QComboBox, QGridLayout, QLabel -from .base_ctrl_widgets import _AbstractCtrlWidget, _AbstractGroupBoxCtrlWidget -from ...config import RoiCombo, RoiFom +from .base_ctrl_widgets import _AbstractGroupBoxCtrlWidget +from ..gui_helpers import invert_dict +from ...config import Normalizer, RoiCombo, RoiFom +from ...database import Metadata as mt class RoiFomCtrlWidget(_AbstractGroupBoxCtrlWidget): """Widget for setting up ROI FOM analysis parameters.""" - _available_norms = _AbstractCtrlWidget._available_norms.copy() - del _available_norms["AUC"] + _available_norms = OrderedDict({ + "": Normalizer.UNDEFINED, + "XGM": Normalizer.XGM, + "DIGITIZER": Normalizer.DIGITIZER, + "ROI": Normalizer.ROI, + }) + _available_norms_inv = invert_dict(_available_norms) _available_combos = OrderedDict({ "ROI1": RoiCombo.ROI1, "ROI2": RoiCombo.ROI2, "ROI1 - ROI2": RoiCombo.ROI1_SUB_ROI2, "ROI1 + ROI2": RoiCombo.ROI1_ADD_ROI2, + "ROI1 / ROI2": RoiCombo.ROI1_DIV_ROI2, }) + _available_combos_inv = invert_dict(_available_combos) _available_types = OrderedDict({ "SUM": RoiFom.SUM, @@ -36,6 +45,7 @@ class RoiFomCtrlWidget(_AbstractGroupBoxCtrlWidget): "MAX": RoiFom.MAX, "MIN": RoiFom.MIN, }) + _available_types_inv = invert_dict(_available_types) def __init__(self, *args, **kwargs): super().__init__("ROI FOM setup", *args, **kwargs) @@ -51,8 +61,8 @@ def __init__(self, *args, **kwargs): self._norm_cb = QComboBox() for v in self._available_norms: self._norm_cb.addItem(v) - # TODO: implement - self._norm_cb.setDisabled(True) + + self._master_slave_cb = QCheckBox("Master-slave") self.initUI() self.initConnections() @@ -76,6 +86,9 @@ def initUI(self): layout.addWidget(QLabel("Norm: "), row, 0, AR) layout.addWidget(self._norm_cb, row, 1) + row += 1 + layout.addWidget(self._master_slave_cb, row, 1) + self.setLayout(layout) def initConnections(self): @@ -91,9 +104,33 @@ def initConnections(self): self._norm_cb.currentTextChanged.connect( lambda x: mediator.onRoiFomNormChange(self._available_norms[x])) + self._master_slave_cb.toggled.connect( + mediator.onRoiFomMasterSlaveModeChange) + self._master_slave_cb.toggled.connect(self.onMasterSlaveModeToggled) + def updateMetaData(self): """Overload.""" self._combo_cb.currentTextChanged.emit(self._combo_cb.currentText()) self._type_cb.currentTextChanged.emit(self._type_cb.currentText()) self._norm_cb.currentTextChanged.emit(self._norm_cb.currentText()) + self._master_slave_cb.toggled.emit( + self._master_slave_cb.isChecked()) return True + + def onMasterSlaveModeToggled(self, state): + if state: + self._combo_cb.setCurrentText("ROI1") + self._combo_cb.setEnabled(False) + else: + self._combo_cb.setEnabled(True) + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.ROI_PROC) + self._combo_cb.setCurrentText( + self._available_combos_inv[int(cfg["fom:combo"])]) + self._type_cb.setCurrentText( + self._available_types_inv[int(cfg["fom:type"])]) + self._norm_cb.setCurrentText( + self._available_norms_inv[int(cfg["fom:norm"])]) + self._master_slave_cb.setChecked(cfg["fom:master_slave"] == 'True') diff --git a/extra_foam/gui/ctrl_widgets/roi_hist_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/roi_hist_ctrl_widget.py index 276fadcd3..0f92ac65c 100644 --- a/extra_foam/gui/ctrl_widgets/roi_hist_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/roi_hist_ctrl_widget.py @@ -15,7 +15,9 @@ from .base_ctrl_widgets import _AbstractGroupBoxCtrlWidget from .smart_widgets import SmartBoundaryLineEdit, SmartLineEdit +from ..gui_helpers import invert_dict from ...config import RoiCombo +from ...database import Metadata as mt class RoiHistCtrlWidget(_AbstractGroupBoxCtrlWidget): @@ -28,6 +30,7 @@ class RoiHistCtrlWidget(_AbstractGroupBoxCtrlWidget): "ROI1 - ROI2": RoiCombo.ROI1_SUB_ROI2, "ROI1 + ROI2": RoiCombo.ROI1_ADD_ROI2, }) + _available_combos_inv = invert_dict(_available_combos) def __init__(self, *args, **kwargs): super().__init__("ROI histogram setup", *args, **kwargs) @@ -84,3 +87,11 @@ def updateMetaData(self): self._n_bins_le.returnPressed.emit() self._bin_range_le.returnPressed.emit() return True + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.ROI_PROC) + self._combo_cb.setCurrentText( + self._available_combos_inv[int(cfg["hist:combo"])]) + self._n_bins_le.setText(cfg["hist:n_bins"]) + self._bin_range_le.setText(cfg["hist:bin_range"][1:-1]) diff --git a/extra_foam/gui/ctrl_widgets/roi_norm_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/roi_norm_ctrl_widget.py index 07dc1d398..3cdcaed40 100644 --- a/extra_foam/gui/ctrl_widgets/roi_norm_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/roi_norm_ctrl_widget.py @@ -13,7 +13,9 @@ from PyQt5.QtWidgets import QComboBox, QGridLayout, QLabel from .base_ctrl_widgets import _AbstractGroupBoxCtrlWidget +from ..gui_helpers import invert_dict from ...config import RoiCombo, RoiFom +from ...database import Metadata as mt class RoiNormCtrlWidget(_AbstractGroupBoxCtrlWidget): @@ -25,6 +27,7 @@ class RoiNormCtrlWidget(_AbstractGroupBoxCtrlWidget): "ROI3 - ROI4": RoiCombo.ROI3_SUB_ROI4, "ROI3 + ROI4": RoiCombo.ROI3_ADD_ROI4, }) + _available_combos_inv = invert_dict(_available_combos) _available_types = OrderedDict({ "SUM": RoiFom.SUM, @@ -33,6 +36,7 @@ class RoiNormCtrlWidget(_AbstractGroupBoxCtrlWidget): "MAX": RoiFom.MAX, "MIN": RoiFom.MIN, }) + _available_types_inv = invert_dict(_available_types) def __init__(self, *args, **kwargs): super().__init__("ROI normalizer setup", *args, **kwargs) @@ -80,3 +84,11 @@ def updateMetaData(self): self._combo_cb.currentTextChanged.emit(self._combo_cb.currentText()) self._type_cb.currentTextChanged.emit(self._type_cb.currentText()) return True + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.ROI_PROC) + self._combo_cb.setCurrentText( + self._available_combos_inv[int(cfg["norm:combo"])]) + self._type_cb.setCurrentText( + self._available_types_inv[int(cfg["norm:type"])]) diff --git a/extra_foam/gui/ctrl_widgets/roi_proj_ctrl_widget.py b/extra_foam/gui/ctrl_widgets/roi_proj_ctrl_widget.py index 51bcadd73..2db8dcf07 100644 --- a/extra_foam/gui/ctrl_widgets/roi_proj_ctrl_widget.py +++ b/extra_foam/gui/ctrl_widgets/roi_proj_ctrl_widget.py @@ -12,15 +12,24 @@ from PyQt5.QtCore import Qt from PyQt5.QtWidgets import QComboBox, QGridLayout, QLabel -from .base_ctrl_widgets import _AbstractCtrlWidget, _AbstractGroupBoxCtrlWidget +from .base_ctrl_widgets import _AbstractGroupBoxCtrlWidget from .smart_widgets import SmartBoundaryLineEdit -from ...config import RoiCombo, RoiProjType +from ..gui_helpers import invert_dict +from ...database import Metadata as mt +from ...config import Normalizer, RoiCombo, RoiProjType class RoiProjCtrlWidget(_AbstractGroupBoxCtrlWidget): """Widget for setting up ROI 1D projection analysis parameters.""" - _available_norms = _AbstractCtrlWidget._available_norms + _available_norms = OrderedDict({ + "": Normalizer.UNDEFINED, + "AUC": Normalizer.AUC, + "XGM": Normalizer.XGM, + "DIGITIZER": Normalizer.DIGITIZER, + "ROI": Normalizer.ROI, + }) + _available_norms_inv = invert_dict(_available_norms) _available_combos = OrderedDict({ "ROI1": RoiCombo.ROI1, @@ -28,11 +37,13 @@ class RoiProjCtrlWidget(_AbstractGroupBoxCtrlWidget): "ROI1 - ROI2": RoiCombo.ROI1_SUB_ROI2, "ROI1 + ROI2": RoiCombo.ROI1_ADD_ROI2, }) + _available_combos_inv = invert_dict(_available_combos) _available_types = OrderedDict({ "SUM": RoiProjType.SUM, "MEAN": RoiProjType.MEAN, }) + _available_types_inv = invert_dict(_available_types) def __init__(self, *args, **kwargs): super().__init__("ROI projection setup", *args, **kwargs) @@ -123,3 +134,16 @@ def updateMetaData(self): self._auc_range_le.returnPressed.emit() self._fom_integ_range_le.returnPressed.emit() return True + + def loadMetaData(self): + """Override.""" + cfg = self._meta.hget_all(mt.ROI_PROC) + self._combo_cb.setCurrentText( + self._available_combos_inv[int(cfg["proj:combo"])]) + self._type_cb.setCurrentText( + self._available_types_inv[int(cfg["proj:type"])]) + self._direct_cb.setCurrentText(cfg["proj:direct"]) + self._norm_cb.setCurrentText( + self._available_norms_inv[int(cfg["proj:norm"])]) + self._auc_range_le.setText(cfg["proj:auc_range"][1:-1]) + self._fom_integ_range_le.setText(cfg["proj:fom_integ_range"][1:-1]) diff --git a/extra_foam/gui/ctrl_widgets/smart_widgets.py b/extra_foam/gui/ctrl_widgets/smart_widgets.py index 50f19ce29..223593a13 100644 --- a/extra_foam/gui/ctrl_widgets/smart_widgets.py +++ b/extra_foam/gui/ctrl_widgets/smart_widgets.py @@ -43,7 +43,7 @@ def onTextChanged(self): self._text_modified = True def setText(self, text): - """'Press enter' after setText. + """'Press enter after setText. This will remove the background color used when modified. """ diff --git a/extra_foam/gui/gui_helpers.py b/extra_foam/gui/gui_helpers.py index b5e9f6830..33f1f56a2 100644 --- a/extra_foam/gui/gui_helpers.py +++ b/extra_foam/gui/gui_helpers.py @@ -129,35 +129,6 @@ def parse_item(v): return sorted(ret) -def parse_table_widget(widget): - """Parse a table widget to a list of list. - - The inner list represents a row of the table. - - :param QTableWidget widget: a table widget. - - :return list: a list of table elements. - - Examples: - - For the following table, - - col1 col2 - row1 1 2 - row2 3 4 - row3 5 6 - - The return value is [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]. - - TODO: add test - """ - n_row, n_col = widget.rowCount(), widget.columnCount() - ret = [] - for i in range(n_col): - ret.append([float(widget.item(j, i).text()) for j in range(n_row)]) - return ret - - def parse_slice(text): """Parse a string into list which can be converted into a slice object. @@ -195,6 +166,46 @@ def parse_slice(text): raise ValueError(err_msg) +def parse_slice_inv(text): + """Parse a string into a slice notation. + + This function inverts the result from 'parse_slice'. + + :param str text: the input string. + + :return str: the slice notation. + + :raise ValueError + + Examples: + + parse_slice_inv('[None, None]') == ":" + parse_slice_inv('[1, 2]') == "1:2" + parse_slice_inv('[0, 10, 2]') == "0:10:2" + """ + err_msg = f"Failed to convert '{text}' to a slice notation." + + if len(text) > 1: + try: + parts = [None if v.strip() == 'None' else int(v) + for v in text[1:-1].split(',')] + except ValueError: + raise ValueError(err_msg) + + if len(parts) == 2: + s0 = '' if parts[0] is None else str(parts[0]) + s1 = '' if parts[1] is None else str(parts[1]) + return f"{s0}:{s1}" + + if len(parts) == 3: + s0 = '' if parts[0] is None else str(parts[0]) + s1 = '' if parts[1] is None else str(parts[1]) + s2 = '' if parts[2] is None else str(parts[2]) + return f"{s0}:{s1}:{s2}" + + raise ValueError(err_msg) + + def create_icon_button(filename, size): """Create a QPushButton with icon. @@ -208,3 +219,11 @@ def create_icon_button(filename, size): btn.setIconSize(QSize(size, size)) btn.setFixedSize(btn.minimumSizeHint()) return btn + + +def invert_dict(mapping): + """Return a dictionary with key and value swapped.""" + ret = dict() + for k, v in mapping.items(): + ret[v] = k + return ret diff --git a/extra_foam/gui/image_tool/corrected_view.py b/extra_foam/gui/image_tool/corrected_view.py index 0795dd08c..f710ac842 100644 --- a/extra_foam/gui/image_tool/corrected_view.py +++ b/extra_foam/gui/image_tool/corrected_view.py @@ -13,7 +13,7 @@ from PyQt5.QtCore import Qt from PyQt5.QtWidgets import ( - QHBoxLayout, QSplitter, QVBoxLayout + QHBoxLayout, QSplitter, QVBoxLayout, QWidget ) from .simple_image_data import _SimpleImageData @@ -110,6 +110,7 @@ def __init__(self, *args, **kwargs): def initUI(self): """Override.""" + ctrl_widget = QWidget() ctrl_layout = QHBoxLayout() AT = Qt.AlignTop ctrl_layout.addWidget(self._roi_ctrl_widget) @@ -117,6 +118,9 @@ def initUI(self): ctrl_layout.addWidget(self._roi_hist_ctrl_widget, alignment=AT) ctrl_layout.addWidget(self._roi_norm_ctrl_widget, alignment=AT) ctrl_layout.addWidget(self._roi_proj_ctrl_widget, alignment=AT) + ctrl_widget.setLayout(ctrl_layout) + ctrl_widget.setFixedHeight( + self._roi_proj_ctrl_widget.minimumSizeHint().height()) subview_splitter = QSplitter(Qt.Vertical) subview_splitter.setHandleWidth(9) @@ -132,7 +136,7 @@ def initUI(self): layout = QVBoxLayout() layout.addWidget(view_splitter) - layout.addLayout(ctrl_layout) + layout.addWidget(ctrl_widget) self.setLayout(layout) def initConnections(self): diff --git a/extra_foam/gui/image_tool/image_tool.py b/extra_foam/gui/image_tool/image_tool.py index f7a92f93d..f9ea5238e 100644 --- a/extra_foam/gui/image_tool/image_tool.py +++ b/extra_foam/gui/image_tool/image_tool.py @@ -232,6 +232,11 @@ def updateMetaData(self): return False return True + def loadMetaData(self): + """Load metadata from Redis and set child control widgets.""" + for widget in self._ctrl_widgets: + widget.loadMetaData() + def reset(self): """Override.""" pass diff --git a/extra_foam/gui/image_tool/tests/test_image_tool.py b/extra_foam/gui/image_tool/tests/test_image_tool.py index c8db0da80..3540ff7bf 100644 --- a/extra_foam/gui/image_tool/tests/test_image_tool.py +++ b/extra_foam/gui/image_tool/tests/test_image_tool.py @@ -113,6 +113,8 @@ def testRoiCtrlWidget(self): proc = self.pulse_worker._image_roi self.assertEqual(4, len(roi_ctrls)) + # test default + proc.update() for i, ctrl in enumerate(roi_ctrls, 1): @@ -121,7 +123,6 @@ def testRoiCtrlWidget(self): list(ctrl._roi.pos())) self.assertListEqual([int(ctrl._width_le.text()), int(ctrl._height_le.text())], list(ctrl._roi.size())) - # test default values self.assertListEqual(RectRoiGeom.INVALID, getattr(proc, f"_geom{i}")) for ctrl in roi_ctrls: @@ -132,61 +133,63 @@ def testRoiCtrlWidget(self): self.assertFalse(ctrl._px_le.isEnabled()) self.assertFalse(ctrl._py_le.isEnabled()) - roi1_ctrl = roi_ctrls[0] - roi1 = self.view._rois[0] - self.assertIs(roi1_ctrl._roi, roi1) - - # activate ROI1 ctrl - QTest.mouseClick(roi1_ctrl._activate_cb, Qt.LeftButton, - pos=QPoint(2, roi1_ctrl._activate_cb.height()/2)) - self.assertTrue(roi1_ctrl._activate_cb.isChecked()) - proc.update() - - self.assertTupleEqual((int(roi1_ctrl._width_le.text()), int(roi1_ctrl._height_le.text())), - tuple(roi1.size())) - self.assertTupleEqual((int(roi1_ctrl._px_le.text()), int(roi1_ctrl._py_le.text())), - tuple(roi1.pos())) - - # use keyClicks to test that the QLineEdit is enabled - roi1_ctrl._width_le.clear() - QTest.keyClicks(roi1_ctrl._width_le, "10") - QTest.keyPress(roi1_ctrl._width_le, Qt.Key_Enter) - roi1_ctrl._height_le.clear() - QTest.keyClicks(roi1_ctrl._height_le, "30") - QTest.keyPress(roi1_ctrl._height_le, Qt.Key_Enter) - self.assertTupleEqual((10, 30), tuple(roi1.size())) - - # ROI can be outside of the image - roi1_ctrl._px_le.clear() - QTest.keyClicks(roi1_ctrl._px_le, "-1") - QTest.keyPress(roi1_ctrl._px_le, Qt.Key_Enter) - roi1_ctrl._py_le.clear() - QTest.keyClicks(roi1_ctrl._py_le, "-3") - QTest.keyPress(roi1_ctrl._py_le, Qt.Key_Enter) - self.assertTupleEqual((-1, -3), tuple(roi1.pos())) + # test activating ROI + + for i, item in enumerate(zip(roi_ctrls, self.view._rois), 1): + ctrl, roi = item + self.assertIs(ctrl._roi, roi) + + QTest.mouseClick(ctrl._activate_cb, Qt.LeftButton, + pos=QPoint(2, ctrl._activate_cb.height()/2)) + self.assertTrue(ctrl._activate_cb.isChecked()) + proc.update() + w_gt, h_gt = int(ctrl._width_le.text()), int(ctrl._height_le.text()) + self.assertTupleEqual((w_gt, h_gt), tuple(roi.size())) + x_gt, y_gt = int(ctrl._px_le.text()), int(ctrl._py_le.text()) + self.assertTupleEqual((x_gt, y_gt), tuple(roi.pos())) + self.assertListEqual([x_gt, y_gt, w_gt, h_gt], getattr(proc, f"_geom{i}")) + + # use keyClicks to test that the QLineEdit is enabled + ctrl._width_le.clear() + QTest.keyClicks(ctrl._width_le, "10") + QTest.keyPress(ctrl._width_le, Qt.Key_Enter) + ctrl._height_le.clear() + QTest.keyClicks(ctrl._height_le, "30") + QTest.keyPress(ctrl._height_le, Qt.Key_Enter) + self.assertTupleEqual((10, 30), tuple(roi.size())) + + # ROI can be outside of the image + ctrl._px_le.clear() + QTest.keyClicks(ctrl._px_le, "-1") + QTest.keyPress(ctrl._px_le, Qt.Key_Enter) + ctrl._py_le.clear() + QTest.keyClicks(ctrl._py_le, "-3") + QTest.keyPress(ctrl._py_le, Qt.Key_Enter) + self.assertTupleEqual((-1, -3), tuple(roi.pos())) + proc.update() + self.assertListEqual([-1, -3, 10, 30], getattr(proc, f"_geom{i}")) + + # lock ROI ctrl + QTest.mouseClick(ctrl._lock_cb, Qt.LeftButton, + pos=QPoint(2, ctrl._lock_cb.height()/2)) + self.assertTrue(ctrl._activate_cb.isChecked()) + self.assertTrue(ctrl._lock_cb.isChecked()) + self.assertFalse(ctrl._width_le.isEnabled()) + self.assertFalse(ctrl._height_le.isEnabled()) + self.assertFalse(ctrl._px_le.isEnabled()) + self.assertFalse(ctrl._py_le.isEnabled()) - proc.update() - self.assertListEqual([-1, -3, 10, 30], proc._geom1) - - # lock ROI ctrl - QTest.mouseClick(roi1_ctrl._lock_cb, Qt.LeftButton, - pos=QPoint(2, roi1_ctrl._lock_cb.height()/2)) - self.assertTrue(roi1_ctrl._activate_cb.isChecked()) - self.assertTrue(roi1_ctrl._lock_cb.isChecked()) - self.assertFalse(roi1_ctrl._width_le.isEnabled()) - self.assertFalse(roi1_ctrl._height_le.isEnabled()) - self.assertFalse(roi1_ctrl._px_le.isEnabled()) - self.assertFalse(roi1_ctrl._py_le.isEnabled()) - - # deactivate ROI ctrl - QTest.mouseClick(roi1_ctrl._activate_cb, Qt.LeftButton, - pos=QPoint(2, roi1_ctrl._activate_cb.height()/2)) - self.assertFalse(roi1_ctrl._activate_cb.isChecked()) - self.assertTrue(roi1_ctrl._lock_cb.isChecked()) - self.assertFalse(roi1_ctrl._width_le.isEnabled()) - self.assertFalse(roi1_ctrl._height_le.isEnabled()) - self.assertFalse(roi1_ctrl._px_le.isEnabled()) - self.assertFalse(roi1_ctrl._py_le.isEnabled()) + # deactivate ROI ctrl + QTest.mouseClick(ctrl._activate_cb, Qt.LeftButton, + pos=QPoint(2, ctrl._activate_cb.height()/2)) + self.assertFalse(ctrl._activate_cb.isChecked()) + self.assertTrue(ctrl._lock_cb.isChecked()) + self.assertFalse(ctrl._width_le.isEnabled()) + self.assertFalse(ctrl._height_le.isEnabled()) + self.assertFalse(ctrl._px_le.isEnabled()) + self.assertFalse(ctrl._py_le.isEnabled()) + proc.update() + self.assertListEqual(RectRoiGeom.INVALID, getattr(proc, f"_geom{i}")) def testMovingAverageQLineEdit(self): # TODO: remove it in the future @@ -196,15 +199,21 @@ def testMovingAverageQLineEdit(self): @patch("extra_foam.gui.plot_widgets.image_views.ImageAnalysis." "onThresholdMaskChange") - @patch("extra_foam.gui.mediator.Mediator.onImageThresholdMaskChange") - def testThresholdMask(self, on_mask_mediator, on_mask): + def testThresholdMask(self, on_mask): widget = self.image_tool._image_ctrl_widget - widget.threshold_mask_le.clear() - QTest.keyClicks(widget.threshold_mask_le, "1, 10") - QTest.keyPress(widget.threshold_mask_le, Qt.Key_Enter) - on_mask.assert_called_once_with((1, 10)) - on_mask_mediator.assert_called_once_with((1, 10)) + with patch.object(widget._mediator, "onImageThresholdMaskChange") as patched: + widget.threshold_mask_le.clear() + QTest.keyClicks(widget.threshold_mask_le, "1, 10") + QTest.keyPress(widget.threshold_mask_le, Qt.Key_Enter) + on_mask.assert_called_once_with((1, 10)) + patched.assert_called_once_with((1, 10)) + + # test loading meta data + mediator = widget._mediator + mediator.onImageThresholdMaskChange((-100, 10000)) + widget.loadMetaData() + self.assertEqual("-100, 10000", widget.threshold_mask_le.text()) def testAutoLevel(self): widget = self.image_tool._image_ctrl_widget @@ -330,10 +339,15 @@ def testDrawMask(self): np.testing.assert_array_equal(mask_gt, proc._image_mask) # test set a mask which has a different shape from the image - mask_gt = np.zeros((2, 2), dtype=np.bool) + mask_gt = np.ones((2, 2), dtype=np.bool) pub.set(mask_gt) with self.assertRaises(ImageProcessingError): proc.process(data) + # an empty image mask with a different shape will be automatically reset + mask_gt = np.zeros((2, 2), dtype=np.bool) + pub.set(mask_gt) + proc.process(data) + np.testing.assert_array_equal(np.zeros((10, 10), dtype=np.bool), proc._image_mask) def testBulletinView(self): processed = ProcessedData(1357) @@ -462,6 +476,20 @@ def _read_constants_side_effect(fn): self.assertTrue(new_offset) self.assertIsNone(offset) + # test loading meta data + mediator = widget._mediator + mediator.onCalDarkAsOffset(True) + mediator.onCalGainCorrection(False) + mediator.onCalOffsetCorrection(False) + mediator.onCalGainSlicerChange([0, None, 2]) + mediator.onCalOffsetSlicerChange([0, None, 4]) + widget.loadMetaData() + self.assertEqual(True, widget._dark_as_offset_cb.isChecked()) + self.assertEqual(False, widget._correct_gain_cb.isChecked()) + self.assertEqual(False, widget._correct_offset_cb.isChecked()) + self.assertEqual("0::2", widget._gain_slicer_le.text()) + self.assertEqual("0::4", widget._offset_slicer_le.text()) + def testAzimuthalInteg1dCtrlWidget(self): from extra_foam.pipeline.processors.azimuthal_integration import energy2wavelength from extra_foam.gui.ctrl_widgets.azimuthal_integ_ctrl_widget import \ @@ -474,6 +502,7 @@ def testAzimuthalInteg1dCtrlWidget(self): proc.update() + # test default self.assertAlmostEqual(config['SAMPLE_DISTANCE'], proc._sample_dist) self.assertAlmostEqual(0.001 * energy2wavelength(config['PHOTON_ENERGY']), proc._wavelength) self.assertEqual(AnalysisType.UNDEFINED, proc.analysis_type) @@ -490,6 +519,7 @@ def testAzimuthalInteg1dCtrlWidget(self): self.assertEqual(0, proc._poni1) self.assertEqual(0, proc._poni2) + # test setting new values widget._photon_energy_le.setText("12.4") widget._sample_dist_le.setText("0.3") widget._integ_method_cb.setCurrentText('nosplit_csr') @@ -516,6 +546,34 @@ def testAzimuthalInteg1dCtrlWidget(self): self.assertEqual(-1000 * 0.000001, proc._poni2) self.assertEqual(1000 * 0.000002, proc._poni1) + # test loading meta data + mediator = widget._mediator + mediator.onPhotonEnergyChange("2.0") + mediator.onSampleDistanceChange("0.2") + mediator.onAiIntegMethodChange("BBox") + mediator.onAiNormChange(Normalizer.XGM) + mediator.onAiIntegPointsChange(512) + mediator.onAiIntegRangeChange((1, 2)) + mediator.onAiAucRangeChange((2, 3)) + mediator.onAiFomIntegRangeChange((3, 4)) + mediator.onAiPixelSizeXChange(0.001) + mediator.onAiPixelSizeYChange(0.002) + mediator.onAiIntegCenterXChange(1) + mediator.onAiIntegCenterYChange(2) + widget.loadMetaData() + self.assertEqual("2.0", widget._photon_energy_le.text()) + self.assertEqual("0.2", widget._sample_dist_le.text()) + self.assertEqual("BBox", widget._integ_method_cb.currentText()) + self.assertEqual("XGM", widget._norm_cb.currentText()) + self.assertEqual("512", widget._integ_pts_le.text()) + self.assertEqual("1, 2", widget._integ_range_le.text()) + self.assertEqual("2, 3", widget._auc_range_le.text()) + self.assertEqual("3, 4", widget._fom_integ_range_le.text()) + self.assertEqual("0.001", widget._px_le.text()) + self.assertEqual("0.002", widget._py_le.text()) + self.assertEqual("1", widget._cx_le.text()) + self.assertEqual("2", widget._cy_le.text()) + def testRoiFomCtrlWidget(self): widget = self.image_tool._corrected_view._roi_fom_ctrl_widget avail_norms = {value: key for key, value in widget._available_norms.items()} @@ -534,13 +592,33 @@ def testRoiFomCtrlWidget(self): widget._combo_cb.setCurrentText(avail_combos[RoiCombo.ROI1_SUB_ROI2]) widget._type_cb.setCurrentText(avail_types[RoiFom.MEDIAN]) widget._norm_cb.setCurrentText(avail_norms[Normalizer.ROI]) - proc.update() - self.assertEqual(RoiCombo.ROI1_SUB_ROI2, proc._fom_combo) self.assertEqual(RoiFom.MEDIAN, proc._fom_type) self.assertEqual(Normalizer.ROI, proc._fom_norm) + # test activate/deactivate master-slave mode + self.assertFalse(widget._master_slave_cb.isChecked()) + widget._master_slave_cb.setChecked(True) + proc.update() + self.assertTrue(proc._roi_fom_master_slave) + self.assertFalse(widget._combo_cb.isEnabled()) + self.assertEqual("ROI1", widget._combo_cb.currentText()) + widget._master_slave_cb.setChecked(False) + self.assertTrue(widget._combo_cb.isEnabled()) + + # test loading meta data + mediator = widget._mediator + mediator.onRoiFomComboChange(RoiCombo.ROI2) + mediator.onRoiFomTypeChange(RoiFom.MEAN) + mediator.onRoiFomNormChange(Normalizer.XGM) + mediator.onRoiFomMasterSlaveModeChange(False) + widget.loadMetaData() + self.assertEqual("ROI2", widget._combo_cb.currentText()) + self.assertEqual("MEAN", widget._type_cb.currentText()) + self.assertEqual("XGM", widget._norm_cb.currentText()) + self.assertFalse(widget._master_slave_cb.isChecked()) + def testRoiHistCtrl(self): widget = self.image_tool._corrected_view._roi_hist_ctrl_widget avail_combos = {value: key for key, value in widget._available_combos.items()} @@ -558,11 +636,19 @@ def testRoiHistCtrl(self): widget._n_bins_le.setText("100") widget._bin_range_le.setText("-1.0, 10.0") proc.update() - self.assertEqual(RoiCombo.ROI1_SUB_ROI2, proc._hist_combo) self.assertEqual(100, proc._hist_n_bins) self.assertEqual((-1.0, 10.0), proc._hist_bin_range) + mediator = widget._mediator + mediator.onRoiHistComboChange(RoiCombo.ROI2) + mediator.onRoiHistNumBinsChange(10) + mediator.onRoiHistBinRangeChange((-3, 3)) + widget.loadMetaData() + self.assertEqual("ROI2", widget._combo_cb.currentText()) + self.assertEqual("10", widget._n_bins_le.text()) + self.assertEqual("-3, 3", widget._bin_range_le.text()) + def testRoiNormCtrlWidget(self): widget = self.image_tool._corrected_view._roi_norm_ctrl_widget avail_combos = {value: key for key, value in widget._available_combos.items()} @@ -578,17 +664,23 @@ def testRoiNormCtrlWidget(self): # test setting new values widget._combo_cb.setCurrentText(avail_combos[RoiCombo.ROI3_ADD_ROI4]) widget._type_cb.setCurrentText(avail_types[RoiFom.MEDIAN]) - proc.update() - self.assertEqual(RoiCombo.ROI3_ADD_ROI4, proc._norm_combo) self.assertEqual(RoiFom.MEDIAN, proc._norm_type) + # test loading meta data + mediator = widget._mediator + mediator.onRoiNormComboChange(RoiCombo.ROI3_SUB_ROI4) + mediator.onRoiNormTypeChange(RoiProjType.SUM) + widget.loadMetaData() + self.assertEqual("ROI3 - ROI4", widget._combo_cb.currentText()) + self.assertEqual("SUM", widget._type_cb.currentText()) + def testRoiProjCtrlWidget(self): widget = self.image_tool._corrected_view._roi_proj_ctrl_widget - avail_norms = {value: key for key, value in widget._available_norms.items()} - avail_combos = {value: key for key, value in widget._available_combos.items()} - avail_types = {value: key for key, value in widget._available_types.items()} + avail_norms_inv = widget._available_norms_inv + avail_combos_inv = widget._available_combos_inv + avail_types_inv = widget._available_types_inv proc = self.train_worker._image_roi proc.update() @@ -602,10 +694,10 @@ def testRoiProjCtrlWidget(self): self.assertEqual((0, math.inf), proc._proj_auc_range) # test setting new values - widget._combo_cb.setCurrentText(avail_combos[RoiCombo.ROI1_SUB_ROI2]) - widget._type_cb.setCurrentText(avail_types[RoiProjType.MEAN]) + widget._combo_cb.setCurrentText(avail_combos_inv[RoiCombo.ROI1_SUB_ROI2]) + widget._type_cb.setCurrentText(avail_types_inv[RoiProjType.MEAN]) widget._direct_cb.setCurrentText('y') - widget._norm_cb.setCurrentText(avail_norms[Normalizer.ROI]) + widget._norm_cb.setCurrentText(avail_norms_inv[Normalizer.ROI]) widget._fom_integ_range_le.setText("10, 20") widget._auc_range_le.setText("30, 40") proc.update() @@ -616,10 +708,27 @@ def testRoiProjCtrlWidget(self): self.assertEqual((10, 20), proc._proj_fom_integ_range) self.assertEqual((30, 40), proc._proj_auc_range) + # test loading meta data + mediator = widget._mediator + mediator.onRoiProjComboChange(RoiCombo.ROI1_ADD_ROI2) + mediator.onRoiProjTypeChange(RoiProjType.SUM) + mediator.onRoiProjDirectChange('x') + mediator.onRoiProjNormChange(Normalizer.XGM) + mediator.onRoiProjAucRangeChange((1, 2)) + mediator.onRoiProjFomIntegRangeChange((-5, 5)) + widget.loadMetaData() + self.assertEqual("ROI1 + ROI2", widget._combo_cb.currentText()) + self.assertEqual("SUM", widget._type_cb.currentText()) + self.assertEqual("x", widget._direct_cb.currentText()) + self.assertEqual("XGM", widget._norm_cb.currentText()) + self.assertEqual("1, 2", widget._auc_range_le.text()) + self.assertEqual("-5, 5", widget._fom_integ_range_le.text()) + def testGeometryCtrlWidget(self): from extra_geom import LPD_1MGeometry as LPD_1MGeometry from extra_foam.geometries import LPD_1MGeometryFast from extra_foam.config import GeomAssembler + from extra_foam.gui.ctrl_widgets.geometry_ctrl_widget import _parse_table_widget cw = self.image_tool._views_tab view = self.image_tool._geometry_view @@ -634,23 +743,40 @@ def testGeometryCtrlWidget(self): self.assertIsInstance(proc._geom, LPD_1MGeometryFast) self.assertFalse(proc._stack_only) self.assertEqual(GeomAssembler.OWN, proc._assembler_type) + self.assertListEqual([list(v) for v in config["QUAD_POSITIONS"]], proc._quad_position) - # test assembler + # test setting new values avail_assemblers = {value: key for key, value in widget._assemblers.items()} widget._assembler_cb.setCurrentText(avail_assemblers[GeomAssembler.EXTRA_GEOM]) - proc.update() - self.assertEqual(GeomAssembler.EXTRA_GEOM, proc._assembler_type) - - # test stack only widget._stack_only_cb.setChecked(True) + widget._geom_file_le.setText("/geometry/file/") + for i in range(4): + for j in range(2): + widget._quad_positions_tb.cellWidget(j, i).setText("0.0") proc.update() self.assertTrue(proc._stack_only) # when "stack only" is checked, "assembler type" setup will be ignored self.assertEqual(GeomAssembler.OWN, proc._assembler_type) + self.assertEqual("/geometry/file/", proc._geom_file) + self.assertListEqual([[0., 0.] for i in range(4)], proc._quad_position) - # test geometry file - widget._geom_file_le.setText("/geometry/file/") - self.assertFalse(widget.updateMetaData()) + widget._stack_only_cb.setChecked(False) + with patch.object(proc, "_load_geometry"): + proc.update() + self.assertEqual(GeomAssembler.EXTRA_GEOM, proc._assembler_type) + + # test loading meta data + mediator = widget._mediator + mediator.onGeomAssemblerChange(GeomAssembler.EXTRA_GEOM) + mediator.onGeomFileChange('geometry/new_file') + mediator.onGeomStackOnlyChange(False) + quad_positions = [[1., 2.], [3., 4.], [5., 6.], [7., 8.]] + mediator.onGeomQuadPositionsChange(quad_positions) + widget.loadMetaData() + self.assertEqual("EXtra-geom", widget._assembler_cb.currentText()) + self.assertEqual('geometry/new_file', widget._geom_file_le.text()) + self.assertFalse(widget._stack_only_cb.isChecked()) + self.assertListEqual(quad_positions, _parse_table_widget((widget._quad_positions_tb))) def testViewTabSwitching(self): tab = self.image_tool._views_tab @@ -730,5 +856,15 @@ def testCalibrationCtrlWidget(self): self.assertFalse(widget._gain_slicer_le.isEnabled()) self.assertFalse(widget._offset_slicer_le.isEnabled()) + # test loading meta data + # test if the meta data is invalid + mediator = widget._mediator + mediator.onCalGainSlicerChange([0, None, 2]) + mediator.onCalOffsetSlicerChange([0, None, 2]) + widget.loadMetaData() + self.assertEqual(":", widget._gain_slicer_le.text()) + self.assertEqual(":", widget._offset_slicer_le.text()) + + if __name__ == '__main__': unittest.main() diff --git a/extra_foam/gui/main_gui.py b/extra_foam/gui/main_gui.py index 5925290d0..52a929b71 100644 --- a/extra_foam/gui/main_gui.py +++ b/extra_foam/gui/main_gui.py @@ -35,7 +35,7 @@ HistogramCtrlWidget, PumpProbeCtrlWidget, TrXasCtrlWidget, ) from .gui_helpers import create_icon_button -from .misc_widgets import GuiLogger +from .misc_widgets import Configurator, GuiLogger from .image_tool import ImageToolWindow from .windows import ( BinningWindow, CorrelationWindow, HistogramWindow, PulseOfInterestWindow, @@ -144,16 +144,6 @@ def __init__(self, pause_ev, close_ev): self._util_panel_container = QWidget() self._util_panel_cw = QTabWidget() - # ************************************************************* - # Menu bar - # ************************************************************* - # self._menu_bar = self.menuBar() - # file_menu = self._menu_bar.addMenu('&Config') - # save_cfg = QAction('Save config', self) - # file_menu.addAction(save_cfg) - # load_cfg = QAction('Load config', self) - # file_menu.addAction(load_cfg) - # ************************************************************* # Tool bar # Note: the order of 'addAction` affect the unittest!!! @@ -227,6 +217,8 @@ def __init__(self, pause_ev, close_ev): self._logger = GuiLogger(parent=self) logging.getLogger().addHandler(self._logger) + self._configurator = Configurator() + self._thread_logger = ThreadLoggerBridge() self.quit_sgn.connect(self._thread_logger.stop) self._thread_logger_t = QThread() @@ -281,6 +273,7 @@ def __init__(self, pause_ev, close_ev): self.initUI() self.initConnections() self.updateMetaData() + self._configurator.onInit() self.setMinimumSize(640, 480) self.resize(self._WIDTH, self._HEIGHT) @@ -359,6 +352,7 @@ def initSpecialAnalysisUI(self): def initUtilUI(self): self._util_panel_cw.addTab(self._logger.widget, "Logger") + self._util_panel_cw.addTab(self._configurator, "Configurator") self._util_panel_cw.setTabPosition(QTabWidget.TabPosition.South) layout = QVBoxLayout() @@ -366,7 +360,7 @@ def initUtilUI(self): self._util_panel_container.setLayout(layout) def initConnections(self): - pass + self._configurator.load_metadata_sgn.connect(self.loadMetaData) def connect_input_to_output(self, output): self._input.connect(output) @@ -516,6 +510,7 @@ def onStart(self): for widget in self._ctrl_widgets: widget.onStart() self._image_tool.onStart() + self._configurator.onStart() self._running = True # starting to update plots self._input_update_ev.set() # notify update @@ -534,6 +529,7 @@ def onStop(self): for widget in self._ctrl_widgets: widget.onStop() self._image_tool.onStop() + self._configurator.onStop() def updateMetaData(self): """Update metadata from all the ctrl widgets. @@ -547,6 +543,12 @@ def updateMetaData(self): return False return self._image_tool.updateMetaData() + def loadMetaData(self): + """Load metadata from Redis and set child control widgets.""" + for widget in self._ctrl_widgets: + widget.loadMetaData() + return self._image_tool.loadMetaData() + @pyqtSlot(str, str) def onLogMsgReceived(self, ch, msg): if ch == 'log:debug': diff --git a/extra_foam/gui/mediator.py b/extra_foam/gui/mediator.py index d0396f68c..4c73e651b 100644 --- a/extra_foam/gui/mediator.py +++ b/extra_foam/gui/mediator.py @@ -68,25 +68,25 @@ def onSourceItemToggled(self, checked: bool, item: object): self._meta.remove_data_source(item) def onCalGainCorrection(self, value: bool): - self._meta.hset(mt.IMAGE_PROC, "correct gain", str(value)) + self._meta.hset(mt.IMAGE_PROC, "correct_gain", str(value)) def onCalOffsetCorrection(self, value: bool): - self._meta.hset(mt.IMAGE_PROC, "correct offset", str(value)) + self._meta.hset(mt.IMAGE_PROC, "correct_offset", str(value)) def onCalGainSlicerChange(self, value: list): - self._meta.hset(mt.IMAGE_PROC, "gain slicer", str(value)) + self._meta.hset(mt.IMAGE_PROC, "gain_slicer", str(value)) def onCalOffsetSlicerChange(self, value: list): - self._meta.hset(mt.IMAGE_PROC, "offset slicer", str(value)) + self._meta.hset(mt.IMAGE_PROC, "offset_slicer", str(value)) def onCalDarkAsOffset(self, value:bool): - self._meta.hset(mt.IMAGE_PROC, "dark as offset", str(value)) + self._meta.hset(mt.IMAGE_PROC, "dark_as_offset", str(value)) def onCalDarkRecording(self, value: bool): - self._meta.hset(mt.IMAGE_PROC, "recording dark", str(value)) + self._meta.hset(mt.IMAGE_PROC, "recording_dark", str(value)) def onCalDarkRemove(self): - self._meta.hset(mt.IMAGE_PROC, "remove dark", 1) + self._meta.hset(mt.IMAGE_PROC, "remove_dark", 1) def onImageThresholdMaskChange(self, value: tuple): self._meta.hset(mt.IMAGE_PROC, "threshold_mask", str(value)) @@ -97,7 +97,7 @@ def onGeomStackOnlyChange(self, value: bool): def onGeomAssemblerChange(self, value: IntEnum): self._meta.hset(mt.GEOMETRY_PROC, "assembler", int(value)) - def onGeomFilenameChange(self, value: str): + def onGeomFileChange(self, value: str): self._meta.hset(mt.GEOMETRY_PROC, "geometry_file", value) def onGeomQuadPositionsChange(self, value: str): @@ -144,10 +144,10 @@ def onAiIntegPointsChange(self, value: int): def onAiIntegRangeChange(self, value: tuple): self._meta.hset(mt.AZIMUTHAL_INTEG_PROC, 'integ_range', str(value)) - def onCurveNormalizerChange(self, value: IntEnum): + def onAiNormChange(self, value: IntEnum): self._meta.hset(mt.AZIMUTHAL_INTEG_PROC, 'normalizer', int(value)) - def onAiAucChangeChange(self, value: tuple): + def onAiAucRangeChange(self, value: tuple): self._meta.hset(mt.AZIMUTHAL_INTEG_PROC, 'auc_range', str(value)) def onAiFomIntegRangeChange(self, value: tuple): @@ -156,11 +156,11 @@ def onAiFomIntegRangeChange(self, value: tuple): def onPpModeChange(self, value: IntEnum): self._meta.hset(mt.PUMP_PROBE_PROC, 'mode', int(value)) - def onPpOnPulseIdsChange(self, value: list): - self._meta.hset(mt.PUMP_PROBE_PROC, 'on_pulse_indices', str(value)) + def onPpOnPulseSlicerChange(self, value: list): + self._meta.hset(mt.PUMP_PROBE_PROC, 'on_pulse_slicer', str(value)) - def onPpOffPulseIdsChange(self, value: list): - self._meta.hset(mt.PUMP_PROBE_PROC, 'off_pulse_indices', str(value)) + def onPpOffPulseSlicerChange(self, value: list): + self._meta.hset(mt.PUMP_PROBE_PROC, 'off_pulse_slicer', str(value)) def onPpAnalysisTypeChange(self, value: IntEnum): self._meta.hset(mt.PUMP_PROBE_PROC, 'analysis_type', int(value)) @@ -174,8 +174,9 @@ def onPpReset(self): self.onResetMa() def onRoiGeometryChange(self, value: tuple): - idx, x, y, w, h = value - self._meta.hset(mt.ROI_PROC, f'geom{idx}', str((x, y, w, h))) + idx, activated, locked, x, y, w, h = value + self._meta.hset(mt.ROI_PROC, f'geom{idx}', + str((int(activated), int(locked), x, y, w, h))) def onRoiFomTypeChange(self, value: IntEnum): self._meta.hset(mt.ROI_PROC, 'fom:type', int(value)) @@ -186,6 +187,9 @@ def onRoiFomComboChange(self, value: IntEnum): def onRoiFomNormChange(self, value: IntEnum): self._meta.hset(mt.ROI_PROC, "fom:norm", int(value)) + def onRoiFomMasterSlaveModeChange(self, value: bool): + self._meta.hset(mt.ROI_PROC, "fom:master_slave", str(value)) + def onRoiHistComboChange(self, value: IntEnum): self._meta.hset(mt.ROI_PROC, "hist:combo", int(value)) @@ -232,7 +236,8 @@ def onCorrelationParamChange(self, value: tuple): pipe.execute() def onCorrelationReset(self): - self._meta.hset(mt.CORRELATION_PROC, "reset", 1) + self._meta.hset(mt.CORRELATION_PROC, "reset1", 1) + self._meta.hset(mt.CORRELATION_PROC, "reset2", 1) def onBinParamChange(self, value: tuple): # index, source, bin_range, number of bins, diff --git a/extra_foam/gui/misc_widgets/__init__.py b/extra_foam/gui/misc_widgets/__init__.py index 02ec64d25..b1add9350 100644 --- a/extra_foam/gui/misc_widgets/__init__.py +++ b/extra_foam/gui/misc_widgets/__init__.py @@ -2,3 +2,4 @@ FColor, SequentialColor, colorMapFactory ) from .gui_logger import GuiLogger, InputDialogWithCheckBox +from .configurator import Configurator diff --git a/extra_foam/gui/misc_widgets/configurator.py b/extra_foam/gui/misc_widgets/configurator.py new file mode 100644 index 000000000..1e569cc4a --- /dev/null +++ b/extra_foam/gui/misc_widgets/configurator.py @@ -0,0 +1,292 @@ +""" +Distributed under the terms of the BSD 3-Clause License. + +The full license is in the file LICENSE, distributed with this software. + +Author: Jun Zhu +Copyright (C) European X-Ray Free-Electron Laser Facility GmbH. +All rights reserved. +""" +from PyQt5.QtCore import Qt, pyqtSlot, pyqtSignal +from PyQt5.QtWidgets import ( + QGridLayout, QHeaderView, QInputDialog, QLineEdit, QMenu, QMessageBox, + QPushButton, QTableWidget, QTableWidgetItem, QWidget, +) + +from ...database import MetaProxy +from ...logger import logger + + +class Configurator(QWidget): + + load_metadata_sgn = pyqtSignal() + + DEFAULT = "default" + LAST_SAVED = "Last saved" + + def __init__(self): + super().__init__() + + self._table = QTableWidget() + self._table.setContextMenuPolicy(Qt.CustomContextMenu) + + self._snapshot_btn = QPushButton("Take snapshot") + self._reset_btn = QPushButton("Reset to default") + self._save_cfg_btn = QPushButton("Save setups in file") + self._load_cfg_btn = QPushButton("Load setups from file") + + self._meta = MetaProxy() + self._config = dict() # key: name, value: # of row + + self.initUI() + self.initConnections() + + def initUI(self): + table = self._table + table.setColumnCount(3) + table.setHorizontalHeaderLabels(['Name', 'Timestamp', 'Description']) + + header = table.horizontalHeader() + header.setSectionResizeMode(QHeaderView.Interactive) + header.setSectionResizeMode(1, QHeaderView.ResizeToContents) + header.setSectionResizeMode(2, QHeaderView.Stretch) + + layout = QGridLayout() + layout.addWidget(table, 0, 0, 1, 4) + layout.addWidget(self._snapshot_btn, 1, 0) + layout.addWidget(self._reset_btn, 1, 1) + layout.addWidget(self._save_cfg_btn, 1, 2) + layout.addWidget(self._load_cfg_btn, 1, 3) + self.setLayout(layout) + + def initConnections(self): + self._snapshot_btn.clicked.connect(lambda x: self._takeSnapshot()) + self._reset_btn.clicked.connect(self._resetToDefault) + self._table.itemDoubleClicked.connect(self.onItemDoubleClicked) + self._table.customContextMenuRequested.connect(self.showContextMenu) + self._save_cfg_btn.clicked.connect(self._askSaveConfiguration) + self._load_cfg_btn.clicked.connect(self._askLoadConfiguration) + + def onInit(self): + """Called by the MainGUI on initialization.""" + # save a default snapshot + self._meta.copy_snapshot(None, self.DEFAULT) + self._loadConfigurations() + + def onStart(self): + self._table.setDisabled(True) + self._snapshot_btn.setDisabled(True) + self._reset_btn.setDisabled(True) + self._save_cfg_btn.setDisabled(True) + self._load_cfg_btn.setDisabled(True) + + def onStop(self): + self._table.setEnabled(True) + self._snapshot_btn.setEnabled(True) + self._reset_btn.setEnabled(True) + self._save_cfg_btn.setEnabled(True) + self._load_cfg_btn.setEnabled(True) + + def _insertConfigurationToList(self, cfg, row=None): + """Insert a row for the new configuration. + + Note: this method does not involve any database operation. + + :param list/tuple cfg: (name, timestamp, description) of the + configuration. + """ + table = self._table + + if cfg[0] == self.LAST_SAVED: + row = 0 + elif row is None: + row = table.rowCount() + + table.insertRow(row) + for col, text in zip(range(table.columnCount()), cfg): + item = QTableWidgetItem() + table.setItem(row, col, item) + flag = Qt.ItemIsEnabled + if cfg[0] != self.LAST_SAVED and col == 2: + flag |= Qt.ItemIsEditable + item.setFlags(flag) + try: + item.setText(text) + except TypeError: + # I hope it will not happen in real life. + item.setText(" ") + logger.error(f"TypeError: Invalid value {text} for column " + f"{col+1} in Configurator!") + + if row != len(self._config): + # adjust the row indices for the other configurations + for k, v in self._config.items(): + if v >= row: + self._config[k] += 1 + + self._config[cfg[0]] = row + + def _removeConfigurationFromList(self, name): + """Remove a row from the table and configuration list. + + :param str name: name of the configuration. + """ + row = self._config[name] + del self._config[name] + self._table.removeRow(row) + + if row != len(self._config): + # adjust the row indices for the other configurations + for k, v in self._config.items(): + if v > row: + self._config[k] -= 1 + + def _copyConfiguration(self, row, new_name): + """Copy a row and insert the new one at the end of the table.""" + cfg = [self._table.item(row, i).text() + for i in range(self._table.columnCount())] + + self._meta.copy_snapshot(cfg[0], new_name) + cfg[0] = new_name + self._insertConfigurationToList(cfg) + + def _removeConfiguration(self, row): + """Remove a row of configuration.""" + table = self._table + name = table.item(row, 0).text() + + self._meta.remove_snapshot(name) + self._removeConfigurationFromList(name) + + def _renameConfiguration(self, row, new_name): + """Rename a row of configuration.""" + table = self._table + name = table.item(row, 0).text() + + self._meta.rename_snapshot(name, new_name) + + del self._config[name] + self._config[new_name] = row + table.item(row, 0).setText(new_name) + + def _askSaveConfiguration(self): + reply = QMessageBox.question( + self, "Save configurations", + "Setups in the file will be lost. Continue?", + QMessageBox.Yes | QMessageBox.No) + + if reply == QMessageBox.Yes: + self._saveConfigurations() + + def _saveConfigurations(self): + """Save all configurations to file.""" + table = self._table + sorted_config = sorted(self._config.items(), key=lambda x: x[1]) + lst = [] + for name, row in sorted_config: + # "description" was not saved in Redis when edited in the table + lst.append((name, table.item(row, 2).text())) + self._meta.dump_configurations(lst) + + def _askLoadConfiguration(self): + reply = QMessageBox.question( + self, "Load configurations", + "Current snapshots will be overwritten in case of name conflict. " + "Continue?", + QMessageBox.Yes | QMessageBox.No) + + if reply == QMessageBox.Yes: + self._loadConfigurations() + + def _loadConfigurations(self): + """Load configurations from file.""" + cfg_list = self._meta.load_configurations() + + for cfg in cfg_list: + # 'self._meta.load_configurations' has already + # write all the configurations into Redis. + if cfg[0] in self._config: + self._removeConfigurationFromList(cfg[0]) + + for cfg in cfg_list: + self._insertConfigurationToList(cfg) + + if self.LAST_SAVED not in self._config: + self._takeSnapshot() + + def _takeSnapshot(self): + """Take a snapshot of the current configuration.""" + table = self._table + cfg = self._meta.take_snapshot(self.LAST_SAVED) + + if table.rowCount() == 0 or table.item(0, 0).text() != self.LAST_SAVED: + self._insertConfigurationToList(cfg, 0) + else: + for i, text in zip(range(table.columnCount()), cfg): + table.item(0, i).setText(text) + + def _resetToDefault(self): + self._meta.load_snapshot(self.DEFAULT) + self.load_metadata_sgn.emit() + + def onItemDoubleClicked(self, item): + """Double-click the name to set the configuration.""" + if self._table.column(item) < 2: + self._meta.load_snapshot(item.text()) + self.load_metadata_sgn.emit() + + def showContextMenu(self, pos): + table = self._table + item = table.itemAt(pos) + if table.column(item) != 0: + # show context menu only when right-clicking on the name + return + + row = self._table.row(item) + menu = QMenu() + copy_action = menu.addAction("Copy snapshot") + if row != 0: + # The first one is always "Last saved" and is not allowed + # to be deleted + delete_action = menu.addAction("Delete snapshot") + rename_action = menu.addAction("Rename snapshot") + + action = menu.exec_(self.mapToGlobal(pos)) + if action == copy_action: + new_name, ok = QInputDialog.getText( + self, "", "New name: ", QLineEdit.Normal, "") + + if not self._checkConfigName(new_name): + return + + if ok: + self._copyConfiguration(row, new_name) + + elif row != 0: + if action == delete_action: + self._removeConfiguration(row) + + elif action == rename_action: + new_name, ok = QInputDialog.getText( + self, "", "New name: ", QLineEdit.Normal, "") + + if not self._checkConfigName(new_name): + return + + if ok: + self._renameConfiguration(row, new_name) + + def _checkConfigName(self, name): + if name in self._config: + logger.error(f"Configuration '{name}' already exists!") + return False + + if name in [self.DEFAULT, self.LAST_SAVED]: + logger.error( + f"'{name}' is not allowed for user-defined configuration!") + return False + + if name == '': + return False + + return True diff --git a/extra_foam/gui/misc_widgets/tests/test_configurator.py b/extra_foam/gui/misc_widgets/tests/test_configurator.py new file mode 100644 index 000000000..5c7b2d3b3 --- /dev/null +++ b/extra_foam/gui/misc_widgets/tests/test_configurator.py @@ -0,0 +1,133 @@ +import unittest +from unittest.mock import MagicMock, patch + +from PyQt5.QtCore import Qt +from PyQt5.QtTest import QSignalSpy, QTest +from PyQt5.QtWidgets import QMessageBox + +from extra_foam.gui import mkQApp +from extra_foam.gui.misc_widgets.configurator import Configurator + +app = mkQApp() + + +class TestConfigurator(unittest.TestCase): + def setUp(self): + self._widget = Configurator() + self._widget._meta = MagicMock() + self._widget._meta.take_snapshot.return_value = ( + self._widget.LAST_SAVED, "2020-01-01 01:01:01", "") + self._widget.onInit() + + def testAddCopyRemoveConfiguration(self): + widget = self._widget + table = widget._table + + self.assertEqual(1, table.rowCount()) + self.assertEqual(widget.LAST_SAVED, table.item(0, 0).text()) + self.assertEqual("2020-01-01 01:01:01", table.item(0, 1).text()) + self.assertEqual("", table.item(0, 2).text()) + + # test add + + cfg = ["abc", "2020-02-02 02:02:02", "abc setup"] + widget._insertConfigurationToList(cfg) + self.assertEqual("abc", table.item(1, 0).text()) + self.assertEqual("2020-02-02 02:02:02", table.item(1, 1).text()) + self.assertEqual("abc setup", table.item(1, 2).text()) + self.assertDictEqual({widget.LAST_SAVED: 0, 'abc': 1}, widget._config) + + # test copy + + widget._copyConfiguration(1, "efg") + self.assertDictEqual({widget.LAST_SAVED: 0, 'abc': 1, 'efg': 2}, widget._config) + + # test remove + + widget._removeConfiguration(1) + self.assertEqual("efg", table.item(1, 0).text()) + self.assertEqual("2020-02-02 02:02:02", table.item(1, 1).text()) + self.assertEqual("abc setup", table.item(1, 2).text()) + self.assertDictEqual({widget.LAST_SAVED: 0, 'efg': 1}, widget._config) + self._widget._meta.remove_snapshot.assert_called_with('abc') + + widget._removeConfiguration(1) + self.assertDictEqual({widget.LAST_SAVED: 0}, widget._config) + self._widget._meta.remove_snapshot.assert_called_with('efg') + + def testRenameConfiguration(self): + widget = self._widget + table = widget._table + + cfg = ["efg", "2020-03-03 03:03:03", "efg setup"] + widget._insertConfigurationToList(cfg) + widget._renameConfiguration(1, "abc") + widget._meta.rename_snapshot.assert_called_with('efg', 'abc') + self.assertEqual("abc", table.item(1, 0).text()) + self.assertDictEqual({widget.LAST_SAVED: 0, 'abc': 1}, widget._config) + + def testSetConfiguration(self): + widget = self._widget + table = widget._table + spy = QSignalSpy(widget.load_metadata_sgn) + spy_count = 0 + + cfg = ["efg", "2020-03-03 03:03:03", "efg setup"] + widget._insertConfigurationToList(cfg) + + for row in range(2): + for i in range(2): + table.itemDoubleClicked.emit(table.item(row, i)) + widget._meta.load_snapshot.assert_called_with(table.item(row, i).text()) + widget._meta.load_snapshot.reset_mock() + spy_count += 1 + self.assertEqual(spy_count, len(spy)) + table.itemDoubleClicked.emit(table.item(row, 2)) + widget._meta.load_snapshot.assert_not_called() + self.assertEqual(spy_count, len(spy)) + + # test reset + spy = QSignalSpy(widget.load_metadata_sgn) + widget._reset_btn.clicked.emit() + widget._meta.load_snapshot.assert_called_with(widget.DEFAULT) + self.assertEqual(1, len(spy)) + + def testSaveLoad(self): + widget = self._widget + table = widget._table + + widget._insertConfigurationToList(["abc", "2020-02-02 02:02:02", "abc setup"]) + widget._insertConfigurationToList(["efg", "2020-03-03 03:03:03", "efg setup"]) + + with patch("PyQt5.QtWidgets.QMessageBox.question", return_value=QMessageBox.No): + widget._save_cfg_btn.clicked.emit() + widget._meta.dump_configurations.assert_not_called() + + with patch("PyQt5.QtWidgets.QMessageBox.question", return_value=QMessageBox.Yes): + widget._save_cfg_btn.clicked.emit() + widget._meta.dump_configurations.assert_called_with( + [(widget.LAST_SAVED, ""), ("abc", "abc setup"), ("efg", "efg setup")]) + + with patch("PyQt5.QtWidgets.QMessageBox.question", return_value=QMessageBox.No): + widget._meta.load_configurations.reset_mock() + widget._load_cfg_btn.clicked.emit() + widget._meta.load_configurations.assert_not_called() + + with patch("PyQt5.QtWidgets.QMessageBox.question", return_value=QMessageBox.Yes): + widget._meta.load_configurations.return_value = [ + (widget.LAST_SAVED, "2020-02-13 04:04:04", ""), + ("abc", "2020-02-12 02:02:02", "abc setup 1"), + ("efg", "2020-03-13 03:03:03", "efg setup 1") + ] + widget._load_cfg_btn.clicked.emit() + widget._meta.load_configurations.assert_called_once() + + self.assertEqual(3, table.rowCount()) + self.assertEqual(widget.LAST_SAVED, table.item(0, 0).text()) + self.assertEqual("2020-02-13 04:04:04", table.item(0, 1).text()) + self.assertEqual("abc", table.item(1, 0).text()) + self.assertEqual("2020-02-12 02:02:02", table.item(1, 1).text()) + self.assertEqual("abc setup 1", table.item(1, 2).text()) + self.assertEqual("efg", table.item(2, 0).text()) + self.assertEqual("2020-03-13 03:03:03", table.item(2, 1).text()) + self.assertEqual("efg setup 1", table.item(2, 2).text()) diff --git a/extra_foam/gui/plot_widgets/plot_items.py b/extra_foam/gui/plot_widgets/plot_items.py index a76000bcc..c0779d45b 100644 --- a/extra_foam/gui/plot_widgets/plot_items.py +++ b/extra_foam/gui/plot_widgets/plot_items.py @@ -423,6 +423,9 @@ def setData(self, x, y, y_min=None, y_max=None, beam=None): self.update() self.informViewBoundsChanged() + def isEmptyGraph(self): + return not bool(len(self._x)) + def preparePath(self): p = QPainterPath() diff --git a/extra_foam/gui/pyqtgraph/graphicsItems/ViewBox/ViewBox.py b/extra_foam/gui/pyqtgraph/graphicsItems/ViewBox/ViewBox.py index 0982cb37c..4a37b82c7 100644 --- a/extra_foam/gui/pyqtgraph/graphicsItems/ViewBox/ViewBox.py +++ b/extra_foam/gui/pyqtgraph/graphicsItems/ViewBox/ViewBox.py @@ -1285,7 +1285,15 @@ def childrenBounds(self, frac=None, orthoRange=(None,None), items=None): useX = True useY = True - + + # FIXME: patch in EXtra-foam + try: + if item.isEmptyGraph(): + continue + except AttributeError: + pass + # FIXME + if hasattr(item, 'dataBounds'): if frac is None: frac = (1.0, 1.0) diff --git a/extra_foam/gui/tests/test_helper_functions.py b/extra_foam/gui/tests/test_helper_functions.py index 96f5d5e34..7cddb14c5 100644 --- a/extra_foam/gui/tests/test_helper_functions.py +++ b/extra_foam/gui/tests/test_helper_functions.py @@ -3,7 +3,9 @@ import numpy as np -from extra_foam.gui.gui_helpers import parse_boundary, parse_id, parse_slice +from extra_foam.gui.gui_helpers import ( + parse_boundary, parse_id, parse_slice, parse_slice_inv +) class TestGUI(unittest.TestCase): @@ -62,24 +64,39 @@ def test_parseslice(self): with self.assertRaises(ValueError): parse_slice("1:5:2:1") - self.assertEqual(slice(2), slice(*parse_slice('2'))) - self.assertEqual(slice(2, 3), slice(*parse_slice('2:3'))) - self.assertEqual(slice(-3, -1), slice(*parse_slice('-3:-1'))) - self.assertEqual(slice(None), slice(*parse_slice(":"))) - self.assertEqual(slice(2, None), slice(*parse_slice("2:"))) - self.assertEqual(slice(None, 3), slice(*parse_slice(':3'))) - self.assertEqual(slice(1, 4, 2), slice(*parse_slice('1:4:2'))) + self.assertListEqual([None, 2], parse_slice('2')) + self.assertListEqual([None, 2], parse_slice(':2')) + self.assertListEqual([2, 3], parse_slice('2:3')) + self.assertListEqual([-3, -1], parse_slice('-3:-1')) + self.assertListEqual([None, None], parse_slice(":")) + self.assertListEqual([2, None], parse_slice("2:")) + self.assertListEqual([None, 3], parse_slice(':3')) + self.assertListEqual([1, 4, 2], parse_slice('1:4:2')) # input with space in between - self.assertEqual(slice(1, 4, 2), slice(*parse_slice(' 1 : 4 : 2 '))) - self.assertEqual(slice(1, -4, 2), slice(*parse_slice('1:-4:2'))) - self.assertEqual(slice(2, None, 4), slice(*parse_slice('2::4'))) - self.assertEqual(slice(1, 3, None), slice(*parse_slice('1:3:'))) - self.assertEqual(slice(None, None, 4), slice(*parse_slice('::4'))) - self.assertEqual(slice(2, None, None), slice(*parse_slice('2::'))) - self.assertEqual(slice(None, None), slice(*parse_slice('::'))) + self.assertListEqual([1, 4, 2], parse_slice(' 1 : 4 : 2 ')) + self.assertListEqual([1, -4, 2], parse_slice('1:-4:2')) + self.assertListEqual([2, None, 4], parse_slice('2::4')) + self.assertListEqual([1, 3, None], parse_slice('1:3:')) + self.assertListEqual([None, None, 4], parse_slice('::4')) + self.assertListEqual([2, None, None], parse_slice('2::')) + self.assertListEqual([None, None, None], parse_slice('::')) with self.assertRaises(ValueError): parse_slice('2.0') with self.assertRaises(ValueError): parse_slice('2:3.0:2.0') + + def testParseSliceInv(self): + with self.assertRaises(ValueError): + parse_slice_inv("") + + with self.assertRaises(ValueError): + parse_slice_inv(":::") + + with self.assertRaises(ValueError): + parse_slice_inv("1:a") + + for s in [':2', '2:3', '-3:-1', ":", "2:", ':3', '1:4:2', + '1:-4:2', '2::4', '1:3:', '::4', '2::', '::']: + self.assertEqual(s, parse_slice_inv(str(parse_slice(s)))) diff --git a/extra_foam/gui/tests/test_main_gui.py b/extra_foam/gui/tests/test_main_gui.py index 4cb2c4678..5e515eba1 100644 --- a/extra_foam/gui/tests/test_main_gui.py +++ b/extra_foam/gui/tests/test_main_gui.py @@ -122,6 +122,16 @@ def testAnalysisCtrlWidget(self): self.assertEqual(index, win._poi_roi_hists[i]._index) win.close() + # test loading meta data + mediator = widget._mediator + mediator.onMaWindowChange(111) + mediator.onPoiIndexChange(0, 22) + mediator.onPoiIndexChange(1, 33) + widget.loadMetaData() + self.assertEqual("111", widget._ma_window_le.text()) + self.assertEqual("22", widget._poi_index_les[0].text()) + self.assertEqual("33", widget._poi_index_les[1].text()) + def testPumpProbeCtrlWidget(self): widget = self.gui.pump_probe_ctrl_widget pp_proc = self.pulse_worker._pp_proc @@ -136,10 +146,8 @@ def testPumpProbeCtrlWidget(self): pp_proc.update() self.assertEqual(PumpProbeMode.UNDEFINED, pp_proc._mode) - self.assertListEqual([-1], pp_proc._indices_on) - self.assertIsInstance(pp_proc._indices_on[0], int) - self.assertListEqual([-1], pp_proc._indices_off) - self.assertIsInstance(pp_proc._indices_off[0], int) + self.assertEqual(slice(None, None), pp_proc._indices_on) + self.assertEqual(slice(None, None), pp_proc._indices_off) # change analysis type pp_proc._reset = False @@ -175,8 +183,8 @@ def testPumpProbeCtrlWidget(self): widget._off_pulse_le.setText('1:10:2') pp_proc.update() self.assertEqual(PumpProbeMode.EVEN_TRAIN_ON, pp_proc._mode) - self.assertListEqual([0, 2, 4, 6, 8], pp_proc._indices_on) - self.assertListEqual([1, 3, 5, 7, 9], pp_proc._indices_off) + self.assertEqual(slice(0, 10, 2), pp_proc._indices_on) + self.assertEqual(slice(1, 10, 2), pp_proc._indices_off) # test reset button pp_proc._reset = False @@ -184,6 +192,20 @@ def testPumpProbeCtrlWidget(self): pp_proc.update() self.assertTrue(pp_proc._reset) + # test loading meta data + mediator = widget._mediator + mediator.onPpAnalysisTypeChange(AnalysisType.AZIMUTHAL_INTEG) + mediator.onPpModeChange(PumpProbeMode.ODD_TRAIN_ON) + mediator.onPpOnPulseSlicerChange([0, None, 2]) + mediator.onPpOffPulseSlicerChange([1, None, 2]) + mediator.onPpAbsDifferenceChange(True) + widget.loadMetaData() + self.assertEqual("azimuthal integ", widget._analysis_type_cb.currentText()) + self.assertEqual("odd/even train", widget._mode_cb.currentText()) + self.assertEqual(True, widget._abs_difference_cb.isChecked()) + self.assertEqual("0::2", widget._on_pulse_le.text()) + self.assertEqual("1::2", widget._off_pulse_le.text()) + def testDataSourceWidget(self): from extra_foam.gui.ctrl_widgets.data_source_widget import DataSourceWidget @@ -228,6 +250,16 @@ def testFomFilterCtrlWidget(self): self.assertEqual(AnalysisType.ROI_FOM, filter_train.analysis_type) self.assertTupleEqual((-2, 2), filter_train._fom_range) + # test loading meta data + mediator = widget._mediator + mediator.onFomFilterAnalysisTypeChange(AnalysisType.UNDEFINED) + mediator.onFomFilterRangeChange((-10, 10)) + mediator.onFomFilterPulseResolvedChange(True) + widget.loadMetaData() + self.assertEqual("", widget._analysis_type_cb.currentText()) + self.assertEqual("-10, 10", widget._fom_range_le.text()) + self.assertEqual(True, widget._pulse_resolved_cb.isChecked()) + def testCorrelationCtrlWidget(self): from extra_foam.gui.ctrl_widgets.correlation_ctrl_widget import ( _N_PARAMS, _DEFAULT_RESOLUTION) @@ -249,74 +281,102 @@ def testCorrelationCtrlWidget(self): combo_lst) train_worker = self.train_worker - proc = train_worker._correlation_proc - - proc.update() + processors = [train_worker._correlation1_proc, train_worker._correlation2_proc] # test default - self.assertEqual(AnalysisType(0), proc.analysis_type) - self.assertEqual([""] * _N_PARAMS, proc._sources) - self.assertEqual([_DEFAULT_RESOLUTION] * _N_PARAMS, proc._resolutions) + for proc in processors: + proc.update() + self.assertEqual(AnalysisType(0), proc.analysis_type) + self.assertEqual("", proc._source) + self.assertEqual(_DEFAULT_RESOLUTION, proc._resolution) # set new FOM - proc._resets = [False] * _N_PARAMS widget._analysis_type_cb.setCurrentText(analysis_types[AnalysisType.ROI_PROJ]) - proc.update() - self.assertEqual(AnalysisType.ROI_PROJ, proc.analysis_type) - self.assertTrue(all(proc._resets)) + for proc in processors: + proc._reset = False + proc.update() + self.assertEqual(AnalysisType.ROI_PROJ, proc.analysis_type) + self.assertTrue(proc._reset) - # change source - for i in range(_N_PARAMS): - proc._resets[i] = False + for idx, proc in enumerate(processors): + # change source + proc._reset = False ctg, device_id, ppt = 'Metadata', "META", "timestamp.tid" - widget._table.cellWidget(i, 0).setCurrentText(ctg) - self.assertEqual(device_id, widget._table.cellWidget(i, 1).currentText()) - self.assertEqual(ppt, widget._table.cellWidget(i, 2).currentText()) + widget._table.cellWidget(idx, 0).setCurrentText(ctg) + self.assertEqual(device_id, widget._table.cellWidget(idx, 1).currentText()) + self.assertEqual(ppt, widget._table.cellWidget(idx, 2).currentText()) proc.update() src = f"{device_id} {ppt}" if device_id and ppt else "" - self.assertEqual(src, proc._sources[i]) - self.assertTrue(proc._resets[i]) + self.assertEqual(src, proc._source) + self.assertTrue(proc._reset) # just test we can set a motor source - proc._resets[i] = False - widget._table.cellWidget(i, 0).setCurrentText("Motor") + proc._reset = False + widget._table.cellWidget(idx, 0).setCurrentText("Motor") proc.update() - self.assertTrue(proc._resets[i]) + self.assertTrue(proc._reset) - proc._resets[i] = False + proc._reset = False ctg, device_id, ppt = USER_DEFINED_KEY, "ABC", "efg" - widget._table.cellWidget(i, 0).setCurrentText(ctg) - self.assertEqual('', widget._table.cellWidget(i, 1).text()) - self.assertEqual('', widget._table.cellWidget(i, 2).text()) - widget._table.cellWidget(i, 1).setText(device_id) - widget._table.cellWidget(i, 2).setText(ppt) + widget._table.cellWidget(idx, 0).setCurrentText(ctg) + self.assertEqual('', widget._table.cellWidget(idx, 1).text()) + self.assertEqual('', widget._table.cellWidget(idx, 2).text()) + widget._table.cellWidget(idx, 1).setText(device_id) + widget._table.cellWidget(idx, 2).setText(ppt) self.assertEqual(device_id, widget._table.cellWidget(0, 1).text()) self.assertEqual(ppt, widget._table.cellWidget(0, 2).text()) proc.update() src = f"{device_id} {ppt}" if device_id and ppt else "" - self.assertEqual(src, proc._sources[i]) - self.assertTrue(proc._resets[i]) - - # change resolution - proc._resets = [False] * _N_PARAMS - for i in range(_N_PARAMS): - self.assertIsInstance(proc._correlations[i], SimplePairSequence) - widget._table.cellWidget(i, 3).setText(str(1.0)) + self.assertEqual(src, proc._source) + self.assertTrue(proc._reset) + + # change resolution + proc._reset = False + self.assertIsInstance(proc._correlation, SimplePairSequence) + self.assertIsInstance(proc._correlation_slave, SimplePairSequence) + widget._table.cellWidget(idx, 3).setText(str(1.0)) proc.update() - self.assertEqual(1.0, proc._resolutions[i]) - self.assertIsInstance(proc._correlations[i], OneWayAccuPairSequence) + self.assertEqual(1.0, proc._resolution) + self.assertIsInstance(proc._correlation, OneWayAccuPairSequence) + self.assertIsInstance(proc._correlation_slave, OneWayAccuPairSequence) # sequence type change will not have 'reset' - self.assertFalse(proc._resets[i]) - widget._table.cellWidget(i, 3).setText(str(2.0)) + self.assertFalse(proc._reset) + widget._table.cellWidget(idx, 3).setText(str(2.0)) proc.update() - self.assertEqual(2.0, proc._resolutions[i]) - self.assertTrue(proc._resets[i]) + self.assertEqual(2.0, proc._resolution) + self.assertTrue(proc._reset) - # test reset button - proc._resets = [False] * _N_PARAMS - widget._reset_btn.clicked.emit() - proc.update() - self.assertTrue(all(proc._resets)) + # test reset button + proc._reset = False + widget._reset_btn.clicked.emit() + proc.update() + self.assertTrue(proc._reset) + + # test loading meta data + mediator = widget._mediator + mediator.onCorrelationAnalysisTypeChange(AnalysisType.UNDEFINED) + if config["TOPIC"] == "FXE": + motor_id = 'FXE_SMS_USR/MOTOR/UM01' + else: + motor_id = 'SCS_ILH_LAS/MOTOR/LT3' + mediator.onCorrelationParamChange((1, f'{motor_id} actualPosition', 0.0)) + mediator.onCorrelationParamChange((2, 'ABC abc', 2.0)) + widget.loadMetaData() + self.assertEqual("", widget._analysis_type_cb.currentText()) + self.assertEqual('Motor', widget._table.cellWidget(0, 0).currentText()) + self.assertEqual(motor_id, widget._table.cellWidget(0, 1).currentText()) + self.assertEqual('actualPosition', widget._table.cellWidget(0, 2).currentText()) + self.assertEqual('0.0', widget._table.cellWidget(0, 3).text()) + self.assertEqual(widget._user_defined_key, widget._table.cellWidget(1, 0).currentText()) + self.assertEqual('ABC', widget._table.cellWidget(1, 1).text()) + self.assertEqual('abc', widget._table.cellWidget(1, 2).text()) + self.assertEqual('2.0', widget._table.cellWidget(1, 3).text()) + + mediator.onCorrelationParamChange((1, f'', 0.0)) + widget.loadMetaData() + self.assertEqual('', widget._table.cellWidget(0, 0).currentText()) + self.assertEqual('', widget._table.cellWidget(0, 1).text()) + self.assertEqual('', widget._table.cellWidget(0, 2).text()) def testBinCtrlWidget(self): from extra_foam.gui.ctrl_widgets.bin_ctrl_widget import ( @@ -327,8 +387,8 @@ def testBinCtrlWidget(self): widget = self.gui.bin_ctrl_widget - analysis_types = {value: key for key, value in widget._analysis_types.items()} - bin_modes = {value: key for key, value in widget._bin_modes.items()} + analysis_types_inv = widget._analysis_types_inv + bin_modes_inv = widget._bin_modes_inv for i in range(_N_PARAMS): combo_lst = [widget._table.cellWidget(i, 0).itemText(j) @@ -353,8 +413,8 @@ def testBinCtrlWidget(self): self.assertEqual(int(_DEFAULT_N_BINS), proc._n_bins2) # test analysis type and mode change - widget._analysis_type_cb.setCurrentText(analysis_types[AnalysisType.PUMP_PROBE]) - widget._mode_cb.setCurrentText(bin_modes[BinMode.ACCUMULATE]) + widget._analysis_type_cb.setCurrentText(analysis_types_inv[AnalysisType.PUMP_PROBE]) + widget._mode_cb.setCurrentText(bin_modes_inv[BinMode.ACCUMULATE]) proc.update() self.assertEqual(AnalysisType.PUMP_PROBE, proc.analysis_type) self.assertEqual(BinMode.ACCUMULATE, proc._mode) @@ -418,6 +478,36 @@ def testBinCtrlWidget(self): self.assertTrue(win._bin2d_count._auto_level) win.close() + # test loading meta data + mediator = widget._mediator + mediator.onBinAnalysisTypeChange(AnalysisType.UNDEFINED) + mediator.onBinModeChange(BinMode.AVERAGE) + if config["TOPIC"] == "FXE": + motor_id = 'FXE_SMS_USR/MOTOR/UM01' + else: + motor_id = 'SCS_ILH_LAS/MOTOR/LT3' + mediator.onBinParamChange((1, f'{motor_id} actualPosition', (-9, 9), 5)) + mediator.onBinParamChange((2, 'ABC abc', (-19, 19), 15)) + widget.loadMetaData() + self.assertEqual("", widget._analysis_type_cb.currentText()) + self.assertEqual("average", widget._mode_cb.currentText()) + self.assertEqual('Motor', widget._table.cellWidget(0, 0).currentText()) + self.assertEqual(motor_id, widget._table.cellWidget(0, 1).currentText()) + self.assertEqual('actualPosition', widget._table.cellWidget(0, 2).currentText()) + self.assertEqual('-9, 9', widget._table.cellWidget(0, 3).text()) + self.assertEqual('5', widget._table.cellWidget(0, 4).text()) + self.assertEqual(widget._user_defined_key, widget._table.cellWidget(1, 0).currentText()) + self.assertEqual('ABC', widget._table.cellWidget(1, 1).text()) + self.assertEqual('abc', widget._table.cellWidget(1, 2).text()) + self.assertEqual('-19, 19', widget._table.cellWidget(1, 3).text()) + self.assertEqual('15', widget._table.cellWidget(1, 4).text()) + + mediator.onBinParamChange((1, f'', (-9, 9), 5)) + widget.loadMetaData() + self.assertEqual('', widget._table.cellWidget(0, 0).currentText()) + self.assertEqual('', widget._table.cellWidget(0, 1).text()) + self.assertEqual('', widget._table.cellWidget(0, 2).text()) + def testHistogramCtrlWidget(self): widget = self.gui.histogram_ctrl_widget train_worker = self.train_worker @@ -455,6 +545,18 @@ def testHistogramCtrlWidget(self): proc.update() self.assertTrue(proc._reset) + # test loading meta data + mediator = widget._mediator + mediator.onHistAnalysisTypeChange(AnalysisType.UNDEFINED) + mediator.onHistBinRangeChange((-10, 10)) + mediator.onHistNumBinsChange(55) + mediator.onHistPulseResolvedChange(True) + widget.loadMetaData() + self.assertEqual("", widget._analysis_type_cb.currentText()) + self.assertEqual("-10, 10", widget._bin_range_le.text()) + self.assertEqual("55", widget._n_bins_le.text()) + self.assertEqual(True, widget._pulse_resolved_cb.isChecked()) + @patch('extra_foam.gui.ctrl_widgets.PumpProbeCtrlWidget.' 'updateMetaData', MagicMock(return_value=True)) @patch('extra_foam.gui.ctrl_widgets.HistogramCtrlWidget.' @@ -706,6 +808,15 @@ def testAnalysisCtrlWidget(self): self.assertFalse(widget._poi_index_les[i].isEnabled()) self.assertEqual(0, idx) # test default values + # test loading meta data + # Test if the meta data is invalid. + mediator = widget._mediator + mediator.onPoiIndexChange(0, 22) + mediator.onPoiIndexChange(1, 33) + widget.loadMetaData() + self.assertEqual("0", widget._poi_index_les[0].text()) + self.assertEqual("0", widget._poi_index_les[1].text()) + def testPumpProbeCtrlWidget(self): widget = self.gui.pump_probe_ctrl_widget pp_proc = self.pulse_worker._pp_proc @@ -713,15 +824,14 @@ def testPumpProbeCtrlWidget(self): self.assertFalse(widget._on_pulse_le.isEnabled()) self.assertFalse(widget._off_pulse_le.isEnabled()) - all_modes = {value: key for key, value in - widget._available_modes.items()} + all_modes = widget._available_modes_inv # we only test train-resolved detector specific configuration pp_proc.update() self.assertEqual(PumpProbeMode.UNDEFINED, pp_proc._mode) - self.assertListEqual([-1], pp_proc._indices_on) - self.assertListEqual([-1], pp_proc._indices_off) + self.assertEqual(slice(None, None), pp_proc._indices_on) + self.assertEqual(slice(None, None), pp_proc._indices_off) spy = QSignalSpy(widget._mode_cb.currentTextChanged) @@ -730,8 +840,8 @@ def testPumpProbeCtrlWidget(self): pp_proc.update() self.assertEqual(PumpProbeMode(PumpProbeMode.EVEN_TRAIN_ON), pp_proc._mode) - self.assertListEqual([-1], pp_proc._indices_on) - self.assertListEqual([-1], pp_proc._indices_off) + self.assertEqual(slice(None, None), pp_proc._indices_on) + self.assertEqual(slice(None, None), pp_proc._indices_off) widget._mode_cb.setCurrentText(all_modes[PumpProbeMode.REFERENCE_AS_OFF]) self.assertEqual(2, len(spy)) @@ -740,19 +850,30 @@ def testPumpProbeCtrlWidget(self): self.assertFalse(widget._on_pulse_le.isEnabled()) # PumpProbeMode.SAME_TRAIN is not available - widget._mode_cb.setCurrentText(all_modes[PumpProbeMode.SAME_TRAIN]) - self.assertEqual(2, len(spy)) + self.assertNotIn(PumpProbeMode.SAME_TRAIN, all_modes) + + # test loading meta data + # test if the meta data is invalid + mediator = widget._mediator + mediator.onPpOnPulseSlicerChange([0, None, 2]) + mediator.onPpOffPulseSlicerChange([0, None, 2]) + widget.loadMetaData() + self.assertEqual(":", widget._on_pulse_le.text()) + self.assertEqual(":", widget._off_pulse_le.text()) + + mediator.onPpModeChange(PumpProbeMode.SAME_TRAIN) + with self.assertRaises(KeyError): + widget.loadMetaData() def testFomFilterCtrlWidget(self): widget = self.gui.fom_filter_ctrl_widget filter_pulse = self.pulse_worker._filter filter_train = self.train_worker._filter - analysis_types = {value: key for key, value in widget._analysis_types.items()} - # test default self.assertFalse(widget._pulse_resolved_cb.isChecked()) + self.assertFalse(widget._pulse_resolved_cb.isEnabled()) filter_pulse.update() self.assertEqual(AnalysisType.UNDEFINED, filter_pulse.analysis_type) @@ -763,7 +884,7 @@ def testFomFilterCtrlWidget(self): # test set new - widget._analysis_type_cb.setCurrentText(analysis_types[AnalysisType.ROI_FOM]) + widget._analysis_type_cb.setCurrentText(widget._analysis_types_inv[AnalysisType.ROI_FOM]) widget._fom_range_le.setText("-2, 2") filter_pulse.update() self.assertEqual(AnalysisType.UNDEFINED, filter_pulse.analysis_type) @@ -772,6 +893,24 @@ def testFomFilterCtrlWidget(self): self.assertEqual(AnalysisType.ROI_FOM, filter_train.analysis_type) self.assertTupleEqual((-2, 2), filter_train._fom_range) + # test loading meta data + # test if the meta data is invalid + mediator = widget._mediator + mediator.onFomFilterPulseResolvedChange(True) + widget.loadMetaData() + self.assertEqual(False, widget._pulse_resolved_cb.isChecked()) + def testHistogramCtrlWidget(self): - # TODO - pass + widget = self.gui.histogram_ctrl_widget + + # test default + + self.assertFalse(widget._pulse_resolved_cb.isChecked()) + self.assertFalse(widget._pulse_resolved_cb.isEnabled()) + + # test loading meta data + # test if the meta data is invalid + mediator = widget._mediator + mediator.onHistPulseResolvedChange(True) + widget.loadMetaData() + self.assertEqual(False, widget._pulse_resolved_cb.isChecked()) diff --git a/extra_foam/gui/windows/correlation_w.py b/extra_foam/gui/windows/correlation_w.py index c585bb09b..f76dd7061 100644 --- a/extra_foam/gui/windows/correlation_w.py +++ b/extra_foam/gui/windows/correlation_w.py @@ -22,9 +22,9 @@ class CorrelationPlot(TimedPlotWidgetF): Widget for displaying correlations between FOM and different parameters. """ _colors = config["GUI_CORRELATION_COLORS"] - _pens = [FColor.mkPen(color) for color in _colors] - _brushes = [FColor.mkBrush(color, alpha=120) for color in _colors] - _opaque_brushes = [FColor.mkBrush(color) for color in _colors] + _pens = [(FColor.mkPen(pair[0]), FColor.mkPen(pair[1])) for pair in _colors] + _brushes = [(FColor.mkBrush(pair[0], alpha=120), + FColor.mkBrush(pair[1], alpha=120)) for pair in _colors] def __init__(self, idx, *, parent=None): """Initialization.""" @@ -41,7 +41,9 @@ def __init__(self, idx, *, parent=None): self.updateLabel() - self._plot = self.plotScatter(brush=self._brushes[self._idx-1]) + brush_pair = self._brushes[self._idx] + self._plot = self.plotScatter(brush=brush_pair[0]) + self._plot_slave = self.plotScatter(brush=brush_pair[1]) def refresh(self): """Override.""" @@ -54,6 +56,7 @@ def refresh(self): resolution = item.resolution y = item.y + y_slave = item.y_slave if resolution == 0: # SimplePairSequence if self._resolution != 0: @@ -61,12 +64,18 @@ def refresh(self): self._resolution = 0 self._plot.setData(item.x, y) + if y_slave is not None: + self._plot_slave.setData(item.x_slave, y_slave) else: # OneWayAccuPairSequence if self._resolution == 0: self._newStatisticsBarPlot(resolution) self._resolution = resolution self._plot.setData(item.x, y.avg, y_min=y.min, y_max=y.max) + if y_slave is not None: + self._plot_slave.setData( + item.x_slave, y_slave.avg, + y_min=y_slave.min, y_max=y_slave.max) def updateLabel(self): src = self._source @@ -80,12 +89,20 @@ def updateLabel(self): def _newScatterPlot(self): self.removeItem(self._plot) - self._plot = self.plotScatter(brush=self._brushes[self._idx-1]) + self.removeItem(self._plot_slave) + + brush_pair = self._brushes[self._idx] + self._plot = self.plotScatter(brush=brush_pair[0]) + self._plot_slave = self.plotScatter(brush=brush_pair[1]) def _newStatisticsBarPlot(self, resolution): self.removeItem(self._plot) - self._plot = self.plotStatisticsBar(beam=resolution, - pen=self._pens[self._idx-1]) + self.removeItem(self._plot_slave) + + pen_pair = self._pens[self._idx] + self._plot = self.plotStatisticsBar(beam=resolution, pen=pen_pair[0]) + self._plot_slave = self.plotStatisticsBar(beam=resolution, + pen=pen_pair[1]) class CorrelationWindow(_AbstractPlotWindow): diff --git a/extra_foam/gui/windows/tests/test_plot_windows.py b/extra_foam/gui/windows/tests/test_plot_windows.py index 9de243c39..fd97e5e62 100644 --- a/extra_foam/gui/windows/tests/test_plot_windows.py +++ b/extra_foam/gui/windows/tests/test_plot_windows.py @@ -235,24 +235,30 @@ def testResolutionSwitch(self): from extra_foam.gui.windows.correlation_w import CorrelationPlot from extra_foam.gui.plot_widgets.plot_items import StatisticsBarItem, pg + # resolution1 = 0.0 and resolution2 > 0.0 data = self.processed_data(1001, (4, 2, 2), correlation=True) widget = CorrelationPlot(0) widget._data = data widget.refresh() - plot_item = widget._plot + plot_item, plot_item_slave = widget._plot, widget._plot_slave self.assertIsInstance(plot_item, pg.ScatterPlotItem) + self.assertIsInstance(plot_item_slave, pg.ScatterPlotItem) widget._idx = 1 # a trick widget.refresh() self.assertNotIn(plot_item, widget._plot_item.items) # being deleted - plot_item = widget._plot + self.assertNotIn(plot_item_slave, widget._plot_item.items) # being deleted + plot_item, plot_item_slave = widget._plot, widget._plot_slave self.assertIsInstance(plot_item, StatisticsBarItem) + self.assertIsInstance(plot_item_slave, StatisticsBarItem) widget._idx = 0 # a trick widget.refresh() self.assertNotIn(plot_item, widget._plot_item.items) # being deleted - self.assertIsInstance(widget._plot, pg.ScatterPlotItem) + self.assertNotIn(plot_item_slave, widget._plot_item.items) # being deleted + self.assertIsInstance(widget._plot, pg.ScatterPlotItem) + self.assertIsInstance(widget._plot_slave, pg.ScatterPlotItem) class testHistogramWidgets(_TestDataMixin, unittest.TestCase): diff --git a/extra_foam/offline/file_server.py b/extra_foam/offline/file_server.py index 3ef685c1f..34fd62c1b 100644 --- a/extra_foam/offline/file_server.py +++ b/extra_foam/offline/file_server.py @@ -225,8 +225,10 @@ def run(self): fast_devices = [("*DET/*CH0:xtdf", "image.data")] elif detector == "JungFrau": fast_devices = [("*/DET/*:daqOutput", "data.adc")] - elif detector == "FastCCD": + elif detector in ["FastCCD"]: fast_devices = [("*/DAQ/*:daqOutput", "data.image.pixels")] + elif detector in ["ePix100"]: + fast_devices = [("*/DET/*:daqOutput", "data.image.pixels")] else: raise NotImplementedError(f"Unknown Detector: {detector}") diff --git a/extra_foam/pipeline/data_model.py b/extra_foam/pipeline/data_model.py index 246a1db3c..dae369fee 100644 --- a/extra_foam/pipeline/data_model.py +++ b/extra_foam/pipeline/data_model.py @@ -329,6 +329,7 @@ class RoiDataTrain(DataItem): Attributes: geom1, geom2, geom3, geom4 (RectRoiGeom): ROI geometry. + fom_slave (float): ROI slave FOM. norm (float): ROI normalizer. proj (RoiProjData): ROI projection data item hist (_HistogramDataItem): ROI histogram data item. @@ -337,7 +338,7 @@ class RoiDataTrain(DataItem): N_ROIS = len(config['GUI_ROI_COLORS']) __slots__ = ['geom1', 'geom2', 'geom3', 'geom4', - 'norm', 'proj', 'hist'] + 'fom_slave', 'norm', 'proj', 'hist'] def __init__(self): super().__init__() @@ -347,6 +348,8 @@ def __init__(self): self.geom3 = RectRoiGeom() self.geom4 = RectRoiGeom() + self.fom_slave = None + self.norm = None self.proj = DataItem() self.hist = _HistogramDataItem() @@ -601,11 +604,13 @@ class CorrelationData(collections.abc.Mapping): class CorrelationDataItem: - __slots__ = ['x', 'y', 'source', 'resolution'] + __slots__ = ['x', 'y', 'x_slave', 'y_slave', 'source', 'resolution'] def __init__(self): self.x = None self.y = None # FOM + self.x_slave = None + self.y_slave = None # FOM slave self.source = "" self.resolution = 0.0 diff --git a/extra_foam/pipeline/processors/correlation.py b/extra_foam/pipeline/processors/correlation.py index f9626053a..d1322f5fb 100644 --- a/extra_foam/pipeline/processors/correlation.py +++ b/extra_foam/pipeline/processors/correlation.py @@ -21,15 +21,18 @@ class CorrelationProcessor(_BaseProcessor): """Add correlation information into processed data. Attributes: + _idx (int): Index of correlation starting from 1. analysis_type (AnalysisType): analysis type. _pp_analysis_type (AnalysisType): pump-probe analysis type. - _n_params (int): number of correlators. - _correlations (list): a list of pair sequences (SimplePairSequence, + _correlation (Sequence): pair sequences (SimplePairSequence, OneWayAccuPairSequence) for storing the history of (correlator, FOM). - _sources (list): a list of sources for slow data correlators. - _resolutions (list): a list of resolutions for correlations. - _resets (list): reset flags for correlation data. + _correlation_slave (Sequence): pair sequences (SimplePairSequence, + OneWayAccuPairSequence) for storing the history of + (correlator, FOM slave). + _source: source of slow data. + _resolution: resolution of correlation. + _reset: reset flag for correlation data. _correlation_pp (SimplePairSequence): store the history of (correlator, FOM) which is displayed in PumpProbeWindow. _pp_fail_flag (int): a flag used to check whether pump-probe FOM is @@ -39,61 +42,65 @@ class CorrelationProcessor(_BaseProcessor): # 10 pulses/train * 60 seconds * 5 minutes = 3000 _MAX_POINTS = 10 * 60 * 5 - def __init__(self): + def __init__(self, index): super().__init__() + self._idx = index + self.analysis_type = AnalysisType.UNDEFINED self._pp_analysis_type = AnalysisType.UNDEFINED - self._n_params = 2 - self._correlations = [] - for i in range(self._n_params): - self._correlations.append( - SimplePairSequence(max_len=self._MAX_POINTS)) - self._sources = [""] * self._n_params - self._resolutions = [0.0] * self._n_params - self._resets = [False] * self._n_params + self._correlation = SimplePairSequence(max_len=self._MAX_POINTS) + self._correlation_slave = SimplePairSequence(max_len=self._MAX_POINTS) + self._source = "" + self._resolution = 0.0 + self._reset = False self._correlation_pp = SimplePairSequence(max_len=self._MAX_POINTS) self._pp_fail_flag = 0 def update(self): """Override.""" + idx = self._idx cfg = self._meta.hget_all(mt.CORRELATION_PROC) if self._update_analysis(AnalysisType(int(cfg['analysis_type']))): - for i in range(self._n_params): - self._resets[i] = True - - for i in range(len(self._correlations)): - src = cfg[f'source{i+1}'] - if self._sources[i] != src: - self._sources[i] = src - self._resets[i] = True - - resolution = float(cfg[f'resolution{i+1}']) - if self._resolutions[i] != 0 and resolution == 0: - self._correlations[i] = SimplePairSequence( - max_len=self._MAX_POINTS) - elif self._resolutions[i] == 0 and resolution != 0: - self._correlations[i] = OneWayAccuPairSequence( - resolution, max_len=self._MAX_POINTS) - elif self._resolutions[i] != resolution: - # In the above two cases, we do not need 'reset' since - # new Sequence object will be constructed. - self._resets[i] = True - self._resolutions[i] = resolution - - if 'reset' in cfg: - self._meta.hdel(mt.CORRELATION_PROC, 'reset') - for i in range(self._n_params): - self._resets[i] = True + self._reset = True + + src = cfg[f'source{idx}'] + if self._source != src: + self._source = src + self._reset = True + + resolution = float(cfg[f'resolution{idx}']) + if self._resolution != 0 and resolution == 0: + self._correlation = SimplePairSequence( + max_len=self._MAX_POINTS) + self._correlation_slave = SimplePairSequence( + max_len=self._MAX_POINTS) + elif self._resolution == 0 and resolution != 0: + self._correlation = OneWayAccuPairSequence( + resolution, max_len=self._MAX_POINTS) + self._correlation_slave = OneWayAccuPairSequence( + resolution, max_len=self._MAX_POINTS) + elif self._resolution != resolution: + # In the above two cases, we do not need 'reset' since + # new Sequence object will be constructed. + self._reset = True + self._resolution = resolution + + reset_key = f'reset{idx}' + if reset_key in cfg: + self._meta.hdel(mt.CORRELATION_PROC, reset_key) + self._reset = True @profiler("Correlation Processor") def process(self, data): """Override.""" self._process_general(data) - self._process_pump_probe(data) + if self._idx == 1: + # process only once + self._process_pump_probe(data) def _process_general(self, data): if self.analysis_type == AnalysisType.UNDEFINED: @@ -104,25 +111,25 @@ def _process_general(self, data): if self.analysis_type == AnalysisType.PUMP_PROBE: pp_analysis_type = processed.pp.analysis_type if self._pp_analysis_type != pp_analysis_type: - for i in range(self._n_params): - self._resets[i] = True + # reset if pump-pobe analysis type changes + self._reset = True self._pp_analysis_type = pp_analysis_type - for i in range(self._n_params): - if self._resets[i]: - self._correlations[i].reset() - self._resets[i] = False + if self._reset: + self._correlation.reset() + self._correlation_slave.reset() + self._reset = False try: self._update_data_point(processed, raw) except ProcessingError as e: logger.error(f"[Correlation] {str(e)}!") - for i in range(self._n_params): - out = processed.corr[i] - out.x, out.y = self._correlations[i].data() - out.source = self._sources[i] - out.resolution = self._resolutions[i] + out = processed.corr[self._idx - 1] + out.x, out.y = self._correlation.data() + out.x_slave, out.y_slave = self._correlation_slave.data() + out.source = self._source + out.resolution = self._resolution def _process_pump_probe(self, data): """Process the correlation in pump-probe analysis. @@ -146,6 +153,7 @@ def _process_pump_probe(self, data): def _update_data_point(self, processed, raw): analysis_type = self.analysis_type + fom_slave = None if analysis_type == AnalysisType.PUMP_PROBE: fom = processed.pp.fom if fom is None: @@ -160,6 +168,7 @@ def _update_data_point(self, processed, raw): self._pp_fail_flag = 0 elif analysis_type == AnalysisType.ROI_FOM: fom = processed.roi.fom + fom_slave = processed.roi.fom_slave if fom is None: raise ProcessingError("ROI FOM is not available") elif analysis_type == AnalysisType.ROI_PROJ: @@ -175,12 +184,12 @@ def _update_data_point(self, processed, raw): raise UnknownParameterError( f"[Correlation] Unknown analysis type: {self.analysis_type}") - for i in range(self._n_params): - v, err = self._fetch_property_data( - processed.tid, raw, self._sources[i]) + v, err = self._fetch_property_data(processed.tid, raw, self._source) - if err: - logger.error(err) + if err: + logger.error(err) - if v is not None: - self._correlations[i].append((v, fom)) + if v is not None: + self._correlation.append((v, fom)) + if fom_slave is not None: + self._correlation_slave.append((v, fom_slave)) diff --git a/extra_foam/pipeline/processors/fom_filter.py b/extra_foam/pipeline/processors/fom_filter.py index c7b4227de..f5b2c2fe6 100644 --- a/extra_foam/pipeline/processors/fom_filter.py +++ b/extra_foam/pipeline/processors/fom_filter.py @@ -53,11 +53,11 @@ def process(self, data): if self.analysis_type == AnalysisType.ROI_FOM_PULSE: fom = processed.pulse.roi.fom if fom is None: - raise ProcessingError(f"[tag] ROI FOM is not available") + raise ProcessingError(f"[{tag}] ROI FOM is not available") else: raise UnknownParameterError( - f"[tag] Unknown analysis type: {self.analysis_type}") + f"[{tag}] Unknown analysis type: {self.analysis_type}") self.filter_pulse_by_vrange(fom, self._fom_range, processed.pidx, tag) @@ -98,9 +98,9 @@ def process(self, data): if self.analysis_type == AnalysisType.ROI_FOM: fom = processed.roi.fom if fom is None: - raise ProcessingError(f"[tag] ROI FOM is not available") + raise ProcessingError(f"[{tag}] ROI FOM is not available") else: raise UnknownParameterError( - f"[tag] Unknown analysis type: {self.analysis_type}") + f"[{tag}] Unknown analysis type: {self.analysis_type}") self.filter_train_by_vrange(fom, self._fom_range, tag) diff --git a/extra_foam/pipeline/processors/histogram.py b/extra_foam/pipeline/processors/histogram.py index 424dd4741..1dbb3e1a5 100644 --- a/extra_foam/pipeline/processors/histogram.py +++ b/extra_foam/pipeline/processors/histogram.py @@ -12,8 +12,8 @@ import numpy as np from .base_processor import _BaseProcessor, SimpleSequence -from ..exceptions import UnknownParameterError -from ...algorithms import hist_with_stats, find_actual_range +from ..exceptions import ProcessingError, UnknownParameterError +from ...algorithms import hist_with_stats from ...ipc import process_logger as logger from ...database import Metadata as mt from ...config import AnalysisType @@ -108,8 +108,11 @@ def process(self, data): data = self._fom.data() if data.size != 0: th = processed.hist - th.hist, th.bin_centers, th.mean, th.median, th.std = \ - hist_with_stats(data, self._bin_range, self._n_bins) + try: + th.hist, th.bin_centers, th.mean, th.median, th.std = \ + hist_with_stats(data, self._bin_range, self._n_bins) + except ValueError as e: + raise ProcessingError(f"[Histogram] {str(e)}") def _process_poi(self, processed): """Calculate histograms of FOMs of POI pulses.""" @@ -121,8 +124,11 @@ def _process_poi(self, processed): poi_fom = self._fom.data()[i::n_pulses] if poi_fom.size != 0: - processed.pulse.hist[i] = hist_with_stats( - poi_fom, self._bin_range, self._n_bins) + try: + processed.pulse.hist[i] = hist_with_stats( + poi_fom, self._bin_range, self._n_bins) + except ValueError as e: + raise ProcessingError(f"[Histogram] {str(e)}") if image_data.poi_indices[1] == image_data.poi_indices[0]: # skip the second one if two POIs have the same index diff --git a/extra_foam/pipeline/processors/image_assembler.py b/extra_foam/pipeline/processors/image_assembler.py index 2bd2be942..0435fa17d 100644 --- a/extra_foam/pipeline/processors/image_assembler.py +++ b/extra_foam/pipeline/processors/image_assembler.py @@ -263,11 +263,14 @@ def update(self): self._stack_only = stack_only self._assembler_type = assembler_type - self._geom_file = geom_file self._quad_position = quad_positions self._geom = None # reset first self._load_geometry(geom_file, quad_positions) + # caveat: if _load_geometry raises, _geom_file will not + # be set. Therefore, _load_geometry will raise + # AssemblingError in the next train. + self._geom_file = geom_file if not stack_only: logger.info(f"Loaded geometry from {geom_file} with " @@ -458,13 +461,26 @@ def _get_modules_file(self, data, src): def _load_geometry(self, filename, quad_positions): """Override.""" if self._assembler_type == GeomAssembler.OWN or self._stack_only: - raise AssemblingError("Not implemented!") + from ...geometries import AGIPD_1MGeometryFast + + if self._stack_only: + self._geom = AGIPD_1MGeometryFast() + else: + try: + # catch any exceptions here since it loads the CFEL + # geometry file with a CFEL function + self._geom = AGIPD_1MGeometryFast.from_crystfel_geom( + filename) + except Exception as e: + raise AssemblingError(e) else: from extra_geom import AGIPD_1MGeometry try: + # catch any exceptions here since it loads the CFEL + # geometry file with a CFEL function self._geom = AGIPD_1MGeometry.from_crystfel_geom(filename) - except (ImportError, ModuleNotFoundError, OSError) as e: + except Exception as e: raise AssemblingError(e) class LpdImageAssembler(BaseAssembler): @@ -511,16 +527,19 @@ def _load_geometry(self, filename, quad_positions): if self._stack_only: self._geom = LPD_1MGeometryFast() else: - self._geom = LPD_1MGeometryFast.from_h5_file_and_quad_positions( - filename, quad_positions) + try: + self._geom = LPD_1MGeometryFast.from_h5_file_and_quad_positions( + filename, quad_positions) + except (OSError, KeyError) as e: + raise AssemblingError(f"[Geometry] {e}") else: from extra_geom import LPD_1MGeometry try: self._geom = LPD_1MGeometry.from_h5_file_and_quad_positions( filename, quad_positions) - except OSError as e: - raise AssemblingError(e) + except (OSError, KeyError) as e: + raise AssemblingError(f"[Geometry] {e}") class DsscImageAssembler(BaseAssembler): @@ -565,16 +584,19 @@ def _load_geometry(self, filename, quad_positions): if self._stack_only: self._geom = DSSC_1MGeometryFast() else: - self._geom = DSSC_1MGeometryFast.from_h5_file_and_quad_positions( - filename, quad_positions) + try: + self._geom = DSSC_1MGeometryFast.from_h5_file_and_quad_positions( + filename, quad_positions) + except (OSError, KeyError) as e: + raise AssemblingError(f"[Geometry] {e}") else: from extra_geom import DSSC_1MGeometry try: self._geom = DSSC_1MGeometry.from_h5_file_and_quad_positions( filename, quad_positions) - except OSError as e: - raise AssemblingError(e) + except (OSError, KeyError) as e: + raise AssemblingError(f"[Geometry] {e}") class JungFrauImageAssembler(BaseAssembler): def _get_modules_bridge(self, data, src): @@ -639,6 +661,35 @@ def _get_modules_file(self, data, src): # return modules_data raise NotImplementedError + class EPix100ImageAssembler(BaseAssembler): + def _get_modules_bridge(self, data, src): + """Override. + + - calibrated, "data.image", (y, x, 1) + - raw, "data.image.data", (1, y, x) + -> (y, x) + """ + img_data = data[src] + dtype = img_data.dtype + + if dtype == _IMAGE_DTYPE: + return img_data.squeeze(axis=-1) + + # raw data of ePix100 has an unexpected dtype int16 + if dtype == np.int16: + return img_data.squeeze(axis=0) + + raise AssemblingError(f"Unknown detector data type: {dtype}!") + + def _get_modules_file(self, data, src): + """Override. + + - calibrated, "data.image.pixels", (y, x) + - raw, "data.image.pixels", (y, x) + -> (y, x) + """ + return data[src] + class FastCCDImageAssembler(BaseAssembler): def _get_modules_bridge(self, data, src): """Override. @@ -703,6 +754,9 @@ def create(cls, detector): if detector == 'FastCCD': return cls.FastCCDImageAssembler() + if detector == 'ePix100': + return cls.EPix100ImageAssembler() + if detector == 'BaslerCamera': return cls.BaslerCameraImageAssembler() diff --git a/extra_foam/pipeline/processors/image_processor.py b/extra_foam/pipeline/processors/image_processor.py index 2cff9b593..901cb00dc 100644 --- a/extra_foam/pipeline/processors/image_processor.py +++ b/extra_foam/pipeline/processors/image_processor.py @@ -93,26 +93,26 @@ def update(self): # image cfg = self._meta.hget_all(mt.IMAGE_PROC) - self._correct_gain = cfg['correct gain'] == 'True' - self._correct_offset = cfg['correct offset'] == 'True' + self._correct_gain = cfg['correct_gain'] == 'True' + self._correct_offset = cfg['correct_offset'] == 'True' - gain_slicer = self.str2slice(cfg['gain slicer']) + gain_slicer = self.str2slice(cfg['gain_slicer']) if gain_slicer != self._gain_slicer: self._compute_gain_mean = True self._gain_slicer = gain_slicer - offset_slicer = self.str2slice(cfg['offset slicer']) + offset_slicer = self.str2slice(cfg['offset_slicer']) if offset_slicer != self._offset_slicer: self._compute_offset_mean = True self._offset_slicer = offset_slicer - dark_as_offset = cfg['dark as offset'] == 'True' + dark_as_offset = cfg['dark_as_offset'] == 'True' if dark_as_offset != self._dark_as_offset: self._compute_offset_mean = True self._dark_as_offset = dark_as_offset - self._recording_dark = cfg['recording dark'] == 'True' - if 'remove dark' in cfg: - self._meta.hdel(mt.IMAGE_PROC, 'remove dark') + self._recording_dark = cfg['recording_dark'] == 'True' + if 'remove_dark' in cfg: + self._meta.hdel(mt.IMAGE_PROC, 'remove_dark') del self._dark self._dark_mean = None @@ -193,16 +193,20 @@ def _record_dark(self, assembled): def _update_image_mask(self, image_shape): image_mask = self._mask_sub.update(self._image_mask, image_shape) if image_mask is not None and image_mask.shape != image_shape: - # This could only happen when the mask is loaded from the files - # and the image shapes in the ImageTool is different from the - # shape of the live images. - # The original image mask remains the same. - raise ImageProcessingError( - f"[Image processor] The shape of the image mask " - f"{image_mask.shape} is different from the shape of the image " - f"{image_shape}!") + if np.sum(image_mask) == 0: + # reset the empty image mask automatically + image_mask = None + else: + # This could only happen when the mask is loaded from the files + # and the image shapes in the ImageTool is different from the + # shape of the live images. + # The original image mask remains the same. + raise ImageProcessingError( + f"[Image processor] The shape of the image mask " + f"{image_mask.shape} is different from the shape of the image " + f"{image_shape}!") - elif image_mask is None: + if image_mask is None: image_mask = np.zeros(image_shape, dtype=np.bool) self._image_mask = image_mask diff --git a/extra_foam/pipeline/processors/image_roi.py b/extra_foam/pipeline/processors/image_roi.py index 45ceec917..a8dc2c450 100644 --- a/extra_foam/pipeline/processors/image_roi.py +++ b/extra_foam/pipeline/processors/image_roi.py @@ -127,10 +127,14 @@ def update(self): """Override.""" cfg = super().update() - self._geom1 = self.str2list(cfg[f'geom1'], handler=int) - self._geom2 = self.str2list(cfg[f'geom2'], handler=int) - self._geom3 = self.str2list(cfg[f'geom3'], handler=int) - self._geom4 = self.str2list(cfg[f'geom4'], handler=int) + geom1 = self.str2list(cfg[f'geom1'], handler=int) + self._geom1 = RectRoiGeom.INVALID if geom1[0] == 0 else geom1[2:] + geom2 = self.str2list(cfg[f'geom2'], handler=int) + self._geom2 = RectRoiGeom.INVALID if geom2[0] == 0 else geom2[2:] + geom3 = self.str2list(cfg[f'geom3'], handler=int) + self._geom3 = RectRoiGeom.INVALID if geom3[0] == 0 else geom3[2:] + geom4 = self.str2list(cfg[f'geom4'], handler=int) + self._geom4 = RectRoiGeom.INVALID if geom4[0] == 0 else geom4[2:] @profiler("ROI Processor (pulse)") def process(self, data): @@ -244,6 +248,9 @@ def _process_fom(self, assembled, processed): processed.pulse.roi.fom = fom1 - fom2 elif self._fom_combo == RoiCombo.ROI1_ADD_ROI2: processed.pulse.roi.fom = fom1 + fom2 + elif self._fom_combo == RoiCombo.ROI1_DIV_ROI2: + # nan and inf will propagate downstream + processed.pulse.roi.fom = fom1 / fom2 else: raise UnknownParameterError( f"[ROI][FOM] Unknown ROI combo: {self._fom_combo}") @@ -271,8 +278,11 @@ def _process_hist(self, processed): if roi is None: continue - processed.pulse.roi.hist[idx] = nanhist_with_stats( - roi, self._hist_bin_range, self._hist_n_bins) + try: + processed.pulse.roi.hist[idx] = nanhist_with_stats( + roi, self._hist_bin_range, self._hist_n_bins) + except ValueError as e: + raise ProcessingError(f"[ROI][histogram] {str(e)}") if image_data.poi_indices[1] == image_data.poi_indices[0]: # skip the second one if two POIs have the same index @@ -283,6 +293,8 @@ class ImageRoiTrain(_RoiProcessorBase): """Train-resolved ROI processor. Attributes: + _roi_fom_master_slave (bool): True for activating the ROI FOM + master-slave mode. _proj_combo (RoiCombo): ROI combination when calculating ROI projection. _proj_type (RoiProjType): ROI projection type. @@ -318,6 +330,8 @@ class ImageRoiTrain(_RoiProcessorBase): def __init__(self): super().__init__() + self._roi_fom_master_slave = False + self._proj_combo = RoiCombo.ROI1 self._proj_type = RoiProjType.SUM self._proj_direct = 'x' @@ -334,6 +348,8 @@ def update(self): cfg = super().update() + self._roi_fom_master_slave = cfg['fom:master_slave'] == 'True' + self._proj_combo = RoiCombo(int(cfg['proj:combo'])) self._proj_type = RoiProjType(int(cfg['proj:type'])) self._proj_direct = cfg['proj:direct'] @@ -450,8 +466,11 @@ def _process_hist(self, processed): if roi is not None: hist = processed.roi.hist - hist.hist, hist.bin_centers, hist.mean, hist.median, hist.std = \ - nanhist_with_stats(roi, self._hist_bin_range, self._hist_n_bins) + try: + hist.hist, hist.bin_centers, hist.mean, hist.median, hist.std = \ + nanhist_with_stats(roi, self._hist_bin_range, self._hist_n_bins) + except ValueError as e: + raise ProcessingError(f"[ROI][histogram] {str(e)}") def _process_norm(self, processed): """Calculate train-resolved ROI normalizer.""" @@ -484,22 +503,31 @@ def _process_fom(self, processed): fom2 = self._compute_fom(self._roi2, self._fom_type) if self._fom_combo == RoiCombo.ROI1: - roi.fom = fom1 + fom = fom1 elif self._fom_combo == RoiCombo.ROI2: - roi.fom = fom2 + fom = fom2 else: if fom1 is None or fom2 is None: return if self._fom_combo == RoiCombo.ROI1_SUB_ROI2: - roi.fom = fom1 - fom2 + fom = fom1 - fom2 elif self._fom_combo == RoiCombo.ROI1_ADD_ROI2: - roi.fom = fom1 + fom2 + fom = fom1 + fom2 + elif self._fom_combo == RoiCombo.ROI1_DIV_ROI2: + # nan and inf will propagate downstream + fom = np.divide(fom1, fom2) else: raise UnknownParameterError( f"[ROI][FOM] Unknown ROI combo: {self._fom_combo}") - # TODO: normalize + try: + if fom is not None: + roi.fom = self._normalize_fom(processed, fom, self._fom_norm) + if self._roi_fom_master_slave and fom2 is not None: + roi.fom_slave = self._normalize_fom(processed, fom2, self._fom_norm) + except ProcessingError as e: + logger.error(repr(e)) def _process_norm_pump_probe(self, processed): """Calculate train-resolved pump-probe ROI normalizers.""" @@ -557,6 +585,10 @@ def _process_fom_pump_probe(self, processed): elif self._fom_combo == RoiCombo.ROI1_ADD_ROI2: fom_on = fom1_on + fom2_on fom_off = fom1_off + fom2_off + elif self._fom_combo == RoiCombo.ROI1_DIV_ROI2: + # nan and inf will propagate downstream + fom_on = np.divide(fom1_on, fom2_on) + fom_off = np.divide(fom1_off, fom2_off) else: raise UnknownParameterError( f"[ROI][FOM] Unknown ROI combo: {self._fom_combo}") @@ -564,9 +596,12 @@ def _process_fom_pump_probe(self, processed): if fom_on is None: return - # TODO: normalize - - pp.fom = fom_on - fom_off + try: + normalized_on, normalized_off = self._normalize_fom_pp( + processed, fom_on, fom_off, self._fom_norm) + pp.fom = normalized_on - normalized_off + except ProcessingError as e: + logger.error(repr(e)) def _compute_proj(self, roi): if roi is None: @@ -597,14 +632,17 @@ def _process_proj(self, processed): x = np.arange(len(proj)) - normalized_proj = self._normalize_fom( - processed, proj, self._proj_norm, x=x, auc_range=self._proj_auc_range) - fom = np.sum(normalized_proj) - - roi = processed.roi - roi.proj.x = x - roi.proj.y = normalized_proj - roi.proj.fom = fom + try: + normalized_proj = self._normalize_fom( + processed, proj, self._proj_norm, x=x, auc_range=self._proj_auc_range) + fom = np.sum(normalized_proj) + + roi = processed.roi + roi.proj.x = x + roi.proj.y = normalized_proj + roi.proj.fom = fom + except ProcessingError as e: + logger.error(repr(e)) def _process_proj_pump_probe(self, processed): """Calculate train-resolved pump-probe ROI projections.""" @@ -627,20 +665,23 @@ def _process_proj_pump_probe(self, processed): x = np.arange(len(y_on)) - normalized_y_on, normalized_y_off = self._normalize_fom_pp( - processed, y_on, y_off, self._proj_norm, - x=x, auc_range=self._proj_auc_range) + try: + normalized_y_on, normalized_y_off = self._normalize_fom_pp( + processed, y_on, y_off, self._proj_norm, + x=x, auc_range=self._proj_auc_range) - normalized_y = normalized_y_on - normalized_y_off + normalized_y = normalized_y_on - normalized_y_off - sliced = slice_curve(normalized_y, x, *self._proj_fom_integ_range)[0] - if pp.abs_difference: - fom = np.sum(np.abs(sliced)) - else: - fom = np.sum(sliced) + sliced = slice_curve(normalized_y, x, *self._proj_fom_integ_range)[0] + if pp.abs_difference: + fom = np.sum(np.abs(sliced)) + else: + fom = np.sum(sliced) - pp.y_on = normalized_y_on - pp.y_off = normalized_y_off - pp.x = x - pp.y = normalized_y - pp.fom = fom + pp.y_on = normalized_y_on + pp.y_off = normalized_y_off + pp.x = x + pp.y = normalized_y + pp.fom = fom + except ProcessingError as e: + logger.error(repr(e)) diff --git a/extra_foam/pipeline/processors/pump_probe.py b/extra_foam/pipeline/processors/pump_probe.py index d5a701838..94dfa423e 100644 --- a/extra_foam/pipeline/processors/pump_probe.py +++ b/extra_foam/pipeline/processors/pump_probe.py @@ -24,8 +24,8 @@ class PumpProbeProcessor(_BaseProcessor): Attributes: analysis_type (AnalysisType): pump-probe analysis type. _mode (PumpProbeMode): pump-probe analysis mode. - _indices_on (list): a list of laser-on pulse indices. - _indices_off (list): a list of laser-off pulse indices. + _indices_on (slice): a slicer for laser-on pulse indices. + _indices_off (slice): a slicer of laser-off pulse indices. _prev_unmasked_on (numpy.ndarray): the most recent on-pulse image. _prev_xi_on (double): the most recent xgm on-intensity. _prev_dpi_on (double): the most recent digitizer on-pulse-integral. @@ -39,8 +39,8 @@ def __init__(self): self.analysis_type = AnalysisType.UNDEFINED self._mode = PumpProbeMode.UNDEFINED - self._indices_on = [] - self._indices_off = [] + self._indices_on = slice(None, None) + self._indices_off = slice(None, None) self._reset = False self._abs_difference = False @@ -72,10 +72,8 @@ def update(self): # reset when commanded by the GUI self._reset = True - self._indices_on = self.str2list( - cfg['on_pulse_indices'], handler=int) - self._indices_off = self.str2list( - cfg['off_pulse_indices'], handler=int) + self._indices_on = self.str2slice(cfg['on_pulse_slicer']) + self._indices_off = self.str2slice(cfg['off_pulse_slicer']) @profiler("Pump-probe processor") def process(self, data): @@ -173,13 +171,13 @@ def _compute_on_off_data(self, tid, assembled, xi, dpi, dropped_indices, *, mode = self._mode if mode != PumpProbeMode.UNDEFINED: - self._parse_on_off_indices(assembled.shape) + indices_on, indices_off = self._parse_on_off_indices(assembled.shape) if assembled.ndim == 3: - self._validate_on_off_indices(assembled.shape[0]) + self._validate_on_off_indices(indices_on, indices_off) - indices_on = list(set(self._indices_on) - set(dropped_indices)) - indices_off = list(set(self._indices_off) - set(dropped_indices)) + indices_on = list(set(indices_on) - set(dropped_indices)) + indices_off = list(set(indices_off) - set(dropped_indices)) # on and off are not from different trains if mode in (PumpProbeMode.REFERENCE_AS_OFF, @@ -286,37 +284,18 @@ def _compute_on_off_data(self, tid, assembled, xi, dpi, dropped_indices, *, def _parse_on_off_indices(self, shape): if len(shape) == 3: - # pulse-resolved - all_indices = list(range(shape[0])) + n_pulses = shape[0] else: - # train-resolved (indeed not used) - all_indices = [0] - - # convert [-1] to a list of indices - if self._indices_on[0] == -1: - self._indices_on = all_indices - if self._indices_off[0] == -1: - self._indices_off = all_indices - - def _validate_on_off_indices(self, n_pulses): - """Check pulse index when on/off pulses in the same train. - - Note: We can not check it in the GUI side since we do not know - how many pulses are there in the train. - """ - # check index range - if self._mode == PumpProbeMode.REFERENCE_AS_OFF: - max_index = max(self._indices_on) - else: - max_index = max(max(self._indices_on), max(self._indices_off)) + n_pulses = 1 - if max_index >= n_pulses: - raise PumpProbeIndexError(f"Index {max_index} is out of range for" - f" a train with {n_pulses} pulses!") + return (list(range(*self._indices_on.indices(n_pulses))), + list(range(*self._indices_off.indices(n_pulses)))) + def _validate_on_off_indices(self, indices_on, indices_off): + """Check pulse index when on/off pulses in the same train.""" if self._mode == PumpProbeMode.SAME_TRAIN: # check pulse index overlap in on- and off- indices - common = set(self._indices_on).intersection(self._indices_off) + common = set(indices_on).intersection(indices_off) if common: raise PumpProbeIndexError( "Pulse indices {} are found in both on- and off- pulses.". diff --git a/extra_foam/pipeline/processors/tests/test_correlation.py b/extra_foam/pipeline/processors/tests/test_correlation.py index 6525e728e..d2ff64140 100644 --- a/extra_foam/pipeline/processors/tests/test_correlation.py +++ b/extra_foam/pipeline/processors/tests/test_correlation.py @@ -12,7 +12,10 @@ import numpy as np -from extra_foam.pipeline.processors import CorrelationProcessor +from extra_foam.pipeline.processors.correlation import ( + CorrelationProcessor, SimplePairSequence, OneWayAccuPairSequence +) + from extra_foam.config import AnalysisType from extra_foam.pipeline.tests import _TestDataMixin @@ -40,69 +43,73 @@ def _set_fom(self, processed, analysis_type, fom): @mock.patch('extra_foam.ipc.ProcessLogger.error') @pytest.mark.parametrize("analysis_type", _analysis_types) - def testGeneral(self, error, analysis_type): + @pytest.mark.parametrize("index", [0, 1]) + def testGeneral(self, error, analysis_type, index): data, processed = self.simple_data(1001, (2, 2)) - data['raw'] = {'A ppt': 1} + corr = processed.corr + + slow_src = f'A{index} ppt' + data['raw'] = {slow_src: 1} - proc = CorrelationProcessor() + proc = CorrelationProcessor(index+1) proc.analysis_type = analysis_type - proc._sources = ['A ppt', ''] - proc._resolutions = [0.0, 0.0] + proc._resolution = 0.0 + proc._correlation = SimplePairSequence() + proc._correlation_slave = SimplePairSequence() + + # source is empty + proc._source = '' empty_arr = np.array([], dtype=np.float64) + proc.process(data) + np.testing.assert_array_equal(empty_arr, corr[index].x) + np.testing.assert_array_equal(empty_arr, corr[index].y) + if analysis_type != AnalysisType.PUMP_PROBE: + error.assert_called_once() + error.reset_mock() + # set FOM and source fom_gt = 10. self._set_fom(processed, analysis_type, fom_gt) + proc._source = slow_src proc.process(data) + assert slow_src == corr[index].source + assert 0.0 == corr[index].resolution + np.testing.assert_array_equal(np.array([1], dtype=np.float64), corr[index].x) + np.testing.assert_array_equal(np.array([10], dtype=np.float64), corr[index].y) - corr = processed.corr - assert 'A ppt' == corr[0].source - assert 0.0 == corr[0].resolution - np.testing.assert_array_equal(np.array([1], dtype=np.float64), corr[0].x) - np.testing.assert_array_equal(np.array([10], dtype=np.float64), corr[0].y) - - assert '' == corr[1].source - assert 0.0 == corr[1].resolution - np.testing.assert_array_equal(empty_arr, corr[1].x) - np.testing.assert_array_equal(empty_arr, corr[1].y) - - if analysis_type == AnalysisType.PUMP_PROBE: + if analysis_type == AnalysisType.PUMP_PROBE and index == 0: + # _process_pump_probe is called only once assert '' == corr.pp.source assert 0.0 == corr.pp.resolution np.testing.assert_array_equal(np.array([1001]), corr.pp.x) np.testing.assert_array_equal(np.array([fom_gt]), corr.pp.y) - # --------------- - # new data arrive - # --------------- - proc._sources = ['A ppt', 'B ppt'] - proc._resolutions = [0.0, 0.0] - - fom_gt = 20. + # ------------------- + # slow data not found + # ------------------- + data['raw'] = {} + fom_gt = 20 self._set_fom(processed, analysis_type, fom_gt) proc.process(data) + np.testing.assert_array_equal(np.array([1], dtype=np.float64), corr[index].x) + np.testing.assert_array_equal(np.array([10], dtype=np.float64), corr[index].y) error.assert_called_once() error.reset_mock() - np.testing.assert_array_equal(np.array([1, 1], dtype=np.float64), corr[0].x) - np.testing.assert_array_equal(np.array([10, 20], dtype=np.float64), corr[0].y) - # ------------------------ - # set slow data source 'B' - # ------------------------ - data['raw'] = {'A ppt': 2, 'B ppt': 5} + # --------------- + # new data arrive + # --------------- + data['raw'] = {slow_src: 2} + fom_gt = 20 + self._set_fom(processed, analysis_type, fom_gt) proc.process(data) - np.testing.assert_array_equal(np.array([1, 1, 2], dtype=np.float64), corr[0].x) - np.testing.assert_array_equal(np.array([10, 20, 20], dtype=np.float64), corr[0].y) - - assert 'B ppt' == corr[1].source - assert 0.0 == corr[1].resolution - np.testing.assert_array_equal(np.array([5], dtype=np.float64), corr[1].x) - np.testing.assert_array_equal(np.array([20], dtype=np.float64), corr[1].y) + np.testing.assert_array_equal(np.array([1, 2], dtype=np.float64), corr[index].x) + np.testing.assert_array_equal(np.array([10, 20], dtype=np.float64), corr[index].y) # ----------- # FOM is None # ----------- - proc._resolutions = [1.0, 0.0] fom_gt = None self._set_fom(processed, analysis_type, fom_gt) proc.process(data) @@ -111,12 +118,8 @@ def testGeneral(self, error, analysis_type): error.reset_mock() else: assert 1 == proc._pp_fail_flag - - np.testing.assert_array_equal(np.array([1, 1, 2], dtype=np.float64), corr[0].x) - np.testing.assert_array_equal(np.array([10, 20, 20], dtype=np.float64), corr[0].y) - - np.testing.assert_array_equal(np.array([5], dtype=np.float64), corr[1].x) - np.testing.assert_array_equal(np.array([20], dtype=np.float64), corr[1].y) + np.testing.assert_array_equal(np.array([1, 2], dtype=np.float64), corr[index].x) + np.testing.assert_array_equal(np.array([10, 20], dtype=np.float64), corr[index].y) # again if analysis_type == AnalysisType.PUMP_PROBE: @@ -124,3 +127,52 @@ def testGeneral(self, error, analysis_type): error.assert_called_once() error.reset_mock() assert 0 == proc._pp_fail_flag + + @pytest.mark.parametrize("index", [0, 1]) + def testMaskSlave(self, index): + data, processed = self.simple_data(1001, (2, 2)) + corr = processed.corr + + slow_src = f'A{index} ppt' + data['raw'] = {slow_src: 1} + + proc = CorrelationProcessor(index+1) + proc.analysis_type = AnalysisType.ROI_FOM + + # first data + processed.roi.fom = 10 + proc._source = slow_src + proc.process(data) + np.testing.assert_array_equal(np.array([1], dtype=np.float64), corr[index].x) + np.testing.assert_array_equal(np.array([10], dtype=np.float64), corr[index].y) + + # second data + processed.roi.fom = 20 + processed.roi.fom_slave = 1 + proc._source = slow_src + proc.process(data) + np.testing.assert_array_equal(np.array([1, 1], dtype=np.float64), corr[index].x) + np.testing.assert_array_equal(np.array([10, 20], dtype=np.float64), corr[index].y) + np.testing.assert_array_equal(np.array([1], dtype=np.float64), corr[index].x_slave) + np.testing.assert_array_equal(np.array([1], dtype=np.float64), corr[index].y_slave) + + # third data + processed.roi.fom = 30 + processed.roi.fom_slave = 2 + proc._source = slow_src + proc.process(data) + np.testing.assert_array_equal(np.array([1, 1, 1], dtype=np.float64), corr[index].x) + np.testing.assert_array_equal(np.array([10, 20, 30], dtype=np.float64), corr[index].y) + np.testing.assert_array_equal(np.array([1, 1], dtype=np.float64), corr[index].x_slave) + np.testing.assert_array_equal(np.array([1, 2], dtype=np.float64), corr[index].y_slave) + + # test reset + proc._reset = True + with mock.patch.object(proc._correlation_pp, "reset") as patched_reset: + proc.process(data) + np.testing.assert_array_equal(np.array([1], dtype=np.float64), corr[index].x) + np.testing.assert_array_equal(np.array([30], dtype=np.float64), corr[index].y) + np.testing.assert_array_equal(np.array([1], dtype=np.float64), corr[index].x_slave) + np.testing.assert_array_equal(np.array([2], dtype=np.float64), corr[index].y_slave) + # correlation_pp has another reset flag + patched_reset.assert_not_called() diff --git a/extra_foam/pipeline/processors/tests/test_histogram.py b/extra_foam/pipeline/processors/tests/test_histogram.py index fa0270429..0b59f7be6 100644 --- a/extra_foam/pipeline/processors/tests/test_histogram.py +++ b/extra_foam/pipeline/processors/tests/test_histogram.py @@ -24,6 +24,7 @@ 'ROI': AnalysisType.ROI_FOM } + class TestHistogramProcessor(_TestDataMixin): @pytest.fixture(autouse=True) def setUp(self): diff --git a/extra_foam/pipeline/processors/tests/test_image_assembler.py b/extra_foam/pipeline/processors/tests/test_image_assembler.py index e5c0c4603..37d6d6638 100644 --- a/extra_foam/pipeline/processors/tests/test_image_assembler.py +++ b/extra_foam/pipeline/processors/tests/test_image_assembler.py @@ -21,7 +21,6 @@ from extra_foam.pipeline.processors.image_assembler import ( _IMAGE_DTYPE, _RAW_IMAGE_DTYPE, ImageAssemblerFactory ) -from extra_foam.pipeline.tests import _TestDataMixin from extra_foam.pipeline.exceptions import AssemblingError from extra_foam.config import GeomAssembler, config, DataSource from extra_foam.database import SourceCatalog, SourceItem @@ -65,19 +64,19 @@ def teardown_module(module): config.ROOT_PATH = module._backup_ROOT_PATH -class TestAgipdAssembler(_TestDataMixin, unittest.TestCase): +class TestAgipdAssembler: @classmethod - def setUpClass(cls): + def setup_class(cls): config.load('AGIPD', random.choice(['SPB', 'MID'])) cls._geom_file = config["GEOMETRY_FILE"] cls._quad_positions = config["QUAD_POSITIONS"] @classmethod - def tearDownClass(cls): + def teardown_class(cls): os.remove(config.config_file) - def setUp(self): + def setup_method(self, method): self._assembler = ImageAssemblerFactory.create("AGIPD") self._assembler._load_geometry(self._geom_file, self._quad_positions) @@ -87,6 +86,18 @@ def _create_catalog(self, src_name, key_name): catalog.add_item(SourceItem('AGIPD', src_name, [], key_name, slice(None, None), None)) return src, catalog + @pytest.mark.parametrize("assembler_type", [GeomAssembler.EXTRA_GEOM, GeomAssembler.OWN]) + def testInvalidGeometryFile(self, assembler_type): + self._assembler._assembler_type = assembler_type + # test file does not exist + with pytest.raises(AssemblingError): + self._assembler._load_geometry("abc", self._quad_positions) + + # test invalid file + with tempfile.NamedTemporaryFile() as fp: + with pytest.raises(AssemblingError): + self._assembler._load_geometry(fp.name, self._quad_positions) + def testGeneral(self): # Note: this test does not need to repeat for each detector key_name = 'image.data' @@ -108,13 +119,13 @@ def testGeneral(self): }, } - with self.assertRaisesRegex(KeyError, "source_type"): + with pytest.raises(KeyError, match="source_type"): self._assembler.process(copy.deepcopy(data)) data['meta'][src]["source_type"] = DataSource.FILE self._assembler.process(data) - self.assertEqual(10001, data['raw']['META timestamp.tid']) - self.assertIsNone(data['raw'][src]) + assert 10001 == data['raw']['META timestamp.tid'] + assert data['raw'][src] is None def testAssembleFileCal(self): self._runAssembleFileTest((4, 512, 128), _IMAGE_DTYPE) @@ -155,7 +166,7 @@ def testAssembleBridge(self): key_name = 'image.data' src, catalog = self._create_catalog('SPB_DET_AGIPD1M-1/CAL/APPEND_CORRECTED', key_name) - with self.assertRaisesRegex(AssemblingError, 'Expected module shape'): + with pytest.raises(AssemblingError, match='Expected module shape'): data = { 'catalog': catalog, 'meta': { @@ -170,11 +181,11 @@ def testAssembleBridge(self): } self._assembler.process(data) - with self.assertRaisesRegex(AssemblingError, 'modules, but'): + with pytest.raises(AssemblingError, match='modules, but'): data['raw'][src] = np.ones((4, 12, 512, 128), dtype=_IMAGE_DTYPE) self._assembler.process(data) - with self.assertRaisesRegex(AssemblingError, 'Number of memory cells'): + with pytest.raises(AssemblingError, match='Number of memory cells'): data['raw'][src] = np.ones((0, 16, 512, 128), dtype=_IMAGE_DTYPE) self._assembler.process(data) @@ -201,7 +212,29 @@ def testAssembleBridge(self): _check_single_module_result(data, src, config["MODULE_SHAPE"]) -class TestLpdAssembler: +class _AssemblerGeometryTest: + @pytest.mark.parametrize("assembler_type", [GeomAssembler.EXTRA_GEOM, GeomAssembler.OWN]) + def testInvalidGeometryFile(self, assembler_type): + import h5py + + self._assembler._assembler_type = assembler_type + # test file does not exist + with pytest.raises(AssemblingError): + self._assembler._load_geometry("abc", self._quad_positions) + + # test invalid file (file signature not found) + with tempfile.TemporaryFile() as fp: + with pytest.raises(AssemblingError): + self._assembler._load_geometry(fp, self._quad_positions) + + # test invalid h5 file (component not found) + with tempfile.NamedTemporaryFile() as fp: + fph5 = h5py.File(fp.name, 'r+') + with pytest.raises(AssemblingError): + self._assembler._load_geometry(fp.name, self._quad_positions) + + +class TestLpdAssembler(_AssemblerGeometryTest): @classmethod def setup_class(cls): config.load('LPD', 'FXE') @@ -431,7 +464,7 @@ def testAssembleDtype(self, assembler_type): self._assembler.process(data) -class TestDSSCAssembler: +class TestDSSCAssembler(_AssemblerGeometryTest): @classmethod def setup_class(cls): config.load('DSSC', 'SCS') @@ -818,6 +851,85 @@ def _runAssembleBridgeTest(self, shape, dtype, key): self._assembler.process(data) +class TestEPix100Assembler(unittest.TestCase): + @classmethod + def setUpClass(cls): + config.load('ePix100', 'MID') + + @classmethod + def tearDownClass(cls): + os.remove(config.config_file) + + def setUp(self): + self._assembler = ImageAssemblerFactory.create("ePix100") + + def _create_catalog(self, src_name, key_name): + catalog = SourceCatalog() + src = f'{src_name} {key_name}' + catalog.add_item(SourceItem('ePix100', src_name, [], key_name, None, None)) + return src, catalog + + def testAssembleFileCal(self): + self._runAssembleFileTest((708, 768), _IMAGE_DTYPE, 'data.image.pixels') + + def testAssembleFileRaw(self): + self._runAssembleFileTest((708, 768), np.int16, 'data.image.pixels') + + def _runAssembleFileTest(self, shape, dtype, key): + src, catalog = self._create_catalog('MID_EXP_EPIX-1/DET/RECEIVER:daqOutput', key) + + data = { + 'catalog': catalog, + 'meta': { + src: { + 'tid': 10001, + 'source_type': DataSource.FILE, + } + }, + 'raw': { + src: np.ones(shape, dtype=dtype) + }, + } + self._assembler.process(data) + self.assertIsNone(data['raw'][src]) + + with self.assertRaisesRegex(AssemblingError, 'Expected module shape'): + data['raw'][src] = np.ones((100, 100)) + self._assembler.process(data) + + def testAssembleBridgeCal(self): + self._runAssembleBridgeTest((708, 768, 1), _IMAGE_DTYPE, "data.image") + + def testAssembleBridgeRaw(self): + self._runAssembleBridgeTest((1, 708, 768), np.int16, "data.image.data") + + def _runAssembleBridgeTest(self, shape, dtype, key): + src, catalog = self._create_catalog('MID_EXP_EPIX-1/DET/RECEIVER:daqOutput', key) + + data = { + 'catalog': catalog, + 'meta': { + src: { + 'tid': 10001, + 'source_type': DataSource.BRIDGE, + } + }, + 'raw': { + src: np.ones(shape, dtype=dtype) + }, + } + + self._assembler.process(data) + self.assertIsNone(data['raw'][src]) + + with self.assertRaisesRegex(AssemblingError, 'Expected module shape'): + if dtype == _IMAGE_DTYPE: + data['raw'][src] = np.ones((100, 100, 1), dtype=dtype) + else: + data['raw'][src] = np.ones((1, 100, 100), dtype=np.int16) + self._assembler.process(data) + + class TestBaslerCameraAssembler(unittest.TestCase): @classmethod def setUpClass(cls): diff --git a/extra_foam/pipeline/processors/tests/test_image_roi.py b/extra_foam/pipeline/processors/tests/test_image_roi.py index a2785f423..cdcc8779a 100644 --- a/extra_foam/pipeline/processors/tests/test_image_roi.py +++ b/extra_foam/pipeline/processors/tests/test_image_roi.py @@ -14,6 +14,7 @@ import numpy as np +from extra_foam.pipeline.exceptions import ProcessingError from extra_foam.pipeline.processors import ImageRoiTrain, ImageRoiPulse from extra_foam.config import AnalysisType, config, Normalizer, RoiCombo, RoiFom, RoiProjType from extra_foam.pipeline.tests import _TestDataMixin @@ -104,33 +105,57 @@ def testRoiNorm(self, norm_type, fom_handler): @pytest.mark.parametrize("fom_type, fom_handler", [(k, v) for k, v in _roi_fom_handlers.items()]) def testRoiFom(self, fom_type, fom_handler): proc = self._proc + proc._fom_norm = Normalizer.UNDEFINED + proc._fom_type = fom_type with patch.object(proc._meta, 'has_analysis', side_effect=lambda x: True if x == AnalysisType.ROI_FOM_PULSE else False): for combo, geom in zip([RoiCombo.ROI1, RoiCombo.ROI2], ['_geom1', '_geom2']): data, processed = self._get_data() proc._fom_combo = combo - proc._fom_type = fom_type - proc._fom_norm = Normalizer.UNDEFINED proc.process(data) s = self._get_roi_slice(getattr(proc, geom)) fom_gt = fom_handler(data['assembled']['sliced'][:, s[0], s[1]], axis=(-1, -2)) np.testing.assert_array_equal(fom_gt, processed.pulse.roi.fom) - for fom_combo in [RoiCombo.ROI1_SUB_ROI2, RoiCombo.ROI1_ADD_ROI2]: + for combo in [RoiCombo.ROI1_SUB_ROI2, RoiCombo.ROI1_ADD_ROI2, RoiCombo.ROI1_DIV_ROI2]: data, processed = self._get_data() - proc._fom_combo = fom_combo - proc._fom_type = fom_type - proc._fom_norm = Normalizer.UNDEFINED + proc._fom_combo = combo proc.process(data) s1 = self._get_roi_slice(proc._geom1) fom1_gt = fom_handler(data['assembled']['sliced'][:, s1[0], s1[1]], axis=(-1, -2)) s2 = self._get_roi_slice(proc._geom2) fom2_gt = fom_handler(data['assembled']['sliced'][:, s2[0], s2[1]], axis=(-1, -2)) - if fom_combo == RoiCombo.ROI1_SUB_ROI2: + if combo == RoiCombo.ROI1_SUB_ROI2: np.testing.assert_array_equal(fom1_gt - fom2_gt, processed.pulse.roi.fom) - else: + elif combo == RoiCombo.ROI1_ADD_ROI2: np.testing.assert_array_equal(fom1_gt + fom2_gt, processed.pulse.roi.fom) + else: + np.testing.assert_array_equal(fom1_gt / fom2_gt, processed.pulse.roi.fom) + + if combo == RoiCombo.ROI1_DIV_ROI2: + with np.warnings.catch_warnings(): + np.warnings.simplefilter("ignore", category=RuntimeWarning) + + # test some of ROI2 FOM are nan + data, processed = self._get_data() + data['assembled']['sliced'][:2, :, :] = np.nan + proc.process(data) + s1 = self._get_roi_slice(proc._geom1) + fom1_gt = fom_handler(data['assembled']['sliced'][:, s1[0], s1[1]], axis=(-1, -2)) + s2 = self._get_roi_slice(proc._geom2) + fom2_gt = fom_handler(data['assembled']['sliced'][:, s2[0], s2[1]], axis=(-1, -2)) + assert np.count_nonzero(~np.isnan(fom1_gt / fom2_gt)) > 0 + np.testing.assert_array_equal(fom1_gt / fom2_gt, processed.pulse.roi.fom) + + # test all of ROI2 FOM are nan + data, processed = self._get_data() + processed.image.image_mask[s2[0], s2[1]] = True + proc.process(data) + if fom_type == RoiFom.SUM: + assert np.count_nonzero(~np.isinf(processed.pulse.roi.fom)) == 0 + else: + assert np.count_nonzero(~np.isnan(processed.pulse.roi.fom)) == 0 with patch.object(proc._meta, 'has_analysis', side_effect=lambda x: False): data, processed = self._get_data() @@ -230,7 +255,8 @@ def _get_data(self): data, processed = self.data_with_assembled(1001, shape) proc = ImageRoiPulse() proc._geom1 = [0, 1, 2, 3] - proc._geom2 = [1, 0, 2, 3] + # In order to pass the test, ROI1 and ROI2 cannot have overlap. + proc._geom2 = [5, 6, 2, 3] proc._geom3 = [1, 2, 2, 3] proc._geom4 = [3, 2, 3, 4] # set processed.roi.geom{1, 2, 3, 4} @@ -243,6 +269,29 @@ def _get_data(self): def _get_roi_slice(self, geom): return slice(geom[1], geom[1] + geom[3]), slice(geom[0], geom[0] + geom[2]) + @patch('extra_foam.ipc.ProcessLogger.error') + def testNormalizationError(self, error): + def side_effect(*args, **kwargs): + raise ProcessingError + + proc = self._proc + + with patch.object(proc, "_normalize_fom", side_effect=side_effect): + # let "_process_fom" raise + with patch.object(proc, "_process_proj") as mocked_p_proj: + data, processed = self._get_data() + proc.process(data) + mocked_p_proj.assert_called_once() + + with patch.object(proc, "_process_fom"): + # let "_process_proj" raise + with patch.object(proc, "_normalize_fom", side_effect=side_effect): + with patch.object(proc, "_process_norm_pump_probe") as mocked_p_norm_pp: + data, processed = self._get_data() + processed.pp.analysis_type = AnalysisType.ROI_PROJ + proc.process(data) + mocked_p_norm_pp.assert_called_once() + @pytest.mark.parametrize("norm_type, fom_handler", [(k, v) for k, v in _roi_fom_handlers.items()]) def testRoiNorm(self, norm_type, fom_handler): proc = self._proc @@ -270,34 +319,63 @@ def testRoiNorm(self, norm_type, fom_handler): assert fom3_gt + fom4_gt == processed.roi.norm @pytest.mark.parametrize("fom_type, fom_handler", [(k, v) for k, v in _roi_fom_handlers.items()]) - def testRoiFom(self, fom_type, fom_handler): + @pytest.mark.parametrize("normalizer, norm", [(Normalizer.UNDEFINED, 1), + (Normalizer.ROI, 2.), + (Normalizer.XGM, 4.), + (Normalizer.DIGITIZER, 8.)]) + def testRoiFom(self, fom_type, fom_handler, normalizer, norm): + def mocked_process_norm(p_data): + if normalizer == Normalizer.ROI: + p_data.roi.norm = 2.0 + elif normalizer == Normalizer.XGM: + p_data.pulse.xgm.intensity = np.array([4., 4., 4., 4.]) # mean = 4.0 + elif normalizer == Normalizer.DIGITIZER: + p_data.pulse.digitizer.ch_normalizer = 'A' + p_data.pulse.digitizer['A'].pulse_integral = np.array([8., 8., 8., 8.]) # mean = 8.0 + proc = self._proc + proc._fom_type = fom_type + proc._fom_norm = normalizer # We do not test all the combinations of parameters. + with patch.object(proc, "_process_norm", side_effect=mocked_process_norm): + for combo, geom in zip([RoiCombo.ROI1, RoiCombo.ROI2], ['geom1', 'geom2']): + data, processed = self._get_data() + proc._fom_combo = combo + proc.process(data) + s = self._get_roi_slice(getattr(processed.roi, geom).geometry) + assert fom_handler(processed.image.masked_mean[s[0], s[1]])/norm == processed.roi.fom - for combo, geom in zip([RoiCombo.ROI1, RoiCombo.ROI2], ['geom1', 'geom2']): - data, processed = self._get_data() - proc._fom_combo = combo - proc._fom_type = fom_type - proc._fom_norm = Normalizer.UNDEFINED - proc.process(data) - s = self._get_roi_slice(getattr(processed.roi, geom).geometry) - assert fom_handler(processed.image.masked_mean[s[0], s[1]]) == processed.roi.fom + for fom_combo in [RoiCombo.ROI1_SUB_ROI2, RoiCombo.ROI1_ADD_ROI2, RoiCombo.ROI1_DIV_ROI2]: + data, processed = self._get_data() + proc._fom_combo = fom_combo + proc.process(data) + s1 = self._get_roi_slice(processed.roi.geom1.geometry) + fom1_gt = fom_handler(processed.image.masked_mean[s1[0], s1[1]]) + s2 = self._get_roi_slice(processed.roi.geom2.geometry) + fom2_gt = fom_handler(processed.image.masked_mean[s2[0], s2[1]]) + if fom_combo == RoiCombo.ROI1_SUB_ROI2: + assert (fom1_gt - fom2_gt) / norm == processed.roi.fom + elif fom_combo == RoiCombo.ROI1_ADD_ROI2: + assert (fom1_gt + fom2_gt) / norm == processed.roi.fom + else: + assert (fom1_gt / fom2_gt) / norm == processed.roi.fom - for fom_combo in [RoiCombo.ROI1_SUB_ROI2, RoiCombo.ROI1_ADD_ROI2]: - data, processed = self._get_data() - proc._fom_combo = fom_combo - proc._fom_type = fom_type - proc._fom_norm = Normalizer.UNDEFINED - proc.process(data) - s1 = self._get_roi_slice(processed.roi.geom1.geometry) - fom1_gt = fom_handler(processed.image.masked_mean[s1[0], s1[1]]) - s2 = self._get_roi_slice(processed.roi.geom2.geometry) - fom2_gt = fom_handler(processed.image.masked_mean[s2[0], s2[1]]) - if fom_combo == RoiCombo.ROI1_SUB_ROI2: - assert fom1_gt - fom2_gt == processed.roi.fom - else: - assert fom1_gt + fom2_gt == processed.roi.fom + if fom_combo == RoiCombo.ROI1_DIV_ROI2: + with np.warnings.catch_warnings(): + np.warnings.simplefilter("ignore", category=RuntimeWarning) + + # test ROI1 FOM is a number and ROI2 FOM equals to 0, which produces inf + s2 = self._get_roi_slice(processed.roi.geom2.geometry) + processed.image.masked_mean[s2[0], s2[1]] = 0 + proc.process(data) + assert np.isinf(processed.roi.fom) + + # both ROI1 FOM and ROI2 FOM are nan + data, processed = self._get_data() + processed.image.masked_mean[:] = np.nan + proc.process(data) + assert np.isnan(processed.roi.fom) def testRoiHist(self): proc = self._proc @@ -476,7 +554,7 @@ def testRoiFomPumpProbe(self, fom_type, fom_handler): fom_off_gt = fom_handler(processed.pp.image_off[s[0], s[1]]) assert fom_on_gt - fom_off_gt == processed.pp.fom - for fom_combo in [RoiCombo.ROI1_SUB_ROI2, RoiCombo.ROI1_ADD_ROI2]: + for fom_combo in [RoiCombo.ROI1_SUB_ROI2, RoiCombo.ROI1_ADD_ROI2, RoiCombo.ROI1_DIV_ROI2]: data, processed = self._get_data() processed.pp.analysis_type = AnalysisType.ROI_FOM proc._fom_combo = fom_combo @@ -491,11 +569,32 @@ def testRoiFomPumpProbe(self, fom_type, fom_handler): if fom_combo == RoiCombo.ROI1_SUB_ROI2: fom_on_gt = fom1_on_gt - fom2_on_gt fom_off_gt = fom1_off_gt - fom2_off_gt - else: + elif fom_combo == RoiCombo.ROI1_ADD_ROI2: fom_on_gt = fom1_on_gt + fom2_on_gt fom_off_gt = fom1_off_gt + fom2_off_gt + else: + fom_on_gt = fom1_on_gt / fom2_on_gt + fom_off_gt = fom1_off_gt / fom2_off_gt assert fom_on_gt - fom_off_gt == processed.pp.fom + if fom_combo == RoiCombo.ROI1_DIV_ROI2: + with np.warnings.catch_warnings(): + np.warnings.simplefilter("ignore", category=RuntimeWarning) + + # test ROI1 FOM is a number and ROI2 FOM equals to 0, which produces inf + s2 = self._get_roi_slice(processed.roi.geom2.geometry) + processed.pp.image_on[s2[0], s2[1]] = 0 + proc.process(data) + assert np.isinf(processed.pp.fom) + + # both ROI1 FOM and ROI2 FOM are nan + data, processed = self._get_data() + processed.pp.analysis_type = AnalysisType.ROI_FOM + processed.pp.image_on[:] = np.nan + processed.pp.image_off[:] = np.nan + proc.process(data) + assert np.isnan(processed.pp.fom) + @patch('extra_foam.ipc.ProcessLogger.error') @pytest.mark.parametrize("proj_type, proj_handler", [(k, v) for k, v in _roi_proj_handlers.items()]) diff --git a/extra_foam/pipeline/processors/tests/test_pump_probe.py b/extra_foam/pipeline/processors/tests/test_pump_probe.py index c0329904b..36363e616 100644 --- a/extra_foam/pipeline/processors/tests/test_pump_probe.py +++ b/extra_foam/pipeline/processors/tests/test_pump_probe.py @@ -20,8 +20,8 @@ class _PumpProbeTestMixin: def _check_pp_params_in_data_model(self, data): self.assertEqual(self._proc._mode, data.pp.mode) - self.assertListEqual(self._proc._indices_on, data.pp.indices_on) - self.assertListEqual(self._proc._indices_off, data.pp.indices_off) + self.assertEqual(self._proc._indices_on, data.pp.indices_on) + self.assertEqual(self._proc._indices_off, data.pp.indices_off) def check_other_none(self, processed): pp = processed.pp @@ -46,8 +46,8 @@ class TestPumpProbeProcessorTr(_PumpProbeTestMixin, _TestDataMixin, unittest.Tes """ def setUp(self): self._proc = PumpProbeProcessor() - self._proc._indices_on = [0] - self._proc._indices_off = [0] + self._proc._indices_on = slice(None, None) + self._proc._indices_off = slice(None, None) def _gen_data(self, tid, with_xgm=True, with_digitizer=True): shape = (3, 2) @@ -194,8 +194,8 @@ class TestPumpProbeProcessorPr(_PumpProbeTestMixin, _TestDataMixin, unittest.Tes """ def setUp(self): self._proc = PumpProbeProcessor() - self._proc._indices_on = [0] - self._proc._indices_off = [0] + self._proc._indices_on = slice(0, 1, 1) + self._proc._indices_off = slice(0, 1, 1) def _gen_data(self, tid, with_xgm=True, with_digitizer=True): shape = (3, 2) @@ -223,42 +223,20 @@ def testFomPulseFilter(self): def testInvalidPulseIndices(self): proc = self._proc - proc._indices_on = [0, 1, 5] - proc._indices_off = [1] - - proc._mode = PumpProbeMode.REFERENCE_AS_OFF - with self.assertRaises(PumpProbeIndexError): - # the maximum index is 4 - data, _ = self._gen_data(1001) - proc.process(data) - - proc._indices_on = [0, 1, 5] - proc._indices_off = [1, 3] - proc._mode = PumpProbeMode.EVEN_TRAIN_ON - with self.assertRaises(PumpProbeIndexError): - data, _ = self._gen_data(1001) - proc.process(data) # raises when the same pulse index was found in both # on- and off- indices - proc._indices_on = [0, 1] - proc._indices_off = [1, 3] + proc._indices_on = slice(None, None) + proc._indices_off = slice(None, None) proc._mode = PumpProbeMode.SAME_TRAIN with self.assertRaises(PumpProbeIndexError): data, _ = self._gen_data(1001) proc.process(data) - # off-indices check is not trigger in REFERENCE_AS_OFF mode - proc._indices_on = [0, 1] - proc._indices_off = [5] - proc._mode = PumpProbeMode.REFERENCE_AS_OFF - data, _ = self._gen_data(1001) - proc.process(data) - def testUndefined(self): proc = self._proc - proc._indices_on = [0, 2] - proc._indices_off = [1, 3] + proc._indices_on = slice(0, None, 2) + proc._indices_off = slice(1, None, 2) proc._threshold_mask = (-np.inf, np.inf) proc._mode = PumpProbeMode.UNDEFINED @@ -273,8 +251,8 @@ def testUndefined(self): def testPredefinedOff(self): proc = self._proc proc._mode = PumpProbeMode.REFERENCE_AS_OFF - proc._indices_on = [0, 2] - proc._indices_off = [1, 3] + proc._indices_on = slice(0, None, 2) + proc._indices_off = slice(1, None, 2) data, processed = self._gen_data(1001) proc.process(data) @@ -322,8 +300,8 @@ def testPredefinedOff(self): def testSameTrain(self): proc = self._proc proc._mode = PumpProbeMode.SAME_TRAIN - proc._indices_on = [0, 2] - proc._indices_off = [1, 3] + proc._indices_on = slice(0, None, 2) + proc._indices_off = slice(1, None, 2) data, processed = self._gen_data(1001) proc.process(data) @@ -377,8 +355,8 @@ def testSameTrain(self): def testEvenOn(self): proc = self._proc proc._mode = PumpProbeMode.EVEN_TRAIN_ON - proc._indices_on = [0, 2] - proc._indices_off = [1, 3] + proc._indices_on = slice(0, None, 2) + proc._indices_off = slice(1, None, 2) # test off will not be acknowledged without on data, processed = self._gen_data(1001) # off @@ -466,8 +444,8 @@ def testEvenOn(self): def testOddOn(self): proc = self._proc proc._mode = PumpProbeMode.ODD_TRAIN_ON - proc._indices_on = [0, 2] - proc._indices_off = [1, 3] + proc._indices_on = slice(0, None, 2) + proc._indices_off = slice(1, None, 2) # test off will not be acknowledged without on data, processed = self._gen_data(1002) # off diff --git a/extra_foam/pipeline/worker.py b/extra_foam/pipeline/worker.py index c14a1b22c..06a0972be 100644 --- a/extra_foam/pipeline/worker.py +++ b/extra_foam/pipeline/worker.py @@ -221,7 +221,8 @@ def __init__(self, pause_ev, close_ev): self._filter = FomTrainFilter() self._histogram = HistogramProcessor() - self._correlation_proc = CorrelationProcessor() + self._correlation1_proc = CorrelationProcessor(1) + self._correlation2_proc = CorrelationProcessor(2) self._binning_proc = BinningProcessor() self._tr_xas = TrXasProcessor() @@ -231,7 +232,8 @@ def __init__(self, pause_ev, close_ev): self._ai_proc, self._filter, self._histogram, - self._correlation_proc, + self._correlation1_proc, + self._correlation2_proc, self._binning_proc, self._tr_xas, ] diff --git a/setup.py b/setup.py index 7b8a2bdd5..903236602 100644 --- a/setup.py +++ b/setup.py @@ -32,6 +32,9 @@ def find_version(): with open(osp.join('extra_foam', '__init__.py')) as fp: for line in fp: m = re.search(r'^__version__ = "(\d+\.\d+\.\d[a-z]*\d*)"', line, re.M) + if m is None: + # could be a hotfix + m = re.search(r'^__version__ = "(\d.){3}\d"', line, re.M) if m is not None: return m.group(1) raise RuntimeError("Unable to find version string.") diff --git a/src/extra_foam/f_geometry.cpp b/src/extra_foam/f_geometry.cpp index ea15da750..1b19df443 100644 --- a/src/extra_foam/f_geometry.cpp +++ b/src/extra_foam/f_geometry.cpp @@ -70,6 +70,8 @@ PYBIND11_MODULE(geometry, m) m.doc() = "Detector geometry."; + declare_1MGeometry(m, "AGIPD"); + declare_1MGeometry(m, "LPD"); declare_1MGeometry(m, "DSSC"); diff --git a/src/extra_foam/include/f_geometry.hpp b/src/extra_foam/include/f_geometry.hpp index 96c87c2e6..955f39efe 100644 --- a/src/extra_foam/include/f_geometry.hpp +++ b/src/extra_foam/include/f_geometry.hpp @@ -264,6 +264,180 @@ void Detector1MGeometryBase::positionModule(M&& src, N& dst, T&& pos) const static_cast(this)->positionModuleImp(std::forward(src), dst, std::forward(pos)); } +/** + * AGIPD-1M geometry + * + * + * Layout of AGIPD-1M: Tile layout for each module: + * (looking along the beam) + * + * Q4M1 | Q1M1 Q1 and Q2: T8 T7 T6 T5 T4 T3 T2 T1 + * Q4M2 | Q1M2 Q3 and q4: T1 T2 T3 T4 T5 T6 T7 T8 + * Q4M3 | Q1M3 + * Q4M4 | Q1M4 + * ----------------------- + * Q3M1 | Q2M1 + * Q3M2 | Q2M2 + * Q3M3 | Q2M3 + * Q3M4 | Q2M4 + * + * The quadrant positions refer to the first pixel + * (top-right corners for Q1, Q2 and bottom-left corners for Q3, Q4) + * of the first module in each quadrant. + * + * For details, please see + * https://extra-geom.readthedocs.io/en/latest/geometry.html#agipd-1m + * + */ +class AGIPD_1MGeometry : public Detector1MGeometryBase +{ +public: + + static const shapeType module_shape; + static const shapeType tile_shape; + static const size_t n_tiles_per_module = 8; // number of tiles per module + static const quadOrientType quad_orientations; +private: + + xt::xtensor_fixed> corner_pos_; + + friend Detector1MGeometryBase; + + template + void positionModuleImp(M&& src, N& dst, T&& pos) const; + +public: + + static const vectorType& pixelSize() + { + static const vectorType pixel_size {2e-4, 2e-4, 1.}; + return pixel_size; + } + + AGIPD_1MGeometry(); + + explicit + AGIPD_1MGeometry(const std::array, n_tiles_per_module>, n_modules>& positions); + + ~AGIPD_1MGeometry() = default; +}; + +// (ss/x, fs/y) This should be called 'data_shape' instead of 'module_shape' +const AGIPD_1MGeometry::shapeType AGIPD_1MGeometry::module_shape {512, 128}; +// (fs/y, ss/x) +const AGIPD_1MGeometry::shapeType AGIPD_1MGeometry::tile_shape {128, 64}; +constexpr size_t AGIPD_1MGeometry::n_tiles_per_module; +const AGIPD_1MGeometry::quadOrientType AGIPD_1MGeometry::quad_orientations { + std::array{1, -1}, + std::array{1, -1}, + std::array{-1, 1}, + std::array{-1, 1} +}; + +AGIPD_1MGeometry::AGIPD_1MGeometry() +{ + // first pixel position of each module + // (upper-right for Q1 and Q2, lower-left for Q3 and Q4) positions + xt::xtensor_fixed> m_pos { + { -512, 512, 0}, + { -512, 384, 0}, + { -512, 256, 0}, + { -512, 128, 0}, + { -512, 0, 0}, + { -512, -128, 0}, + { -512, -256, 0}, + { -512, -384, 0}, + { 512, -128, 0}, + { 512, -256, 0}, + { 512, -384, 0}, + { 512, -512, 0}, + { 512, 384, 0}, + { 512, 256, 0}, + { 512, 128, 0}, + { 512, 0, 0} + }; + + xt::xtensor_fixed> positions; + auto ht = static_cast(tile_shape[0]); + auto wt = static_cast(tile_shape[1]); + for (int im = 0; im < n_modules; ++im) + { + auto orient = quad_orientations[im / 4]; + for (int it = 0; it < n_tiles_per_module; ++it) + { + positions(im, it, 0) = m_pos(im, 0) + orient[0] * it * wt; + positions(im, it, 1) = m_pos(im, 1); + positions(im, it, 2) = m_pos(im, 2); + } + } + + positions *= pixelSize(); + + for (int im = 0; im < n_modules; ++im) + { + auto orient = quad_orientations[im / 4]; + for (int it = 0; it < n_tiles_per_module; ++it) + { + for (int j = 0; j < 3; ++j) corner_pos_(im, it, 0, j) = positions(im, it, j); + // calculate the position of the diagonal corner + corner_pos_(im, it, 1, 0) = positions(im, it, 0) + orient[0] * wt * pixelSize()(0); + corner_pos_(im, it, 1, 1) = positions(im, it, 1) + orient[1] * ht * pixelSize()(1); + corner_pos_(im, it, 1, 2) = 0.0; + } + } +} + +AGIPD_1MGeometry::AGIPD_1MGeometry( + const std::array, n_tiles_per_module>, n_modules>& positions) +{ + auto ht = static_cast(tile_shape[0]); + auto wt = static_cast(tile_shape[1]); + for (int im = 0; im < n_modules; ++im) + { + auto orient = quad_orientations[im / 4]; + + for (int it = 0; it < n_tiles_per_module; ++it) + { + for (int j = 0; j < 3; ++j) corner_pos_(im, it, 0, j) = positions[im][it][j]; + // calculate the position of the diagonal corner + corner_pos_(im, it, 1, 0) = positions[im][it][0] + orient[0] * wt * pixelSize()(0); + corner_pos_(im, it, 1, 1) = positions[im][it][1] + orient[1] * ht * pixelSize()(1); + corner_pos_(im, it, 1, 2) = 0.0; + } + } +} + +template +void AGIPD_1MGeometry::positionModuleImp(M&& src, N& dst, T&& pos) const +{ + auto center = assembledDim().second; + auto shape = src.shape(); // caveat: shape has layout (y, x) + size_t n_tiles = n_tiles_per_module; + size_t wt = tile_shape[1]; + size_t ht = tile_shape[0]; + for (int it = 0; it < n_tiles; ++it) + { + auto x0 = pos(it, 0, 0); + auto y0 = pos(it, 0, 1); + + int ix_dir = (pos(it, 1, 0) - x0 > 0) ? 1 : -1; + int iy_dir = (pos(it, 1, 1) - y0 > 0) ? 1 : -1; + + size_t ix0 = it * wt; + size_t iy0 = 0; + + size_t ix0_dst = ix_dir > 0 ? std::floor(x0 + center[0]) : std::ceil(x0 + center[0]) - 1; + size_t iy0_dst = iy_dir > 0 ? std::floor(y0 + center[1]) : std::ceil(y0 + center[1]) - 1; + for (size_t iy = iy0, iy_dst = iy0_dst; iy < iy0 + ht; ++iy, iy_dst += iy_dir) + { + for (size_t ix = ix0, ix_dst = ix0_dst; ix < ix0 + wt; ++ix, ix_dst += ix_dir) + { + dst(iy_dst, ix_dst) = src(ix, iy); // (fs/y, ss/x) + } + } + } +} + /** * LPD-1M geometry * @@ -329,7 +503,10 @@ const LPD_1MGeometry::shapeType LPD_1MGeometry::module_shape {256, 256}; const LPD_1MGeometry::shapeType LPD_1MGeometry::tile_shape {32, 128}; constexpr size_t LPD_1MGeometry::n_tiles_per_module; const LPD_1MGeometry::quadOrientType LPD_1MGeometry::quad_orientations { - std::array{1, 1}, std::array{1, 1}, std::array{1, 1}, std::array{1, 1} + std::array{1, 1}, + std::array{1, 1}, + std::array{1, 1}, + std::array{1, 1} }; LPD_1MGeometry::LPD_1MGeometry() @@ -359,7 +536,7 @@ LPD_1MGeometry::LPD_1MGeometry() auto wt = static_cast(tile_shape[1]); for (size_t im = 0; im < n_modules; ++im) { - for (size_t it = 0; it < 16; ++it) + for (size_t it = 0; it < n_tiles_per_module; ++it) { positions(im, it, 0) = m_pos(im, 0) - (1 - it / 8) * wt; positions(im, it, 1) = m_pos(im, 1) - (it < 8 ? (it % 8) * ht : (7 - it % 8) * ht); @@ -498,7 +675,10 @@ const DSSC_1MGeometry::shapeType DSSC_1MGeometry::module_shape {128, 512}; const DSSC_1MGeometry::shapeType DSSC_1MGeometry::tile_shape {128, 256}; constexpr size_t DSSC_1MGeometry::n_tiles_per_module; const DSSC_1MGeometry::quadOrientType DSSC_1MGeometry::quad_orientations { - std::array{-1, 1}, std::array{-1, 1}, std::array{1, -1}, std::array{1, -1} + std::array{-1, 1}, + std::array{-1, 1}, + std::array{1, -1}, + std::array{1, -1} }; DSSC_1MGeometry::DSSC_1MGeometry() diff --git a/test/test_geometry.cpp b/test/test_geometry.cpp index 2ba43dd29..309d3a4ea 100644 --- a/test/test_geometry.cpp +++ b/test/test_geometry.cpp @@ -39,7 +39,7 @@ class Test1MGeometry : public ::testing::Test -using Geometry1MTypes = ::testing::Types; +using Geometry1MTypes = ::testing::Types; TYPED_TEST_CASE(Test1MGeometry, Geometry1MTypes);