Implement the Correlator special suite

.github/dependabot/constraints.txt

@@ -36,7 +36,9 @@ jupyterlab-pygments==0.2.2
 jupyterlab-widgets==1.1.0
 karabo-bridge==0.6.2
 kiwisolver==1.4.2
+libcst==0.4.1
 locket==1.0.0
+lttbc==0.2.1
 MarkupSafe==2.1.1
 matplotlib==3.5.2
 mistune==2.0.2

docs/special_suite.rst

@@ -148,6 +148,69 @@ instance.**
     checkbox is checked.
 
 
+.. _Correlator:
+
+Correlator
+""""""""""
+
+.. image:: images/special_suite_correlator.jpg
+   :width: 800
+
+The Correlator is a special suite that embeds `metropc
+<https://desy.de/~schmidtp/metropc-docs/>`_. The purpose is to allow the user to
+write custom analysis code on-the-fly and have it visualized immediately. The
+suite supports reading either raw data from a Karabo bridge, or processed data
+from a main EXtra-foam instance.
+
+The interface is organized by tabs, in the first tab there is a text editor
+where you can write the context code. All other tabs are for visualizing the
+outputs of metropc, called `views`.
+
+.. image:: images/special_suite_correlator_view_split.jpg
+   :width: 800
+
+If you select a view to visualize it will be displayed in the tab. By right
+clicking on the plot and selecting a split direction option from the `Split
+frame` menu, you can split the frame in any direction to display multiple plots
+on the same tab. It's also possible to undock a tab into a separate window by
+clicking on the undock button (little square icon next to the close tab
+button). If you then close the tab window it will redock to the main window.
+
+The usual workflow is:
+
+1. Set the source, EXtra-foam or Karabo bridge, and configure the hostname and
+   port appropriately.
+2. Modify the context file as desired (you can also load your own from a file).
+3. Click the `Reload context` button to apply your changes (don't forget to save
+   it too with the `Save context` button). If there were errors with the context
+   (e.g. a syntax error), they will be displayed in the log panel (bottom left
+   corner). Other errors with the context (e.g. runtime errors) will be
+   displayed in the terminal.
+4. Have a look at the views in a tab to see the results.
+
+Possible inputs to the views are displayed in the `Data paths` table on the
+left:
+
+.. image:: images/special_suite_correlator_data_paths.jpg
+   :width: 800
+
+As usual with metropc, you can access data from Karabo and the internal
+variables with the :code:`karabo#` and :code:`internal#` paths. To access data
+from EXtra-foam you'll need to use the :code:`foam#` path. For example, to get
+the drawn image mask from EXtra-foam (can be seen in the screenshot) you would
+write :code:`foam#image.image_mask` and get an array of booleans with shape
+:code:`(129, 1092)`. Note that this is more for advanced users as it requires
+some knowledge of EXtra-foams internals, contact the DA team if you want to do
+some analysis on top of what EXtra-foam already does (e.g. some special
+processing for a detector).
+
+To help with visualization beyond what metropc already supports, there are some
+extra tools in :code:`extra_foam.utils` that can be used:
+
+.. autofunction:: extra_foam.utils.rich_output
+
+.. autodata:: extra_foam.utils.Series
+
 Special purposed apps
 ---------------------
 

environment.yml

@@ -19,3 +19,6 @@ dependencies:
   - gtest==1.10.0
   - gmock==1.10.0
   - pip
+  - pyqt
+  - pyqt5-sip
+  - qscintilla2

extra_foam/algorithms/curve_fitting.py

@@ -8,20 +8,20 @@
 All rights reserved.
 """
 from abc import abstractmethod
-from enum import IntEnum
+from enum import Enum
 
 import numpy as np
 from scipy.optimize import curve_fit
 from scipy import special
 
 
-class FittingType(IntEnum):
-    UNDEFINED = 0
-    LINEAR = 1
-    CUBIC = 2
-    GAUSSIAN = 11
-    LORENTZIAN = 21
-    ERF = 31
+class FittingType(Enum):
+    UNDEFINED = "None"
+    LINEAR = "Linear"
+    CUBIC = "Cubic"
+    GAUSSIAN = "Gaussian"
+    LORENTZIAN = "Lorentzian"
+    ERF = "erf"
 
 
 class CurveFitting:
@@ -34,9 +34,21 @@ def fit(self, x, y, *, p0=None):
             popt, _ = curve_fit(self, x, y, p0=p0, check_finite=True)
             return popt
 
+        def parameters(self):
+            """
+            Return a list of parameters by name. Intended for display in GUIs etc.
+            """
+            raise NotImplementedError()
+
+        def guess_p0(self, x, y):
+            """
+            Return initial guesses for all required parameters.
+            """
+            raise NotImplementedError()
+
         @staticmethod
         def format(*args):
-            raise NotImplementedError
+            raise NotImplementedError()
 
     class Linear(_BaseCurveFitting):
         def __init__(self):
@@ -46,6 +58,15 @@ def __init__(self):
         def __call__(self, x, a, b):
             return a + b * x
 
+        def parameters(self):
+            return ["Offset", "Slope"]
+
+        def guess_p0(self, x, y):
+            offset = np.nanmin(y)
+            slope = (y[-1] - y[0]) / (x[-1] - x[0])
+
+            return offset, slope
+
         @staticmethod
         def format(a, b):
             """Override."""
@@ -72,6 +93,18 @@ def __init__(self):
         def __call__(self, x, a, b, c, d):
             return a * np.exp(-(x - b)**2 / (2 * c**2)) + d
 
+        def parameters(self):
+            return ["Amplitude", "Mean", "Std dev.", "Offset"]
+
+        def guess_p0(self, x, y):
+            max_idx = np.nanargmax(y)
+            mean = x[max_idx]
+            amplitude = max(1, y[max_idx])
+            stdev = len(y) / 4
+            offset = np.nanmin(y)
+
+            return [amplitude, mean, stdev, offset]
+
         @staticmethod
         def format(a, b, c, d):
             return f"y = a * exp(-(x - b)^2 / (2 * c^2)) + d, \n" \

extra_foam/algorithms/data_structures.py

@@ -93,6 +93,9 @@ def __len__(self):
         """Override."""
         return self._len
 
+    def capacity(self):
+        return self._max_len
+
     @abstractmethod
     def data(self):
         """Return all the data."""
@@ -125,7 +128,8 @@ class SimpleSequence(_AbstractSequence):
     def __init__(self, *, max_len=100000, dtype=np.float64):
         super().__init__(max_len=max_len)
 
-        self._x = np.zeros(self._OVER_CAPACITY * max_len, dtype=dtype)
+        self._dtype = dtype
+        self.resize(max_len)
 
     def __getitem__(self, index):
         """Override."""
@@ -151,6 +155,11 @@ def extend(self, items):
         for item in items:
             self.append(item)
 
+    def resize(self, max_len):
+        self._max_len = max_len
+        self._x = np.zeros(self._OVER_CAPACITY * max_len, dtype=self._dtype)
+        self.reset()
+
     def reset(self):
         """Override."""
         self._i0 = 0
@@ -325,6 +334,10 @@ def __init__(self, resolution, *,
 
         self._last = 0
 
+    @property
+    def resolution(self):
+        return self._resolution
+
     def __getitem__(self, index):
         """Override."""
         s = slice(self._i0, self._i0 + self._len)

extra_foam/gui/gui_helpers.py

@@ -206,16 +206,19 @@ def parse_slice_inv(text):
     raise ValueError(err_msg)
 
 
+def get_icon_path(filename):
+    root_dir = osp.dirname(osp.abspath(__file__))
+    return osp.join(root_dir, "icons", filename)
+
 def create_icon_button(filename, size, *, description=""):
     """Create a QPushButton with icon.
 
     :param str filename: name of the icon file.
     :param int size: size of the icon (button).
     :param str description: tool tip of the button.
     """
-    root_dir = osp.dirname(osp.abspath(__file__))
     btn = QPushButton()
-    icon = QIcon(osp.join(root_dir, "icons/" + filename))
+    icon = QIcon(get_icon_path(filename))
     btn.setIcon(icon)
     btn.setIconSize(QSize(size, size))
     btn.setFixedSize(btn.minimumSizeHint())
@@ -246,6 +249,6 @@ def center_window(window, resize=True):
         max_width = screen_size.width()
         max_height = screen_size.height()
 
-        window.resize(max_width * 0.8, max_height * 0.8)
+        window.resize(int(max_width * 0.8), int(max_height * 0.8))
 
     window.move(screen.geometry().center() - window.frameGeometry().center())

extra_foam/gui/plot_widgets/__init__.py

@@ -1,5 +1,6 @@
 from .image_items import RectROI, RingItem
 from .image_view_base import ImageViewF, TimedImageViewF
 from .image_views import ImageAnalysis, RoiImageView
+from .plot_items import LinearROI
 from .plot_widget_base import HistMixin, PlotWidgetF, TimedPlotWidgetF
 from .graphics_widgets import Crosshair