From 3fa90fc249aa4cae6d47a6f84c9afa3fbe836424 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 27 Mar 2023 18:06:23 +0200
Subject: [PATCH 01/44] Fix PosixPath on Windows

---
 hercules/constants.py  | 6 +++---
 hercules/simulation.py | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/hercules/constants.py b/hercules/constants.py
index 557105b..1b6b1e9 100644
--- a/hercules/constants.py
+++ b/hercules/constants.py
@@ -8,7 +8,7 @@
 
 __all__ = []
 
-from pathlib import Path, PosixPath
+from pathlib import Path, PurePosixPath
 
 from .configuration import Configuration
 
@@ -16,8 +16,8 @@
 HEXBUG_DIR = MODULE_DIR / 'hexbug'
 #container is running linux
 #-> make sure it's PosixPath when run from windows
-HEXBUG_DIR_CONTAINER = PosixPath('/') / 'tmp'
-OUTPUT_DIR_CONTAINER = PosixPath('/') / 'home' 
+HEXBUG_DIR_CONTAINER = PurePosixPath('/') / 'tmp'
+OUTPUT_DIR_CONTAINER = PurePosixPath('/') / 'home' 
 LOCUST_CONFIG_NAME_P2 = 'LocustPhase2Template.json'
 KASS_CONFIG_NAME_P2 = 'Project8Phase2_electrons.xml'
 LOCUST_CONFIG_NAME_P3 = 'LocustPhase3Template.json'
diff --git a/hercules/simulation.py b/hercules/simulation.py
index ca21352..3bd4994 100644
--- a/hercules/simulation.py
+++ b/hercules/simulation.py
@@ -8,7 +8,7 @@
 
 __all__ = ['KassLocustP3']
 
-from pathlib import Path, PosixPath
+from pathlib import Path, PurePosixPath
 import subprocess
 from abc import ABC, abstractmethod
 import concurrent.futures as cf
@@ -160,8 +160,8 @@ class AbstractKassLocustP3(ABC):
     """An abstract base class for all KassLocust simulations."""
         
     #configuration parameters
-    _p8_locust_dir = PosixPath(CONFIG.locust_path) / CONFIG.locust_version
-    _p8_compute_dir = PosixPath(CONFIG.p8compute_path) / CONFIG.p8compute_version
+    _p8_locust_dir = PurePosixPath(CONFIG.locust_path) / CONFIG.locust_version
+    _p8_compute_dir = PurePosixPath(CONFIG.p8compute_path) / CONFIG.p8compute_version
         
     def __init__(self, working_dir, use_locust=True, use_kass=False,
                         python_script=None, direct=True):
@@ -248,7 +248,7 @@ def factory(name, working_dir, use_locust=True, use_kass=False,
 class KassLocustP3Desktop(AbstractKassLocustP3):
     """A class for running KassLocust on a desktop."""
     
-    _working_dir_container = PosixPath('/') / 'workingdir'
+    _working_dir_container = PurePosixPath('/') / 'workingdir'
     _command_script_name = 'locustcommands.sh'
     _container = CONFIG.container
     _max_workers = int(CONFIG.desktop_parallel_jobs)

From 87bd8310c2b3246f2c641ca15529d2907319f0d8 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Thu, 30 Mar 2023 13:09:43 +0200
Subject: [PATCH 02/44] Update hexbug

---
 hercules/hexbug | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hercules/hexbug b/hercules/hexbug
index 324aa54..8cb8231 160000
--- a/hercules/hexbug
+++ b/hercules/hexbug
@@ -1 +1 @@
-Subproject commit 324aa540d65df531fb1ea45056bb6810f488e411
+Subproject commit 8cb8231a0ab220541827b88ca8196727f6783791

From 90aebfdad39a82b20bd0cd2a3c14ea84e95021a9 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Fri, 31 Mar 2023 20:51:54 +0200
Subject: [PATCH 03/44] Update hexbug

---
 hercules/hexbug | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hercules/hexbug b/hercules/hexbug
index 8cb8231..edc4ed8 160000
--- a/hercules/hexbug
+++ b/hercules/hexbug
@@ -1 +1 @@
-Subproject commit 8cb8231a0ab220541827b88ca8196727f6783791
+Subproject commit edc4ed8d05d43d03b9c0552636af90901d3d9b21

From 9bcbf99dcfa17a5ba5a9c1937ebcf7422b4a8a58 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Fri, 4 Aug 2023 15:07:42 +0200
Subject: [PATCH 04/44] Add n_cpus as parameter to the job submission

---
 hercules/simulation.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hercules/simulation.py b/hercules/simulation.py
index 3bd4994..8616367 100644
--- a/hercules/simulation.py
+++ b/hercules/simulation.py
@@ -430,6 +430,7 @@ def _submit_job(self, **kwargs):
         module = 'module load dSQ;'
         
         n_cpus = 2 if self._use_locust else 1
+        n_cpus = n_cpus if 'n_cpus' not in kwargs else kwargs['n_cpus']
         memory = CONFIG.job_memory if 'memory' not in kwargs else kwargs['memory']
         timelimit = CONFIG.job_timelimit if 'timelimit' not in kwargs else kwargs['timelimit']
         

From d1271db931b6edf08b9a6fcbe44880881199100d Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Fri, 4 Aug 2023 15:25:47 +0200
Subject: [PATCH 05/44] Modify config file generation

Kass and Locust config files are only generated now when the job also
runs them
---
 hercules/simconfig.py  | 18 +++++++++++++-----
 hercules/simulation.py | 17 +++++++++++++++--
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/hercules/simconfig.py b/hercules/simconfig.py
index a03fd04..2060157 100644
--- a/hercules/simconfig.py
+++ b/hercules/simconfig.py
@@ -1154,8 +1154,18 @@ def help(cls):
         print()
         print('Note that all keyword arguments are optional and take default values from the config files!')
         
-    def make_config_file(self, filename_locust, filename_kass):
-        """Create the final Kassiopeia and Locust config files.
+    def make_kass_config_file(self, filename_kass):
+        """Create the final Kassiopeia file.
+        
+        Parameters
+        ----------
+        filename_kass : str
+            the path to the output Kassiopeia config file
+        """
+        self._kass_config.make_config_file(filename_kass)
+
+    def make_locust_config_file(self, filename_locust, filename_kass):
+        """Create the final Locust config file.
         
         Parameters
         ----------
@@ -1164,7 +1174,5 @@ def make_config_file(self, filename_locust, filename_kass):
         filename_kass : str
             the path to the output Kassiopeia config file
         """
-        
         self._locust_config.set_xml(filename_kass)
-        self._locust_config.make_config_file(filename_locust)
-        self._kass_config.make_config_file(filename_kass)
+        self._locust_config.make_config_file(filename_locust)
\ No newline at end of file
diff --git a/hercules/simulation.py b/hercules/simulation.py
index 8616367..cb73032 100644
--- a/hercules/simulation.py
+++ b/hercules/simulation.py
@@ -298,8 +298,13 @@ def _submit(self, sim_config: SimConfig):
         kass_file = output_dir / KASS_CONFIG_NAME
         config_dump = output_dir / SIM_CONFIG_NAME
 
-        sim_config.make_config_file(locust_file, kass_file)
         sim_config.to_json(config_dump)
+
+        if self._use_locust:
+            sim_config.make_locust_config_file(locust_file, kass_file)
+
+        if self._use_kass:
+            sim_config.make_kass_config_file(kass_file)
         
         if self._use_locust or self._use_kass:
             self._gen_command_script(output_dir)
@@ -460,8 +465,16 @@ def _add_job(self, sim_config):
         kass_file = output_dir / KASS_CONFIG_NAME
         config_dump = output_dir / SIM_CONFIG_NAME
 
-        sim_config.make_config_file(locust_file, kass_file)
         sim_config.to_json(config_dump)
+
+        if self._use_locust:
+            sim_config.make_locust_config_file(locust_file, kass_file)
+
+        if self._use_kass:
+            sim_config.make_kass_config_file(kass_file)
+        
+        if self._use_locust or self._use_kass:
+            self._gen_locust_script(output_dir)
         
         self._gen_locust_script(output_dir)
         cmd = self._assemble_command(output_dir)

From 93a9c7cedcb24edeadbdabbb8069e73cee6baf2b Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Fri, 4 Aug 2023 16:12:39 +0200
Subject: [PATCH 06/44] Add some improvements for pickling of the dataset

---
 hercules/dataset.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index d213b3d..a8d0138 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -121,11 +121,18 @@ def get_path(self, energy, pitch, r, phi, z, interpolation=True):
         return parameters, self.directory / sim_path
         
     def dump(self):
-        pickle.dump(self, open(self.directory/'index.he', "wb"), protocol=4)
+        with open(self.directory/'index.he', "wb") as f:
+            pickle.dump(self, f, protocol=4)
         
     @classmethod
     def load(cls, path):
         path_p = Path(path)
-        instance = pickle.load(open(path_p/'index.he', "rb"))
+
+        with open(path_p/'index.he', "rb") as f:
+            instance = pickle.load(f)
+
+        if type(instance) is not cls:
+            raise TypeError('Path does not point to a hercules dataset')
+
         instance.directory = path_p
         return instance

From 9ca76bc19a1298b7657c86c0611ed3cb52f803e8 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Fri, 4 Aug 2023 17:48:28 +0200
Subject: [PATCH 07/44] Add new class SimpleSimConfig

---
 hercules/simconfig.py  | 108 ++++++++++++++++++++++++++++++++++++++++-
 hercules/simulation.py |  15 +++++-
 2 files changed, 120 insertions(+), 3 deletions(-)

diff --git a/hercules/simconfig.py b/hercules/simconfig.py
index 2060157..ef81f48 100644
--- a/hercules/simconfig.py
+++ b/hercules/simconfig.py
@@ -1175,4 +1175,110 @@ def make_locust_config_file(self, filename_locust, filename_kass):
             the path to the output Kassiopeia config file
         """
         self._locust_config.set_xml(filename_kass)
-        self._locust_config.make_config_file(filename_locust)
\ No newline at end of file
+        self._locust_config.make_config_file(filename_locust)
+
+
+class SimpleSimConfig:
+    """A class for a more general simulation configuration
+    
+    This class is intended for the use with more general python scripts.
+    It supports an arbitrary number of keyword arguments. Arguments with the prefix 'meta_'
+    are treated as meta parameters. Meta parameters are expected to be the same for an entire dataset.
+    All other parameters are considered regular configuration parameters that should vary over a dataset.
+    
+    Attributes
+    ----------
+    sim_name : str
+        Name of the simulation
+    """
+    
+    def __init__(self, sim_name, **kwargs):
+        """
+        Parameters
+        ----------
+        sim_name : str
+            Name of the simulation
+        **kwargs :
+            Arbitrary number of keyword arguments.
+        
+        """
+        
+        self._sim_name = sim_name
+        self._extract_kwargs(kwargs)
+
+    def _extract_kwargs(self, kwargs):
+
+        self._meta_data = {}
+        self._regular_data = {}
+        for e in kwargs:
+            if e.startswith('meta_'):
+                new_key = e.removeprefix('meta_')
+                self._meta_data[new_key] = kwargs[e]
+            else:
+                self._regular_data[e] = kwargs[e]
+    
+    @property
+    def sim_name(self):
+        return self._sim_name
+    
+    def to_json(self, file_name):
+        """Write a json file with the entire simulation configuration."""
+        
+        with open(file_name, 'w') as outfile:
+            json.dump({ 'sim-name': self._sim_name,
+                        'meta-data': self._meta_data, 
+                        'config-data': self._regular_data}, outfile, 
+                        indent=2)#, default=lambda x: x.config_dict)
+ 
+                            
+    def to_dict(self):
+        """Return a dictionary with the entire simulation configuration.
+        
+        Returns
+        -------
+        dict
+            Nested dictionary with the simulation configuration
+        """
+        
+        return {'sim-name': self._sim_name,
+                'meta-data': self._meta_data, 
+                'config-data': self._regular_data}
+            
+    @classmethod
+    def from_json(cls, file_name):
+        """Return a SimpleSimConfig from a json file.
+        
+        Creates a new instance of a SimpleSimConfig from the contents of a json file.
+        This should only be used with a json file that was created by the 
+        `to_json` method. No checks applied for the validity of the json file.
+        
+        Returns
+        -------
+        SimpleSimConfig
+            The new SimpleSimConfig instance
+        """
+        
+        with open(file_name, 'r') as infile:
+            config = json.load(infile)
+            
+            sim_name = config['sim-name']
+            
+            instance = cls(sim_name)
+            
+            instance._meta_data = config['meta-data']
+            instance._regular_data = config['config-data']
+            
+        return instance
+    
+    @classmethod
+    def help(cls):
+        """Print documentation about the SimConfig.
+        
+        Prints the docstrings of the class and the __init__ method as well as
+        additional information about the accepted parameters in the keyword
+        arguments. The latter is provided via the two wrapped configurations.
+        
+        """
+        print(cls.__doc__)
+        print(cls.__init__.__doc__)
+        
\ No newline at end of file
diff --git a/hercules/simulation.py b/hercules/simulation.py
index cb73032..1deda47 100644
--- a/hercules/simulation.py
+++ b/hercules/simulation.py
@@ -16,7 +16,7 @@
 from math import sqrt, atan2
 import pickle
 
-from hercules.simconfig import SimConfig
+from hercules.simconfig import SimConfig, SimpleSimConfig
 from .dataset import Dataset
 from .constants import (HEXBUG_DIR, HEXBUG_DIR_CONTAINER, OUTPUT_DIR_CONTAINER,
                         LOCUST_CONFIG_NAME, KASS_CONFIG_NAME, SIM_CONFIG_NAME, 
@@ -286,7 +286,7 @@ def run(self, sim_config_list, **kwargs):
             for future in tqdm(cf.as_completed(futures), total=len(futures)):
                 future.result()
     
-    def _submit(self, sim_config: SimConfig):
+    def _submit(self, sim_config):
         #Submit the job with the given SimConfig
         #Creates all the necessary configuration files, directories and the
         #json output
@@ -573,6 +573,9 @@ def __init__(self, working_dir, use_locust=True, use_kass=False,
         working_dir : str
             The string for the path to the working directory
         """
+
+        self._use_locust = use_locust
+        self._use_kass = use_kass
         
         self._kass_locust = AbstractKassLocustP3.factory(CONFIG.env, 
                                                           working_dir,
@@ -590,5 +593,13 @@ def __call__(self, config_list, **kwargs):
         """
         if type(config_list) is not list:
             config_list = [config_list]
+        
+        for config in config_list:
+            if type(config) is not type(config_list[0]):
+                raise TypeError('All configurations in the configuration list have to be of the same type!')
+            
+        if type(config_list[0]) is SimpleSimConfig and (self._use_kass or self._use_locust):
+            raise TypeError('SimpleSimConfig is not compatible with the use of Locust or Kassiopeia!')
+
         return self._kass_locust(config_list, **kwargs)
     

From be42f35dd15be9f35abe0c2348d876440cd44082 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Fri, 4 Aug 2023 17:50:22 +0200
Subject: [PATCH 08/44] Add SimpleSimConfig to imports

---
 hercules/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hercules/__init__.py b/hercules/__init__.py
index 542d164..49c8a22 100644
--- a/hercules/__init__.py
+++ b/hercules/__init__.py
@@ -7,6 +7,6 @@
 """
 
 from .simulation import KassLocustP3
-from .simconfig import SimConfig
+from .simconfig import SimConfig, SimpleSimConfig
 from .eggreader import LocustP3File
 from .dataset import Dataset

From 88c3b6b050826cb9a3ddc5318044508d0f022419 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Fri, 4 Aug 2023 18:27:57 +0200
Subject: [PATCH 09/44] Add getters for metadata and configdata of SimConfigs

---
 hercules/simconfig.py | 73 ++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 66 insertions(+), 7 deletions(-)

diff --git a/hercules/simconfig.py b/hercules/simconfig.py
index ef81f48..a5d558a 100644
--- a/hercules/simconfig.py
+++ b/hercules/simconfig.py
@@ -11,9 +11,8 @@
 import time
 import json
 import re
-from pathlib import Path, PosixPath
-from abc import ABC, abstractmethod
 from copy import deepcopy
+from math import sqrt, atan2
 
 from .constants import (HEXBUG_DIR, HEXBUG_DIR_CONTAINER, OUTPUT_DIR_CONTAINER,
                         LOCUST_CONFIG_NAME_P2, KASS_CONFIG_NAME_P2,
@@ -1177,6 +1176,60 @@ def make_locust_config_file(self, filename_locust, filename_kass):
         self._locust_config.set_xml(filename_kass)
         self._locust_config.make_config_file(filename_locust)
 
+    def get_meta_data(self):
+        
+        #maybe incomplete
+        #add more when you realize you need more metadata
+        meta_data = {'trap': self._kass_config['geometry'],
+                     'transfer-function': self._locust_config[LocustConfig._array_signal_key][LocustConfig._tf_receiver_filename_key],
+                     'n-channels': self._locust_config[LocustConfig._sim_key][LocustConfig._n_channels_key],
+                     'acquisition-rate': self._locust_config[LocustConfig._sim_key][LocustConfig._acq_rate_key],
+                     'lo-frequency': self._locust_config[LocustConfig._array_signal_key][LocustConfig._lo_frequency_key],
+                     }
+        
+        return meta_data
+    
+    def get_config_data(self):
+        
+        config_data = {}
+
+        x_min = self._kass_config._config_dict['x_min']
+        y_min = self._kass_config._config_dict['y_min']
+        z_min = self._kass_config._config_dict['z_min']
+        pitch_min = self._kass_config._config_dict['theta_min']
+
+        x_max = self._kass_config._config_dict['x_max']
+        y_max = self._kass_config._config_dict['y_max']
+        z_max = self._kass_config._config_dict['z_max']
+        pitch_max = self._kass_config._config_dict['theta_max']
+
+        energy = self._kass_config._config_dict['energy']
+        
+        r_min = sqrt(x_min**2 + y_min**2)
+        phi_min = atan2(y_min, x_min)
+
+        r_max = sqrt(x_max**2 + y_max**2)
+        phi_max = atan2(y_max, x_max)
+
+        if x_min==x_max and y_min==y_max and z_min==z_max and pitch_min==pitch_max:
+            config_data['r'] = r_min
+            config_data['phi'] = phi_min
+            config_data['z'] = z_min
+            config_data['pitch'] = pitch_min
+        else:
+            config_data['r_min'] = r_min
+            config_data['phi_min'] = phi_min
+            config_data['z_min'] = z_min
+            config_data['pitch_min'] = pitch_min
+            config_data['r_max'] = r_max
+            config_data['phi_max'] = phi_max
+            config_data['z_max'] = z_max
+            config_data['pitch_max'] = pitch_max
+
+        config_data['energy'] = energy
+
+        return config_data
+
 
 class SimpleSimConfig:
     """A class for a more general simulation configuration
@@ -1209,13 +1262,13 @@ def __init__(self, sim_name, **kwargs):
     def _extract_kwargs(self, kwargs):
 
         self._meta_data = {}
-        self._regular_data = {}
+        self._config_data = {}
         for e in kwargs:
             if e.startswith('meta_'):
                 new_key = e.removeprefix('meta_')
                 self._meta_data[new_key] = kwargs[e]
             else:
-                self._regular_data[e] = kwargs[e]
+                self._config_data[e] = kwargs[e]
     
     @property
     def sim_name(self):
@@ -1227,7 +1280,7 @@ def to_json(self, file_name):
         with open(file_name, 'w') as outfile:
             json.dump({ 'sim-name': self._sim_name,
                         'meta-data': self._meta_data, 
-                        'config-data': self._regular_data}, outfile, 
+                        'config-data': self._config_data}, outfile, 
                         indent=2)#, default=lambda x: x.config_dict)
  
                             
@@ -1242,7 +1295,7 @@ def to_dict(self):
         
         return {'sim-name': self._sim_name,
                 'meta-data': self._meta_data, 
-                'config-data': self._regular_data}
+                'config-data': self._config_data}
             
     @classmethod
     def from_json(cls, file_name):
@@ -1266,7 +1319,7 @@ def from_json(cls, file_name):
             instance = cls(sim_name)
             
             instance._meta_data = config['meta-data']
-            instance._regular_data = config['config-data']
+            instance._config_data = config['config-data']
             
         return instance
     
@@ -1281,4 +1334,10 @@ def help(cls):
         """
         print(cls.__doc__)
         print(cls.__init__.__doc__)
+
+    def get_meta_data(self):
+        return self._meta_data
+    
+    def get_config_data(self):
+        return self._config_data
         
\ No newline at end of file

From d623db4b69fea25e9e22db390751a745946dcdf1 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Fri, 4 Aug 2023 21:03:10 +0200
Subject: [PATCH 10/44] Fix version conflict of pickled dataset classes

---
 hercules/dataset.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index a8d0138..2374a32 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -26,11 +26,12 @@ def __call__(self, x):
 
 class Dataset:
     
-    __version = '1.0'
+    _class_version = '2.0'
     
     def __init__(self, directory):
         
         self.directory = Path(directory)
+        self._version = self._class_version
         
     def make_index(self, config_list):
         
@@ -133,6 +134,14 @@ def load(cls, path):
 
         if type(instance) is not cls:
             raise TypeError('Path does not point to a hercules dataset')
+        
+        if '_version' not in dir(instance):
+            instance_version = '1.0'
+        else:
+            instance_version = instance._version
+        
+        if instance_version != cls._class_version:
+            raise RuntimeError(f'Tried to load a version {instance_version} hercules dataset with version {cls._class_version}! To open this file you need an older hercules release')
 
         instance.directory = path_p
-        return instance
+        return instance
\ No newline at end of file

From a3720198b180e898f3adb34df3f9f75e953f8122 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Fri, 4 Aug 2023 22:13:14 +0200
Subject: [PATCH 11/44] Add safety checks for config_list

---
 hercules/simulation.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/hercules/simulation.py b/hercules/simulation.py
index 1deda47..5dfc3da 100644
--- a/hercules/simulation.py
+++ b/hercules/simulation.py
@@ -598,6 +598,12 @@ def __call__(self, config_list, **kwargs):
             if type(config) is not type(config_list[0]):
                 raise TypeError('All configurations in the configuration list have to be of the same type!')
             
+            if config.get_meta_data() != config_list[0].get_meta_data():
+                raise RuntimeError('All configurations in the configuration list need the same metadata')
+            
+            if config.get_config_data().keys() != config_list[0].get_config_data().keys():
+                raise RuntimeError('All configurations in the configuration list need the same configuration data keys')
+            
         if type(config_list[0]) is SimpleSimConfig and (self._use_kass or self._use_locust):
             raise TypeError('SimpleSimConfig is not compatible with the use of Locust or Kassiopeia!')
 

From ebb78ac0d9d675619c1c8b6d7713ed5a2abbc97d Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Fri, 4 Aug 2023 23:07:51 +0200
Subject: [PATCH 12/44] Add ConfigList class as safety layer

---
 hercules/simconfig.py  | 67 +++++++++++++++++++++++++++++++++++++-----
 hercules/simulation.py | 20 ++++---------
 2 files changed, 65 insertions(+), 22 deletions(-)

diff --git a/hercules/simconfig.py b/hercules/simconfig.py
index a5d558a..21db815 100644
--- a/hercules/simconfig.py
+++ b/hercules/simconfig.py
@@ -1037,6 +1037,7 @@ def __init__(self, sim_name, phase = 'Phase3', kass_file_name = None,
         
         self._sim_name = sim_name
         self._phase = phase
+        self._extra_meta_data = {}
         
         self._locust_config = LocustConfig(phase = phase, 
                                            locust_file_name = locust_file_name,
@@ -1187,8 +1188,13 @@ def get_meta_data(self):
                      'lo-frequency': self._locust_config[LocustConfig._array_signal_key][LocustConfig._lo_frequency_key],
                      }
         
+        meta_data.update(self._extra_meta_data)
+        
         return meta_data
     
+    def add_meta_data(self, meta_data):
+        self._extra_meta_data = meta_data
+    
     def get_config_data(self):
         
         config_data = {}
@@ -1263,12 +1269,13 @@ def _extract_kwargs(self, kwargs):
 
         self._meta_data = {}
         self._config_data = {}
-        for e in kwargs:
-            if e.startswith('meta_'):
-                new_key = e.removeprefix('meta_')
-                self._meta_data[new_key] = kwargs[e]
-            else:
-                self._config_data[e] = kwargs[e]
+        #for e in kwargs:
+        #    if e.startswith('meta_'):
+        #        new_key = e.removeprefix('meta_')
+        #        self._meta_data[new_key] = kwargs[e]
+        #    else:
+        #        self._config_data[e] = kwargs[e]
+        self._config_data = kwargs
     
     @property
     def sim_name(self):
@@ -1340,4 +1347,50 @@ def get_meta_data(self):
     
     def get_config_data(self):
         return self._config_data
-        
\ No newline at end of file
+    
+    def add_meta_data(self, meta_data):
+        self._meta_data = meta_data
+
+
+class ConfigList:
+
+    def __init__(self, **kwargs):
+        self._config_list = []
+        self._meta_data = kwargs
+        self._extra_meta_data = None
+        self._config_list_type = None
+        self._config_data_keys = None
+
+    def add_config(self, config):
+
+        if len(self._config_list) == 0:
+
+            common_keys = set(self._meta_data.keys()).intersection(config.get_meta_data().keys())
+
+            if len(common_keys)>0:
+                print('Warning, adding a config with metadata that overwrites an existing metadata entry! This might not be what you want.')
+
+            self._extra_meta_data = config.get_meta_data()
+            self._meta_data.update(config.get_meta_data())
+            self._config_list_type = type(config)
+            self._config_data_keys = config.get_config_data().keys()
+
+        if type(config) is not self._config_list_type:
+            raise TypeError('All configurations in the configuration list have to be of the same type!')
+        
+        if config.get_meta_data() != self._extra_meta_data:
+            raise RuntimeError('All configurations in the configuration list need the same metadata')
+        
+        if config.get_config_data().keys() != self._config_data_keys:
+            raise RuntimeError('All configurations in the configuration list need the same configuration data keys')
+
+        config.add_meta_data(self._meta_data)
+        self._config_list.append(config)
+
+    def get_config_list(self):
+        return self._config_list
+    
+    def get_list_type(self):
+        return self._config_list_type
+
+
diff --git a/hercules/simulation.py b/hercules/simulation.py
index 5dfc3da..14b1b4a 100644
--- a/hercules/simulation.py
+++ b/hercules/simulation.py
@@ -16,7 +16,7 @@
 from math import sqrt, atan2
 import pickle
 
-from hercules.simconfig import SimConfig, SimpleSimConfig
+from hercules.simconfig import ConfigList, SimpleSimConfig
 from .dataset import Dataset
 from .constants import (HEXBUG_DIR, HEXBUG_DIR_CONTAINER, OUTPUT_DIR_CONTAINER,
                         LOCUST_CONFIG_NAME, KASS_CONFIG_NAME, SIM_CONFIG_NAME, 
@@ -591,21 +591,11 @@ def __call__(self, config_list, **kwargs):
         config_list : list or SimConfig
             Either a single SimConfig object or a list
         """
-        if type(config_list) is not list:
-            config_list = [config_list]
-        
-        for config in config_list:
-            if type(config) is not type(config_list[0]):
-                raise TypeError('All configurations in the configuration list have to be of the same type!')
-            
-            if config.get_meta_data() != config_list[0].get_meta_data():
-                raise RuntimeError('All configurations in the configuration list need the same metadata')
-            
-            if config.get_config_data().keys() != config_list[0].get_config_data().keys():
-                raise RuntimeError('All configurations in the configuration list need the same configuration data keys')
+        if type(config_list) is not ConfigList:
+            raise TypeError('Needs an instance of ConfigList')
             
-        if type(config_list[0]) is SimpleSimConfig and (self._use_kass or self._use_locust):
+        if config_list.get_list_type() is SimpleSimConfig and (self._use_kass or self._use_locust):
             raise TypeError('SimpleSimConfig is not compatible with the use of Locust or Kassiopeia!')
 
-        return self._kass_locust(config_list, **kwargs)
+        return self._kass_locust(config_list.get_config_list(), **kwargs)
     

From 0b9652594b9e1ab52dd861f0d8eff716248ab0f9 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 00:28:37 +0200
Subject: [PATCH 13/44] Small modification for ConfigList usage

ConfigList object now is passed through inside the AbstractKassLocustP3
class instead of being unpacked for its constructor. This way it can get
passed through to make_index. Unpacking now happens for run method.
---
 hercules/simconfig.py  | 2 +-
 hercules/simulation.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/hercules/simconfig.py b/hercules/simconfig.py
index 21db815..ff4fdf9 100644
--- a/hercules/simconfig.py
+++ b/hercules/simconfig.py
@@ -1387,7 +1387,7 @@ def add_config(self, config):
         config.add_meta_data(self._meta_data)
         self._config_list.append(config)
 
-    def get_config_list(self):
+    def get_internal_list(self):
         return self._config_list
     
     def get_list_type(self):
diff --git a/hercules/simulation.py b/hercules/simulation.py
index 14b1b4a..a3ddf1b 100644
--- a/hercules/simulation.py
+++ b/hercules/simulation.py
@@ -184,12 +184,12 @@ def __call__(self, config_list, **kwargs):
         
         Parameters
         ----------
-        config_list : list
-            A list of SimConfig objects
+        config_list : ConfigList
+            A ConfigList object
         """
         
         self.make_index(config_list)
-        self.run(config_list, **kwargs)
+        self.run(config_list.get_internal_list(), **kwargs)
         
     def make_index(self, config_list):
         
@@ -597,5 +597,5 @@ def __call__(self, config_list, **kwargs):
         if config_list.get_list_type() is SimpleSimConfig and (self._use_kass or self._use_locust):
             raise TypeError('SimpleSimConfig is not compatible with the use of Locust or Kassiopeia!')
 
-        return self._kass_locust(config_list.get_config_list(), **kwargs)
+        return self._kass_locust(config_list, **kwargs)
     

From c36ae7b22c887439c2704c13ab3242b682abacad Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 00:41:28 +0200
Subject: [PATCH 14/44] Modify naming of the simulation config objects

Now naming is not done by the user anymore. The ConfigList incrementally
names the individual configs now.
---
 hercules/simconfig.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/hercules/simconfig.py b/hercules/simconfig.py
index ff4fdf9..bc6aece 100644
--- a/hercules/simconfig.py
+++ b/hercules/simconfig.py
@@ -982,7 +982,7 @@ class SimConfig:
         Name of the simulation
     """
     
-    def __init__(self, sim_name, phase = 'Phase3', kass_file_name = None, 
+    def __init__(self, phase = 'Phase3', kass_file_name = None, 
                     kass_unknown_args_translation = {},
                     locust_file_name = None,
                     locust_unknown_args_translation = {}, **kwargs):
@@ -1035,7 +1035,7 @@ def __init__(self, sim_name, phase = 'Phase3', kass_file_name = None,
             If phase is not 'Phase2' or 'Phase3'.
         """
         
-        self._sim_name = sim_name
+        self._sim_name = None
         self._phase = phase
         self._extra_meta_data = {}
         
@@ -1084,6 +1084,10 @@ def _trigger_unknown_parameter_warnings(self, kwargs):
     def sim_name(self):
         return self._sim_name
     
+    @sim_name.setter
+    def sim_name(self, sim_name):
+        self._sim_name = sim_name
+    
     def to_json(self, file_name):
         """Write a json file with the entire simulation configuration."""
         
@@ -1251,7 +1255,7 @@ class SimpleSimConfig:
         Name of the simulation
     """
     
-    def __init__(self, sim_name, **kwargs):
+    def __init__(self, **kwargs):
         """
         Parameters
         ----------
@@ -1262,7 +1266,7 @@ def __init__(self, sim_name, **kwargs):
         
         """
         
-        self._sim_name = sim_name
+        self._sim_name = None
         self._extract_kwargs(kwargs)
 
     def _extract_kwargs(self, kwargs):
@@ -1281,6 +1285,10 @@ def _extract_kwargs(self, kwargs):
     def sim_name(self):
         return self._sim_name
     
+    @sim_name.setter
+    def sim_name(self, sim_name):
+        self._sim_name = sim_name
+    
     def to_json(self, file_name):
         """Write a json file with the entire simulation configuration."""
         
@@ -1363,7 +1371,9 @@ def __init__(self, **kwargs):
 
     def add_config(self, config):
 
-        if len(self._config_list) == 0:
+        n = len(self._config_list)
+
+        if n == 0:
 
             common_keys = set(self._meta_data.keys()).intersection(config.get_meta_data().keys())
 
@@ -1385,6 +1395,7 @@ def add_config(self, config):
             raise RuntimeError('All configurations in the configuration list need the same configuration data keys')
 
         config.add_meta_data(self._meta_data)
+        config.sim_name = f'run{n}'
         self._config_list.append(config)
 
     def get_internal_list(self):
@@ -1393,4 +1404,6 @@ def get_internal_list(self):
     def get_list_type(self):
         return self._config_list_type
 
+    def get_meta_data(self):
+        return self._meta_data
 

From f0e14487b63aef9faf9770f327f26f6f29482f1a Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 02:51:18 +0200
Subject: [PATCH 15/44] Modify the Dataset class

Significant modifications to make the dataset more generic
---
 hercules/dataset.py   | 101 +++++++++++++++++++++---------------------
 hercules/simconfig.py |   3 ++
 2 files changed, 53 insertions(+), 51 deletions(-)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index 2374a32..a365e34 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -34,52 +34,45 @@ def __init__(self, directory):
         self._version = self._class_version
         
     def make_index(self, config_list):
+        """Create the index dictionary.
+        
+        Parameters
+        ----------
+        config_list : ConfigList
+            A ConfigList object
+        """
         
         print('Making file index')
         
-        self.index = {}
-        r_np = np.empty(len(config_list))
-        phi_np = np.empty(len(config_list))
-        z_np = np.empty(len(config_list))
-        pitch_np = np.empty(len(config_list))
-        energy_np = np.empty(len(config_list))
+        self._index = {}
+        self._meta_data = config_list.get_meta_data()
+        self._config_data_keys = config_list.get_config_data_keys()
+        config_list_internal = config_list.get_internal_list()
+
+        self._axes_dict = {k: np.empty(len(config_list_internal)) for k in self._config_data_keys}
         
-        for i, sim_config in enumerate(config_list):
+        for i, sim_config in enumerate(config_list_internal):
             path = sim_config.sim_name
-            x = sim_config._kass_config._config_dict['x_min']
-            y = sim_config._kass_config._config_dict['y_min']
-            z = sim_config._kass_config._config_dict['z_min']
-            pitch = sim_config._kass_config._config_dict['theta_min']
-            energy = sim_config._kass_config._config_dict['energy']
-            
-            r = sqrt(x**2 + y**2)
-            phi = atan2(y, x)
-            
-            self.index[energy, pitch, r, phi, z] = path
-            
-            r_np[i] = r
-            phi_np[i] = phi
-            z_np[i] = z
-            pitch_np[i] = pitch
-            energy_np[i] = energy
+            config_data = sim_config.get_config_data()
+
+            for k in config_data:
+                self._axes_dict[k][i] = config_data[k]
             
-        self.r = np.sort(np.unique(r_np))
-        self.phi = np.sort(np.unique(phi_np))
-        self.z = np.sort(np.unique(z_np))
-        self.pitch = np.sort(np.unique(pitch_np))
-        self.energy = np.sort(np.unique(energy_np))
+            self._index[tuple(config_data.values())] = path
+
+        for k in self._axes_dict:
+            self._axes_dict[k] = np.sort(np.unique(self._axes_dict[k]))
         
         self.interpolate_all()
         
     def interpolate_all(self):
         
         print('Making interpolation')
-        
-        self.r_int = self.interpolate(self.r)
-        self.phi_int = self.interpolate(self.phi)
-        self.z_int = self.interpolate(self.z)
-        self.pitch_int = self.interpolate(self.pitch)
-        self.energy_int = self.interpolate(self.energy)      
+
+        self._axes_dict_int = {}
+
+        for k in self._axes_dict:
+            self._axes_dict_int[k] = self.interpolate(self._axes_dict[k])    
         
     def interpolate(self, x):
         
@@ -90,9 +83,9 @@ def interpolate(self, x):
             
         return x_int
         
-    def get_data(self, energy, pitch, r, phi, z, interpolation=True):
+    def get_data(self, params, interpolation=True):
         
-        parameters, sim_path = self.get_path(energy, pitch, r, phi, z, interpolation=interpolation)
+        parameters, sim_path = self.get_path(params, interpolation=interpolation)
         
         path = sim_path.relative_to(self.directory)
         
@@ -101,29 +94,35 @@ def get_data(self, energy, pitch, r, phi, z, interpolation=True):
     def load_sim(self, path):
         return np.load(self.directory / path / PY_DATA_NAME)
         
-    def get_path(self, energy, pitch, r, phi, z, interpolation=True):
+    def get_path(self, params, interpolation=True):
         
         if interpolation:
-            energy_i = self.energy_int(energy).item()
-            pitch_i = self.pitch_int(pitch).item()
-            r_i = self.r_int(r).item()
-            phi_i = self.phi_int(phi).item()
-            z_i = self.z_int(z).item()
+            key = [self._axes_dict_int[i](params[i]).item() for i in range(len(params))]
         else:
-            energy_i = energy
-            pitch_i = pitch
-            r_i = r
-            phi_i = phi
-            z_i = z
-            
-        parameters = (energy_i, pitch_i, r_i, phi_i, z_i)
-        sim_path = self.index[parameters]
+            key = [self._axes_dict[i] for i in range(len(params))]
+
+        parameters = tuple(key)
+
+        sim_path = self._index[parameters]
         
         return parameters, self.directory / sim_path
         
     def dump(self):
         with open(self.directory/'index.he', "wb") as f:
             pickle.dump(self, f, protocol=4)
+
+        with open(self.directory/'info.txt', "w") as f:
+            f.write(f'Hercules dataset version {self._version}\n')
+            f.write('Metadata:\n')
+            f.write(str(self._meta_data))
+            f.write('\n\n')
+            f.write('Dataset has following configurations:\n')
+
+            for k in self._axes_dict:
+                n = len(self._axes_dict[k])
+                lower = self._axes_dict[k][0]
+                upper = self._axes_dict[k][-1]
+                f.write(f'{k}: {n} values in [{lower},{upper}] \n')
         
     @classmethod
     def load(cls, path):
@@ -133,7 +132,7 @@ def load(cls, path):
             instance = pickle.load(f)
 
         if type(instance) is not cls:
-            raise TypeError('Path does not point to a hercules dataset')
+            raise RuntimeError('Path does not point to a hercules dataset')
         
         if '_version' not in dir(instance):
             instance_version = '1.0'
diff --git a/hercules/simconfig.py b/hercules/simconfig.py
index bc6aece..ceef581 100644
--- a/hercules/simconfig.py
+++ b/hercules/simconfig.py
@@ -1406,4 +1406,7 @@ def get_list_type(self):
 
     def get_meta_data(self):
         return self._meta_data
+    
+    def get_config_data_keys(self):
+        return self._config_data_keys
 

From 53f919da010d47f332f39f9712044d887b3f4d2f Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 13:15:20 +0200
Subject: [PATCH 16/44] Modify get_path of dataset

Added another possible case of using it and more sanity checks
---
 hercules/dataset.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index a365e34..9a0021f 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -94,16 +94,26 @@ def get_data(self, params, interpolation=True):
     def load_sim(self, path):
         return np.load(self.directory / path / PY_DATA_NAME)
         
-    def get_path(self, params, interpolation=True):
-        
-        if interpolation:
+    def get_path(self, params, method='interpolated'):
+
+        if len(params) != len(self._axes_dict):
+            raise ValueError(f'params has len {len(params)} but dataset expects len {len(self._axes_dict)}!')
+
+        if method == 'interpolated':
             key = [self._axes_dict_int[i](params[i]).item() for i in range(len(params))]
+        elif method == 'index':
+            key = [self._axes_dict[i][params[i]] for i in range(len(params))]
+        elif method == 'exact':
+            key = params
         else:
-            key = [self._axes_dict[i] for i in range(len(params))]
+            raise ValueError("method can only take values 'interpolated', 'index' or 'exact'!")
 
         parameters = tuple(key)
 
-        sim_path = self._index[parameters]
+        sim_path = self._index.get(parameters) # self._index[parameters]
+
+        if sim_path is None:
+            raise KeyError(f'{parameters} is not part of the dataset!')
         
         return parameters, self.directory / sim_path
         

From 60d35050301ad8f0d18a59ed93a7297a2ff1616f Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 13:20:34 +0200
Subject: [PATCH 17/44] Add getters to Dataset

---
 hercules/dataset.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index 9a0021f..5772291 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -116,6 +116,18 @@ def get_path(self, params, method='interpolated'):
             raise KeyError(f'{parameters} is not part of the dataset!')
         
         return parameters, self.directory / sim_path
+    
+    @property
+    def config_data_keys(self):
+        return self._config_data_keys
+    
+    @property
+    def axes_dict(self):
+        return self._axes_dict
+    
+    @property
+    def shape(self):
+        return tuple(len(self._axes_dict[k] for k in self._axes_dict))
         
     def dump(self):
         with open(self.directory/'index.he', "wb") as f:

From bd03553c0768cfc7816aaa20d81ae270c75d74a1 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 13:22:46 +0200
Subject: [PATCH 18/44] Move make_index to constructor of Dataset

---
 hercules/dataset.py    | 7 ++++---
 hercules/simulation.py | 3 +--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index 5772291..a4da1e0 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -10,7 +10,6 @@
 from scipy.interpolate import interp1d
 import pickle
 from pathlib import Path
-from math import sqrt, atan2
 
 from .constants import PY_DATA_NAME
 
@@ -24,16 +23,18 @@ def __init__(self, x):
     def __call__(self, x):
         return self.x
 
+
 class Dataset:
     
     _class_version = '2.0'
     
-    def __init__(self, directory):
+    def __init__(self, directory, config_list):
         
         self.directory = Path(directory)
         self._version = self._class_version
+        self._make_index(config_list)
         
-    def make_index(self, config_list):
+    def _make_index(self, config_list):
         """Create the index dictionary.
         
         Parameters
diff --git a/hercules/simulation.py b/hercules/simulation.py
index a3ddf1b..d8865c1 100644
--- a/hercules/simulation.py
+++ b/hercules/simulation.py
@@ -193,8 +193,7 @@ def __call__(self, config_list, **kwargs):
         
     def make_index(self, config_list):
         
-        dataset = Dataset(self._working_dir)
-        dataset.make_index(config_list)
+        dataset = Dataset(self._working_dir, config_list)
         dataset.dump()
     
     @abstractmethod

From 9357a6315a193ba19e903fd3b138bb07ff6bd88f Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 14:07:55 +0200
Subject: [PATCH 19/44] Add iterator for Dataset

---
 hercules/dataset.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index a4da1e0..6c80f37 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -118,6 +118,34 @@ def get_path(self, params, method='interpolated'):
         
         return parameters, self.directory / sim_path
     
+    def __iter__(self):
+        self._it_index = tuple(0 for i in range(len(self.shape)))
+        self._it_stop = False
+        return self
+
+    def __next__(self):
+
+        if not self._it_stop:
+
+            new_index = list(self._it_index)
+
+            for i in reversed(range(len(self._it_index))):
+                new_index[i] += 1
+                if new_index[i] == self.shape[i]:
+                    new_index[i] = 0
+                else:
+                    break
+
+            old_index = self._it_index
+            self._it_index = tuple(new_index)
+
+            if self._it_index == tuple(0 for i in range(len(self.shape))):
+                self._it_stop = True
+
+            return self.get_path(old_index, method='index')
+        else:
+            raise StopIteration
+    
     @property
     def config_data_keys(self):
         return self._config_data_keys

From cfd3138c6d54dff95b2a4b55efe83ccdee6c17f6 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 14:13:54 +0200
Subject: [PATCH 20/44] Add ConfigList to hercules imports

---
 hercules/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hercules/__init__.py b/hercules/__init__.py
index 49c8a22..4d517a3 100644
--- a/hercules/__init__.py
+++ b/hercules/__init__.py
@@ -7,6 +7,6 @@
 """
 
 from .simulation import KassLocustP3
-from .simconfig import SimConfig, SimpleSimConfig
+from .simconfig import SimConfig, SimpleSimConfig, ConfigList
 from .eggreader import LocustP3File
 from .dataset import Dataset

From fb6698f7b39dc8d59e73ad3dd6cc9c0819e97ff8 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 14:31:19 +0200
Subject: [PATCH 21/44] Small fixes for dataset

---
 hercules/dataset.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index 6c80f37..a3c8a42 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -8,7 +8,7 @@
 
 import numpy as np
 from scipy.interpolate import interp1d
-import pickle
+import dill as pickle
 from pathlib import Path
 
 from .constants import PY_DATA_NAME
@@ -31,6 +31,7 @@ class Dataset:
     def __init__(self, directory, config_list):
         
         self.directory = Path(directory)
+        self.directory.mkdir(parents=True, exist_ok=True)
         self._version = self._class_version
         self._make_index(config_list)
         

From 4ce8745b3df5983fc128c3d53e3d459bbcd6fe93 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 14:31:44 +0200
Subject: [PATCH 22/44] Add getter for metadata of Dataset

---
 hercules/dataset.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index a3c8a42..63b3ede 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -158,6 +158,10 @@ def axes_dict(self):
     @property
     def shape(self):
         return tuple(len(self._axes_dict[k] for k in self._axes_dict))
+    
+    @property
+    def meta_data(self):
+        return self._meta_data
         
     def dump(self):
         with open(self.directory/'index.he', "wb") as f:

From 96901fe761807799d18a9869c3ef32cda7a6756b Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 14:35:46 +0200
Subject: [PATCH 23/44] Fix wrong parenthesis position

---
 hercules/dataset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index 63b3ede..82d809d 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -157,7 +157,7 @@ def axes_dict(self):
     
     @property
     def shape(self):
-        return tuple(len(self._axes_dict[k] for k in self._axes_dict))
+        return tuple(len(self._axes_dict[k]) for k in self._axes_dict)
     
     @property
     def meta_data(self):

From 1af775e89eaee6846f7ed092a180a2f98e1b9ab3 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 14:38:46 +0200
Subject: [PATCH 24/44] Fix private members of Dataset

---
 hercules/dataset.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index 82d809d..e4abe26 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -30,8 +30,8 @@ class Dataset:
     
     def __init__(self, directory, config_list):
         
-        self.directory = Path(directory)
-        self.directory.mkdir(parents=True, exist_ok=True)
+        self._directory = Path(directory)
+        self._directory.mkdir(parents=True, exist_ok=True)
         self._version = self._class_version
         self._make_index(config_list)
         
@@ -67,7 +67,7 @@ def _make_index(self, config_list):
         
         self.interpolate_all()
         
-    def interpolate_all(self):
+    def _interpolate_all(self):
         
         print('Making interpolation')
 
@@ -76,7 +76,7 @@ def interpolate_all(self):
         for k in self._axes_dict:
             self._axes_dict_int[k] = self.interpolate(self._axes_dict[k])    
         
-    def interpolate(self, x):
+    def _interpolate(self, x):
         
         if len(x)>1:
             x_int = interp1d(x, x, kind='nearest', bounds_error=None, fill_value='extrapolate')
@@ -89,12 +89,12 @@ def get_data(self, params, interpolation=True):
         
         parameters, sim_path = self.get_path(params, interpolation=interpolation)
         
-        path = sim_path.relative_to(self.directory)
+        path = sim_path.relative_to(self._directory)
         
         return parameters, self.load_sim(path)
         
     def load_sim(self, path):
-        return np.load(self.directory / path / PY_DATA_NAME)
+        return np.load(self._directory / path / PY_DATA_NAME)
         
     def get_path(self, params, method='interpolated'):
 
@@ -117,7 +117,7 @@ def get_path(self, params, method='interpolated'):
         if sim_path is None:
             raise KeyError(f'{parameters} is not part of the dataset!')
         
-        return parameters, self.directory / sim_path
+        return parameters, self._directory / sim_path
     
     def __iter__(self):
         self._it_index = tuple(0 for i in range(len(self.shape)))
@@ -164,10 +164,10 @@ def meta_data(self):
         return self._meta_data
         
     def dump(self):
-        with open(self.directory/'index.he', "wb") as f:
+        with open(self._directory/'index.he', "wb") as f:
             pickle.dump(self, f, protocol=4)
 
-        with open(self.directory/'info.txt', "w") as f:
+        with open(self._directory/'info.txt', "w") as f:
             f.write(f'Hercules dataset version {self._version}\n')
             f.write('Metadata:\n')
             f.write(str(self._meta_data))
@@ -198,5 +198,5 @@ def load(cls, path):
         if instance_version != cls._class_version:
             raise RuntimeError(f'Tried to load a version {instance_version} hercules dataset with version {cls._class_version}! To open this file you need an older hercules release')
 
-        instance.directory = path_p
+        instance._directory = path_p
         return instance
\ No newline at end of file

From 0555f43464d9281bf22b4b16791c56858ed39d9c Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 14:41:22 +0200
Subject: [PATCH 25/44] Fix wrong access of private members of Dataset

---
 hercules/dataset.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index e4abe26..259e25c 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -65,7 +65,7 @@ def _make_index(self, config_list):
         for k in self._axes_dict:
             self._axes_dict[k] = np.sort(np.unique(self._axes_dict[k]))
         
-        self.interpolate_all()
+        self._interpolate_all()
         
     def _interpolate_all(self):
         
@@ -74,7 +74,7 @@ def _interpolate_all(self):
         self._axes_dict_int = {}
 
         for k in self._axes_dict:
-            self._axes_dict_int[k] = self.interpolate(self._axes_dict[k])    
+            self._axes_dict_int[k] = self._interpolate(self._axes_dict[k])    
         
     def _interpolate(self, x):
         

From cf66f1d3b45fcaa991b91746bb2fe8410b4904f9 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 15:49:21 +0200
Subject: [PATCH 26/44] Fix dictionaries that should have been lists

---
 hercules/dataset.py | 42 ++++++++++++++++++++++--------------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index 259e25c..bb92d62 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -48,22 +48,23 @@ def _make_index(self, config_list):
         
         self._index = {}
         self._meta_data = config_list.get_meta_data()
-        self._config_data_keys = config_list.get_config_data_keys()
+        self._config_data_keys = list(config_list.get_config_data_keys()) # maps a list index to a key
         config_list_internal = config_list.get_internal_list()
 
-        self._axes_dict = {k: np.empty(len(config_list_internal)) for k in self._config_data_keys}
+        self._axes = [np.empty(len(config_list_internal)) for k in self._config_data_keys]
         
         for i, sim_config in enumerate(config_list_internal):
             path = sim_config.sim_name
             config_data = sim_config.get_config_data()
 
             for k in config_data:
-                self._axes_dict[k][i] = config_data[k]
+                k_ind = self._config_data_keys.index(k)
+                self._axes[k_ind][i] = config_data[k]
             
             self._index[tuple(config_data.values())] = path
 
-        for k in self._axes_dict:
-            self._axes_dict[k] = np.sort(np.unique(self._axes_dict[k]))
+        for i in range(len(self._axes)):
+            self._axes[i] = np.sort(np.unique(self._axes[i]))
         
         self._interpolate_all()
         
@@ -71,10 +72,10 @@ def _interpolate_all(self):
         
         print('Making interpolation')
 
-        self._axes_dict_int = {}
+        self._axes_int = []
 
-        for k in self._axes_dict:
-            self._axes_dict_int[k] = self._interpolate(self._axes_dict[k])    
+        for ax in self._axes:
+            self._axes_int.append(self._interpolate(ax))
         
     def _interpolate(self, x):
         
@@ -98,13 +99,13 @@ def load_sim(self, path):
         
     def get_path(self, params, method='interpolated'):
 
-        if len(params) != len(self._axes_dict):
-            raise ValueError(f'params has len {len(params)} but dataset expects len {len(self._axes_dict)}!')
+        if len(params) != len(self._axes):
+            raise ValueError(f'params has len {len(params)} but dataset expects len {len(self._axes)}!')
 
         if method == 'interpolated':
-            key = [self._axes_dict_int[i](params[i]).item() for i in range(len(params))]
+            key = [self._axes_int[i](params[i]).item() for i in range(len(params))]
         elif method == 'index':
-            key = [self._axes_dict[i][params[i]] for i in range(len(params))]
+            key = [self._axes[i][params[i]] for i in range(len(params))]
         elif method == 'exact':
             key = params
         else:
@@ -152,12 +153,12 @@ def config_data_keys(self):
         return self._config_data_keys
     
     @property
-    def axes_dict(self):
-        return self._axes_dict
+    def axes(self):
+        return self._axes
     
     @property
     def shape(self):
-        return tuple(len(self._axes_dict[k]) for k in self._axes_dict)
+        return tuple(len(ax) for ax in self._axes)
     
     @property
     def meta_data(self):
@@ -174,11 +175,12 @@ def dump(self):
             f.write('\n\n')
             f.write('Dataset has following configurations:\n')
 
-            for k in self._axes_dict:
-                n = len(self._axes_dict[k])
-                lower = self._axes_dict[k][0]
-                upper = self._axes_dict[k][-1]
-                f.write(f'{k}: {n} values in [{lower},{upper}] \n')
+            for i, ax in enumerate(self._axes):
+                n = len(ax)
+                lower = ax[0]
+                upper = ax[-1]
+                ax_name = self._config_data_keys[i]
+                f.write(f'{ax_name}: {n} values in [{lower},{upper}] \n')
         
     @classmethod
     def load(cls, path):

From 469e5f44531f9fcf53c45865b3dd209032e3e25a Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 17:14:27 +0200
Subject: [PATCH 27/44] Add unittest for Dataset

---
 test/test_dataset.py | 112 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 112 insertions(+)
 create mode 100644 test/test_dataset.py

diff --git a/test/test_dataset.py b/test/test_dataset.py
new file mode 100644
index 0000000..0372f3a
--- /dev/null
+++ b/test/test_dataset.py
@@ -0,0 +1,112 @@
+
+"""
+
+Author: F. Thomas
+Date: Aug 05, 2023
+
+"""
+
+from hercules import SimpleSimConfig, Dataset, ConfigList
+from pathlib import Path
+import unittest
+import numpy as np
+import shutil
+
+module_dir = Path(__file__).parent.absolute()
+test_dataset_name = 'test_directory'
+test_path = module_dir / test_dataset_name
+
+class DatasetTest(unittest.TestCase):
+
+    def setUp(self) -> None:
+
+        clist = ConfigList(sr=200., info='hello')
+
+        y = 3.
+        for x in range(10):
+            for z in range(5,7,1):
+                clist.add_config(SimpleSimConfig(x=x, y=y, z=z))
+
+        self.d = Dataset(test_path, clist)
+
+    def tearDown(self) -> None:
+        shutil.rmtree(test_path)
+
+    def test_axes(self) -> None:
+        expected_result = [np.array([0., 1., 2., 3., 4., 5., 6., 7., 8., 9.]), np.array([3.]), np.array([5., 6.])]
+        axes = self.d.axes
+        equals = min([np.array_equal(axes[i], expected_result[i]) for i in range(len(expected_result))])
+        self.assertTrue(equals)
+
+    def test_config_data_keys(self) -> None:
+        expected_result = ['x', 'y', 'z']
+        self.assertTrue(expected_result==self.d.config_data_keys)
+
+    def test_shape(self) -> None:
+        expected_result = (10, 1, 2)
+        self.assertTrue(expected_result==self.d.shape)
+
+    def test_meta_data(self) -> None:
+        expected_result = {'sr': 200.0, 'info': 'hello'}
+        self.assertTrue(expected_result==self.d.meta_data)
+
+    def test_get_path(self) -> None:
+
+        expected_result_1 = ((9.0, 3.0, 6.0), Path('test_directory/run19').absolute())
+        expected_result_2 = ((1.0, 3.0, 5.0), Path('test_directory/run2').absolute())
+        expected_result_3 = ((4.0, 3.0, 6.0), Path('test_directory/run9').absolute())
+
+        self.assertTrue(expected_result_1==self.d.get_path([100, 100, 100], method='interpolated'))
+        self.assertTrue(expected_result_2==self.d.get_path([1., 3., 5.], method='exact'))
+        self.assertTrue(expected_result_3==self.d.get_path([4, 0, 1], method='index'))
+
+        with self.assertRaises(ValueError) as cm:
+            self.d.get_path([4, 0, 1], method='inde')
+
+    def test_iterator(self) -> None:
+
+        expected_result = [((0.0, 3.0, 5.0), Path('test_directory/run0').absolute()),
+                            ((0.0, 3.0, 6.0), Path('test_directory/run1').absolute()),
+                            ((1.0, 3.0, 5.0), Path('test_directory/run2').absolute()),
+                            ((1.0, 3.0, 6.0), Path('test_directory/run3').absolute()),
+                            ((2.0, 3.0, 5.0), Path('test_directory/run4').absolute()),
+                            ((2.0, 3.0, 6.0), Path('test_directory/run5').absolute()),
+                            ((3.0, 3.0, 5.0), Path('test_directory/run6').absolute()),
+                            ((3.0, 3.0, 6.0), Path('test_directory/run7').absolute()),
+                            ((4.0, 3.0, 5.0), Path('test_directory/run8').absolute()),
+                            ((4.0, 3.0, 6.0), Path('test_directory/run9').absolute()),
+                            ((5.0, 3.0, 5.0), Path('test_directory/run10').absolute()),
+                            ((5.0, 3.0, 6.0), Path('test_directory/run11').absolute()),
+                            ((6.0, 3.0, 5.0), Path('test_directory/run12').absolute()),
+                            ((6.0, 3.0, 6.0), Path('test_directory/run13').absolute()),
+                            ((7.0, 3.0, 5.0), Path('test_directory/run14').absolute()),
+                            ((7.0, 3.0, 6.0), Path('test_directory/run15').absolute()),
+                            ((8.0, 3.0, 5.0), Path('test_directory/run16').absolute()),
+                            ((8.0, 3.0, 6.0), Path('test_directory/run17').absolute()),
+                            ((9.0, 3.0, 5.0), Path('test_directory/run18').absolute()),
+                            ((9.0, 3.0, 6.0), Path('test_directory/run19').absolute())]
+        
+        result = []
+
+        for entry in self.d:
+            result.append(entry)
+
+        self.assertTrue(result==expected_result)
+
+    def test_dump_load(self) -> None:
+
+        self.d.dump()
+        d = Dataset.load(test_path)
+
+        axes_self = self.d.axes
+        axes_load = d.axes
+        equals = min([np.array_equal(axes_self[i], axes_load[i]) for i in range(len(axes_self))])
+        self.assertTrue(equals)
+        self.assertTrue(self.d.meta_data == d.meta_data)
+        self.assertTrue(self.d.config_data_keys == d.config_data_keys)
+        self.assertTrue(self.d._directory == d._directory)
+        self.assertTrue(self.d._index == d._index)
+
+
+if __name__ == '__main__':
+    unittest.main()

From 772d0cb88e28412f9879e7101e3882510c5f444d Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 19:57:04 +0200
Subject: [PATCH 28/44] Fix sim_name usage in from_json of SimConfig classes

---
 hercules/simconfig.py | 35 +++++++++++++++++++++++------------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/hercules/simconfig.py b/hercules/simconfig.py
index ceef581..53597d6 100644
--- a/hercules/simconfig.py
+++ b/hercules/simconfig.py
@@ -1130,10 +1130,10 @@ def from_json(cls, file_name):
         with open(file_name, 'r') as infile:
             config = json.load(infile)
             
-            sim_name = config['sim-name']
             phase = config['phase']
             
-            instance = cls(sim_name, phase=phase)
+            instance = cls(phase=phase)
+            instance.sim_name = config['sim-name']
             
             instance._locust_config._config_dict = config['locust-config']
             instance._kass_config._config_dict = config['kass-config']
@@ -1185,12 +1185,25 @@ def get_meta_data(self):
         
         #maybe incomplete
         #add more when you realize you need more metadata
-        meta_data = {'trap': self._kass_config['geometry'],
-                     'transfer-function': self._locust_config[LocustConfig._array_signal_key][LocustConfig._tf_receiver_filename_key],
-                     'n-channels': self._locust_config[LocustConfig._sim_key][LocustConfig._n_channels_key],
-                     'acquisition-rate': self._locust_config[LocustConfig._sim_key][LocustConfig._acq_rate_key],
-                     'lo-frequency': self._locust_config[LocustConfig._array_signal_key][LocustConfig._lo_frequency_key],
-                     }
+
+        tf_file_name = self._locust_config._config_dict[LocustConfig._array_signal_key].get(LocustConfig._tf_receiver_filename_key)
+        n_channels = self._locust_config._config_dict[LocustConfig._sim_key].get(LocustConfig._n_channels_key)
+        acq_rate = self._locust_config._config_dict[LocustConfig._sim_key].get(LocustConfig._acq_rate_key)
+        lo_f = self._locust_config._config_dict[LocustConfig._array_signal_key].get(LocustConfig._lo_frequency_key)
+
+        meta_data = {'trap': self._kass_config._config_dict['geometry']}
+
+        if tf_file_name is not None:
+            meta_data.update({'transfer-function': tf_file_name})
+
+        if n_channels is not None:
+            meta_data.update({'n-channels': n_channels})
+
+        if acq_rate is not None:
+            meta_data.update({'acquisition-rate': acq_rate})
+
+        if lo_f is not None:
+            meta_data.update({'lo-frequency': lo_f})
         
         meta_data.update(self._extra_meta_data)
         
@@ -1329,10 +1342,8 @@ def from_json(cls, file_name):
         with open(file_name, 'r') as infile:
             config = json.load(infile)
             
-            sim_name = config['sim-name']
-            
-            instance = cls(sim_name)
-            
+            instance = cls()
+            instance.sim_name = config['sim-name']            
             instance._meta_data = config['meta-data']
             instance._config_data = config['config-data']
             

From 036fc3e200f320f9a1b35238ff55cf518429b835 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 20:33:23 +0200
Subject: [PATCH 29/44] Add unittest for SimConfig

---
 test/test_config.py | 160 ++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 156 insertions(+), 4 deletions(-)

diff --git a/test/test_config.py b/test/test_config.py
index 47e9853..9ae6b50 100644
--- a/test/test_config.py
+++ b/test/test_config.py
@@ -9,12 +9,164 @@
 FILE_DIR = Path(__file__).parent.absolute()
 
 import unittest
-import hercules as he
+from hercules import SimConfig, SimpleSimConfig, ConfigList
 import numpy as np
 
-class ConfigTest(unittest.TestCase):
+
+class SimConfigTest(unittest.TestCase):
+
+    def setUp(self) -> None:
+        
+        n_channels = 3
+        x = 1.
+        y = 0.
+        theta = 90.
+
+        self.config = SimConfig(
+                    n_channels=n_channels,
+                    seed_locust=42,
+                    seed_kass=43,
+                    egg_filename="simulation.egg",
+                    x_min=x,
+                    x_max=x,
+                    y_min=y,
+                    y_max=y,
+                    z_min=0,
+                    z_max=0,
+                    theta_min=theta,
+                    theta_max=theta,
+                    t_max=5e-6,
+                    v_range=3.0e-7,
+                    geometry='FreeSpaceGeometry_V00_00_10.xml')
+        
+        self.file_name_locust = Path('locust.json')
+        self.file_name_kass = Path('kass.xml') 
+        self.file_name_json = Path('test.json')
+
+    def tearDown(self) -> None:
+        
+        if self.file_name_json.exists():
+            self.file_name_json.unlink()
+
+        if self.file_name_locust.exists():
+            self.file_name_locust.unlink()
+
+        if self.file_name_kass.exists():
+            self.file_name_kass.unlink()
+        
+    def test_meta_data(self):
+        expected = {'trap': '[config_path]/Trap/FreeSpaceGeometry_V00_00_10.xml',
+                    'transfer-function': '/tmp/Phase3/TransferFunctions/FiveSlotTF.txt',
+                    'n-channels': 3,
+                    'lo-frequency': 25878100000.0}
+        
+        self.assertTrue(expected==self.config.get_meta_data())
+
+    def test_config_data(self):
+        expected = {'r': 1.0, 'phi': 0.0, 'z': 0, 'pitch': 90.0, 'energy': 18600.0}
+
+        self.assertTrue(expected==self.config.get_config_data())
+
+    def test_to_json(self):
+
+        self.config.to_json(self.file_name_json)
+        config_loaded = SimConfig.from_json(self.file_name_json)
+        self.assertTrue(config_loaded.to_dict()==self.config.to_dict())
+
+    def test_make_config_file(self):
+
+        self.config.make_kass_config_file(self.file_name_kass)
+        self.config.make_locust_config_file(self.file_name_locust, self.file_name_kass)
+
+        self.assertTrue(self.file_name_kass.exists())
+        self.assertTrue(self.file_name_locust.exists())
+
+    def test_add_metadata(self):
+
+        additional_meta_data = {'trap': 'this should be overwritten', 
+                                'info': 'this is some additional info', 
+                                'acquisition-rate': 'this will not be overwritten because SimConfig does not set it. Adding metadata CANNOT overwrite the config!'}
+
+        expected = {'trap': '[config_path]/Trap/FreeSpaceGeometry_V00_00_10.xml',
+                    'info': 'this is some additional info',
+                    'acquisition-rate': 'this will not be overwritten because SimConfig does not set it. Adding metadata CANNOT overwrite the config!',
+                    'transfer-function': '/tmp/Phase3/TransferFunctions/FiveSlotTF.txt',
+                    'n-channels': 3,
+                    'lo-frequency': 25878100000.0}
+
+        self.config.add_meta_data(additional_meta_data)
+        self.assertTrue(self.config.get_meta_data()==expected)
+
+
+"""
+class ConfigListTest(unittest.TestCase):
+        
+    def setUp(self) -> None:
+            
+        self.configlist = ConfigList(acquisition_rate=1., info='hello', trap='nonsense')
+
+        n_channels = 3
+        r_range = np.linspace(0.002, 0.008, 8)
+        theta_range = np.linspace(89.7, 90.0, 30)
+
+        r_phi_range = np.linspace(0, 2 * np.pi / n_channels, 1)
+
+        for theta in theta_range:
+            for r_phi in r_phi_range:
+                for r in r_range:
+                    x = r * np.cos(r_phi)
+                    y = r * np.sin(r_phi)
+                    r_phi_deg = np.rad2deg(r_phi)
+
+                    config = SimConfig(
+                                        n_channels=n_channels,
+                                        seed_locust=42,
+                                        seed_kass=43,
+                                        egg_filename="simulation.egg",
+                                        x_min=x,
+                                        x_max=x,
+                                        y_min=y,
+                                        y_max=y,
+                                        z_min=0,
+                                        z_max=0,
+                                        theta_min=theta,
+                                        theta_max=theta,
+                                        t_max=5e-6,
+                                        v_range=3.0e-7,
+                                        geometry='FreeSpaceGeometry_V00_00_10.xml')
+                    
+                    self.clist.add_config(config)
+
+                # json_file = FILE_DIR/'SimConfig.json'
+                # locust_file = FILE_DIR/'Locust.json'
+                    # Note kass file name has to be the following for dict comparison
+                # kass_file = FILE_DIR/'LocustKassElectrons.xml'
+
+                # config.to_json(json_file)
+
+                    #config2 = he.SimConfig.from_json(json_file)
+                    #config2.make_config_file(locust_file, kass_file)
+                    #self.assertDictEqual(config.to_dict(), config2.to_dict())
+
+                    #config_list.append(config)
+
+    def test_meta_data(self) -> None:
+    
+        expected = {'acquisition_rate': 1.0, 'info': 'hello', 'trap': '[config_path]/Trap/FreeSpaceGeometry_V00_00_10.xml', 
+                    'transfer-function': '/tmp/Phase3/TransferFunctions/FiveSlotTF.txt', 'n-channels': 3, 'lo-frequency': 25878100000.0}
+        
+        self.assertTrue(self.clist.get_meta_data()==expected)
+        
+        for config in self.clist.get_internal_list():
+             self.assertTrue(config.get_meta_data()==expected)
+
+    def test_config_data(self) ->None:
+
+        
 
     def test_generate_configs(self) -> None:
+
+        clist = ConfigList(acquisition_rate=1., info='hello')
         # Generates a list of config
         n_channels = 3
         r_range = np.linspace(0.002, 0.008, 8)
@@ -32,7 +184,7 @@ def test_generate_configs(self) -> None:
                     r_phi_deg = np.rad2deg(r_phi)
                     name = "Sim_theta_{:.4f}_R_{:.4f}_phi_{:.4f}".format(
                         theta, r, r_phi_deg)
-                    config = he.SimConfig(name,
+                    config = he.SimConfig(
                                         n_channels=n_channels,
                                         seed_locust=42,
                                         seed_kass=43,
@@ -72,7 +224,7 @@ def tearDown(self) -> None:
             # Note f is a Path object
             # Deletes the file
             f.unlink()
-
+"""
 
 if __name__ == '__main__':
     unittest.main()

From dfdc775fb4f85a91d5d85dc24a9691863ed10f93 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 21:14:58 +0200
Subject: [PATCH 30/44] Fix wrong path in test_dataset.py

---
 test/test_dataset.py | 46 ++++++++++++++++++++++----------------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/test/test_dataset.py b/test/test_dataset.py
index 0372f3a..a7813f6 100644
--- a/test/test_dataset.py
+++ b/test/test_dataset.py
@@ -52,9 +52,9 @@ def test_meta_data(self) -> None:
 
     def test_get_path(self) -> None:
 
-        expected_result_1 = ((9.0, 3.0, 6.0), Path('test_directory/run19').absolute())
-        expected_result_2 = ((1.0, 3.0, 5.0), Path('test_directory/run2').absolute())
-        expected_result_3 = ((4.0, 3.0, 6.0), Path('test_directory/run9').absolute())
+        expected_result_1 = ((9.0, 3.0, 6.0), (test_path / 'run19').absolute())
+        expected_result_2 = ((1.0, 3.0, 5.0), (test_path /'run2').absolute())
+        expected_result_3 = ((4.0, 3.0, 6.0), (test_path / 'run9').absolute())
 
         self.assertTrue(expected_result_1==self.d.get_path([100, 100, 100], method='interpolated'))
         self.assertTrue(expected_result_2==self.d.get_path([1., 3., 5.], method='exact'))
@@ -65,26 +65,26 @@ def test_get_path(self) -> None:
 
     def test_iterator(self) -> None:
 
-        expected_result = [((0.0, 3.0, 5.0), Path('test_directory/run0').absolute()),
-                            ((0.0, 3.0, 6.0), Path('test_directory/run1').absolute()),
-                            ((1.0, 3.0, 5.0), Path('test_directory/run2').absolute()),
-                            ((1.0, 3.0, 6.0), Path('test_directory/run3').absolute()),
-                            ((2.0, 3.0, 5.0), Path('test_directory/run4').absolute()),
-                            ((2.0, 3.0, 6.0), Path('test_directory/run5').absolute()),
-                            ((3.0, 3.0, 5.0), Path('test_directory/run6').absolute()),
-                            ((3.0, 3.0, 6.0), Path('test_directory/run7').absolute()),
-                            ((4.0, 3.0, 5.0), Path('test_directory/run8').absolute()),
-                            ((4.0, 3.0, 6.0), Path('test_directory/run9').absolute()),
-                            ((5.0, 3.0, 5.0), Path('test_directory/run10').absolute()),
-                            ((5.0, 3.0, 6.0), Path('test_directory/run11').absolute()),
-                            ((6.0, 3.0, 5.0), Path('test_directory/run12').absolute()),
-                            ((6.0, 3.0, 6.0), Path('test_directory/run13').absolute()),
-                            ((7.0, 3.0, 5.0), Path('test_directory/run14').absolute()),
-                            ((7.0, 3.0, 6.0), Path('test_directory/run15').absolute()),
-                            ((8.0, 3.0, 5.0), Path('test_directory/run16').absolute()),
-                            ((8.0, 3.0, 6.0), Path('test_directory/run17').absolute()),
-                            ((9.0, 3.0, 5.0), Path('test_directory/run18').absolute()),
-                            ((9.0, 3.0, 6.0), Path('test_directory/run19').absolute())]
+        expected_result = [((0.0, 3.0, 5.0), (test_path / 'run0').absolute()),
+                            ((0.0, 3.0, 6.0), (test_path / 'run1').absolute()),
+                            ((1.0, 3.0, 5.0), (test_path / 'run2').absolute()),
+                            ((1.0, 3.0, 6.0), (test_path / 'run3').absolute()),
+                            ((2.0, 3.0, 5.0), (test_path / 'run4').absolute()),
+                            ((2.0, 3.0, 6.0), (test_path / 'run5').absolute()),
+                            ((3.0, 3.0, 5.0), (test_path / 'run6').absolute()),
+                            ((3.0, 3.0, 6.0), (test_path / 'run7').absolute()),
+                            ((4.0, 3.0, 5.0), (test_path / 'run8').absolute()),
+                            ((4.0, 3.0, 6.0), (test_path / 'run9').absolute()),
+                            ((5.0, 3.0, 5.0), (test_path / 'run10').absolute()),
+                            ((5.0, 3.0, 6.0), (test_path / 'run11').absolute()),
+                            ((6.0, 3.0, 5.0), (test_path / 'run12').absolute()),
+                            ((6.0, 3.0, 6.0), (test_path / 'run13').absolute()),
+                            ((7.0, 3.0, 5.0), (test_path / 'run14').absolute()),
+                            ((7.0, 3.0, 6.0), (test_path / 'run15').absolute()),
+                            ((8.0, 3.0, 5.0), (test_path / 'run16').absolute()),
+                            ((8.0, 3.0, 6.0), (test_path / 'run17').absolute()),
+                            ((9.0, 3.0, 5.0), (test_path / 'run18').absolute()),
+                            ((9.0, 3.0, 6.0), (test_path / 'run19').absolute())]
         
         result = []
 

From 6550b6c4619930beb7d7e20be2dedaba4bb16dc9 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 21:18:28 +0200
Subject: [PATCH 31/44] Fix relative paths in test_config.py

---
 test/test_config.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/test/test_config.py b/test/test_config.py
index 9ae6b50..ee263ab 100644
--- a/test/test_config.py
+++ b/test/test_config.py
@@ -6,7 +6,8 @@
 """
 
 from pathlib import Path
-FILE_DIR = Path(__file__).parent.absolute()
+
+module_dir = Path(__file__).parent.absolute()
 
 import unittest
 from hercules import SimConfig, SimpleSimConfig, ConfigList
@@ -39,9 +40,9 @@ def setUp(self) -> None:
                     v_range=3.0e-7,
                     geometry='FreeSpaceGeometry_V00_00_10.xml')
         
-        self.file_name_locust = Path('locust.json')
-        self.file_name_kass = Path('kass.xml') 
-        self.file_name_json = Path('test.json')
+        self.file_name_locust = module_dir / 'locust.json'
+        self.file_name_kass = module_dir / 'kass.xml' 
+        self.file_name_json = module_dir / 'test.json'
 
     def tearDown(self) -> None:
         

From 5f54beca0f17dc5977292f6365119411274d49cf Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 21:19:14 +0200
Subject: [PATCH 32/44] Fix wrong order for meta_data update in SimConfig

---
 hercules/simconfig.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/hercules/simconfig.py b/hercules/simconfig.py
index 53597d6..3365001 100644
--- a/hercules/simconfig.py
+++ b/hercules/simconfig.py
@@ -1191,7 +1191,10 @@ def get_meta_data(self):
         acq_rate = self._locust_config._config_dict[LocustConfig._sim_key].get(LocustConfig._acq_rate_key)
         lo_f = self._locust_config._config_dict[LocustConfig._array_signal_key].get(LocustConfig._lo_frequency_key)
 
-        meta_data = {'trap': self._kass_config._config_dict['geometry']}
+        meta_data = {}
+        meta_data.update(self._extra_meta_data)
+
+        meta_data.update({'trap': self._kass_config._config_dict['geometry']})
 
         if tf_file_name is not None:
             meta_data.update({'transfer-function': tf_file_name})
@@ -1205,8 +1208,6 @@ def get_meta_data(self):
         if lo_f is not None:
             meta_data.update({'lo-frequency': lo_f})
         
-        meta_data.update(self._extra_meta_data)
-        
         return meta_data
     
     def add_meta_data(self, meta_data):

From 2d0dac4929e23750224ce4e107fbc1f755884dcb Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Sat, 5 Aug 2023 21:27:07 +0200
Subject: [PATCH 33/44] Add unittest for SimpleSimConfig

---
 test/test_config.py | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/test/test_config.py b/test/test_config.py
index ee263ab..4549ff3 100644
--- a/test/test_config.py
+++ b/test/test_config.py
@@ -98,6 +98,43 @@ def test_add_metadata(self):
         self.config.add_meta_data(additional_meta_data)
         self.assertTrue(self.config.get_meta_data()==expected)
 
+class SimpleSimConfigTest(unittest.TestCase):
+
+    def setUp(self) -> None:
+        
+        self.config = SimpleSimConfig(x=1., y=2., z=3.)
+        self.file_name_json = module_dir / 'test.json'
+
+    def tearDown(self) -> None:
+        
+        if self.file_name_json.exists():
+            self.file_name_json.unlink()
+        
+    def test_meta_data(self):
+        expected = {}
+        
+        self.assertTrue(expected==self.config.get_meta_data())
+
+    def test_config_data(self):
+        expected = {'x': 1., 'y': 2., 'z': 3.}
+
+        self.assertTrue(expected==self.config.get_config_data())
+
+    def test_to_json(self):
+
+        self.config.to_json(self.file_name_json)
+        config_loaded = SimpleSimConfig.from_json(self.file_name_json)
+        self.assertTrue(config_loaded.to_dict()==self.config.to_dict())
+
+    def test_add_metadata(self):
+
+        meta_data = {'info1': 2, 
+                                'info2': 'this is some additional info'}
+
+        self.config.add_meta_data(meta_data)
+        self.assertTrue(self.config.get_meta_data()==meta_data)
+
+
 
 """
 class ConfigListTest(unittest.TestCase):

From b5a1be14066aad2b52f158906c6718be22687660 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 7 Aug 2023 14:30:40 +0200
Subject: [PATCH 34/44] Add unittest for the ConfigList class

---
 test/test_config.py | 160 +++++++++++++++++++++++---------------------
 1 file changed, 84 insertions(+), 76 deletions(-)

diff --git a/test/test_config.py b/test/test_config.py
index 4549ff3..dfa1b3a 100644
--- a/test/test_config.py
+++ b/test/test_config.py
@@ -135,15 +135,17 @@ def test_add_metadata(self):
         self.assertTrue(self.config.get_meta_data()==meta_data)
 
 
-
-"""
 class ConfigListTest(unittest.TestCase):
         
     def setUp(self) -> None:
             
+        n_channels = 3
         self.configlist = ConfigList(acquisition_rate=1., info='hello', trap='nonsense')
+        self.configlist_simple = ConfigList(acquisition_rate=1., info='hello simple', trap='nonsense', n_channels=n_channels)
+
+        self.configlist_l = []
+        self.configlist_simple_l = []
 
-        n_channels = 3
         r_range = np.linspace(0.002, 0.008, 8)
         theta_range = np.linspace(89.7, 90.0, 30)
 
@@ -154,9 +156,8 @@ def setUp(self) -> None:
                 for r in r_range:
                     x = r * np.cos(r_phi)
                     y = r * np.sin(r_phi)
-                    r_phi_deg = np.rad2deg(r_phi)
 
-                    config = SimConfig(
+                    sim_config = SimConfig(
                                         n_channels=n_channels,
                                         seed_locust=42,
                                         seed_kass=43,
@@ -173,96 +174,103 @@ def setUp(self) -> None:
                                         v_range=3.0e-7,
                                         geometry='FreeSpaceGeometry_V00_00_10.xml')
                     
-                    self.clist.add_config(config)
+                    self.configlist_l.append(sim_config)
 
-                # json_file = FILE_DIR/'SimConfig.json'
-                # locust_file = FILE_DIR/'Locust.json'
-                    # Note kass file name has to be the following for dict comparison
-                # kass_file = FILE_DIR/'LocustKassElectrons.xml'
+                    simple_config = SimpleSimConfig(r=r, phi=r_phi, z=0., energy=0., pitch=90.)
+                    self.configlist_simple_l.append(simple_config)
 
-                # config.to_json(json_file)
+    def test_add_config(self):
+        
+        self.configlist.add_config(self.configlist_l[0])
+        self.configlist.add_config(self.configlist_l[1])
 
-                    #config2 = he.SimConfig.from_json(json_file)
-                    #config2.make_config_file(locust_file, kass_file)
-                    #self.assertDictEqual(config.to_dict(), config2.to_dict())
+        #adding wrong type
+        with self.assertRaises(TypeError) as cm:
+            self.configlist.add_config(self.configlist_simple_l[0])
 
-                    #config_list.append(config)
+        self.configlist_simple.add_config(self.configlist_simple_l[0])
+        self.configlist_simple.add_config(self.configlist_simple_l[1])
 
-    def test_meta_data(self) -> None:
-    
-        expected = {'acquisition_rate': 1.0, 'info': 'hello', 'trap': '[config_path]/Trap/FreeSpaceGeometry_V00_00_10.xml', 
-                    'transfer-function': '/tmp/Phase3/TransferFunctions/FiveSlotTF.txt', 'n-channels': 3, 'lo-frequency': 25878100000.0}
-        
-        self.assertTrue(self.clist.get_meta_data()==expected)
+        #adding non matching config data
+        with self.assertRaises(RuntimeError) as cm:
+            self.configlist_simple.add_config(SimpleSimConfig(x='foo'))
+
+    def test_type(self):
+
+        self.configlist.add_config(self.configlist_l[0])
+
+        self.configlist_simple.add_config(self.configlist_simple_l[0])
+
+        self.assertEqual(self.configlist.get_list_type(), SimConfig)
+        self.assertEqual(self.configlist_simple.get_list_type(), SimpleSimConfig)
+
+
+    def add_all(self):
+        for config in self.configlist_l:
+            self.configlist.add_config(config)
+
+        for config in self.configlist_simple_l:
+            self.configlist_simple.add_config(config)
+
+    def test_add_all(self):
+
+        self.add_all()
+
+        self.assertEqual(len(self.configlist.get_internal_list()), len(self.configlist_l))
+        self.assertEqual(len(self.configlist_simple.get_internal_list()), len(self.configlist_simple_l))
+
+    def test_meta_data(self):
+
+        self.add_all()
+
+        expected = {'acquisition_rate': 1.0,
+                    'info': 'hello',
+                    'trap': '[config_path]/Trap/FreeSpaceGeometry_V00_00_10.xml',
+                    'transfer-function': '/tmp/Phase3/TransferFunctions/FiveSlotTF.txt',
+                    'n-channels': 3,
+                    'lo-frequency': 25878100000.0}
         
-        for config in self.clist.get_internal_list():
-             self.assertTrue(config.get_meta_data()==expected)
+        self.assertEqual(expected, self.configlist.get_meta_data())
 
-    def test_config_data(self) ->None:
+        for config in self.configlist.get_internal_list():
+            self.assertEqual(expected, config.get_meta_data())
 
+        expected = {'acquisition_rate': 1.0,
+                    'info': 'hello simple',
+                    'trap': 'nonsense',
+                    'n_channels': 3}
         
+        self.assertEqual(expected, self.configlist_simple.get_meta_data())
 
-    def test_generate_configs(self) -> None:
+        for config in self.configlist_simple.get_internal_list():
+            self.assertEqual(expected, config.get_meta_data())
 
-        clist = ConfigList(acquisition_rate=1., info='hello')
-        # Generates a list of config
-        n_channels = 3
-        r_range = np.linspace(0.002, 0.008, 8)
-        theta_range = np.linspace(89.7, 90.0, 30)
+    def test_config_data(self):
 
-        r_phi_range = np.linspace(0, 2 * np.pi / n_channels, 1)
+        self.add_all()
+        
+        expected = ['energy', 'phi', 'pitch', 'r', 'z']
 
-        config_list = []
+        self.assertEqual(sorted(list(self.configlist.get_config_data_keys())), expected)
+        self.assertEqual(sorted(list(self.configlist_simple.get_config_data_keys())), expected)
 
-        for theta in theta_range:
-            for r_phi in r_phi_range:
-                for r in r_range:
-                    x = r * np.cos(r_phi)
-                    y = r * np.sin(r_phi)
-                    r_phi_deg = np.rad2deg(r_phi)
-                    name = "Sim_theta_{:.4f}_R_{:.4f}_phi_{:.4f}".format(
-                        theta, r, r_phi_deg)
-                    config = he.SimConfig(
-                                        n_channels=n_channels,
-                                        seed_locust=42,
-                                        seed_kass=43,
-                                        egg_filename="simulation.egg",
-                                        x_min=x,
-                                        x_max=x,
-                                        y_min=y,
-                                        y_max=y,
-                                        z_min=0,
-                                        z_max=0,
-                                        theta_min=theta,
-                                        theta_max=theta,
-                                        t_max=5e-6,
-                                        v_range=3.0e-7,
-                                        geometry='FreeSpaceGeometry_V00_00_10.xml')
+        for config in self.configlist.get_internal_list():
+            config_data_keys = config.get_config_data().keys()
+            self.assertEqual(sorted(list(config_data_keys)), expected)
 
-                    json_file = FILE_DIR/'SimConfig.json'
-                    locust_file = FILE_DIR/'Locust.json'
-                    # Note kass file name has to be the following for dict comparison
-                    kass_file = FILE_DIR/'LocustKassElectrons.xml'
+        for config in self.configlist_simple.get_internal_list():
+            config_data_keys = config.get_config_data().keys()
+            self.assertEqual(sorted(list(config_data_keys)), expected)
 
-                    config.to_json(json_file)
+    def test_names(self):
 
-                    config2 = he.SimConfig.from_json(json_file)
-                    config2.make_config_file(locust_file, kass_file)
-                    self.assertDictEqual(config.to_dict(), config2.to_dict())
+        self.add_all()
 
-                    config_list.append(config)
+        for i, config in enumerate(self.configlist.get_internal_list()):
+            self.assertEqual(config.sim_name,f'run{i}')
 
-    def tearDown(self) -> None:
-        # Clean up
-        files_json = FILE_DIR.glob('*.json')
-        files_xml = FILE_DIR.glob('*.xml')
-        filtered_files = sorted(files_json) + sorted(files_xml)
-
-        for f in filtered_files:
-            # Note f is a Path object
-            # Deletes the file
-            f.unlink()
-"""
+        for i, config in enumerate(self.configlist_simple.get_internal_list()):
+            self.assertEqual(config.sim_name,f'run{i}')
 
 if __name__ == '__main__':
     unittest.main()

From aafaddb3bea330c72911778d8acc792119f42411 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 7 Aug 2023 14:52:39 +0200
Subject: [PATCH 35/44] Fix the locust unittest

---
 test/test_locustP2.py | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/test/test_locustP2.py b/test/test_locustP2.py
index 65bb3dd..1a51632 100644
--- a/test/test_locustP2.py
+++ b/test/test_locustP2.py
@@ -6,12 +6,15 @@
 
 """
 
-import hercules as he
+from hercules import KassLocustP3, ConfigList, SimConfig
 from pathlib import Path
 import numpy as np
 import unittest
+import shutil
 
 module_dir = Path(__file__).parent.absolute()
+test_dataset_name = 'working_directory'
+test_path = module_dir / test_dataset_name
 
 class LocustP2Test(unittest.TestCase):
     def setUp(self) -> None:
@@ -20,19 +23,15 @@ def setUp(self) -> None:
         theta_range = np.linspace(89.7, 90.0, 2)
         r_phi_range = np.linspace(0, 2 * np.pi, 1)
 
-        config_list = []
-        sim = he.KassLocustP3(str(module_dir) + '/out_test_locust_P2')
+        sim = KassLocustP3(test_path)
+        configlist = ConfigList(acquisition_rate=1., info='hello', trap='nonsense')
         
         for theta in theta_range:
             for r_phi in r_phi_range:
                 for r in r_range:
                     x = r * np.cos(r_phi)
                     y = r * np.sin(r_phi)
-                    r_phi_deg = np.rad2deg(r_phi)
-                    name = "Sim_theta_{:.4f}_R_{:.4f}_phi_{:.4f}".format(
-                        theta, r, r_phi_deg)
-                    config = he.SimConfig(
-                        name,
+                    config = SimConfig(
                         phase = 'Phase2',
                         seed_locust=42,
                         seed_kass=43,
@@ -49,14 +48,26 @@ def setUp(self) -> None:
                         record_size=3000,
                         v_range=3.0e-6,
                         geometry='Trap_3.xml')
-                    config_list.append(config)
+                    configlist.add_config(config)
         
-        sim(config_list)
+        sim(configlist)
+
+    def tearDown(self) -> None:
+        shutil.rmtree(test_path)
 
     def test_locust(self) -> None:
-        # Test sim generation (done in setUp), substitutes regular locust test
-        # This test is always true.
-        self.assertTrue(True)
+
+        paths =[test_path / 'index.he',
+                test_path / 'info.txt',
+                test_path / 'run0' / 'simulation.egg',
+                test_path / 'run1' / 'simulation.egg',
+                test_path / 'run2' / 'simulation.egg',
+                test_path / 'run3' / 'simulation.egg'
+                ]
+        
+        for path in paths:
+            self.assertTrue(path.exists())
+                
 
 if __name__ == '__main__':
     unittest.main()

From 3caeb0dc8d85c89f3be999e8ab74fecef799009f Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 7 Aug 2023 16:44:46 +0200
Subject: [PATCH 36/44] Fix wrong arguments for Locust unittest

---
 test/test_locustP2.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_locustP2.py b/test/test_locustP2.py
index 1a51632..4e03ea0 100644
--- a/test/test_locustP2.py
+++ b/test/test_locustP2.py
@@ -23,7 +23,7 @@ def setUp(self) -> None:
         theta_range = np.linspace(89.7, 90.0, 2)
         r_phi_range = np.linspace(0, 2 * np.pi, 1)
 
-        sim = KassLocustP3(test_path)
+        sim = KassLocustP3(test_path, use_kass=True, use_locust=True)
         configlist = ConfigList(acquisition_rate=1., info='hello', trap='nonsense')
         
         for theta in theta_range:

From e72d0c453717584cdf26c13d52716926293ae221 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 7 Aug 2023 17:13:43 +0200
Subject: [PATCH 37/44] Fix the eggreader unittest

---
 test/test_eggreader.py | 138 ++++++++++++++++++++---------------------
 1 file changed, 69 insertions(+), 69 deletions(-)

diff --git a/test/test_eggreader.py b/test/test_eggreader.py
index 65cd522..53867d0 100644
--- a/test/test_eggreader.py
+++ b/test/test_eggreader.py
@@ -1,6 +1,6 @@
 """
 
-Author: Mingyu (Charles) Li
+Author: Mingyu (Charles) Li, Florian Thomas
 Date: Apr. 12, 2021
 
 """
@@ -10,39 +10,43 @@
 
 from matplotlib.figure import Figure
 
-FILE_DIR = Path(__file__).parent.absolute()
-
 import matplotlib
 import unittest
-import hercules as he
+from hercules import KassLocustP3, SimConfig, ConfigList, Dataset, LocustP3File
 import numpy as np
 import matplotlib.pyplot as plt
+import shutil
+
+#matplotlib.use("Agg")  # No GUI
 
-matplotlib.use("Agg")  # No GUI
+module_dir = Path(__file__).parent.absolute()
+test_dataset_name = 'egg_reader_test'
+test_path = module_dir / test_dataset_name
+egg_filename = 'simulation.egg'
 
 
 class EggReaderTest(unittest.TestCase):
-    def setUp(self) -> None:
+
+    @classmethod
+    def setUpClass(cls) -> None:
         n_ch = 3
         r_range = np.linspace(0.000, 0.010, 1)
-        theta_range = np.linspace(85.0, 90.0, 1)
+        theta_range = np.linspace(85.0, 90.0, 2)
         r_phi_range = np.linspace(0, 2 * np.pi / n_ch, 1)
 
-        test_data_dict = {}
+        sim = KassLocustP3(test_path, use_kass=True, use_locust=True)
+        configlist = ConfigList()
+        
         for theta in theta_range:
             for r_phi in r_phi_range:
                 for r in r_range:
                     x = r * np.cos(r_phi)
                     y = r * np.sin(r_phi)
-                    r_phi_deg = np.rad2deg(r_phi)
-                    name = "Sim_theta_{:.4f}_R_{:.4f}_phi_{:.4f}".format(
-                        theta, r, r_phi_deg)
-                    config = he.SimConfig(
-                        name,
+                    config = SimConfig(
                         n_channels=n_ch,
                         seed_locust=42,
                         seed_kass=43,
-                        egg_filename="simulation.egg",
+                        egg_filename=egg_filename,
                         x_min=x,
                         x_max=x,
                         y_min=y,
@@ -57,44 +61,41 @@ def setUp(self) -> None:
                         lo_frequency=25.8881e9,
                         acq_rate=250.0,
                         geometry='FreeSpaceGeometry_V00_00_10.xml')
-                    test_data_dict[name] = config
-
-        self.test_data_dict = test_data_dict
-
-        # Check if the test dir exists and names are correct
-        self.test_data_dir = test_data_dir = FILE_DIR / "test_dir"
-        test_data_dir.mkdir(exist_ok=True)
-        sub_dir_list = [
-            d.parts[-1] for d in test_data_dir.iterdir() if d.is_dir()
-        ]
-        missing_dir_list = list(
-            set(self.test_data_dict.keys()) - set(sub_dir_list))
-        print(
-            "The following test data are missing: {}".format(missing_dir_list))
-
-        # Create missing test data
-        # Note: if data is missing, must run the tests in cmd line to create the data
-        if len(missing_dir_list) != 0:
-            print("Creating missing data...")
-            missing_data_config = [test_data_dict[l] for l in missing_dir_list]
-            sim = he.KassLocustP3(str(test_data_dir))
-            sim(missing_data_config)
-
-    def test_locust(self) -> None:
-        # Test sim generation (done in setUp), substitutes regular locust test
-        # This test is always true.
-        self.assertTrue(True)
+                    configlist.add_config(config)
+
+        sim(configlist)
+
+        cls.dataset = Dataset.load(test_path)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        pass
+        #shutil.rmtree(test_path)
+
+    def test_0(self) -> None:
+
+        paths =[test_path / 'index.he',
+                test_path / 'info.txt',
+                test_path / 'run0' / 'simulation.egg',
+                test_path / 'run1' / 'simulation.egg'
+                ]
+        print('Testing simulation successful')
+        for path in paths:
+            self.assertTrue(path.exists())
 
     def test_ts_stream(self) -> None:
-        for k in self.test_data_dict.keys():
-            self._load_ts_stream(k)
-            self._quick_load_ts_stream(k)
+        for _, path in self.dataset:
+            print(path)
+            self._load_ts_stream(path)
+            self._quick_load_ts_stream(path)
+
+        ok = input('Plots look ok? (y/n): ').lower().strip() == 'y'
+        self.assertTrue(ok)
 
     def _load_ts_stream(self, name) -> None:
         # Plot some specific test data
-        file_name = self.test_data_dir.joinpath(name).joinpath(
-            "simulation.egg")
-        file = he.LocustP3File(str(file_name))
+
+        file = LocustP3File(str(name / egg_filename))
         n_streams = file.n_streams
 
         for s in range(n_streams):
@@ -122,13 +123,12 @@ def _load_ts_stream(self, name) -> None:
             ax.set_xlabel(r"Time $[s]$")
             ax.set_ylabel(r"DAQ V $[V]$")
             ax.legend(loc="best")
-            self._save_fig(fig, title)
+            #self._save_fig(fig, f'{name}/plot_{s}.pdf', title)
+            plt.show()
 
     def _quick_load_ts_stream(self, name) -> None:
         # Plot some specific test data
-        file_name = self.test_data_dir.joinpath(name).joinpath(
-            "simulation.egg")
-        file = he.LocustP3File(str(file_name))
+        file = LocustP3File(str(name / egg_filename))
         n_streams = file.n_streams
         n_ch = file.n_channels
 
@@ -159,18 +159,21 @@ def _quick_load_ts_stream(self, name) -> None:
             ax.set_xlabel(r"Time $[s]$")
             ax.set_ylabel(r"DAQ V $[V]$")
             ax.legend(loc="best")
-            self._save_fig(fig, title)
+            #self._save_fig(fig, f'{name}/plot_quick_{s}.pdf', title)
+            plt.show()
 
     def test_fft_stream(self) -> None:
-        for k in self.test_data_dict.keys():
-            self._load_fft_stream(k)
-            self._quick_load_fft_stream(k)
+        for _, path in self.dataset:
+            print(path)
+            self._load_fft_stream(path)
+            self._quick_load_fft_stream(path)
+
+        ok = input('Plots look ok? (y/n): ').lower().strip() == 'y'
+        self.assertTrue(ok)
 
     def _load_fft_stream(self, name) -> None:
-        # Plot some specific test data
-        file_name = self.test_data_dir.joinpath(name).joinpath(
-            "simulation.egg")
-        file = he.LocustP3File(str(file_name))
+
+        file = LocustP3File(str(name / egg_filename))
         n_streams = file.n_streams
 
         for s in range(n_streams):
@@ -192,13 +195,12 @@ def _load_fft_stream(self, name) -> None:
             ax.set_xlabel(r"Frequency $[Hz]$")
             ax.set_ylabel(r"FFT")
             ax.legend(loc="best")
-            self._save_fig(fig, title)
+            #self._save_fig(fig, title)
+            plt.show()
 
     def _quick_load_fft_stream(self, name) -> None:
         # Plot some specific test data
-        file_name = self.test_data_dir.joinpath(name).joinpath(
-            "simulation.egg")
-        file = he.LocustP3File(str(file_name))
+        file = LocustP3File(str(name / egg_filename))
         n_streams = file.n_streams
         n_ch = file.n_channels
 
@@ -225,9 +227,10 @@ def _quick_load_fft_stream(self, name) -> None:
             ax.set_xlabel(r"Frequency $[Hz]$")
             ax.set_ylabel(r"FFT")
             ax.legend(loc="best")
-            self._save_fig(fig, title)
+            #self._save_fig(fig, title)
+            plt.show()
 
-    def _save_fig(self, fig: Figure, title: str = None) -> None:
+    def _save_fig(self, fig: Figure, out_file: str, title: str = None) -> None:
         # Saves the figure to data/images with title and closes the figure.
         # Default title to axes title if there's only one axes
         if title is None:
@@ -236,11 +239,8 @@ def _save_fig(self, fig: Figure, title: str = None) -> None:
                 title = ax_list[0].get_title()
             else:
                 raise ValueError("title cannot be None")
-        out_file = os.path.join(
-            str(self.test_data_dir),
-            title.replace(' ', '_').replace('.', '-') + ".pdf")
         fig.savefig(out_file, bbox_inches='tight', dpi=300)
-        plt.close(fig)
+        plt.close(fig) 
 
 
 if __name__ == '__main__':

From deb880887414e9e55bcba17953ae00f08f54ab0f Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 7 Aug 2023 17:15:25 +0200
Subject: [PATCH 38/44] Forgot to uncomment a line

---
 test/test_eggreader.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/test_eggreader.py b/test/test_eggreader.py
index 53867d0..ea4a2c9 100644
--- a/test/test_eggreader.py
+++ b/test/test_eggreader.py
@@ -69,8 +69,7 @@ def setUpClass(cls) -> None:
 
     @classmethod
     def tearDownClass(cls) -> None:
-        pass
-        #shutil.rmtree(test_path)
+        shutil.rmtree(test_path)
 
     def test_0(self) -> None:
 

From 99df0b1e0c5f3e33472451ae65f823739f0303d4 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 7 Aug 2023 17:37:52 +0200
Subject: [PATCH 39/44] Update old example scripts

---
 examples/locustP2_example.py | 13 ++++++-------
 examples/locustP3_example.py | 14 +++++++-------
 2 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/examples/locustP2_example.py b/examples/locustP2_example.py
index c920b24..eacf6ad 100644
--- a/examples/locustP2_example.py
+++ b/examples/locustP2_example.py
@@ -6,15 +6,15 @@
 
 """
 
-import hercules as he
+from hercules import KassLocustP3, SimConfig, ConfigList
 from pathlib import Path
 import numpy as np
 
 module_dir = Path(__file__).parent.absolute()
 
 #just an example
-config_list = []
-sim = he.KassLocustP3(str(module_dir) + '/workingDir')
+sim = KassLocustP3(str(module_dir) + '/workingDir', use_kass=True, use_locust=True)
+configlist = ConfigList()
 
 n_r = 1
 r_vals = np.linspace(0.0, 0.03, n_r)
@@ -24,13 +24,12 @@
 
 for i, r in enumerate(r_vals):
     for j, theta in enumerate(theta_vals):
-        config = he.SimConfig('someDirName_{0:1.3f}_{1:1.3f}'.format(r, theta), 
-                                phase='Phase2', egg_filename='someFileName.egg', 
+        config = SimConfig(     phase='Phase2', egg_filename='someFileName.egg', 
                                 theta_min = theta, theta_max = theta,
                                 x_min = r, x_max = r,
                                 v_range=1.0e-6, record_size=20000, t_max=7.5e-5, 
                                 geometry='Trap_3.xml')
                                 
-        config_list.append(config)
+        configlist.add_config(config)
 
-sim(config_list)
+sim(configlist)
diff --git a/examples/locustP3_example.py b/examples/locustP3_example.py
index a9c779a..881bcc7 100644
--- a/examples/locustP3_example.py
+++ b/examples/locustP3_example.py
@@ -6,24 +6,24 @@
 
 """
 
-import hercules as he
+from hercules import SimConfig, KassLocustP3, ConfigList
 from pathlib import Path
 import numpy as np
 
 module_dir = Path(__file__).parent.absolute()
 
 #just an example
-config_list = []
-sim = he.KassLocustP3(str(module_dir) + '/workingDir')
+sim = KassLocustP3(str(module_dir) + '/workingDirP3', use_kass=True, use_locust=True)
+configlist = ConfigList()
 
-simulations = 20
+simulations = 5
 r_vals = np.linspace(0.0, 0.03, simulations)
 
 for i, r in enumerate(r_vals):
-    config = he.SimConfig('someDirName_{0:1.3f}'.format(r), n_channels=2, seed_locust=1, 
+    config = SimConfig(n_channels=2, seed_locust=1, 
                             v_range=7.0, egg_filename='someFileName.egg', 
                             x_min=r, x_max=r, t_max=0.5e-6,
                             geometry='FreeSpaceGeometry_V00_00_10.xml')
-    config_list.append(config)
+    configlist.add_config(config)
 
-sim(config_list)
+sim(configlist)

From 7ed1764f83ca2a0d545ad9c94fc70e8242911286 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 7 Aug 2023 19:06:12 +0200
Subject: [PATCH 40/44] Make load_sim in Dataset private

---
 hercules/dataset.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hercules/dataset.py b/hercules/dataset.py
index bb92d62..ee3d4f4 100644
--- a/hercules/dataset.py
+++ b/hercules/dataset.py
@@ -94,7 +94,7 @@ def get_data(self, params, interpolation=True):
         
         return parameters, self.load_sim(path)
         
-    def load_sim(self, path):
+    def _load_sim(self, path):
         return np.load(self._directory / path / PY_DATA_NAME)
         
     def get_path(self, params, method='interpolated'):

From fc8f16075555a05bcd3af45dcd5d880a51d34584 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 7 Aug 2023 19:28:45 +0200
Subject: [PATCH 41/44] Add example for handling a Dataset

---
 examples/dataset_example.py | 50 +++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 examples/dataset_example.py

diff --git a/examples/dataset_example.py b/examples/dataset_example.py
new file mode 100644
index 0000000..70f0ef5
--- /dev/null
+++ b/examples/dataset_example.py
@@ -0,0 +1,50 @@
+
+"""
+
+Author: F. Thomas
+Date: August 07, 2023
+
+"""
+
+from hercules import Dataset, LocustP3File
+from pathlib import Path
+
+module_dir = Path(__file__).parent.absolute()
+dataset_name = 'workingDirP3'
+
+path = module_dir / dataset_name
+
+#load the hercules dataset
+dataset = Dataset.load(path)
+
+#print the metadata which is valid for all entries in the dataset
+print(dataset.meta_data)
+
+#print the axes of the dataset grid
+axes = dataset.axes
+for i, ax in enumerate(axes):
+    print(f'{dataset.config_data_keys[i]} -> {ax}' )
+
+#get path to data by index of all data axes
+param, path = dataset.get_path([0, 0, 0, 0, 0], method='index')
+print(param, path)
+
+#use path
+data = LocustP3File(path / 'someFileName.egg')
+
+#get path to data by exact value
+param, path = dataset.get_path([0., 0., 0., 87., 18600.], method='exact')
+print(param, path)
+
+#get path to data by exact value
+param, path = dataset.get_path([axes[0][3], axes[1][0], axes[2][0], axes[3][0], axes[4][0]], method='exact')
+print(param, path)
+
+#get path to data by value using nearest neighbor interpolation
+param, path = dataset.get_path([0.0, 0.0, 0.0, 0.0, 0.0], method='interpolated')
+print(param, path)
+
+#iterate over all data in dataset
+for param, path in dataset:
+    print(param, path)
+    data = LocustP3File(path / 'someFileName.egg')
\ No newline at end of file

From ac61e5eddf97bd21ccea6df9f9807b69456ab544 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 7 Aug 2023 20:36:55 +0200
Subject: [PATCH 42/44] Update README

---
 README.md | 93 ++++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 82 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index d5673a0..296cad1 100644
--- a/README.md
+++ b/README.md
@@ -25,30 +25,101 @@ Note: If you are using the system wide python environment instead of a virtual e
 Example script:
 
 ```python
-import hercules as he
+from hercules import KassLocustP3, SimConfig, ConfigList
 
-sim = he.KassLocustP3('/path/to/your/workingDir')
+sim = KassLocustP3('/path/to/your/workingDir', use_kass=True, use_locust=True)
+configlist = ConfigList(info='Additional meta data')
 #just an example
-config = he.SimConfig('yourSimulationName', phase='Phase3', kass_file_name='someXMLFile.xml', 
+config = SimConfig(phase='Phase3', kass_file_name='someXMLFile.xml', 
                     locust_file_name='someJSONFile.json', 
-                    nChannels=2, seedLocust=1, vRange=7.0,
-                    eggFilename='someFileName.egg', seedKass =12534, xMin=-0.1e-5, 
-                    xMax=0.1e-5, tMax=0.5e-6,
+                    n_channels=2, seed_locust=1, v_range=7.0,
+                    egg_filename='someFileName.egg', seed_kass =12534, x_min=0.1e-5, 
+                    x_max=0.1e-5, t_max=0.5e-6,
                     geometry='FreeSpaceGeometry_V00_00_10.xml')
 
 
-sim(config) #can also take a list of configs
+configlist.add_config(config)
+#runs the simulations
+sim(configlist)
 
 ```
-The example above runs a single phase 3 Kassiopeia-Locust simulation with the given parameters. All parameters except for the simulation name are optional. Omitted parameters in general take on default values with the exception being the seeds which are generated on the fly. The phase parameter can take the values 'Phase2' or 'Phase3' (default). Hercules generates the config files for Kassiopeia and Locust based on the inputs, the selected phase and the template config files which also provide the default values. All config files will be taken from the [hexbug](https://github.com/project8/hexbug/tree/459dffe30eea7d8bab9ddff78b63fda5198041ad) repository. Once hercules is installed you can run the script from anywhere specifying any working directory that you want and it will always be able to find the config files. Config files from hexbug (including Transfer functions and trap geometries) are passed by just their names as demonstrated above. Hercules will look for them in the appropriate directory of hexbug depending on the phase. In most cases you want to use the defaults for 'kass_file_name' and 'loucst_file_name'.  
+The example above runs a single phase 3 Kassiopeia-Locust simulation with the given parameters. Hercules is most useful if you run `config=...` and `configlist.add_config(config)` in an arbitrary python loop. The line `sim(configlist)` will always run all the simulation configurations from the list. All parameters are optional. Omitted parameters in general take on default values with the exception being the seeds which are generated on the fly. The phase parameter can take the values 'Phase2' or 'Phase3' (default). Hercules generates the config files for Kassiopeia and Locust based on the inputs, the selected phase and the template config files which also provide the default values. All config files will be taken from the [hexbug](https://github.com/project8/hexbug/tree/459dffe30eea7d8bab9ddff78b63fda5198041ad) repository. Once hercules is installed you can run the script from anywhere specifying any working directory that you want and it will always be able to find the config files. Config files from hexbug (including Transfer functions and trap geometries) are passed by just their names as demonstrated above. Hercules will look for them in the appropriate directory of hexbug depending on the phase. In most cases you want to use the defaults for 'kass_file_name' and 'loucst_file_name'.  
 If you need the full list of simulation parameters you can ask hercules for a help message. `he.SimConfig.help()` will print a full list of all available keyword arguments with a short explanation for each one.  
-You can find example scripts in [examples](./examples). Hercules scripts work in a desktop environment as well as on the grace cluster without requiring any modifications. 
+You can find example scripts in [examples](./examples). Hercules scripts work in a desktop environment as well as on the grace cluster without requiring any modifications.
+
+Running hercules with the example from above will create a `hercules.Dataset` in the specified working directory. A `hercules.Dataset` is a dataformat in the form of an indexed directory structure which handles like a normal directory. For each configuration in the `configlist` hercules creates a subdirectory in the working directory that is called `run{i}` with `i` being the incremental number of configurations. In addition to that the directory contains a text file `info.txt` with some info about the dataset (meta data and parameter axes) and the very important file `index.he`. The latter is the pickled `hercules.Dataset` python object. Its core is a hashmap to map configuration parameters to the corresponding paths in the working directory. The class implements some utility for convenient recovery of any data stored in the subdirectories. In the example below a loop is used to recover all egg files but note that under each path other data than just the egg file can be found (with Locust as default you get at least log files and a json file with the configuration), which can be accessed with that path. See [dataset_example](./examples/dataset_example.py) for more details on how to utilize the `Dataset` class.
+
+```python
+from hercules import Dataset, LocustP3File
+
+dataset = Dataset.load('/path/to/your/workingDir')
+
+for param, path in dataset:
+    data = LocustP3File(path / 'someFileName.egg')
+```
+
+Another interesting feature is the use of python scripts for post-processing. Any python script located in [hexbug/CRESana](./hercules/hexbug/CRESana/) (was implemented for the use with [CRESana](https://github.com/MCFlowMace/CRESana)) can be passed by its name and for each configuration it will be run after Kassiopeia and Locust. This represents another way of producing more data for a single configuration which can be retrieved via the path from the `Dataset`.
+
+```python
+sim = KassLocustP3('/path/to/your/workingDir', use_kass=True, use_locust=True, python_script='post_processing.py')
+```
+
+The sole command line argument of these python scripts is the path of the result. More parameters can be used in the python script by importing the `SimConfig` object of the configuration via json. Thus the top of these scripts should be like
+
+```python
+from hercules import SimConfig
+
+path = sys.argv[1]
+config = SimConfig.from_json(path + '/SimConfig.json').to_dict()
+#get pitch angle
+pitch = config['kass-config']['theta_min']
+
+```
+
+Finally it is important to mention that the use of Kassiopeia and Locust is optional and can both be turned off as seen below. It depends on the configuration if it makes sense to run like that.
+
+```python
+sim = KassLocustP3('/path/to/your/workingDir', use_kass=True, use_locust=False, python_script='run_no_locust.py')
+```
+
+Without both Locust and Kassiopeia hercules turns into a simple convenience tool for running python scripts on a parameter grid on the grace cluster with the `hercules.Dataset` as output. In this case the simpler and more flexible `SimpleSimConfig` can be used. It supports any configuration parameters as passed via keyword arguments, which can be passed to the script. Together with the metadata (`info`) which is added on creation of the `ConfigList` they will be represented in the final `Dataset`. Example:
+
+```python
+sim = KassLocustP3('/path/to/your/workingDir', use_kass=False, use_locust=False, python_script='run_no_kass_no_locust.py')
+configlist = ConfigList(info='Additional meta data')
+config = SimpleSimConfig(x=2., some_exotic_data_name='interesting_value')
+configlist.add_config(config)
+sim(configlist)
+```
+
+Corresponding script head of `run_no_kass_no_locust.py`:
+
+```python
+from hercules import SimpleSimConfig
+
+path = sys.argv[1]
+config = SimpleSimConfig.from_json(path + '/SimConfig.json').to_dict()
+
+#get config parameters
+x = config['x']
+some_exotic_data_name = config['some_exotic_data_name']
+```
+
+## Running on grace cluster
+
+For running on the grace cluster there are a couple of extra keyword arguments for the `KassLocustP3` class.
+
+```python
+sim(config_list, memory='1000', timelimit='01:00:00', n_cpus=8, batch_size=3)
+```
+
+Setting `timelimit` and `memory` only as high as required for the job is good practice and should theoretically help with job scheduling. `batch_size` determines how many entries in `config_list` are combined into a single job, you need to make sure to have a `batch_size` that gets you job run times >10 minutes since the cluster is not well suited for high job throughput. `n_cpus` defaults to 2 for the use with Locust and Kass+Locust alone does not profit from using more than that. Setting it to higher values is only useful if your postprocessing python script uses muliple processes.
 
 ## Tests
 
-To test whether Docker is working on desktop, the `test_eggreader.py` provides a separate test. Run the following in cmd line:
+To test whether Docker is working on your device, the unittest `test_locustP2.py` can be used. Run the following in cmd line:
 
 ```sh
 cd ./test
-python -m unittest test_eggreader.EggReaderTest.test_locust
+python -m unittest test_locustP2.py
 ```

From d0b098a1b70c59b6c430328ef110b92829281e4b Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 7 Aug 2023 20:37:04 +0200
Subject: [PATCH 43/44] Update version

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 4f85a08..35f5a22 100644
--- a/setup.py
+++ b/setup.py
@@ -19,7 +19,7 @@
 
 setuptools.setup(
     name="hercules",
-    version="0.4.3",
+    version="0.5.0",
     author="Florian Thomas, Mingyu (Charles) Li",
     author_email="fthomas@uni-mainz.de, mingyuli@mit.edu",
     description="https://github.com/project8/hercules",

From fc945ce21d3c19107fc5be7a81cb6882a43e50d3 Mon Sep 17 00:00:00 2001
From: Florian Thomas <fthomas@uni-mainz.de>
Date: Mon, 7 Aug 2023 21:00:10 +0200
Subject: [PATCH 44/44] Add dill to list of required packages

---
 setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 35f5a22..f64f2e4 100644
--- a/setup.py
+++ b/setup.py
@@ -14,7 +14,8 @@
     "h5py",
     "numpy",
     "scipy",
-    "tqdm"
+    "tqdm",
+    "dill"
 ]
 
 setuptools.setup(