Merge branch 'main' of https://github.com/XENONnT/saltax into g4_fuse…

…_input

.pre-commit-config.yaml

@@ -10,7 +10,7 @@ repos:
     -   id: check-added-large-files
 
 -   repo: https://github.com/psf/black
-    rev: 24.4.0
+    rev: 24.4.2
     hooks:
     -   id: black
         args: [--safe, --line-length=100, --preview]

jobs/README.md

@@ -10,6 +10,15 @@ Update `config.ini` and run this in a container
 ```
 python submit.py
 ```
+
+## Logging
+When submitting jobs, you might want to have a bit more information for debugging. To do that, please do this before job submission:
+- Copy the default xenon config to your home or somewhere safe `/project2/lgrandi/xenonnt/xenon.config`
+- Keep everything else the same, and then modify the logging level there by `logging_level=debug` (or `info` should also work). See [here](https://github.com/XENONnT/utilix/blob/b94ef41851e437efa35ae9dc82c6fcdfca77b88c/utilix/config.py#L95) for details.
+- Then export the config by `export XENON_CONFIG=<YOUR_DIR_TO_NEW_CONFIG>`.
+- Submit jobs as usual.
+
+
 ## Tips
 - You need to download yourself `raw_records` and `raw_records_aqmon` before submission!
 - Unless you are working on AmBe, put `mem_per_cpu = 45000` (MB) should be enough. Otherwise please do `55000`.

jobs/config.ini

@@ -4,16 +4,16 @@ t_sleep = 1
 
 [job]
 container = xenonnt-montecarlo-development.simg
-runids = 48692,48693,48699,49028,49077,49079,49080,49432,49433
-output_folder = /scratch/midway2/yuanlq/salt/rn220
+runids = 51938,51914,51907,51905,51908,51904,51906,51909,51911,51913,51939,51910
+output_folder = /scratch/midway2/yuanlq/download/ambe
 package = fuse
 saltax_mode = salt
 simu_config_version = sr1_dev
 generator_name = flat
-recoil = 8
+recoil = 0
 simu_mode = all
 rate = 100
-en_range = 0.2,15
+en_range = 0.2,45
 process_data = False
 process_simu = True
 skip_records = False
@@ -24,8 +24,8 @@ storage_to_patch = /project/lgrandi/yuanlq/salt/raw_records,/scratch/midway2/yua
 username = yuanlq
 account = pi-lgrandi
 job_title = sprinkling
-partition = lgrandi
-qos = lgrandi
+partition = broadwl
+qos = broadwl
 mem_per_cpu = 45000
 cpus_per_task = 1
-log_dir = /home/yuanlq/.tmp_job_submission/saltax/rn220
+log_dir = /home/yuanlq/.tmp_job_submission/saltax/ambe

jobs/job.py

@@ -28,7 +28,7 @@
     "event_ambience",
     "event_n_channel",
     "veto_intervals",
-    "cuts_basic",
+#    "cuts_basic",
 ]
 TO_PROCESS_DTYPES_SE = [
     "peaklets",

saltax/__init__.py

@@ -1,3 +1,5 @@
+__version__ = "0.1.5"
+
 from . import instructions
 from .instructions import *
 

saltax/contexts.py

@@ -9,7 +9,7 @@
 from utilix import xent_collection
 
 
-logging.basicConfig(handlers=[logging.StreamHandler()])
+logging.basicConfig(level=logging.INFO, handlers=[logging.StreamHandler()])
 log = logging.getLogger("fuse.context")
 
 
@@ -49,7 +49,6 @@
 # straxen XENONnT options/configuration
 XNT_COMMON_OPTS = straxen.contexts.xnt_common_opts.copy()
 XNT_COMMON_CONFIG = straxen.contexts.xnt_common_config.copy()
-XNT_SIMULATION_CONFIG = straxen.contexts.xnt_simulation_config.copy()
 
 # wfsim based saltax options overrides
 SXNT_COMMON_OPTS_REGISTER = XNT_COMMON_OPTS["register"].copy()
@@ -138,7 +137,6 @@ def xenonnt_salted_fuse(
     saltax_mode="salt",
     output_folder="./fuse_data",
     cut_list=cutax.BasicCuts,
-    auto_register_cuts=True,
     corrections_version=DEFAULT_XEDOCS_VERSION,
     simu_config_version="sr1_dev",
     run_id_specific_config={
@@ -164,8 +162,6 @@ def xenonnt_salted_fuse(
     :param corrections_version: XENONnT documentation version to use,
         defaults to DEFAULT_XEDOCS_VERSION
     :param cut_list: Cut list to use, defaults to cutax.BasicCuts
-    :param auto_register_cuts: Whether to automatically register cuts,
-        defaults to True
     :param simu_config_version: simulation configuration version to use,
         defaults to "sr1_dev"
     :param run_id_specific_config: Mapping of run_id specific config
@@ -232,8 +228,6 @@ def xenonnt_salted_fuse(
     st.set_config(dict(saltax_mode=saltax_mode))
 
     # Register cuts plugins
-    if auto_register_cuts:
-        st.register_cuts()
     if cut_list is not None:
         st.register_cut_list(cut_list)
 
@@ -253,7 +247,7 @@ def xenonnt_salted_fuse(
             log.info("Loaded instructions from file", instr_file_name)
         except:
             log.info(f"Instruction file {instr_file_name} not found. Generating instructions...")
-            instr = generator_func(runid=runid, **kwargs)
+            instr = generator_func(runid=runid, recoil=recoil, **kwargs)
             pd.DataFrame(instr).to_csv(instr_file_name, index=False)
             log.info(f"Instructions saved to {instr_file_name}")
 
@@ -275,7 +269,6 @@ def xenonnt_salted_wfsim(
     output_folder="./strax_data",
     corrections_version=DEFAULT_XEDOCS_VERSION,
     cut_list=cutax.BasicCuts,
-    auto_register_cuts=True,
     simu_config_version="sr0_v4",
     cmt_version="global_v9",
     cmt_run_id="026000",
@@ -295,8 +288,6 @@ def xenonnt_salted_wfsim(
     :param corrections_version: XENONnT documentation version to use,
         defaults to DEFAULT_XEDOCS_VERSION
     :param cut_list: Cut list to use, defaults to cutax.BasicCuts
-    :param auto_register_cuts: Whether to automatically register cuts,
-        defaults to True
     :param simu_config_version: (for simulation) fax configuration
         version to use, defaults to "sr0_v4"
     :param cmt_version: (for simulation) CMT version to use, defaults to
@@ -365,8 +356,6 @@ def xenonnt_salted_wfsim(
             "g2": "bodega://g2?bodega_version=v5",
         }
     )
-    if auto_register_cuts:
-        st.register_cuts()
     if cut_list is not None:
         st.register_cut_list(cut_list)
 
@@ -425,7 +414,6 @@ def fxenonnt(
     saltax_mode="salt",
     output_folder="./fuse_data",
     cut_list=cutax.BasicCuts,
-    auto_register_cuts=True,
     corrections_version=DEFAULT_XEDOCS_VERSION,
     simu_config_version="sr1_dev",
     run_id_specific_config={
@@ -453,8 +441,6 @@ def fxenonnt(
         with cutax latest
     :param cut_list: List of cuts to register, default is
         cutax.BasicCuts
-    :param auto_register_cuts: Whether to automatically register cuts,
-        defaults to True
     :param simu_config_version: fax config version to use, default is
         synced with cutax latest
     :param run_id_specific_config: Mapping of run_id specific config
@@ -482,7 +468,6 @@ def fxenonnt(
         saltax_mode=saltax_mode,
         output_folder=output_folder,
         corrections_version=corrections_version,
-        auto_register_cuts=auto_register_cuts,
         cut_list=cut_list,
         simu_config_version=simu_config_version,
         run_id_specific_config=run_id_specific_config,
@@ -500,7 +485,6 @@ def sxenonnt(
     output_folder="./strax_data",
     corrections_version=DEFAULT_XEDOCS_VERSION,
     cut_list=cutax.BasicCuts,
-    auto_register_cuts=True,
     simu_config_version="sr0_v4",
     cmt_version="global_v9",
     cmt_run_id="026000",
@@ -522,8 +506,6 @@ def sxenonnt(
         with cutax latest
     :param cut_list: List of cuts to register, default is
         cutax.BasicCuts
-    :param auto_register_cuts: Whether to auto register cuts, default
-        True
     :param simu_config_version: fax config version to use, default is
         synced with cutax latest
     :param cmt_version: cmt version to use, default is synced with cutax
@@ -552,7 +534,6 @@ def sxenonnt(
         output_folder=output_folder,
         corrections_version=corrections_version,
         cut_list=cut_list,
-        auto_register_cuts=auto_register_cuts,
         simu_config_version=simu_config_version,
         cmt_version=cmt_version,
         cmt_run_id=cmt_run_id,

saltax/instructions/generator.py

@@ -24,11 +24,11 @@
     method="RegularGridInterpolator",
 )
 SE_INSTRUCTIONS_DIR = "/project/lgrandi/yuanlq/salt/se_instructions/"
+YBE_INSTRUCTIONS_FILE = "/project2/lgrandi/ghusheng/ybe_instrutions/ybe_wfsim_instructions_6806_events_time_modified.csv"
 #AMBE_INSTRUCTIONS_FILE = "/project/lgrandi/yuanlq/salt/ambe_instructions/minghao_aptinput.csv"
 AMBE_INSTRUCTIONS_FILE = "/project2/lgrandi/jjakob/AmBeSprinkling/AmBe_fuse_input.csv"
 #AMBE_INSTRUCTIONS_TYPE = "wfsim" 
 AMBE_INSTRUCTIONS_TYPE = "fuse"
-# BASE_DIR = "/project2/lgrandi/yuanlq/shared/saltax_instr/"
 BASE_DIR = os.path.abspath(__file__)[:-12] + "../../generated/"
 
 FUSE_DTYPE = [
@@ -438,6 +438,67 @@ def generator_ambe(
     return instr
 
 
+def generator_ybe(
+    runid,
+    n_tot=None,
+    rate=1e9 / SALT_TIME_INTERVAL,
+    time_mode="uniform",
+    ybe_instructions_file=YBE_INSTRUCTIONS_FILE,
+    **kwargs
+):
+    """Generate instructions for a run with YBe source.
+
+    YBe instruction was first generated by full-chain simulation, and
+    then passing the post-epix instruction to feed this function. Each
+    event with a certain event_id in the fed instructions will be
+    shifted in time based on the time_mode you specified.
+    :param runid: run number in integer
+    :param n_tot: total number of events to generate, default: None i.e.
+        generate events until end_time
+    :param rate: rate of events in Hz, default: 1e9/SALT_TIME_INTERVAL
+    :param time_mode: 'uniform' or 'realistic', default: 'uniform'
+    :param ybe_instructions_file: file containing ybe instructions,
+        default: YBE_INSTRUCTIONS_FILE
+    :return: instructions in numpy array
+    """
+    # determine time offsets to shift ybe instructions
+    start_time, end_time = get_run_start_end(runid)
+    times_offset = generate_times(start_time, end_time, size=n_tot, rate=rate, time_mode=time_mode)
+    n_tot = len(times_offset)
+
+    # bootstrap instructions
+    ybe_instructions = pd.read_csv(ybe_instructions_file)
+    ybe_event_numbers = np.random.choice(
+        np.unique(ybe_instructions.event_number), n_tot, replace=True
+    )
+
+    # assign instructions
+    instr = np.zeros(0, dtype=wfsim.instruction_dtype)
+    for i in tqdm(range(n_tot)):
+        # bootstrapped ybe instruction
+        selected_ybe = ybe_instructions[ybe_instructions["event_number"] == ybe_event_numbers[i]]
+        # instruction for i-th event
+        instr_i = np.zeros(len(selected_ybe), dtype=wfsim.instruction_dtype)
+        instr_i["time"] = times_offset[i] + selected_ybe["time"]
+        instr_i["event_number"] = i + 1
+        instr_i["type"] = selected_ybe["type"]
+        instr_i["x"] = selected_ybe["x"]
+        instr_i["y"] = selected_ybe["y"]
+        instr_i["z"] = selected_ybe["z"]
+        instr_i["recoil"] = selected_ybe["recoil"]
+        instr_i["e_dep"] = selected_ybe["e_dep"]
+        instr_i["amp"] = selected_ybe["amp"]
+        instr_i["n_excitons"] = selected_ybe["n_excitons"]
+
+        # concatenate instr
+        instr = np.concatenate((instr, instr_i))
+
+    # Filter out 0 amplitudes
+    instr = instr[instr["amp"] > 0]
+
+    return instr
+
+
 def generator_flat(
     runid,
     en_range=DEFAULT_EN_RANGE,