From af1172d8ae0e6535370dace954f66d5a1cbe98b1 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Thu, 18 Apr 2024 13:32:01 -0500 Subject: [PATCH 01/37] renamed energy deposit --- saltax/instructions/generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/saltax/instructions/generator.py b/saltax/instructions/generator.py index ff28d83..ff2a434 100644 --- a/saltax/instructions/generator.py +++ b/saltax/instructions/generator.py @@ -354,6 +354,7 @@ def generator_ambe( instr_i["z"] = selected_ambe["z"] instr_i["recoil"] = selected_ambe["recoil"] instr_i["e_dep"] = selected_ambe["e_dep"] + instr_i["ed"] = selected_ambe["e_dep"] instr_i["amp"] = selected_ambe["amp"] instr_i["n_excitons"] = selected_ambe["n_excitons"] @@ -416,6 +417,7 @@ def generator_flat( ens = np.random.uniform(en_range[0], en_range[1], size=n_tot) instr["recoil"][:] = recoil instr["e_dep"][:] = ens.repeat(2) + instr["ed"][:] = ens.repeat(2) # Getting local field from field map instr["local_field"] = fmap(np.array([np.sqrt(x**2 + y**2), z]).T).repeat(2) From e674b7459bfaf2bc00bc54218e769f2b7a280cdb Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Thu, 18 Apr 2024 13:38:51 -0500 Subject: [PATCH 02/37] revert; it's wrong --- saltax/instructions/generator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/saltax/instructions/generator.py b/saltax/instructions/generator.py index ff2a434..ff28d83 100644 --- a/saltax/instructions/generator.py +++ b/saltax/instructions/generator.py @@ -354,7 +354,6 @@ def generator_ambe( instr_i["z"] = selected_ambe["z"] instr_i["recoil"] = selected_ambe["recoil"] instr_i["e_dep"] = selected_ambe["e_dep"] - instr_i["ed"] = selected_ambe["e_dep"] instr_i["amp"] = selected_ambe["amp"] instr_i["n_excitons"] = selected_ambe["n_excitons"] @@ -417,7 +416,6 @@ def generator_flat( ens = np.random.uniform(en_range[0], en_range[1], size=n_tot) instr["recoil"][:] = recoil instr["e_dep"][:] = ens.repeat(2) - instr["ed"][:] = ens.repeat(2) # Getting local field from field map instr["local_field"] = fmap(np.array([np.sqrt(x**2 + y**2), z]).T).repeat(2) From 6800f92176830a8182d8f47d8813756dfae65d5a Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Thu, 18 Apr 2024 13:51:09 -0500 Subject: [PATCH 03/37] renamed records and peaklets --- saltax/plugins/README.md | 4 ++-- saltax/plugins/__init__.py | 8 ++++---- saltax/plugins/{s_peaklets.py => peaklets.py} | 2 +- saltax/plugins/{s_records.py => records.py} | 0 4 files changed, 7 insertions(+), 7 deletions(-) rename saltax/plugins/{s_peaklets.py => peaklets.py} (99%) rename saltax/plugins/{s_records.py => records.py} (100%) diff --git a/saltax/plugins/README.md b/saltax/plugins/README.md index 89adcf3..49fee44 100644 --- a/saltax/plugins/README.md +++ b/saltax/plugins/README.md @@ -3,8 +3,8 @@ Modified `straxen` [data structure](https://straxen.readthedocs.io/en/latest/reference/datastructure_nT.html) up to `peaklets`. Everything above is designed to be exactly the same as `straxen`. ## Structure - `s_raw_records.py`: Modified `raw_records` specifically for simulated `raw_records`, which determine chunking time range based on the ones' from `raw_reocrds` -- `s_records.py`: Modified `records`, which combines `raw_reords` and `raw_records_simu` together. The latter one's channel starts at constant variable `SCHANNEL_STARTS_AT`. -- `s_peaklets.py`: Modified `peaklets`, which build peaks as there are `2*n_tpc_pmts` channels, but sum up per-channel information (`area_per_channel` and top-bottom specific fields) as if there are only `n_tpc_pmts`. +- `records.py`: Modified `records`, which combines `raw_reords` and `raw_records_simu` together. The latter one's channel starts at constant variable `SCHANNEL_STARTS_AT`. +- `peaklets.py`: Modified `peaklets`, which build peaks as there are `2*n_tpc_pmts` channels, but sum up per-channel information (`area_per_channel` and top-bottom specific fields) as if there are only `n_tpc_pmts`. ## Data Structure image diff --git a/saltax/plugins/__init__.py b/saltax/plugins/__init__.py index 0985e22..cdb0630 100644 --- a/saltax/plugins/__init__.py +++ b/saltax/plugins/__init__.py @@ -1,8 +1,8 @@ from . import s_raw_records from .s_raw_records import * -from . import s_records -from .s_records import * +from . import records +from .records import * -from . import s_peaklets -from .s_peaklets import * +from . import peaklets +from .peaklets import * diff --git a/saltax/plugins/s_peaklets.py b/saltax/plugins/peaklets.py similarity index 99% rename from saltax/plugins/s_peaklets.py rename to saltax/plugins/peaklets.py index 544d309..4f6db9a 100644 --- a/saltax/plugins/s_peaklets.py +++ b/saltax/plugins/peaklets.py @@ -6,7 +6,7 @@ from immutabledict import immutabledict from strax.processing.general import _touching_windows from strax.dtypes import DIGITAL_SUM_WAVEFORM_CHANNEL -from .s_records import SCHANNEL_STARTS_AT +from .records import SCHANNEL_STARTS_AT import straxen from straxen.plugins.peaklets.peaklets import hit_max_sample, get_tight_coin, drop_data_top_field diff --git a/saltax/plugins/s_records.py b/saltax/plugins/records.py similarity index 100% rename from saltax/plugins/s_records.py rename to saltax/plugins/records.py From 1b2dd2bd67fdd7488b23ab099c279a48169f9f94 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 13:53:56 -0500 Subject: [PATCH 04/37] scratch --- saltax/plugins/f_raw_records.py | 196 ++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 saltax/plugins/f_raw_records.py diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py new file mode 100644 index 0000000..a1b3e41 --- /dev/null +++ b/saltax/plugins/f_raw_records.py @@ -0,0 +1,196 @@ +import logging +from typing import Tuple + +import numpy as np +import pandas as pd +import strax +import straxen + +from fuse.plugin import FuseBasePlugin +from .s_raw_records import NS_NO_INSTRUCTION_BEFORE_CHUNK_END, NS_NO_INSTRUCTION_AFTER_CHUNK_START + +export, __all__ = strax.exporter() + +logging.basicConfig(handlers=[logging.StreamHandler()]) +log = logging.getLogger("fuse.detector_physics.csv_input") + + +@export +class SChunkCsvInput(FuseBasePlugin): + """Plugin which reads a CSV file containing instructions for the detector + physics simulation (in wfsim format) and returns the data in chunks.""" + + __version__ = "0.0.0" + + depends_on: Tuple = tuple("raw_records") + provides = "microphysics_summary" + data_kind = "interactions_in_roi" + + save_when = strax.SaveWhen.TARGET + + source_done = False + + dtype = [ + (("x position of the cluster [cm]", "x"), np.float32), + (("y position of the cluster [cm]", "y"), np.float32), + (("z position of the cluster [cm]", "z"), np.float32), + (("Number of photons at interaction position", "photons"), np.int32), + (("Number of electrons at interaction position", "electrons"), np.int32), + (("Number of excitons at interaction position", "excitons"), np.int32), + (("Electric field value at the cluster position [V/cm]", "e_field"), np.float32), + (("Energy of the cluster [keV]", "ed"), np.float32), + (("NEST interaction type", "nestid"), np.int8), + (("ID of the cluster", "cluster_id"), np.int32), + ] + dtype = dtype + strax.time_fields + + # Config options + input_file = straxen.URLConfig( + track=False, + infer_type=False, + help="CSV file (in wfsim format) to read.", + ) + ns_no_instruction_after_chunk_start = straxen.URLConfig( + default=5e7, + track=False, + type=(int), + help="No instruction this amount of time after a chunk starts will be used, " + "as a safeguard for not getting raw_records_simu out of raw_records chunk time range. " + ) + ns_no_instruction_before_chunk_end = straxen.URLConfig( + default=5e7, + track=False, + type=(int), + help="No instruction this amount of time before a chunk ends will be used, " + "as a safeguard for not getting raw_records_simu out of raw_records chunk time range. " + ) + + + def setup(self): + super().setup() + self.csv_file_reader = SCsvFileLoader( + input_file=self.input_file, + random_number_generator=self.rng, + ns_no_instruction_before_chunk_end=self.ns_no_instruction_before_chunk_end, + ns_no_instruction_after_chunk_start=self.ns_no_instruction_after_chunk_start, + debug=self.debug, + ) + + def compute(self, raw_records, start, end): + try: + chunk_data = self.csv_file_reader(start, end) + chunk_data["endtime"] = self.chunk_data["time"] + data = np.zeros(len(chunk_data), dtype=self.dtype) + strax.copy_to_buffer(self.chunk_data, data, "_bring_data_into_correct_format") + + self.source_done = True + + # Stick rigorously with raw_records time range + return self.chunk( + start=start, end=end, data=data, data_type="geant4_interactions" + ) + + except StopIteration: + raise RuntimeError("Bug in chunk building!") + + def source_finished(self): + return self.source_done + + def is_ready(self, chunk_i): + """Overwritten to mimic online input plugin. + + Returns False to check source finished; Returns True to get next + chunk. + """ + if "ready" not in self.__dict__: + self.ready = False + self.ready ^= True # Flip + return self.ready + + +class SCsvFileLoader: + """Class to load a CSV file (in wfsim format) with detector simulation instructions""" + def __init__( + self, + input_file, + random_number_generator, + ns_no_instruction_before_chunk_end=NS_NO_INSTRUCTION_BEFORE_CHUNK_END, + ns_no_instruction_after_chunk_start=NS_NO_INSTRUCTION_AFTER_CHUNK_START, + debug=False, + ): + self.input_file = input_file + self.rng = random_number_generator + self.ns_no_instruction_before_chunk_end = ns_no_instruction_before_chunk_end + self.ns_no_instruction_after_chunk_start = ns_no_instruction_after_chunk_start + self.debug = debug + + self.dtype = [ + (("x position of the cluster [cm]", "x"), np.float32), + (("y position of the cluster [cm]", "y"), np.float32), + (("z position of the cluster [cm]", "z"), np.float32), + (("Number of photons at interaction position", "photons"), np.int32), + (("Number of electrons at interaction position", "electrons"), np.int32), + (("Number of excitons at interaction position", "excitons"), np.int32), + (("Electric field value at the cluster position [V/cm]", "e_field"), np.float32), + (("Energy of the cluster [keV]", "ed"), np.float32), + (("NEST interaction type", "nestid"), np.int8), + (("ID of the cluster", "cluster_id"), np.int32), + (("Time of the interaction", "t"), np.int64), + (("Geant4 event ID", "eventid"), np.int32), + ] + self.dtype = self.dtype + strax.time_fields + + # The csv file needs to have these columns: + self.columns = [ + "x", + "y", + "z", + "photons", + "electrons", + "excitons", + "e_field", + "ed", + "nestid", + "t", + "eventid", + "cluster_id", + ] + + def output_chunk(self, chunk_start, chunk_end): + # Load the csv file in wfsim format + log.debug("Loaded detector simulation instructions from a csv file in wfsim format!") + instructions = self._load_csv_file() + + # Translate the wfsim instructions to the fuse format + log.debug("Translating the wfsim instructions to the fuse format!") + instructions = self.translate_wfsim_to_fuse(instructions) + + # truncate instructions to the chunk time range + log.debug("Truncating instructions to the chunk time range!") + log.debug("We will further truncate the instructions to the range [%d, %d]", + chunk_start + self.ns_no_instruction_after_chunk_start, + chunk_end - self.ns_no_instruction_before_chunk_end) + mask = (instructions["t"] >= chunk_start + self.ns_no_instruction_after_chunk_start) + mask &= (instructions["t"] < chunk_end - self.ns_no_instruction_before_chunk_end) + instructions = instructions[mask] + + return instructions + + + def _load_csv_file(self): + log.debug("Loading detector simulation instructions from a csv file in wfsim format!") + df = pd.read_csv(self.input_file) + + # Check if all needed columns are in place: + if not set(self.columns).issubset(df.columns): + log.warning("Not all needed columns provided!") + + instructions = np.zeros(len(df), dtype=self.dtype) + for column in df.columns: + instructions[column] = df[column] + + return instructions + + @staticmethod + def translate_wfsim_to_fuse(instructions): + """Translate the wfsim instructions to the fuse format""" From b2b2ff064e3eb56e732d05a906aa368f9e91f9ce Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 14:11:26 -0500 Subject: [PATCH 05/37] translator from scratch --- saltax/plugins/f_raw_records.py | 44 ++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index a1b3e41..6855887 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -18,7 +18,10 @@ @export class SChunkCsvInput(FuseBasePlugin): """Plugin which reads a CSV file containing instructions for the detector - physics simulation (in wfsim format) and returns the data in chunks.""" + physics simulation (in wfsim format) and returns the data in chunks. + Modified from csv_file_loader: https://github.com/XENONnT/fuse/blob/e538d32a5a0735757a77b1dce31d6f95a379bf4e/fuse/plugins/detector_physics/csv_input.py#L118 + Similar to the case event_rate=0 there: we don't reassign times. + """ __version__ = "0.0.0" @@ -79,6 +82,7 @@ def setup(self): def compute(self, raw_records, start, end): try: chunk_data = self.csv_file_reader(start, end) + chunk_data["time"] = chunk_data["t"] chunk_data["endtime"] = self.chunk_data["time"] data = np.zeros(len(chunk_data), dtype=self.dtype) strax.copy_to_buffer(self.chunk_data, data, "_bring_data_into_correct_format") @@ -156,6 +160,9 @@ def __init__( "cluster_id", ] + # Translator to translate the wfsim instructions to the fuse format + self.translator = InstrTranslator(input_format="wfsim", output_format="fuse") + def output_chunk(self, chunk_start, chunk_end): # Load the csv file in wfsim format log.debug("Loaded detector simulation instructions from a csv file in wfsim format!") @@ -163,7 +170,7 @@ def output_chunk(self, chunk_start, chunk_end): # Translate the wfsim instructions to the fuse format log.debug("Translating the wfsim instructions to the fuse format!") - instructions = self.translate_wfsim_to_fuse(instructions) + instructions = self.translator.translate(instructions) # truncate instructions to the chunk time range log.debug("Truncating instructions to the chunk time range!") @@ -176,7 +183,6 @@ def output_chunk(self, chunk_start, chunk_end): return instructions - def _load_csv_file(self): log.debug("Loading detector simulation instructions from a csv file in wfsim format!") df = pd.read_csv(self.input_file) @@ -192,5 +198,35 @@ def _load_csv_file(self): return instructions @staticmethod - def translate_wfsim_to_fuse(instructions): + def _translate_wfsim_to_fuse(instructions): """Translate the wfsim instructions to the fuse format""" + + +class InstrTranslator: + """Class to translate instructions between wfsim and fuse formats""" + def __init__(self, input_format="wfsim", output_format="fuse"): + self.input_format = input_format + self.output_format = output_format + + assert self.input_format in ["wfsim", "fuse"], "Unknown input format! Choose 'wfsim' or 'fuse'!" + assert self.output_format in ["wfsim", "fuse"], "Unknown output format! Choose 'wfsim' or 'fuse'!" + + self.translator = self.translator() + + def translator(self): + if self.input_format == "wfsim" and self.output_format == "fuse": + return self.translate_wfsim_to_fuse + elif self.input_format == "fuse" and self.output_format == "wfsim": + return self.translate_fuse_to_wfsim + else: + raise NotImplementedError("Translation from {} to {} is not implemented yet!".format(self.input_format, self.output_format)) + + def translate(self, instructions): + return self.translator(instructions) + + def translate_wfsim_to_fuse(self, instructions): + """Translate the wfsim instructions to the fuse format""" + + def translate_fuse_to_wfsim(self, instructions): + """Translate the fuse instructions to the wfsim format""" + raise NotImplementedError("Not implemented yet!") \ No newline at end of file From 086fa3f9d00944368bfee71db9fd85933b6abe7d Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 14:15:48 -0500 Subject: [PATCH 06/37] removed redundant translator --- saltax/plugins/f_raw_records.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 6855887..aa15cb0 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -171,6 +171,7 @@ def output_chunk(self, chunk_start, chunk_end): # Translate the wfsim instructions to the fuse format log.debug("Translating the wfsim instructions to the fuse format!") instructions = self.translator.translate(instructions) + log.debug("Instructions translated to the fuse format!") # truncate instructions to the chunk time range log.debug("Truncating instructions to the chunk time range!") @@ -197,10 +198,6 @@ def _load_csv_file(self): return instructions - @staticmethod - def _translate_wfsim_to_fuse(instructions): - """Translate the wfsim instructions to the fuse format""" - class InstrTranslator: """Class to translate instructions between wfsim and fuse formats""" @@ -211,6 +208,7 @@ def __init__(self, input_format="wfsim", output_format="fuse"): assert self.input_format in ["wfsim", "fuse"], "Unknown input format! Choose 'wfsim' or 'fuse'!" assert self.output_format in ["wfsim", "fuse"], "Unknown output format! Choose 'wfsim' or 'fuse'!" + log.debug("Translating instructions from %s to %s", self.input_format, self.output_format) self.translator = self.translator() def translator(self): From 13086011bf1f05f85039a18052fdae58f5c11ee9 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 14:19:03 -0500 Subject: [PATCH 07/37] debug indent --- saltax/plugins/f_raw_records.py | 69 +++++++++++++++++---------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index aa15cb0..7b3e640 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -163,40 +163,40 @@ def __init__( # Translator to translate the wfsim instructions to the fuse format self.translator = InstrTranslator(input_format="wfsim", output_format="fuse") - def output_chunk(self, chunk_start, chunk_end): - # Load the csv file in wfsim format - log.debug("Loaded detector simulation instructions from a csv file in wfsim format!") - instructions = self._load_csv_file() - - # Translate the wfsim instructions to the fuse format - log.debug("Translating the wfsim instructions to the fuse format!") - instructions = self.translator.translate(instructions) - log.debug("Instructions translated to the fuse format!") - - # truncate instructions to the chunk time range - log.debug("Truncating instructions to the chunk time range!") - log.debug("We will further truncate the instructions to the range [%d, %d]", - chunk_start + self.ns_no_instruction_after_chunk_start, - chunk_end - self.ns_no_instruction_before_chunk_end) - mask = (instructions["t"] >= chunk_start + self.ns_no_instruction_after_chunk_start) - mask &= (instructions["t"] < chunk_end - self.ns_no_instruction_before_chunk_end) - instructions = instructions[mask] - - return instructions - - def _load_csv_file(self): - log.debug("Loading detector simulation instructions from a csv file in wfsim format!") - df = pd.read_csv(self.input_file) - - # Check if all needed columns are in place: - if not set(self.columns).issubset(df.columns): - log.warning("Not all needed columns provided!") - - instructions = np.zeros(len(df), dtype=self.dtype) - for column in df.columns: - instructions[column] = df[column] - - return instructions + def output_chunk(self, chunk_start, chunk_end): + # Load the csv file in wfsim format + log.debug("Loaded detector simulation instructions from a csv file in wfsim format!") + instructions = self._load_csv_file() + + # Translate the wfsim instructions to the fuse format + log.debug("Translating the wfsim instructions to the fuse format!") + instructions = self.translator.translate(instructions) + log.debug("Instructions translated to the fuse format!") + + # truncate instructions to the chunk time range + log.debug("Truncating instructions to the chunk time range!") + log.debug("We will further truncate the instructions to the range [%d, %d]", + chunk_start + self.ns_no_instruction_after_chunk_start, + chunk_end - self.ns_no_instruction_before_chunk_end) + mask = (instructions["t"] >= chunk_start + self.ns_no_instruction_after_chunk_start) + mask &= (instructions["t"] < chunk_end - self.ns_no_instruction_before_chunk_end) + instructions = instructions[mask] + + return instructions + + def _load_csv_file(self): + log.debug("Loading detector simulation instructions from a csv file in wfsim format!") + df = pd.read_csv(self.input_file) + + # Check if all needed columns are in place: + if not set(self.columns).issubset(df.columns): + log.warning("Not all needed columns provided!") + + instructions = np.zeros(len(df), dtype=self.dtype) + for column in df.columns: + instructions[column] = df[column] + + return instructions class InstrTranslator: @@ -224,6 +224,7 @@ def translate(self, instructions): def translate_wfsim_to_fuse(self, instructions): """Translate the wfsim instructions to the fuse format""" + def translate_fuse_to_wfsim(self, instructions): """Translate the fuse instructions to the wfsim format""" From 75df43f057a785020773688b3b9fe4b97a6a1bbf Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 15:28:09 -0500 Subject: [PATCH 08/37] draft for translator --- saltax/plugins/f_raw_records.py | 57 ++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 7b3e640..35cea7c 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -224,7 +224,62 @@ def translate(self, instructions): def translate_wfsim_to_fuse(self, instructions): """Translate the wfsim instructions to the fuse format""" - + # Sort time for sanity + instructions = instructions.sort_values(by="time") + + # Find cluster and events row by row + previous_time = 0 + previous_event_number = -1 + cluster_id = 0 + event_id = 0 + for i, row in instructions.iterrows(): + new_cluster = False + new_event = False + + # Check if we have a new cluster or event + if row["time"] > previous_time: + new_cluster = True + if row["event_number"] != previous_event_number: + new_event = True + if new_event and (not new_cluster): + raise ValueError("New event without new cluster at time %s!?"%(row['time'])) + + # Update the previous event number + if new_event: + event_id += 1 + + # Update the cluster as a new row + if new_cluster: + cluster_id += 1 + new_row = { + "x": row["x"], + "y": row["y"], + "z": row["z"], + "e_field": row["local_field"], + "ed": row["e_dep"], + "nestid": row["recoil"], + "t": row["time"], + "cluster_id": cluster_id, + "eventid": event_id, + } + last_row = new_row + + # Assign the number of photons, excitons and electrons to the last row + if row["type"] == 1: + last_row["photons"] = row["amp"] + last_row["excitons"] = row["n_excitons"] + elif row["type"] == 2: + last_row["electrons"] = row["amp"] + else: + raise ValueError("Unknown type %s!"%(row["type"])) + + # Concatenate the last row to the new instructions + if i==1: + translated = pd.DataFrame([last_row]) + else: + translated = pd.concat([translated, pd.DataFrame([last_row])]) + + return translated def translate_fuse_to_wfsim(self, instructions): """Translate the fuse instructions to the wfsim format""" From a2f7563c03d239ccd28eb0805cbc1e72f545fab2 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 15:33:03 -0500 Subject: [PATCH 09/37] faster --- saltax/plugins/f_raw_records.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 35cea7c..6b3a532 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -275,11 +275,11 @@ def translate_wfsim_to_fuse(self, instructions): # Concatenate the last row to the new instructions if i==1: - translated = pd.DataFrame([last_row]) + rows = [last_row] else: - translated = pd.concat([translated, pd.DataFrame([last_row])]) - - return translated + rows.append(last_row) + + return pd.DataFrame(rows) def translate_fuse_to_wfsim(self, instructions): """Translate the fuse instructions to the wfsim format""" From 34bda2e970638f3a0aad98082aa380ec1ef91fb6 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 15:34:29 -0500 Subject: [PATCH 10/37] debug --- saltax/plugins/f_raw_records.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 6b3a532..3edf7a5 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -276,7 +276,7 @@ def translate_wfsim_to_fuse(self, instructions): # Concatenate the last row to the new instructions if i==1: rows = [last_row] - else: + elif new_cluster: rows.append(last_row) return pd.DataFrame(rows) From 46cba941b792bca2f59450da3f7f9a66a9e9a9f7 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 15:39:24 -0500 Subject: [PATCH 11/37] debug --- saltax/plugins/f_raw_records.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 3edf7a5..09042de 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -243,6 +243,8 @@ def translate_wfsim_to_fuse(self, instructions): new_event = True if new_event and (not new_cluster): raise ValueError("New event without new cluster at time %s!?"%(row['time'])) + previous_time = row["time"] + previous_event_number = row["event_number"] # Update the previous event number if new_event: @@ -257,10 +259,10 @@ def translate_wfsim_to_fuse(self, instructions): "z": row["z"], "e_field": row["local_field"], "ed": row["e_dep"], - "nestid": row["recoil"], - "t": row["time"], - "cluster_id": cluster_id, - "eventid": event_id, + "nestid": int(row["recoil"]), + "t": int(row["time"]), + "cluster_id": int(cluster_id), + "eventid": int(event_id), } last_row = new_row From 6393aca9160828dd2b61100f6c7705311bc17bf7 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 15:59:18 -0500 Subject: [PATCH 12/37] fix types --- saltax/plugins/f_raw_records.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 09042de..eae255a 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -254,24 +254,24 @@ def translate_wfsim_to_fuse(self, instructions): if new_cluster: cluster_id += 1 new_row = { - "x": row["x"], - "y": row["y"], - "z": row["z"], - "e_field": row["local_field"], - "ed": row["e_dep"], - "nestid": int(row["recoil"]), - "t": int(row["time"]), - "cluster_id": int(cluster_id), - "eventid": int(event_id), + "x": np.float32(row["x"]), + "y": np.float32(row["y"]), + "z": np.float32(row["z"]), + "e_field": np.float32(row["local_field"]), + "ed": np.float32(row["e_dep"]), + "nestid": np.int8(row["recoil"]), + "t": np.int64(row["time"]), + "cluster_id": np.int32(cluster_id), + "eventid": np.int32(event_id), } last_row = new_row # Assign the number of photons, excitons and electrons to the last row if row["type"] == 1: - last_row["photons"] = row["amp"] - last_row["excitons"] = row["n_excitons"] + last_row["photons"] = np.int32(row["amp"]) + last_row["excitons"] = np.int32(row["n_excitons"]) elif row["type"] == 2: - last_row["electrons"] = row["amp"] + last_row["electrons"] = np.int32(row["amp"]) else: raise ValueError("Unknown type %s!"%(row["type"])) From 7ab28901c1acccdeb8eea4b7ebc35b4b7f26be4d Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 16:06:12 -0500 Subject: [PATCH 13/37] first row --- saltax/plugins/f_raw_records.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index eae255a..18b2ed7 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -232,7 +232,8 @@ def translate_wfsim_to_fuse(self, instructions): previous_event_number = -1 cluster_id = 0 event_id = 0 - for i, row in instructions.iterrows(): + first_row = True + for _, row in instructions.iterrows(): new_cluster = False new_event = False @@ -276,8 +277,9 @@ def translate_wfsim_to_fuse(self, instructions): raise ValueError("Unknown type %s!"%(row["type"])) # Concatenate the last row to the new instructions - if i==1: + if first_row: rows = [last_row] + first_row = False elif new_cluster: rows.append(last_row) From fb1bf3e67d7d27b3b2e40b4ba3ec09aabbb27e83 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 16:22:58 -0500 Subject: [PATCH 14/37] debug and acceleration --- saltax/plugins/f_raw_records.py | 38 ++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 18b2ed7..8d607b5 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -233,19 +233,19 @@ def translate_wfsim_to_fuse(self, instructions): cluster_id = 0 event_id = 0 first_row = True - for _, row in instructions.iterrows(): + for row in instructions.itertuples(): new_cluster = False new_event = False # Check if we have a new cluster or event - if row["time"] > previous_time: + if row.time > previous_time: new_cluster = True - if row["event_number"] != previous_event_number: + if row.event_number != previous_event_number: new_event = True if new_event and (not new_cluster): - raise ValueError("New event without new cluster at time %s!?"%(row['time'])) - previous_time = row["time"] - previous_event_number = row["event_number"] + raise ValueError("New event without new cluster at time %s!?"%(row.time)) + previous_time = row.time + previous_event_number = row.event_number # Update the previous event number if new_event: @@ -255,26 +255,26 @@ def translate_wfsim_to_fuse(self, instructions): if new_cluster: cluster_id += 1 new_row = { - "x": np.float32(row["x"]), - "y": np.float32(row["y"]), - "z": np.float32(row["z"]), - "e_field": np.float32(row["local_field"]), - "ed": np.float32(row["e_dep"]), - "nestid": np.int8(row["recoil"]), - "t": np.int64(row["time"]), + "x": row.x, + "y": row.y, + "z": row.z, + "e_field": row.local_field, + "ed": row.e_dep, + "nestid": row.recoil, + "t": row.time, "cluster_id": np.int32(cluster_id), "eventid": np.int32(event_id), } last_row = new_row # Assign the number of photons, excitons and electrons to the last row - if row["type"] == 1: - last_row["photons"] = np.int32(row["amp"]) - last_row["excitons"] = np.int32(row["n_excitons"]) - elif row["type"] == 2: - last_row["electrons"] = np.int32(row["amp"]) + if row.type == 1: + last_row["photons"] = np.int32(row.amp) + last_row["excitons"] = np.int32(row.n_excitons) + elif row.type == 2: + last_row["electrons"] = np.int32(row.amp) else: - raise ValueError("Unknown type %s!"%(row["type"])) + raise ValueError("Unknown type %s!"%(row.type)) # Concatenate the last row to the new instructions if first_row: From 964512ae9f5511cae7fa0156092d57158d762f2a Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 16:38:54 -0500 Subject: [PATCH 15/37] debug dtype --- saltax/plugins/f_raw_records.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 8d607b5..716b0ac 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -255,15 +255,18 @@ def translate_wfsim_to_fuse(self, instructions): if new_cluster: cluster_id += 1 new_row = { - "x": row.x, - "y": row.y, - "z": row.z, - "e_field": row.local_field, - "ed": row.e_dep, - "nestid": row.recoil, - "t": row.time, + "x": np.float32(row.x), + "y": np.float32(row.y), + "z": np.float32(row.z), + "e_field": np.float32(row.local_field), + "ed": np.float32(row.e_dep), + "nestid": np.int8(row.recoil), + "t": np.int64(row.time), "cluster_id": np.int32(cluster_id), "eventid": np.int32(event_id), + "photons": np.int32(0), + "electrons": np.int32(0), + "excitons": np.int32(0), } last_row = new_row From ecdfeaea28d1bcc537cc42b09f0fd0e358b87a00 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 17:16:47 -0500 Subject: [PATCH 16/37] avoid splitting --- saltax/contexts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/saltax/contexts.py b/saltax/contexts.py index 9cae0f9..61392bc 100644 --- a/saltax/contexts.py +++ b/saltax/contexts.py @@ -40,6 +40,9 @@ TRUTH_INFORMATION_PLUGINS, ] +# ~Infinite raw_records file size to avoid downchunking +MAX_RAW_RECORDS_FILE_SIZE_MB = 1e9 + # fuse placeholder parameters CORRECTION_RUN_ID_DEFAULT = "046477" @@ -233,7 +236,7 @@ def xenonnt_salted_fuse( st.set_config( { "input_file": instr_file_name, - # "n_interactions_per_chunk": 50, #TODO: Surgery needed here + "raw_records_file_size_target": MAX_RAW_RECORDS_FILE_SIZE_MB, } ) From aa1e4cd082dcc7e3e50465f65c05eb11a783fe01 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 17:19:16 -0500 Subject: [PATCH 17/37] init --- saltax/plugins/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/saltax/plugins/__init__.py b/saltax/plugins/__init__.py index cdb0630..50bab23 100644 --- a/saltax/plugins/__init__.py +++ b/saltax/plugins/__init__.py @@ -1,3 +1,6 @@ +from . import f_raw_records +from .f_raw_records import * + from . import s_raw_records from .s_raw_records import * From e336df3509b327a9ed1881c608e7cad56a4b4586 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 17:25:01 -0500 Subject: [PATCH 18/37] docstr --- saltax/plugins/f_raw_records.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 716b0ac..87fe6d4 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -164,6 +164,10 @@ def __init__( self.translator = InstrTranslator(input_format="wfsim", output_format="fuse") def output_chunk(self, chunk_start, chunk_end): + """ + Load the detector simulation instructions from the csv file in wfsim format and then + translate them to the fuse format. Truncate the instructions to the chunk time range. + """ # Load the csv file in wfsim format log.debug("Loaded detector simulation instructions from a csv file in wfsim format!") instructions = self._load_csv_file() From 78a936f486bd93eb54bff78b13050508813ed775 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 17:25:21 -0500 Subject: [PATCH 19/37] docstr --- saltax/plugins/f_raw_records.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 87fe6d4..df3612e 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -165,7 +165,7 @@ def __init__( def output_chunk(self, chunk_start, chunk_end): """ - Load the detector simulation instructions from the csv file in wfsim format and then + Load the simulation instructions from the csv file in wfsim format and then translate them to the fuse format. Truncate the instructions to the chunk time range. """ # Load the csv file in wfsim format @@ -189,6 +189,9 @@ def output_chunk(self, chunk_start, chunk_end): return instructions def _load_csv_file(self): + """ + Load the simulation instructions from a csv file in wfsim format. + """ log.debug("Loading detector simulation instructions from a csv file in wfsim format!") df = pd.read_csv(self.input_file) From 17241d23f41a9d758d4cebead5a4e0d4a71ebb30 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 17:45:15 -0500 Subject: [PATCH 20/37] debug --- saltax/plugins/f_raw_records.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index df3612e..576ad08 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -25,7 +25,7 @@ class SChunkCsvInput(FuseBasePlugin): __version__ = "0.0.0" - depends_on: Tuple = tuple("raw_records") + depends_on: Tuple = ("raw_records") provides = "microphysics_summary" data_kind = "interactions_in_roi" From 97a3f651ff0824bbf4c983738dd054a2bc795a60 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 17:56:39 -0500 Subject: [PATCH 21/37] naming bug --- saltax/contexts.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/saltax/contexts.py b/saltax/contexts.py index 61392bc..9ddfee2 100644 --- a/saltax/contexts.py +++ b/saltax/contexts.py @@ -58,11 +58,11 @@ saltax.SPulseProcessing, saltax.SRawRecordsFromFaxNT, ] + SXNT_COMMON_OPTS_REGISTER -XNT_COMMON_OPTS_OVERRIDE = dict( +SXNT_COMMON_OPTS_OVERRIDE = dict( register=SXNT_COMMON_OPTS_REGISTER, ) SXNT_COMMON_OPTS = XNT_COMMON_OPTS.copy() -SXNT_COMMON_OPTS["register"] = XNT_COMMON_OPTS_OVERRIDE["register"] +SXNT_COMMON_OPTS["register"] = SXNT_COMMON_OPTS_OVERRIDE["register"] # saltax configuration overrides @@ -239,6 +239,9 @@ def xenonnt_salted_fuse( "raw_records_file_size_target": MAX_RAW_RECORDS_FILE_SIZE_MB, } ) + + # register the csv input plugin + st.register(saltax.SChunkCsvInput) return st From d05b07d6ada3648dba6f4437fe1f940f3fbd8bfc Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:02:10 -0500 Subject: [PATCH 22/37] raw_records_simu --- saltax/plugins/f_raw_records.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 576ad08..ae21db8 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -7,6 +7,7 @@ import straxen from fuse.plugin import FuseBasePlugin +from fuse.plugins.pmt_and_daq.pmt_response_and_daq import PMTResponseAndDAQ from .s_raw_records import NS_NO_INSTRUCTION_BEFORE_CHUNK_END, NS_NO_INSTRUCTION_AFTER_CHUNK_START export, __all__ = strax.exporter() @@ -15,6 +16,12 @@ log = logging.getLogger("fuse.detector_physics.csv_input") +@export +class SPMTResponseAndDAQ(PMTResponseAndDAQ): + __version__ = "0.0.0" + provides = "raw_records_simu" + + @export class SChunkCsvInput(FuseBasePlugin): """Plugin which reads a CSV file containing instructions for the detector From dd9a8e3d92137b8e5885493bdf5f9d2b1c1ef620 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:04:42 -0500 Subject: [PATCH 23/37] raw_records_simu --- saltax/plugins/f_raw_records.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index ae21db8..6b0c651 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -20,7 +20,7 @@ class SPMTResponseAndDAQ(PMTResponseAndDAQ): __version__ = "0.0.0" provides = "raw_records_simu" - + data_kind = "raw_records_simu" @export class SChunkCsvInput(FuseBasePlugin): From a4c1e146712d8bd4376e499a2c2e8e938713b9be Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:17:51 -0500 Subject: [PATCH 24/37] sanity --- saltax/contexts.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/saltax/contexts.py b/saltax/contexts.py index 9ddfee2..0730f40 100644 --- a/saltax/contexts.py +++ b/saltax/contexts.py @@ -25,7 +25,7 @@ # Plugins to merge delayed and regular electrons DELAYED_ELECTRON_MERGER_PLUGINS = fuse.context.delayed_electron_merger_plugins # Plugins to simulate PMTs and DAQ -# TODO: this plugin fuse.pmt_and_daq.PMTResponseAndDAQ will be replaced +# NB: this plugin fuse.pmt_and_daq.PMTResponseAndDAQ will be replaced later in context PMT_AND_DAQ_PLUGINS = fuse.context.pmt_and_daq_plugins # Plugins to get truth information TRUTH_INFORMATION_PLUGINS = fuse.context.truth_information_plugins @@ -51,8 +51,9 @@ XNT_COMMON_CONFIG = straxen.contexts.xnt_common_config.copy() XNT_SIMULATION_CONFIG = straxen.contexts.xnt_simulation_config.copy() -# saltax options overrides +# wfsim based saltax options overrides SXNT_COMMON_OPTS_REGISTER = XNT_COMMON_OPTS["register"].copy() +SXNT_COMMON_OPTS_REGISTER.remove(straxen.Peaklets) SXNT_COMMON_OPTS_REGISTER.remove(straxen.PulseProcessing) SXNT_COMMON_OPTS_REGISTER = [ saltax.SPulseProcessing, @@ -64,6 +65,20 @@ SXNT_COMMON_OPTS = XNT_COMMON_OPTS.copy() SXNT_COMMON_OPTS["register"] = SXNT_COMMON_OPTS_OVERRIDE["register"] +# fuse based saltax options overrides +FXNT_COMMON_OPTS_REGISTER = XNT_COMMON_OPTS["register"].copy() +FXNT_COMMON_OPTS_REGISTER.remove(straxen.Peaklets) +FXNT_COMMON_OPTS_REGISTER.remove(straxen.PulseProcessing) +FXNT_COMMON_OPTS_REGISTER = [ + saltax.SPeaklets, + saltax.SPulseProcessing, + saltax.SPMTResponseAndDAQ +] + FXNT_COMMON_OPTS_REGISTER +FXNT_COMMON_OPTS_OVERRIDE = dict( + register=SXNT_COMMON_OPTS_REGISTER, +) +FXNT_COMMON_OPTS = XNT_COMMON_OPTS.copy() +FXNT_COMMON_OPTS["register"] = FXNT_COMMON_OPTS_OVERRIDE["register"] # saltax configuration overrides SCHANNEL_STARTS_AT = 3000 @@ -164,13 +179,13 @@ def xenonnt_salted_fuse( "Take the context defined in cutax if you want to run XENONnT simulations." ) - st = strax.Context(storage=strax.DataDirectory(output_folder), **XNT_COMMON_OPTS) + st = strax.Context(storage=strax.DataDirectory(output_folder)) st.config.update( dict( # detector='XENONnT', check_raw_record_overlaps=True, - **XNT_COMMON_OPTS, + **FXNT_COMMON_OPTS, **FXNT_COMMON_CONFIG, ) ) From c0064d27d47ad3bfc579d8e9b04f3672080fac68 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:21:04 -0500 Subject: [PATCH 25/37] don't register --- saltax/contexts.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/saltax/contexts.py b/saltax/contexts.py index 0730f40..9d422f6 100644 --- a/saltax/contexts.py +++ b/saltax/contexts.py @@ -25,7 +25,6 @@ # Plugins to merge delayed and regular electrons DELAYED_ELECTRON_MERGER_PLUGINS = fuse.context.delayed_electron_merger_plugins # Plugins to simulate PMTs and DAQ -# NB: this plugin fuse.pmt_and_daq.PMTResponseAndDAQ will be replaced later in context PMT_AND_DAQ_PLUGINS = fuse.context.pmt_and_daq_plugins # Plugins to get truth information TRUTH_INFORMATION_PLUGINS = fuse.context.truth_information_plugins @@ -39,6 +38,10 @@ PMT_AND_DAQ_PLUGINS, TRUTH_INFORMATION_PLUGINS, ] +FUSE_DONT_REGISTER = [ + fuse.plugins.micro_physics.microphysics_summary.MicroPhysicsSummary, + fuse.plugins.pmt_and_daq.pmt_response_and_daq.PMTResponseAndDAQ +] # ~Infinite raw_records file size to avoid downchunking MAX_RAW_RECORDS_FILE_SIZE_MB = 1e9 @@ -192,7 +195,8 @@ def xenonnt_salted_fuse( for plugin_list in FUSED_PLUGINS: for plugin in plugin_list: - st.register(plugin) + if plugin not in FUSE_DONT_REGISTER: + st.register(plugin) if corrections_version is not None: st.apply_xedocs_configs(version=corrections_version) From ebd2d1828ac8b5dae939f8c802c8231d649133c9 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:22:36 -0500 Subject: [PATCH 26/37] debug --- saltax/contexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saltax/contexts.py b/saltax/contexts.py index 9d422f6..fc6a6f8 100644 --- a/saltax/contexts.py +++ b/saltax/contexts.py @@ -78,7 +78,7 @@ saltax.SPMTResponseAndDAQ ] + FXNT_COMMON_OPTS_REGISTER FXNT_COMMON_OPTS_OVERRIDE = dict( - register=SXNT_COMMON_OPTS_REGISTER, + register=FXNT_COMMON_OPTS_REGISTER, ) FXNT_COMMON_OPTS = XNT_COMMON_OPTS.copy() FXNT_COMMON_OPTS["register"] = FXNT_COMMON_OPTS_OVERRIDE["register"] From 6407481a39cd14cf1a073658de3c3bb450d63643 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:27:49 -0500 Subject: [PATCH 27/37] fixed options --- saltax/contexts.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/saltax/contexts.py b/saltax/contexts.py index fc6a6f8..e9baa5e 100644 --- a/saltax/contexts.py +++ b/saltax/contexts.py @@ -182,14 +182,18 @@ def xenonnt_salted_fuse( "Take the context defined in cutax if you want to run XENONnT simulations." ) - st = strax.Context(storage=strax.DataDirectory(output_folder)) + if kwargs is not None: + context_options = dict(**FXNT_COMMON_OPTS, **kwargs) + else: + context_options = FXNT_COMMON_OPTS.copy() + + st = strax.Context(storage=strax.DataDirectory(output_folder), **context_options) st.config.update( dict( # detector='XENONnT', check_raw_record_overlaps=True, **FXNT_COMMON_OPTS, - **FXNT_COMMON_CONFIG, ) ) From ea27a14a56484debc3588d9deaa60730b0273930 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:46:41 -0500 Subject: [PATCH 28/37] fixed options --- saltax/contexts.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/saltax/contexts.py b/saltax/contexts.py index e9baa5e..808a933 100644 --- a/saltax/contexts.py +++ b/saltax/contexts.py @@ -73,6 +73,7 @@ FXNT_COMMON_OPTS_REGISTER.remove(straxen.Peaklets) FXNT_COMMON_OPTS_REGISTER.remove(straxen.PulseProcessing) FXNT_COMMON_OPTS_REGISTER = [ + saltax.SChunkCsvInput, saltax.SPeaklets, saltax.SPulseProcessing, saltax.SPMTResponseAndDAQ @@ -186,16 +187,13 @@ def xenonnt_salted_fuse( context_options = dict(**FXNT_COMMON_OPTS, **kwargs) else: context_options = FXNT_COMMON_OPTS.copy() - - st = strax.Context(storage=strax.DataDirectory(output_folder), **context_options) - - st.config.update( - dict( - # detector='XENONnT', - check_raw_record_overlaps=True, - **FXNT_COMMON_OPTS, - ) + context_config = dict( + detector='XENONnT', + check_raw_record_overlaps=True, + **FXNT_COMMON_CONFIG, ) + st = strax.Context(storage=strax.DataDirectory(output_folder), + config=context_config, **context_options) for plugin_list in FUSED_PLUGINS: for plugin in plugin_list: @@ -262,9 +260,6 @@ def xenonnt_salted_fuse( "raw_records_file_size_target": MAX_RAW_RECORDS_FILE_SIZE_MB, } ) - - # register the csv input plugin - st.register(saltax.SChunkCsvInput) return st From eaed028eb687c32494f7f973d42f993965d59f1c Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:48:59 -0500 Subject: [PATCH 29/37] debug --- saltax/plugins/f_raw_records.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 6b0c651..6f8b8f0 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -88,7 +88,7 @@ def setup(self): def compute(self, raw_records, start, end): try: - chunk_data = self.csv_file_reader(start, end) + chunk_data = self.output_chunk(start, end) chunk_data["time"] = chunk_data["t"] chunk_data["endtime"] = self.chunk_data["time"] data = np.zeros(len(chunk_data), dtype=self.dtype) From 4142414c12fab86bea1b930f42e05eb209498962 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:49:38 -0500 Subject: [PATCH 30/37] debug --- saltax/plugins/f_raw_records.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 6f8b8f0..50ce4c6 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -88,7 +88,7 @@ def setup(self): def compute(self, raw_records, start, end): try: - chunk_data = self.output_chunk(start, end) + chunk_data = self.csv_file_reader.output_chunk(start, end) chunk_data["time"] = chunk_data["t"] chunk_data["endtime"] = self.chunk_data["time"] data = np.zeros(len(chunk_data), dtype=self.dtype) From 5dfafdb7731fdc3c320468f0660621226c99a64b Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:53:28 -0500 Subject: [PATCH 31/37] debug --- saltax/plugins/f_raw_records.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 50ce4c6..58cbf2a 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -206,11 +206,7 @@ def _load_csv_file(self): if not set(self.columns).issubset(df.columns): log.warning("Not all needed columns provided!") - instructions = np.zeros(len(df), dtype=self.dtype) - for column in df.columns: - instructions[column] = df[column] - - return instructions + return df class InstrTranslator: From 2c369e0c4264096f5ed2ef01d4c07caa2ee6ec96 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:54:17 -0500 Subject: [PATCH 32/37] debug --- saltax/plugins/f_raw_records.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 58cbf2a..3b4f6e3 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -90,9 +90,9 @@ def compute(self, raw_records, start, end): try: chunk_data = self.csv_file_reader.output_chunk(start, end) chunk_data["time"] = chunk_data["t"] - chunk_data["endtime"] = self.chunk_data["time"] + chunk_data["endtime"] = chunk_data["time"] data = np.zeros(len(chunk_data), dtype=self.dtype) - strax.copy_to_buffer(self.chunk_data, data, "_bring_data_into_correct_format") + strax.copy_to_buffer(chunk_data, data, "_bring_data_into_correct_format") self.source_done = True From a56d477649fad008abbbdc21db6be82f96070d34 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 18:59:56 -0500 Subject: [PATCH 33/37] debug --- saltax/plugins/f_raw_records.py | 1 + 1 file changed, 1 insertion(+) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index 3b4f6e3..a2d8ccb 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -91,6 +91,7 @@ def compute(self, raw_records, start, end): chunk_data = self.csv_file_reader.output_chunk(start, end) chunk_data["time"] = chunk_data["t"] chunk_data["endtime"] = chunk_data["time"] + chunk_data = chunk_data.to_records(index=False) data = np.zeros(len(chunk_data), dtype=self.dtype) strax.copy_to_buffer(chunk_data, data, "_bring_data_into_correct_format") From 2eee3956ea8d9a84546322fef8ee29053078427a Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 19:01:41 -0500 Subject: [PATCH 34/37] debug --- saltax/plugins/f_raw_records.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index a2d8ccb..cd4dc2e 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -203,10 +203,6 @@ def _load_csv_file(self): log.debug("Loading detector simulation instructions from a csv file in wfsim format!") df = pd.read_csv(self.input_file) - # Check if all needed columns are in place: - if not set(self.columns).issubset(df.columns): - log.warning("Not all needed columns provided!") - return df From c2004c33e6000e2ba7274ea7096a1861f41a8a33 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 19:21:58 -0500 Subject: [PATCH 35/37] debug --- saltax/plugins/f_raw_records.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index cd4dc2e..aed92e0 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -38,7 +38,7 @@ class SChunkCsvInput(FuseBasePlugin): save_when = strax.SaveWhen.TARGET - source_done = False + #source_done = False dtype = [ (("x position of the cluster [cm]", "x"), np.float32), @@ -95,7 +95,7 @@ def compute(self, raw_records, start, end): data = np.zeros(len(chunk_data), dtype=self.dtype) strax.copy_to_buffer(chunk_data, data, "_bring_data_into_correct_format") - self.source_done = True + #self.source_done = True # Stick rigorously with raw_records time range return self.chunk( @@ -190,11 +190,19 @@ def output_chunk(self, chunk_start, chunk_end): log.debug("We will further truncate the instructions to the range [%d, %d]", chunk_start + self.ns_no_instruction_after_chunk_start, chunk_end - self.ns_no_instruction_before_chunk_end) + + # See if we have any instructions after the chunk end + mask_next = (instructions["t"] > chunk_end) + if np.any(mask_next): + source_done = False + else: + source_done = True + mask = (instructions["t"] >= chunk_start + self.ns_no_instruction_after_chunk_start) mask &= (instructions["t"] < chunk_end - self.ns_no_instruction_before_chunk_end) instructions = instructions[mask] - return instructions + return instructions, source_done def _load_csv_file(self): """ From 18e850179d37395d680f06bf060af70ecaf15e9c Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Sun, 21 Apr 2024 19:22:49 -0500 Subject: [PATCH 36/37] debug --- saltax/plugins/f_raw_records.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index aed92e0..af50699 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -88,14 +88,14 @@ def setup(self): def compute(self, raw_records, start, end): try: - chunk_data = self.csv_file_reader.output_chunk(start, end) + chunk_data, source_done = self.csv_file_reader.output_chunk(start, end) chunk_data["time"] = chunk_data["t"] chunk_data["endtime"] = chunk_data["time"] chunk_data = chunk_data.to_records(index=False) data = np.zeros(len(chunk_data), dtype=self.dtype) strax.copy_to_buffer(chunk_data, data, "_bring_data_into_correct_format") - #self.source_done = True + self.source_done = source_done # Stick rigorously with raw_records time range return self.chunk( @@ -196,6 +196,7 @@ def output_chunk(self, chunk_start, chunk_end): if np.any(mask_next): source_done = False else: + log.debug("This is the last chunk! No more instructions available!") source_done = True mask = (instructions["t"] >= chunk_start + self.ns_no_instruction_after_chunk_start) From c66a06330a117c2bc86306976bb0f245c6d8c80c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 22 Apr 2024 00:40:53 +0000 Subject: [PATCH 37/37] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- saltax/contexts.py | 11 ++-- saltax/plugins/f_raw_records.py | 101 ++++++++++++++++++-------------- 2 files changed, 63 insertions(+), 49 deletions(-) diff --git a/saltax/contexts.py b/saltax/contexts.py index 808a933..3ad09d7 100644 --- a/saltax/contexts.py +++ b/saltax/contexts.py @@ -40,7 +40,7 @@ ] FUSE_DONT_REGISTER = [ fuse.plugins.micro_physics.microphysics_summary.MicroPhysicsSummary, - fuse.plugins.pmt_and_daq.pmt_response_and_daq.PMTResponseAndDAQ + fuse.plugins.pmt_and_daq.pmt_response_and_daq.PMTResponseAndDAQ, ] # ~Infinite raw_records file size to avoid downchunking @@ -76,7 +76,7 @@ saltax.SChunkCsvInput, saltax.SPeaklets, saltax.SPulseProcessing, - saltax.SPMTResponseAndDAQ + saltax.SPMTResponseAndDAQ, ] + FXNT_COMMON_OPTS_REGISTER FXNT_COMMON_OPTS_OVERRIDE = dict( register=FXNT_COMMON_OPTS_REGISTER, @@ -188,12 +188,13 @@ def xenonnt_salted_fuse( else: context_options = FXNT_COMMON_OPTS.copy() context_config = dict( - detector='XENONnT', + detector="XENONnT", check_raw_record_overlaps=True, **FXNT_COMMON_CONFIG, ) - st = strax.Context(storage=strax.DataDirectory(output_folder), - config=context_config, **context_options) + st = strax.Context( + storage=strax.DataDirectory(output_folder), config=context_config, **context_options + ) for plugin_list in FUSED_PLUGINS: for plugin in plugin_list: diff --git a/saltax/plugins/f_raw_records.py b/saltax/plugins/f_raw_records.py index af50699..0155c87 100644 --- a/saltax/plugins/f_raw_records.py +++ b/saltax/plugins/f_raw_records.py @@ -22,23 +22,25 @@ class SPMTResponseAndDAQ(PMTResponseAndDAQ): provides = "raw_records_simu" data_kind = "raw_records_simu" + @export class SChunkCsvInput(FuseBasePlugin): """Plugin which reads a CSV file containing instructions for the detector physics simulation (in wfsim format) and returns the data in chunks. + Modified from csv_file_loader: https://github.com/XENONnT/fuse/blob/e538d32a5a0735757a77b1dce31d6f95a379bf4e/fuse/plugins/detector_physics/csv_input.py#L118 Similar to the case event_rate=0 there: we don't reassign times. """ __version__ = "0.0.0" - depends_on: Tuple = ("raw_records") + depends_on: Tuple = "raw_records" provides = "microphysics_summary" data_kind = "interactions_in_roi" save_when = strax.SaveWhen.TARGET - #source_done = False + # source_done = False dtype = [ (("x position of the cluster [cm]", "x"), np.float32), @@ -65,17 +67,16 @@ class SChunkCsvInput(FuseBasePlugin): track=False, type=(int), help="No instruction this amount of time after a chunk starts will be used, " - "as a safeguard for not getting raw_records_simu out of raw_records chunk time range. " + "as a safeguard for not getting raw_records_simu out of raw_records chunk time range. ", ) ns_no_instruction_before_chunk_end = straxen.URLConfig( default=5e7, track=False, type=(int), help="No instruction this amount of time before a chunk ends will be used, " - "as a safeguard for not getting raw_records_simu out of raw_records chunk time range. " + "as a safeguard for not getting raw_records_simu out of raw_records chunk time range. ", ) - def setup(self): super().setup() self.csv_file_reader = SCsvFileLoader( @@ -85,7 +86,7 @@ def setup(self): ns_no_instruction_after_chunk_start=self.ns_no_instruction_after_chunk_start, debug=self.debug, ) - + def compute(self, raw_records, start, end): try: chunk_data, source_done = self.csv_file_reader.output_chunk(start, end) @@ -98,16 +99,14 @@ def compute(self, raw_records, start, end): self.source_done = source_done # Stick rigorously with raw_records time range - return self.chunk( - start=start, end=end, data=data, data_type="geant4_interactions" - ) + return self.chunk(start=start, end=end, data=data, data_type="geant4_interactions") except StopIteration: raise RuntimeError("Bug in chunk building!") - def source_finished(self): + def source_finished(self): return self.source_done - + def is_ready(self, chunk_i): """Overwritten to mimic online input plugin. @@ -118,10 +117,12 @@ def is_ready(self, chunk_i): self.ready = False self.ready ^= True # Flip return self.ready - + class SCsvFileLoader: - """Class to load a CSV file (in wfsim format) with detector simulation instructions""" + """Class to load a CSV file (in wfsim format) with detector simulation + instructions.""" + def __init__( self, input_file, @@ -148,7 +149,7 @@ def __init__( (("NEST interaction type", "nestid"), np.int8), (("ID of the cluster", "cluster_id"), np.int32), (("Time of the interaction", "t"), np.int64), - (("Geant4 event ID", "eventid"), np.int32), + (("Geant4 event ID", "eventid"), np.int32), ] self.dtype = self.dtype + strax.time_fields @@ -172,9 +173,10 @@ def __init__( self.translator = InstrTranslator(input_format="wfsim", output_format="fuse") def output_chunk(self, chunk_start, chunk_end): - """ - Load the simulation instructions from the csv file in wfsim format and then - translate them to the fuse format. Truncate the instructions to the chunk time range. + """Load the simulation instructions from the csv file in wfsim format + and then translate them to the fuse format. + + Truncate the instructions to the chunk time range. """ # Load the csv file in wfsim format log.debug("Loaded detector simulation instructions from a csv file in wfsim format!") @@ -187,59 +189,70 @@ def output_chunk(self, chunk_start, chunk_end): # truncate instructions to the chunk time range log.debug("Truncating instructions to the chunk time range!") - log.debug("We will further truncate the instructions to the range [%d, %d]", - chunk_start + self.ns_no_instruction_after_chunk_start, - chunk_end - self.ns_no_instruction_before_chunk_end) - + log.debug( + "We will further truncate the instructions to the range [%d, %d]", + chunk_start + self.ns_no_instruction_after_chunk_start, + chunk_end - self.ns_no_instruction_before_chunk_end, + ) + # See if we have any instructions after the chunk end - mask_next = (instructions["t"] > chunk_end) + mask_next = instructions["t"] > chunk_end if np.any(mask_next): source_done = False else: log.debug("This is the last chunk! No more instructions available!") source_done = True - mask = (instructions["t"] >= chunk_start + self.ns_no_instruction_after_chunk_start) - mask &= (instructions["t"] < chunk_end - self.ns_no_instruction_before_chunk_end) + mask = instructions["t"] >= chunk_start + self.ns_no_instruction_after_chunk_start + mask &= instructions["t"] < chunk_end - self.ns_no_instruction_before_chunk_end instructions = instructions[mask] return instructions, source_done def _load_csv_file(self): - """ - Load the simulation instructions from a csv file in wfsim format. - """ + """Load the simulation instructions from a csv file in wfsim format.""" log.debug("Loading detector simulation instructions from a csv file in wfsim format!") df = pd.read_csv(self.input_file) return df - + class InstrTranslator: - """Class to translate instructions between wfsim and fuse formats""" + """Class to translate instructions between wfsim and fuse formats.""" + def __init__(self, input_format="wfsim", output_format="fuse"): self.input_format = input_format self.output_format = output_format - assert self.input_format in ["wfsim", "fuse"], "Unknown input format! Choose 'wfsim' or 'fuse'!" - assert self.output_format in ["wfsim", "fuse"], "Unknown output format! Choose 'wfsim' or 'fuse'!" + assert self.input_format in [ + "wfsim", + "fuse", + ], "Unknown input format! Choose 'wfsim' or 'fuse'!" + assert self.output_format in [ + "wfsim", + "fuse", + ], "Unknown output format! Choose 'wfsim' or 'fuse'!" log.debug("Translating instructions from %s to %s", self.input_format, self.output_format) self.translator = self.translator() - + def translator(self): if self.input_format == "wfsim" and self.output_format == "fuse": return self.translate_wfsim_to_fuse elif self.input_format == "fuse" and self.output_format == "wfsim": return self.translate_fuse_to_wfsim else: - raise NotImplementedError("Translation from {} to {} is not implemented yet!".format(self.input_format, self.output_format)) - + raise NotImplementedError( + "Translation from {} to {} is not implemented yet!".format( + self.input_format, self.output_format + ) + ) + def translate(self, instructions): return self.translator(instructions) def translate_wfsim_to_fuse(self, instructions): - """Translate the wfsim instructions to the fuse format""" + """Translate the wfsim instructions to the fuse format.""" # Sort time for sanity instructions = instructions.sort_values(by="time") @@ -257,12 +270,12 @@ def translate_wfsim_to_fuse(self, instructions): if row.time > previous_time: new_cluster = True if row.event_number != previous_event_number: - new_event = True + new_event = True if new_event and (not new_cluster): - raise ValueError("New event without new cluster at time %s!?"%(row.time)) + raise ValueError("New event without new cluster at time %s!?" % (row.time)) previous_time = row.time previous_event_number = row.event_number - + # Update the previous event number if new_event: event_id += 1 @@ -281,7 +294,7 @@ def translate_wfsim_to_fuse(self, instructions): "cluster_id": np.int32(cluster_id), "eventid": np.int32(event_id), "photons": np.int32(0), - "electrons": np.int32(0), + "electrons": np.int32(0), "excitons": np.int32(0), } last_row = new_row @@ -293,17 +306,17 @@ def translate_wfsim_to_fuse(self, instructions): elif row.type == 2: last_row["electrons"] = np.int32(row.amp) else: - raise ValueError("Unknown type %s!"%(row.type)) - + raise ValueError("Unknown type %s!" % (row.type)) + # Concatenate the last row to the new instructions if first_row: rows = [last_row] first_row = False elif new_cluster: rows.append(last_row) - + return pd.DataFrame(rows) def translate_fuse_to_wfsim(self, instructions): - """Translate the fuse instructions to the wfsim format""" - raise NotImplementedError("Not implemented yet!") \ No newline at end of file + """Translate the fuse instructions to the wfsim format.""" + raise NotImplementedError("Not implemented yet!")