From b833675b417c02fe5733d4f345ea981b4a62c41d Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Tue, 12 Mar 2024 22:57:17 -0500 Subject: [PATCH] Skip records and tune SE generator (#78) * new options * new options * process simu * skip records and delete records * debug info better * debug * default rate increase for se --- jobs/config.ini | 3 + jobs/job.py | 100 ++++++++++++++++++++++--------- saltax/instructions/generator.py | 4 +- 3 files changed, 78 insertions(+), 29 deletions(-) diff --git a/jobs/config.ini b/jobs/config.ini index ad5b7de..c293307 100644 --- a/jobs/config.ini +++ b/jobs/config.ini @@ -16,6 +16,9 @@ generator_name = flat recoil = 8 mode = all process_data = False +process_simu = True +skip_records = True +delete_records = True storage_to_patch = /project/lgrandi/yuanlq/salt/raw_records,/scratch/midway3/yuanlq/salt/raw_records [slurm] diff --git a/jobs/job.py b/jobs/job.py index 58d646b..72221b1 100644 --- a/jobs/job.py +++ b/jobs/job.py @@ -5,6 +5,7 @@ from datetime import datetime import sys import gc +import os straxen.print_versions() now = datetime.now() @@ -14,23 +15,29 @@ config = configparser.ConfigParser() config.read('config.ini') -output_folder = config.get('job', 'output_folder') +output_folder = str(config.get('job', 'output_folder')) saltax_mode = config.get('job', 'saltax_mode') faxconf_version = config.get('job', 'faxconf_version') generator_name = config.get('job', 'generator_name') recoil = config.getint('job', 'recoil') mode = config.get('job', 'mode') process_data = config.getboolean('job', 'process_data') +process_simu = config.getboolean('job', 'process_simu') +skip_records = config.getboolean('job', 'skip_records') storage_to_patch = config.get('job', 'storage_to_patch').split(',') +delete_records = config.getboolean('job', 'delete_records') -to_process_dtypes = ['records', 'peaklets', 'merged_s2s', 'peak_basics', +to_process_dtypes = ['peaklets', 'merged_s2s', 'peak_basics', 'events', 'event_basics', 'event_info', 'event_pattern_fit', 'event_shadow', 'event_ambience', 'event_n_channel','veto_intervals', 'cuts_basic'] +if not skip_records: + to_process_dtypes = ['records'] + to_process_dtypes print("Used time:", datetime.now() - now) now = datetime.now() +print('====================') print("Finished importing and config loading, now start to load context.") print("Now starting %s context for run %d"%(saltax_mode, runid)) st = saltax.contexts.sxenonnt(runid = runid, @@ -56,13 +63,23 @@ now = datetime.now() print("Finished making all the computation for run %d in \ - saltax mode %s. "%(runid, saltax_mode)) + saltax mode salt. "%(runid)) +if delete_records: + print("Deleting records.") + records_name = str(st.key_for(strrunid, 'records')) + records_path = os.path.join(output_folder, records_name) + if os.path.exists(records_path): + os.rmdir(records_path) + gc.collect() + print("Deleted records for run %d in saltax mode salt. "%(runid)) +print('====================') -if saltax_mode == 'salt': - print("Since you specified saltax_mode = salt, \ - we will also compute simulation-only and data-only.") +if saltax_mode == 'salt': if process_data: + print('====================') + print("Since you specified saltax_mode = salt, \ + we will also compute simulation-only and data-only.") print("Now starting data-only context for run %d"%(runid)) st = saltax.contexts.sxenonnt(runid = runid, saltax_mode = 'data', output_folder = output_folder, @@ -85,32 +102,61 @@ print("Finished making all the computation for run %d in \ saltax mode %s. "%(runid, 'data')) + + print("Finished making all the computation for run %d in \ + saltax mode data. "%(runid)) + if delete_records: + print("Deleting records.") + records_name = str(st.key_for(strrunid, 'records')) + records_path = os.path.join(output_folder, records_name) + if os.path.exists(records_path): + os.rmdir(records_path) + gc.collect() + print("Deleted records for run %d in saltax mode data. "%(runid)) + print('====================') else: print("You specified process_data = False, so we will not process data.") - st = saltax.contexts.sxenonnt(runid = runid, - saltax_mode = 'simu', - output_folder = output_folder, - faxconf_version = faxconf_version, - generator_name = generator_name, - recoil = recoil, - mode = mode) - if len(storage_to_patch) and storage_to_patch[0] != "": - for d in st.storage: - st.storage.append(strax.DataDirectory(d, readonly=True)) - - st.make(strrunid, 'raw_records_simu') - gc.collect() - for dt in to_process_dtypes: - print("Making %s. "%dt) - st.make(strrunid, dt, save=(dt)) - print("Done with %s. "%dt) + if process_simu: + print('====================') + print("Now starting simu-only context for run %d"%(runid)) + st = saltax.contexts.sxenonnt(runid = runid, + saltax_mode = 'simu', + output_folder = output_folder, + faxconf_version = faxconf_version, + generator_name = generator_name, + recoil = recoil, + mode = mode) + if len(storage_to_patch) and storage_to_patch[0] != "": + for d in st.storage: + st.storage.append(strax.DataDirectory(d, readonly=True)) + + st.make(strrunid, 'raw_records_simu') gc.collect() + for dt in to_process_dtypes: + print("Making %s. "%dt) + st.make(strrunid, dt, save=(dt)) + print("Done with %s. "%dt) + gc.collect() - print("Used time:", datetime.now() - now) - now = datetime.now() + print("Used time:", datetime.now() - now) + now = datetime.now() - print("Finished making all the computation for run %d in \ - saltax mode %s. "%(runid, 'simu')) + print("Finished making all the computation for run %d in \ + saltax mode %s. "%(runid, 'simu')) + + print("Finished making all the computation for run %d in \ + saltax mode simu. "%(runid)) + if delete_records: + print("Deleting records.") + records_name = str(st.key_for(strrunid, 'records')) + records_path = os.path.join(output_folder, records_name) + if os.path.exists(records_path): + os.rmdir(records_path) + gc.collect() + print("Deleted records for run %d in saltax mode simu. "%(runid)) + print('====================') + else: + print("You specified process_simu = False, so we will not process simu.") print("Finished all. Exiting.") diff --git a/saltax/instructions/generator.py b/saltax/instructions/generator.py index b9984e6..aa441a6 100644 --- a/saltax/instructions/generator.py +++ b/saltax/instructions/generator.py @@ -157,7 +157,7 @@ def instr_file_name(runid, instr, recoil, generator_name, mode, rate=1e9/SALT_TI return filename def generator_se(runid, - n_tot=None, rate=1e9/SALT_TIME_INTERVAL, + n_tot=None, rate=1e10/SALT_TIME_INTERVAL, r_range=R_RANGE, z_range=Z_RANGE, time_mode="uniform", *args): """ @@ -193,7 +193,7 @@ def generator_se(runid, return instr -def generator_flat(runid, en_range=(0.2, 15.0), recoil=7, +def generator_flat(runid, en_range=(0.2, 15.0), recoil=8, n_tot=None, rate=1e9/SALT_TIME_INTERVAL, fmap=FIELD_MAP, nc=NC, r_range=R_RANGE, z_range=Z_RANGE,