From ffe580381ff516444167d8e8c4b9795641a9399a Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Tue, 12 Mar 2024 18:30:40 -0500 Subject: [PATCH 1/7] new options --- jobs/config.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jobs/config.ini b/jobs/config.ini index ad5b7de..519d7b3 100644 --- a/jobs/config.ini +++ b/jobs/config.ini @@ -16,6 +16,8 @@ generator_name = flat recoil = 8 mode = all process_data = False +skip_records = True +delete_records = True storage_to_patch = /project/lgrandi/yuanlq/salt/raw_records,/scratch/midway3/yuanlq/salt/raw_records [slurm] From af9e1c0836ddd3e21264326d792871c7e22bc8fa Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Tue, 12 Mar 2024 18:31:29 -0500 Subject: [PATCH 2/7] new options --- jobs/config.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/jobs/config.ini b/jobs/config.ini index 519d7b3..c293307 100644 --- a/jobs/config.ini +++ b/jobs/config.ini @@ -16,6 +16,7 @@ generator_name = flat recoil = 8 mode = all process_data = False +process_simu = True skip_records = True delete_records = True storage_to_patch = /project/lgrandi/yuanlq/salt/raw_records,/scratch/midway3/yuanlq/salt/raw_records From a7ec24065ff2c804016713e8b16dae8eadd359ac Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Tue, 12 Mar 2024 18:32:26 -0500 Subject: [PATCH 3/7] process simu --- jobs/job.py | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/jobs/job.py b/jobs/job.py index 58d646b..46823ee 100644 --- a/jobs/job.py +++ b/jobs/job.py @@ -21,6 +21,7 @@ recoil = config.getint('job', 'recoil') mode = config.get('job', 'mode') process_data = config.getboolean('job', 'process_data') +process_simu = config.getboolean('job', 'process_simu') storage_to_patch = config.get('job', 'storage_to_patch').split(',') to_process_dtypes = ['records', 'peaklets', 'merged_s2s', 'peak_basics', @@ -88,29 +89,32 @@ else: print("You specified process_data = False, so we will not process data.") - st = saltax.contexts.sxenonnt(runid = runid, - saltax_mode = 'simu', - output_folder = output_folder, - faxconf_version = faxconf_version, - generator_name = generator_name, - recoil = recoil, - mode = mode) - if len(storage_to_patch) and storage_to_patch[0] != "": - for d in st.storage: - st.storage.append(strax.DataDirectory(d, readonly=True)) - - st.make(strrunid, 'raw_records_simu') - gc.collect() - for dt in to_process_dtypes: - print("Making %s. "%dt) - st.make(strrunid, dt, save=(dt)) - print("Done with %s. "%dt) + if process_simu: + st = saltax.contexts.sxenonnt(runid = runid, + saltax_mode = 'simu', + output_folder = output_folder, + faxconf_version = faxconf_version, + generator_name = generator_name, + recoil = recoil, + mode = mode) + if len(storage_to_patch) and storage_to_patch[0] != "": + for d in st.storage: + st.storage.append(strax.DataDirectory(d, readonly=True)) + + st.make(strrunid, 'raw_records_simu') gc.collect() + for dt in to_process_dtypes: + print("Making %s. "%dt) + st.make(strrunid, dt, save=(dt)) + print("Done with %s. "%dt) + gc.collect() - print("Used time:", datetime.now() - now) - now = datetime.now() + print("Used time:", datetime.now() - now) + now = datetime.now() - print("Finished making all the computation for run %d in \ - saltax mode %s. "%(runid, 'simu')) + print("Finished making all the computation for run %d in \ + saltax mode %s. "%(runid, 'simu')) + else: + print("You specified process_simu = False, so we will not process simu.") print("Finished all. Exiting.") From 0c0e428c54d04e18ca43441a613e28f10bc6cbb9 Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Tue, 12 Mar 2024 18:46:01 -0500 Subject: [PATCH 4/7] skip records and delete records --- jobs/job.py | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/jobs/job.py b/jobs/job.py index 46823ee..597bc36 100644 --- a/jobs/job.py +++ b/jobs/job.py @@ -5,6 +5,7 @@ from datetime import datetime import sys import gc +import os straxen.print_versions() now = datetime.now() @@ -22,12 +23,16 @@ mode = config.get('job', 'mode') process_data = config.getboolean('job', 'process_data') process_simu = config.getboolean('job', 'process_simu') +skip_records = config.getboolean('job', 'skip_records') storage_to_patch = config.get('job', 'storage_to_patch').split(',') +delete_records = config.getboolean('job', 'delete_records') -to_process_dtypes = ['records', 'peaklets', 'merged_s2s', 'peak_basics', +to_process_dtypes = ['peaklets', 'merged_s2s', 'peak_basics', 'events', 'event_basics', 'event_info', 'event_pattern_fit', 'event_shadow', 'event_ambience', 'event_n_channel','veto_intervals', 'cuts_basic'] +if not skip_records: + to_process_dtypes = ['records'] + to_process_dtypes print("Used time:", datetime.now() - now) now = datetime.now() @@ -57,7 +62,15 @@ now = datetime.now() print("Finished making all the computation for run %d in \ - saltax mode %s. "%(runid, saltax_mode)) + saltax mode salt. "%(runid)) +if delete_records: + print("Deleting records.") + records_name = st.key_for('records') + records_path = os.path.join(output_folder, records_name) + if os.path.exists(records_path): + os.rmdir(records_path) + gc.collect() + print("Deleted records for run %d in saltax mode salt. "%(runid)) if saltax_mode == 'salt': print("Since you specified saltax_mode = salt, \ @@ -86,6 +99,17 @@ print("Finished making all the computation for run %d in \ saltax mode %s. "%(runid, 'data')) + + print("Finished making all the computation for run %d in \ + saltax mode data. "%(runid)) + if delete_records: + print("Deleting records.") + records_name = st.key_for('records') + records_path = os.path.join(output_folder, records_name) + if os.path.exists(records_path): + os.rmdir(records_path) + gc.collect() + print("Deleted records for run %d in saltax mode data. "%(runid)) else: print("You specified process_data = False, so we will not process data.") @@ -114,6 +138,17 @@ print("Finished making all the computation for run %d in \ saltax mode %s. "%(runid, 'simu')) + + print("Finished making all the computation for run %d in \ + saltax mode simu. "%(runid)) + if delete_records: + print("Deleting records.") + records_name = st.key_for('records') + records_path = os.path.join(output_folder, records_name) + if os.path.exists(records_path): + os.rmdir(records_path) + gc.collect() + print("Deleted records for run %d in saltax mode simu. "%(runid)) else: print("You specified process_simu = False, so we will not process simu.") From 15e355bf3e93f8942a8abd0140b3e323cc0b1d3a Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Tue, 12 Mar 2024 18:48:11 -0500 Subject: [PATCH 5/7] debug info better --- jobs/job.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/jobs/job.py b/jobs/job.py index 597bc36..23bb1fb 100644 --- a/jobs/job.py +++ b/jobs/job.py @@ -37,6 +37,7 @@ print("Used time:", datetime.now() - now) now = datetime.now() +print('====================') print("Finished importing and config loading, now start to load context.") print("Now starting %s context for run %d"%(saltax_mode, runid)) st = saltax.contexts.sxenonnt(runid = runid, @@ -71,12 +72,14 @@ os.rmdir(records_path) gc.collect() print("Deleted records for run %d in saltax mode salt. "%(runid)) +print('====================') -if saltax_mode == 'salt': - print("Since you specified saltax_mode = salt, \ - we will also compute simulation-only and data-only.") +if saltax_mode == 'salt': if process_data: + print('====================') + print("Since you specified saltax_mode = salt, \ + we will also compute simulation-only and data-only.") print("Now starting data-only context for run %d"%(runid)) st = saltax.contexts.sxenonnt(runid = runid, saltax_mode = 'data', output_folder = output_folder, @@ -110,10 +113,13 @@ os.rmdir(records_path) gc.collect() print("Deleted records for run %d in saltax mode data. "%(runid)) + print('====================') else: print("You specified process_data = False, so we will not process data.") if process_simu: + print('====================') + print("Now starting simu-only context for run %d"%(runid)) st = saltax.contexts.sxenonnt(runid = runid, saltax_mode = 'simu', output_folder = output_folder, @@ -149,6 +155,7 @@ os.rmdir(records_path) gc.collect() print("Deleted records for run %d in saltax mode simu. "%(runid)) + print('====================') else: print("You specified process_simu = False, so we will not process simu.") From 120b282cf3d877192a280015710516c5195755eb Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Tue, 12 Mar 2024 19:08:07 -0500 Subject: [PATCH 6/7] debug --- jobs/job.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jobs/job.py b/jobs/job.py index 23bb1fb..72221b1 100644 --- a/jobs/job.py +++ b/jobs/job.py @@ -15,7 +15,7 @@ config = configparser.ConfigParser() config.read('config.ini') -output_folder = config.get('job', 'output_folder') +output_folder = str(config.get('job', 'output_folder')) saltax_mode = config.get('job', 'saltax_mode') faxconf_version = config.get('job', 'faxconf_version') generator_name = config.get('job', 'generator_name') @@ -66,7 +66,7 @@ saltax mode salt. "%(runid)) if delete_records: print("Deleting records.") - records_name = st.key_for('records') + records_name = str(st.key_for(strrunid, 'records')) records_path = os.path.join(output_folder, records_name) if os.path.exists(records_path): os.rmdir(records_path) @@ -107,7 +107,7 @@ saltax mode data. "%(runid)) if delete_records: print("Deleting records.") - records_name = st.key_for('records') + records_name = str(st.key_for(strrunid, 'records')) records_path = os.path.join(output_folder, records_name) if os.path.exists(records_path): os.rmdir(records_path) @@ -149,7 +149,7 @@ saltax mode simu. "%(runid)) if delete_records: print("Deleting records.") - records_name = st.key_for('records') + records_name = str(st.key_for(strrunid, 'records')) records_path = os.path.join(output_folder, records_name) if os.path.exists(records_path): os.rmdir(records_path) From 9ae06f9d05448b460058b357d0324010fe79eb4c Mon Sep 17 00:00:00 2001 From: Lanqing Yuan Date: Tue, 12 Mar 2024 22:46:24 -0500 Subject: [PATCH 7/7] default rate increase for se --- saltax/instructions/generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/saltax/instructions/generator.py b/saltax/instructions/generator.py index b9984e6..aa441a6 100644 --- a/saltax/instructions/generator.py +++ b/saltax/instructions/generator.py @@ -157,7 +157,7 @@ def instr_file_name(runid, instr, recoil, generator_name, mode, rate=1e9/SALT_TI return filename def generator_se(runid, - n_tot=None, rate=1e9/SALT_TIME_INTERVAL, + n_tot=None, rate=1e10/SALT_TIME_INTERVAL, r_range=R_RANGE, z_range=Z_RANGE, time_mode="uniform", *args): """ @@ -193,7 +193,7 @@ def generator_se(runid, return instr -def generator_flat(runid, en_range=(0.2, 15.0), recoil=7, +def generator_flat(runid, en_range=(0.2, 15.0), recoil=8, n_tot=None, rate=1e9/SALT_TIME_INTERVAL, fmap=FIELD_MAP, nc=NC, r_range=R_RANGE, z_range=Z_RANGE,