- © Copyright 2018-2020, momentoscope team
- © Copyright 2018-2020, momentoscope team
@property
def metadata_dict(self):
- try:
- md = self.metadata
- except AttributeError:
- md = {}
- try:
- md['run'] = self.runInfo
- except AttributeError:
- md['run'] = {
- 'runNumber': self.runNumber,
- 'pulseIdInterval': self.pulseIdInterval,
- }
- md['processor'] = {'n_cores': self.N_CORES,
- 'chunk_size': self.CHUNK_SIZE,
- }
- md['calibration'] = {'TOF_STEP_TO_NS': self.TOF_STEP_TO_NS,
- 'ET_CONV_E_OFFSET': self.ET_CONV_E_OFFSET,
- 'ET_CONV_T_OFFSET': self.ET_CONV_T_OFFSET,
- 'ET_CONV_L': self.ET_CONV_L,
- 'TOF_IN_NS': self.TOF_IN_NS,
- }
- md['paths'] = {
- 'DATA_RAW_DIR': self.DATA_RAW_DIR,
- 'DATA_H5_DIR': self.DATA_H5_DIR,
- 'DATA_PARQUET_DIR': self.DATA_PARQUET_DIR,
- 'DATA_RESULTS_DIR': self.DATA_RESULTS_DIR,
- 'LOG_DIR': self.LOG_DIR,
- 'PAH_MODULE_DIR': self.PAH_MODULE_DIR,
- }
- # md['DAQ channels'] =
- md['sample'] = self.sample
+ if self.metadata is None:
+ md = self.get_metadata()
self.metadata = md
+ else:
+ md = self.metadata
return md
+ def __repr__(self):
+ run = f'- Run Number: {self.runNumber}'
+ trainInterval = f'- MacrobunchIds: {self.pulseIdInterval}'
+ settings = f'- Settings loaded from {self._settings_file}'
+ bins = '- Bins:'
+ for i,name,rng in zip(range(len(self.binNameList)),self.binNameList,self.binRangeList):
+ bins += f'\n\t{i}. {name}: {len(rng)} points. From {min(rng):.2f} to {max(rng):.2f} in steps of {rng[1]-rng[0]:.2f}'
+ string = f'DldFlashProcessor.\n'
+ for s in [run,trainInterval,settings,bins]:
+ string += s + '\n'
+ return string
+
@property
def settings(self):
""" Easy access to settings.ini file
@@ -344,8 +327,10 @@ Source code for processor.DldProcessor
# Hard coded class attributes which can be overwritten by settings files.
self.N_CORES = int(max(os.cpu_count() - 1, 1))
self.UBID_OFFSET = int(0)
+ self.CORRECT_MB_ID = bool(True)
self.CHUNK_SIZE = int(100000)
self.TOF_STEP_TO_NS = np.float64(0.020576131995767355)
+ self.DLD_ID_BITS = int(3)
self.ET_CONV_E_OFFSET = np.float64(357.7)
self.ET_CONV_T_OFFSET = np.float64(82.7)
self.ET_CONV_L = np.float64(.75)
@@ -368,7 +353,7 @@ Source code for processor.DldProcessor
self.USE_BAM = bool(True)
self.USE_STREAK = bool(False)
- self.SECTOR_CORRECTION = [int(i) for i in [0,0,0,0,0,0,0,0]]
+ self.SECTOR_CORRECTION = [int(i) for i in [0,1,2,3,4,5,6,7]]
self.DATA_RAW_DIR = str('/gpfs/pg2/current/raw/hdf')
self.DATA_H5_DIR = str('/home/pg2user/data/h5')
@@ -595,17 +580,17 @@ Source code for processor.DldProcessor
except AttributeError:
pass
try:
- i = self.metadata_dict['run']
+ i = self.metadata['runInfo']
except:
pass
if i is None:
print('no run info available.')
else:
- print(f'Run {i["runNumber"]}')
+ print(f"Run {i['runNumber']}")
try:
- print(f"Started at {i['timeStart']}, finished at {i['timeStop']}, "
- f"total duration {i['timeDuration']:,} s")
+ print(f"Started at {i['timestampStart']}, finished at {i['timestampStop']}, "
+ f"total duration {i['timestampDuration']:,} s")
except:
pass
print(f"Macrobunches: {i['numberOfMacrobunches']:,} "
@@ -747,7 +732,7 @@ Source code for processor.DldProcessor
# self.ddMicrobunches['delayStageDirection']*backLash
[docs] def calibrateEnergy(self, toffset=None, eoffset=None, l=None, useAvgSampleBias=True, k_shift_func=None,
- k_shift_parameters=None, applyJitter=True, jitterAmplitude=4, jitterType='uniform',
+ k_shift_parameters=None, applyJitter=True, jitterAmplitude=None, jitterType='uniform',
useAvgMonochormatorEnergy=True, useAvgToFEnergy=True,
sampleBias=None, monochromatorPhotonEnergy=None, tofVoltage=None):
""" Add calibrated energy axis to dataframe
@@ -789,6 +774,10 @@ Source code for processor.DldProcessor
if tofVoltage is None:
tofVoltage = np.nanmean(self.dd['tofVoltage'].values)
+ if jitterAmplitude is None:
+ jitterAmplitude = np.floor(np.power(2,self.DLD_ID_BITS-1))
+
+ # sector dependent shift. this is necessary due to bit structure and due to photon peak shift among sectors
self.dd['dldTime_corrected'] = self.dd['dldTime']
if 'dldSectorId' in self.dd.columns:
# Converts the SECTOR_CORRECTION list to a dask array so things can be kept lazy
@@ -1327,21 +1316,21 @@ Source code for processor.DldProcessor
"""
# write the parameters to the bin list:
- if name in ['dldTime'] and self.TOF_IN_NS:
+ if name in ['dldTime','dldTime_corrected'] and self.TOF_IN_NS:
start = round(start/self.TOF_STEP_TO_NS)
end = round(end/self.TOF_STEP_TO_NS)
if useStepSize is True:
# division by 8 is necessary since the first 3 bits of the channel where these values are
# taken from is used for other purpouses. Therefore the real tof step is:
- steps = round(steps/self.TOF_STEP_TO_NS/8)*8
- steps = max(steps,8)
+ steps = round(steps/self.TOF_STEP_TO_NS/np.power(2,self.DLD_ID_BITS))*np.power(2,self.DLD_ID_BITS)
+ steps = max(steps,np.power(2,self.DLD_ID_BITS))
bins = self.genBins(start, end, steps, useStepSize, forceEnds, include_last, force_legacy)
self.binNameList.append(name)
self.binRangeList.append(bins)
axes = np.array([np.mean((x, y)) for x, y in zip(bins[:-1], bins[1:])])
- if name in ['dldTime'] and self.TOF_IN_NS:
+ if name in ['dldTime','dldTime_corrected'] and self.TOF_IN_NS:
axes *= self.TOF_STEP_TO_NS
# TODO: could be improved for nonlinear scales
@@ -1466,20 +1455,20 @@ Source code for processor.DldProcessor
# print("computing partitions " + str(i) + " to " + str(i + j) + " of " + str(
# self.dd.npartitions) + ". partitions calculated in parallel: " + str(
# len(resultsToCalculate)))
- results = dask.compute(*resultsToCalculate)
- total = np.zeros_like(results[0])
- for result in results:
- total = total + result
- calculatedResults.append(total)
- del total
+ rs = dask.compute(*resultsToCalculate)
+ # we now need to add them all up (single core):
+ try:
+ result
+ for r in rs:
+ result = result + r
+ except NameError:
+ result = np.zeros_like(rs[0])
+ for r in rs:
+ result = result + r
+
+ del rs
del resultsToCalculate
- # we now need to add them all up (single core):
- result = np.zeros_like(calculatedResults[0])
- for r in calculatedResults:
- r = np.nan_to_num(r)
- result = result + r
-
if force_64bit:
result = result.astype(np.float64)
@@ -1515,38 +1504,42 @@ Source code for processor.DldProcessor
"""
print('Generating metadata...')
metadata = {}
- try:
+ if hasattr(self,'startEndTime'):
start, stop = self.startEndTime[0], self.startEndTime[1]
- except AttributeError:
- if not fast_mode:
- start, stop = self.dd['timeStamp'].min().compute(), self.dd['timeStamp'].max().compute()
- else:
- start, stop = 0, 1
+ elif hasattr(self, 'dd') and not fast_mode:
+ start, stop = self.dd['timeStamp'].min().compute(), self.dd['timeStamp'].max().compute()
+ else:
+ start, stop = 0, 1
metadata['timing'] = {'acquisition start': datetime.fromtimestamp(start).strftime('%Y-%m-%d %H:%M:%S'),
'acquisition stop': datetime.fromtimestamp(stop).strftime('%Y-%m-%d %H:%M:%S'),
- 'acquisition duration': stop - start,
+ 'acquisition duration': int(stop - start),
# 'bin array creation': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
}
metadata['sample'] = self.sample
- metadata['settings'] = self.settings['processor']#misc.parse_category('processor')
- metadata['DAQ channels'] = self.settings['DAQ channels']#misc.parse_category('DAQ channels')
- if self.pulseIdInterval is None and not fast_mode:
- pulseIdFrom = self.dd['macroBunchPulseId'].min().compute()
- pulseIdTo = self.dd['macroBunchPulseId'].max().compute()
- else:
- pulseIdFrom, pulseIdTo = self.pulseIdInterval[0], self.pulseIdInterval[1]
+ metadata['settings'] = dict(self.settings._sections['processor'])#misc.parse_category('processor')
+ metadata['DAQ channels'] = dict(self.settings._sections['DAQ channels'])#misc.parse_category('DAQ channels')
+
- metadata['run'] = {
- 'runNumber': self.runNumber,
- 'macroBunchPulseIdInterval': [pulseIdFrom, pulseIdTo],
- 'nMacrobunches': pulseIdTo - pulseIdFrom,
- }
try:
- metadata['run']['nElectrons'] = self.numOfElectrons
- metadata['run']['electronsPerMacrobunch'] = self.electronsPerMacrobunch,
+ metadata['runInfo']=self.runInfo
except:
- pass # TODO: find smarter solution
+ if self.pulseIdInterval is None and not fast_mode:
+ pulseIdFrom = self.dd['macroBunchPulseId'].min().compute()
+ pulseIdTo = self.dd['macroBunchPulseId'].max().compute()
+ else:
+ pulseIdFrom, pulseIdTo = self.pulseIdInterval[0], self.pulseIdInterval[1]
+
+ metadata['runInfo'] = {
+ 'runNumber': self.runNumber,
+ 'pulseIdInterval': [pulseIdFrom, pulseIdTo],
+ 'numberOfMacrobunches': pulseIdTo - pulseIdFrom,
+ }
+ try:
+ metadata['runInfo']['numberOfElectrons'] = self.numOfElectrons
+ metadata['runInfo']['electronsPerMacrobunch'] = self.electronsPerMacrobunch,
+ except:
+ pass # TODO: find smarter solution
metadata['histograms'] = {}
@@ -2090,19 +2083,11 @@ Source code for processor.DldProcessor
-
© Copyright 2018-2020, momentoscope team
-
-
-
- Built with Sphinx using a
-
- theme
-
- provided by Read the Docs.
+ Built with Sphinx using a theme provided by Read the Docs.
@@ -2114,6 +2099,7 @@ Source code for processor.DldProcessor
+
@@ -35,6 +30,10 @@
+
+
+
+
@@ -50,7 +49,7 @@
- hextof-processor
+ hextof-processor
@@ -60,7 +59,7 @@
- 1.0.3
+ 1.0.4
@@ -77,7 +76,6 @@
-
@@ -118,7 +116,6 @@
-
@@ -157,7 +154,7 @@
- - »
+ - Docs »
- Module code »
@@ -185,6 +182,8 @@ Source code for processor.pah
import os
import sys
+import h5py
+
try:
from camp.pah.beamtimedaqaccess import BeamtimeDaqAccess as _BeamtimeDaqAccess, H5FileDataAccess as _H5FileDataAccess, \
@@ -260,7 +259,32 @@ Source code for processor.pah
or channelName.startswith('/Photon Diagnostic/') \
or channelName.startswith('/Electron Diagnostic/') \
or channelName.startswith('/Beamlines/') \
- or channelName.startswith('/Timing/') # <--for datasets before 08-2018
+ or channelName.startswith('/Timing/') # <--for datasets before 08-2018
+
+ def filledInDesiredDataSets(self, sortedDesiredDataSets):
+ assert sortedDesiredDataSets, "Precond.: desiredDataSets not empty"
+
+ # finding largest datumsShape
+ largestShape = 0
+ for currentDataset in sortedDesiredDataSets:
+ with h5py.File(currentDataset.fileMeta.fileName(), 'r') as h5file:
+ if h5file[currentDataset.channelName].shape[1] > largestShape:
+ largestDataset = currentDataset
+ largestShape = h5file[currentDataset.channelName].shape[1]
+ largestDataset.desiredDatasetInitializedFromFile()
+
+ # all the result parts need to be the same shape. Therefore, they need to be the largest possible
+ result= [sortedDesiredDataSets[0].desiredDatasetInitializedFromSample(largestDataset)]
+ preceedingDataset= result[0]
+
+ for currentDataset in sortedDesiredDataSets[1:]:
+ if currentDataset.pulseIdInterval[0] - preceedingDataset.pulseIdInterval[1] > 0:
+ nanDataSet= _H5FileDataAccess.NaNDesiredDataSet((preceedingDataset.pulseIdInterval[1], currentDataset.pulseIdInterval[0]), preceedingDataset)
+ result.append(nanDataSet)
+ result.append(currentDataset.desiredDatasetInitializedFromSample(preceedingDataset))
+ preceedingDataset= currentDataset
+ assert self.hasNoGapsBetween(result), "Postcond.: No gaps between file chunks."
+ return result
[docs]class H5FileManager(_H5FileManager):
@@ -281,19 +305,11 @@ Source code for processor.pah
-
© Copyright 2018-2020, momentoscope team
-
-
-
- Built with Sphinx using a
-
- theme
-
- provided by Read the Docs.
+ Built with Sphinx using a theme provided by Read the Docs.
@@ -305,6 +321,7 @@ Source code for processor.pah
+
@@ -35,6 +30,10 @@
+
+
+
+
@@ -50,7 +49,7 @@
- hextof-processor
+ hextof-processor
@@ -60,7 +59,7 @@
- 1.0.3
+ 1.0.4
@@ -77,7 +76,6 @@
-
@@ -118,7 +116,6 @@
-
@@ -157,7 +154,7 @@
- - »
+ - Docs »
- Module code »
@@ -186,6 +183,50 @@ Source code for processor.utilities.calibration
<
import numpy as np
+[docs]def gen_sector_correction(prc, energies, eref, tofVoltage=None, sampleBias=None, monoEnergy=None):
+ """
+ This function is helpful in generating the sector_correction list.
+ This takes into account the time shift caused by the bit stealing hack
+ plus is keeps track of the time shift due to detector misalignment by making sure
+ all values of energies are at eref.
+
+ Usage: use the function to create the sector_correction list and assign it to prc.SECTOR_CORRECTION
+ or paste it into the settings with no brackets
+
+ :param prc:
+ :param energies:
+ :param eref:
+ :return:
+ """
+
+ if sampleBias is None:
+ sampleBias = np.nanmean(prc.dd['sampleBias'].values)
+ if monoEnergy is None:
+ monoEnergy = np.nanmean(prc.dd['monochromatorPhotonEnergy'].values)
+ if tofVoltage is None:
+ tofVoltage = np.nanmean(prc.dd['tofVoltage'].values)
+
+ # Here, the most basic sector_correction list is generated, where the bit stealing hack gets corrected
+ n_sectors=prc.dd['dldSectorId'].values.compute().max().astype(int)+1
+ sector_correction = np.floor(np.array(range(n_sectors))*np.power(2,prc.DLD_ID_BITS)/n_sectors)
+ # e.g. for 3 stolen bits, 8 detector ids (new s8 data), this will look like [0,1,2,3,4,5,6,7]
+ # e.g. for 1 stolen bit, 8 detector ids (old modified s8 data), this will look like [0,0,0,0,1,1,1,1]
+ # e.g. for 1 stolen bit, 2 detector ids (old s8 data), this will look like [0,1]
+
+
+ t_ref = energy2tof(eref, l=0.965, eoffset=-2.64-sampleBias+tofVoltage+monoEnergy)
+
+ times=[]
+ for ee in energies:
+ times.append((energy2tof(ee, l=0.965, eoffset=-2.64-47+30+monoEnergy)-t_ref)/prc.TOF_STEP_TO_NS)
+
+ sector_correction=sector_correction+np.array(times)
+ # here the sectors are shifter by the required time to align the energies
+
+ return sector_correction
+
+
+
# %% Energy calibration
""" The following functions convert between binding energy (Eb) in eV (negative convention)
@@ -489,19 +530,11 @@ Source code for processor.utilities.calibration
<
-
© Copyright 2018-2020, momentoscope team
-
-
-
- Built with Sphinx using a
-
- theme
-
- provided by Read the Docs.
+ Built with Sphinx using a theme provided by Read the Docs.
@@ -513,6 +546,7 @@ Source code for processor.utilities.calibration
<
+
@@ -35,6 +30,10 @@
+
+
+
+
@@ -50,7 +49,7 @@
- hextof-processor
+ hextof-processor
@@ -60,7 +59,7 @@
- 1.0.3
+ 1.0.4
@@ -77,7 +76,6 @@
-
@@ -118,7 +116,6 @@
-
@@ -157,7 +154,7 @@
- - »
+ - Docs »
- Module code »
@@ -365,19 +362,11 @@ Source code for processor.utilities.diagnostics
<
-
© Copyright 2018-2020, momentoscope team
-
-
-
- Built with Sphinx using a
-
- theme
-
- provided by Read the Docs.
+ Built with Sphinx using a theme provided by Read the Docs.
@@ -389,6 +378,7 @@ Source code for processor.utilities.diagnostics
<
+
@@ -35,6 +30,10 @@
+
+
+
+
@@ -50,7 +49,7 @@
- hextof-processor
+ hextof-processor
@@ -60,7 +59,7 @@
- 1.0.3
+ 1.0.4
@@ -77,7 +76,6 @@
-
@@ -118,7 +116,6 @@
-
@@ -157,7 +154,7 @@
- - »
+ - Docs »
- Module code »
@@ -446,19 +443,11 @@ Source code for processor.utilities.io
-
© Copyright 2018-2020, momentoscope team
-
-
-
- Built with Sphinx using a
-
- theme
-
- provided by Read the Docs.
+ Built with Sphinx using a theme provided by Read the Docs.
@@ -470,6 +459,7 @@ Source code for processor.utilities.io
+
@@ -35,6 +30,10 @@
+
+
+
+
@@ -50,7 +49,7 @@
- hextof-processor
+ hextof-processor
@@ -60,7 +59,7 @@
- 1.0.3
+ 1.0.4
@@ -77,7 +76,6 @@
-
@@ -118,7 +116,6 @@
-
@@ -157,7 +154,7 @@
- - »
+ - Docs »
- Module code »
@@ -853,19 +850,11 @@ Source code for processor.utilities.misc
-
© Copyright 2018-2020, momentoscope team
-
-
-
- Built with Sphinx using a
-
- theme
-
- provided by Read the Docs.
+ Built with Sphinx using a theme provided by Read the Docs.
@@ -877,6 +866,7 @@ Source code for processor.utilities.misc
+
@@ -35,6 +30,10 @@
+
+
+
+
@@ -50,7 +49,7 @@
- hextof-processor
+ hextof-processor
@@ -60,7 +59,7 @@
- 1.0.3
+ 1.0.4
@@ -77,7 +76,6 @@
-
@@ -118,7 +116,6 @@
-
@@ -157,7 +154,7 @@
- - »
+ - Docs »
- Module code »
@@ -604,19 +601,11 @@ Source code for processor.utilities.vis
-
© Copyright 2018-2020, momentoscope team
-
-
-
- Built with Sphinx using a
-
- theme
-
- provided by Read the Docs.
+ Built with Sphinx using a theme provided by Read the Docs.
@@ -628,6 +617,7 @@ Source code for processor.utilities.vis
+
@@ -35,6 +30,10 @@
+
+
+
+
@@ -52,7 +51,7 @@
- hextof-processor
+ hextof-processor
@@ -62,7 +61,7 @@
- 1.0.3
+ 1.0.4
@@ -79,7 +78,6 @@
-
@@ -124,7 +122,6 @@
-
@@ -163,7 +160,7 @@
- - »
+ - Docs »
- 1. Read DAQ data
@@ -242,19 +239,11 @@ 1.2. Run the postProcess me
-
© Copyright 2018-2020, momentoscope team
-
-
-
- Built with Sphinx using a
-
- theme
-
- provided by Read the Docs.
+ Built with Sphinx using a theme provided by Read the Docs.
@@ -266,6 +255,7 @@ 1.2. Run the postProcess me
+
@@ -35,6 +30,10 @@
+
+
+
+
@@ -52,7 +51,7 @@
- hextof-processor
+ hextof-processor
@@ -62,7 +61,7 @@
- 1.0.3
+ 1.0.4
@@ -79,7 +78,6 @@
-
@@ -120,7 +118,6 @@
-
- © Copyright 2018-2020, momentoscope team
- © Copyright 2018-2020, momentoscope team
- © Copyright 2018-2020, momentoscope team
- - - - Built with Sphinx using a - - theme - - provided by Read the Docs. + Built with Sphinx using a theme provided by Read the Docs. @@ -350,6 +339,7 @@- © Copyright 2018-2020, momentoscope team
- - - - Built with Sphinx using a - - theme - - provided by Read the Docs. + Built with Sphinx using a theme provided by Read the Docs. @@ -229,6 +218,7 @@- © Copyright 2018-2020, momentoscope team
- - - - Built with Sphinx using a - - theme - - provided by Read the Docs. + Built with Sphinx using a theme provided by Read the Docs. @@ -287,6 +276,7 @@