Skip to content

Commit

Permalink
TOF matching multi-threaded
Browse files Browse the repository at this point in the history
  • Loading branch information
@noferini
noferini committed Nov 19, 2023
1 parent 1170977 commit 61c8244
Show file tree
Hide file tree
Showing 5 changed files with 51 additions and 31 deletions.
13 changes: 8 additions & 5 deletions Detectors/GlobalTracking/include/GlobalTracking/MatchTOF.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,8 @@ class MatchTOF

void storeMatchable(bool val = true) { mStoreMatchable = val; }

void setNlanes(int lanes) { mNlanes = lanes; }

private:
bool prepareFITData();
int prepareInteractionTimes();
Expand All @@ -215,9 +217,9 @@ class MatchTOF

void doMatching(int sec);
void doMatchingForTPC(int sec);
void selectBestMatches();
void BestMatches(std::vector<o2::dataformats::MatchInfoTOFReco>& matchedTracksPairs, std::vector<o2::dataformats::MatchInfoTOF>* matchedTracks, std::vector<int>* matchedTracksIndex, int* matchedClustersIndex, const gsl::span<const o2::ft0::RecPoints>& FITRecPoints, const std::vector<Cluster>& TOFClusWork, const std::vector<matchTrack>* TracksWork, std::vector<o2::dataformats::CalibInfoTOF>& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer<o2::MCCompLabel>* TOFClusLabels, const std::vector<o2::MCCompLabel>* TracksLblWork, std::vector<o2::MCCompLabel>* OutTOFLabels, float calibMaxChi2);
void BestMatchesHP(std::vector<o2::dataformats::MatchInfoTOFReco>& matchedTracksPairs, std::vector<o2::dataformats::MatchInfoTOF>* matchedTracks, std::vector<int>* matchedTracksIndex, int* matchedClustersIndex, const gsl::span<const o2::ft0::RecPoints>& FITRecPoints, const std::vector<Cluster>& TOFClusWork, std::vector<o2::dataformats::CalibInfoTOF>& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer<o2::MCCompLabel>* TOFClusLabels, const std::vector<o2::MCCompLabel>* TracksLblWork, std::vector<o2::MCCompLabel>* OutTOFLabels);
void selectBestMatches(int sec);
void BestMatches(int sec, std::vector<o2::dataformats::MatchInfoTOFReco>& matchedTracksPairs, std::vector<o2::dataformats::MatchInfoTOF>* matchedTracks, std::vector<int>* matchedTracksIndex, int* matchedClustersIndex, const gsl::span<const o2::ft0::RecPoints>& FITRecPoints, const std::vector<Cluster>& TOFClusWork, const std::vector<matchTrack>* TracksWork, std::vector<o2::dataformats::CalibInfoTOF>& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer<o2::MCCompLabel>* TOFClusLabels, const std::vector<o2::MCCompLabel>* TracksLblWork, std::vector<o2::MCCompLabel>* OutTOFLabels, float calibMaxChi2);
void BestMatchesHP(int sec, std::vector<o2::dataformats::MatchInfoTOFReco>& matchedTracksPairs, std::vector<o2::dataformats::MatchInfoTOF>* matchedTracks, std::vector<int>* matchedTracksIndex, int* matchedClustersIndex, const gsl::span<const o2::ft0::RecPoints>& FITRecPoints, const std::vector<Cluster>& TOFClusWork, std::vector<o2::dataformats::CalibInfoTOF>& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer<o2::MCCompLabel>* TOFClusLabels, const std::vector<o2::MCCompLabel>* TracksLblWork, std::vector<o2::MCCompLabel>* OutTOFLabels);
bool propagateToRefX(o2::track::TrackParCov& trc, float xRef /*in cm*/, float stepInCm /*in cm*/, o2::track::TrackLTIntegral& intLT);
bool propagateToRefXWithoutCov(o2::track::TrackParCov& trc, float xRef /*in cm*/, float stepInCm /*in cm*/, float bz);

Expand All @@ -232,6 +234,8 @@ class MatchTOF
const o2::globaltracking::RecoContainer* mRecoCont = nullptr;
o2::InteractionRecord mStartIR{0, 0}; ///< IR corresponding to the start of the TF

int mNlanes = 3; ///< for multi-threading in matching

// TOF matching params (work in progress)
const MatchTOFParams* mMatchParams = nullptr;

Expand Down Expand Up @@ -313,8 +317,7 @@ class MatchTOF
///< per sector indices of TOF cluster entry in mTOFClusWork
std::array<std::vector<int>, o2::constants::math::NSectors> mTOFClusSectIndexCache;

///<array of track-TOFCluster pairs from the matching
std::vector<o2::dataformats::MatchInfoTOFReco> mMatchedTracksPairs;
///< array of track-TOFCluster pairs from the matching
std::vector<o2::dataformats::MatchInfoTOFReco> mMatchedTracksPairsSec[o2::constants::math::NSectors];

///<array of TOFChannel calibration info
Expand Down
54 changes: 33 additions & 21 deletions Detectors/GlobalTracking/src/MatchTOF.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@
#include "DataFormatsGlobalTracking/RecoContainerCreateTracksVariadic.h"
#include "TOFBase/Utils.h"

#include <omp.h>

using namespace o2::globaltracking;
using evGIdx = o2::dataformats::EvIndex<int, o2::dataformats::GlobalTrackID>;
using trkType = o2::dataformats::MatchInfoTOFReco::TrackType;
Expand Down Expand Up @@ -127,25 +129,35 @@ void MatchTOF::run(const o2::globaltracking::RecoContainer& inp, unsigned long f

mTimerTot.Start();
std::array<uint32_t, 18> nMatches = {0};
// run matching (this can be multi-thread)
mTimerMatchITSTPC.Start();

for (int sec = o2::constants::math::NSectors; sec--;) {
mMatchedTracksPairs.clear(); // new sector
LOG(debug) << "Doing matching for sector " << sec << "...";
if (mIsITSTPCused || mIsTPCTRDused || mIsITSTPCTRDused) {
mTimerMatchITSTPC.Start(sec == o2::constants::math::NSectors - 1);
mMatchedTracksPairsSec[sec].clear(); // new sector
}

o2::tof::Geo::Init();

if (mIsITSTPCused || mIsTPCTRDused || mIsITSTPCTRDused) {
#pragma omp parallel for schedule(dynamic) num_threads(mNlanes)
for (int sec = o2::constants::math::NSectors - 1; sec > -1; sec--) {
doMatching(sec);
mTimerMatchITSTPC.Stop();
}
if (mIsTPCused) {
mTimerMatchTPC.Start(sec == o2::constants::math::NSectors - 1);
}
mTimerMatchITSTPC.Stop();

mTimerMatchTPC.Start();
if (mIsTPCused) {
#pragma omp parallel for schedule(dynamic) num_threads(mNlanes)
for (int sec = o2::constants::math::NSectors - 1; sec > -1; sec--) {
doMatchingForTPC(sec);
mTimerMatchTPC.Stop();
}
}
mTimerMatchTPC.Stop();

// finalize
for (int sec = o2::constants::math::NSectors; sec--;) {
if (mStoreMatchable) {
// fill per sector
mMatchedTracksPairsSec[sec].clear();
mMatchedTracksPairsSec[sec].insert(mMatchedTracksPairsSec[sec].end(), mMatchedTracksPairs.begin(), mMatchedTracksPairs.end());

// if MC check if good or fake matches
if (mMCTruthON) {
for (auto& matchingPair : mMatchedTracksPairsSec[sec]) {
Expand All @@ -169,8 +181,8 @@ void MatchTOF::run(const o2::globaltracking::RecoContainer& inp, unsigned long f
}

LOG(debug) << "...done. Now check the best matches";
nMatches[sec] = mMatchedTracksPairs.size();
selectBestMatches();
nMatches[sec] = mMatchedTracksPairsSec[sec].size();
selectBestMatches(sec);
}
std::string nMatchesStr = "Number of pairs matched per sector: ";
for (int sec = o2::constants::math::NSectors; sec--;) {
Expand Down Expand Up @@ -850,7 +862,7 @@ void MatchTOF::doMatching(int sec)
foundCluster = true;
// set event indexes (to be checked)
int eventIndexTOFCluster = mTOFClusSectIndexCache[indices[0]][itof];
mMatchedTracksPairs.emplace_back(cacheTrk[itrk], eventIndexTOFCluster, mTOFClusWork[cacheTOF[itof]].getTime(), chi2, trkLTInt[iPropagation], mTrackGid[type][cacheTrk[itrk]], type, (trefTOF.getTime() - (minTrkTime + maxTrkTime - 100E3) * 0.5) * 1E-6, 0., resX, resZ); // subracting 100 ns to max track which was artificially added
mMatchedTracksPairsSec[sec].emplace_back(cacheTrk[itrk], eventIndexTOFCluster, mTOFClusWork[cacheTOF[itof]].getTime(), chi2, trkLTInt[iPropagation], mTrackGid[type][cacheTrk[itrk]], type, (trefTOF.getTime() - (minTrkTime + maxTrkTime - 100E3) * 0.5) * 1E-6, 0., resX, resZ); // subracting 100 ns to max track which was artificially added
}
}
}
Expand Down Expand Up @@ -1179,7 +1191,7 @@ void MatchTOF::doMatchingForTPC(int sec)
foundCluster = true;
// set event indexes (to be checked)
int eventIndexTOFCluster = mTOFClusSectIndexCache[indices[0]][itof];
mMatchedTracksPairs.emplace_back(cacheTrk[itrk], eventIndexTOFCluster, mTOFClusWork[cacheTOF[itof]].getTime(), chi2, trkLTInt[ibc][iPropagation], mTrackGid[trkType::UNCONS][cacheTrk[itrk]], trkType::UNCONS, resZ / mTPCVDrift * side, trefTOF.getZ(), resX, resZ); // TODO: check if this is correct!
mMatchedTracksPairsSec[sec].emplace_back(cacheTrk[itrk], eventIndexTOFCluster, mTOFClusWork[cacheTOF[itof]].getTime(), chi2, trkLTInt[ibc][iPropagation], mTrackGid[trkType::UNCONS][cacheTrk[itrk]], trkType::UNCONS, resZ / mTPCVDrift * side, trefTOF.getZ(), resX, resZ); // TODO: check if this is correct!
}
}
}
Expand Down Expand Up @@ -1231,16 +1243,16 @@ int MatchTOF::findFITIndex(int bc, const gsl::span<const o2::ft0::RecPoints>& FI
return index;
}
//______________________________________________
void MatchTOF::selectBestMatches()
void MatchTOF::selectBestMatches(int sec)
{
if (mSetHighPurity) {
BestMatchesHP(mMatchedTracksPairs, mMatchedTracks, mMatchedTracksIndex, mMatchedClustersIndex, mFITRecPoints, mTOFClusWork, mCalibInfoTOF, mTimestamp, mMCTruthON, mTOFClusLabels, mTracksLblWork, mOutTOFLabels);
BestMatchesHP(sec, mMatchedTracksPairsSec[sec], mMatchedTracks, mMatchedTracksIndex, mMatchedClustersIndex, mFITRecPoints, mTOFClusWork, mCalibInfoTOF, mTimestamp, mMCTruthON, mTOFClusLabels, mTracksLblWork, mOutTOFLabels);
return;
}
BestMatches(mMatchedTracksPairs, mMatchedTracks, mMatchedTracksIndex, mMatchedClustersIndex, mFITRecPoints, mTOFClusWork, mTracksWork, mCalibInfoTOF, mTimestamp, mMCTruthON, mTOFClusLabels, mTracksLblWork, mOutTOFLabels, mMatchParams->calibMaxChi2);
BestMatches(sec, mMatchedTracksPairsSec[sec], mMatchedTracks, mMatchedTracksIndex, mMatchedClustersIndex, mFITRecPoints, mTOFClusWork, mTracksWork, mCalibInfoTOF, mTimestamp, mMCTruthON, mTOFClusLabels, mTracksLblWork, mOutTOFLabels, mMatchParams->calibMaxChi2);
}
//______________________________________________
void MatchTOF::BestMatches(std::vector<o2::dataformats::MatchInfoTOFReco>& matchedTracksPairs, std::vector<o2::dataformats::MatchInfoTOF>* matchedTracks, std::vector<int>* matchedTracksIndex, int* matchedClustersIndex, const gsl::span<const o2::ft0::RecPoints>& FITRecPoints, const std::vector<Cluster>& TOFClusWork, const std::vector<matchTrack>* TracksWork, std::vector<o2::dataformats::CalibInfoTOF>& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer<o2::MCCompLabel>* TOFClusLabels, const std::vector<o2::MCCompLabel>* TracksLblWork, std::vector<o2::MCCompLabel>* OutTOFLabels, float calibMaxChi2)
void MatchTOF::BestMatches(int sec, std::vector<o2::dataformats::MatchInfoTOFReco>& matchedTracksPairs, std::vector<o2::dataformats::MatchInfoTOF>* matchedTracks, std::vector<int>* matchedTracksIndex, int* matchedClustersIndex, const gsl::span<const o2::ft0::RecPoints>& FITRecPoints, const std::vector<Cluster>& TOFClusWork, const std::vector<matchTrack>* TracksWork, std::vector<o2::dataformats::CalibInfoTOF>& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer<o2::MCCompLabel>* TOFClusLabels, const std::vector<o2::MCCompLabel>* TracksLblWork, std::vector<o2::MCCompLabel>* OutTOFLabels, float calibMaxChi2)
{
///< define the track-TOFcluster pair per sector

Expand Down Expand Up @@ -1355,7 +1367,7 @@ void MatchTOF::BestMatches(std::vector<o2::dataformats::MatchInfoTOFReco>& match
}
}
//______________________________________________
void MatchTOF::BestMatchesHP(std::vector<o2::dataformats::MatchInfoTOFReco>& matchedTracksPairs, std::vector<o2::dataformats::MatchInfoTOF>* matchedTracks, std::vector<int>* matchedTracksIndex, int* matchedClustersIndex, const gsl::span<const o2::ft0::RecPoints>& FITRecPoints, const std::vector<Cluster>& TOFClusWork, std::vector<o2::dataformats::CalibInfoTOF>& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer<o2::MCCompLabel>* TOFClusLabels, const std::vector<o2::MCCompLabel>* TracksLblWork, std::vector<o2::MCCompLabel>* OutTOFLabels)
void MatchTOF::BestMatchesHP(int sec, std::vector<o2::dataformats::MatchInfoTOFReco>& matchedTracksPairs, std::vector<o2::dataformats::MatchInfoTOF>* matchedTracks, std::vector<int>* matchedTracksIndex, int* matchedClustersIndex, const gsl::span<const o2::ft0::RecPoints>& FITRecPoints, const std::vector<Cluster>& TOFClusWork, std::vector<o2::dataformats::CalibInfoTOF>& CalibInfoTOF, unsigned long Timestamp, bool MCTruthON, const o2::dataformats::MCTruthContainer<o2::MCCompLabel>* TOFClusLabels, const std::vector<o2::MCCompLabel>* TracksLblWork, std::vector<o2::MCCompLabel>* OutTOFLabels)
{
///< define the track-TOFcluster pair per sector
float chi2SeparationCut = 2;
Expand Down
2 changes: 1 addition & 1 deletion Detectors/GlobalTrackingWorkflow/include/GlobalTrackingWorkflow/TOFMatcherSpec.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ namespace globaltracking
{

/// create a processor spec
framework::DataProcessorSpec getTOFMatcherSpec(o2::dataformats::GlobalTrackID::mask_t src, bool useMC, bool useFIT, bool tpcRefit, bool strict, float extratolerancetrd, bool pushMatchable, int lumiType);
framework::DataProcessorSpec getTOFMatcherSpec(o2::dataformats::GlobalTrackID::mask_t src, bool useMC, bool useFIT, bool tpcRefit, bool strict, float extratolerancetrd, bool pushMatchable, int lumiType, int nlanes = 1);

} // namespace globaltracking
} // namespace o2
Expand Down
8 changes: 5 additions & 3 deletions Detectors/GlobalTrackingWorkflow/src/TOFMatcherSpec.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ namespace globaltracking
class TOFMatcherSpec : public Task
{
public:
TOFMatcherSpec(std::shared_ptr<DataRequest> dr, std::shared_ptr<o2::base::GRPGeomRequest> gr, bool useMC, bool useFIT, bool tpcRefit, bool strict, bool pushMatchable) : mDataRequest(dr), mGGCCDBRequest(gr), mUseMC(useMC), mUseFIT(useFIT), mDoTPCRefit(tpcRefit), mStrict(strict), mPushMatchable(pushMatchable) {}
TOFMatcherSpec(std::shared_ptr<DataRequest> dr, std::shared_ptr<o2::base::GRPGeomRequest> gr, bool useMC, bool useFIT, bool tpcRefit, bool strict, bool pushMatchable, int lanes = 1) : mDataRequest(dr), mGGCCDBRequest(gr), mUseMC(useMC), mUseFIT(useFIT), mDoTPCRefit(tpcRefit), mStrict(strict), mPushMatchable(pushMatchable), mNlanes(lanes) {}
~TOFMatcherSpec() override = default;
void init(InitContext& ic) final;
void run(ProcessingContext& pc) final;
Expand All @@ -77,6 +77,7 @@ class TOFMatcherSpec : public Task
bool mStrict = false;
bool mPushMatchable = false;
float mExtraTolTRD = 0.;
int mNlanes = 1;
MatchTOF mMatcher; ///< Cluster finder
TStopwatch mTimer;
};
Expand All @@ -92,6 +93,7 @@ void TOFMatcherSpec::init(InitContext& ic)
mTPCCorrMapsLoader.init(ic);
mMatcher.storeMatchable(mPushMatchable);
mMatcher.setExtraTimeToleranceTRD(mExtraTolTRD);
mMatcher.setNlanes(mNlanes);
}

void TOFMatcherSpec::updateTimeDependentParams(ProcessingContext& pc)
Expand Down Expand Up @@ -237,7 +239,7 @@ void TOFMatcherSpec::endOfStream(EndOfStreamContext& ec)
mTimer.CpuTime(), mTimer.RealTime(), mTimer.Counter() - 1);
}

DataProcessorSpec getTOFMatcherSpec(GID::mask_t src, bool useMC, bool useFIT, bool tpcRefit, bool strict, float extratolerancetrd, bool pushMatchable, int lumiType)
DataProcessorSpec getTOFMatcherSpec(GID::mask_t src, bool useMC, bool useFIT, bool tpcRefit, bool strict, float extratolerancetrd, bool pushMatchable, int lumiType, int nlanes)
{
uint32_t ss = o2::globaltracking::getSubSpec(strict ? o2::globaltracking::MatchingType::Strict : o2::globaltracking::MatchingType::Standard);
Options opts;
Expand Down Expand Up @@ -314,7 +316,7 @@ DataProcessorSpec getTOFMatcherSpec(GID::mask_t src, bool useMC, bool useFIT, bo
"tof-matcher",
dataRequest->inputs,
outputs,
AlgorithmSpec{adaptFromTask<TOFMatcherSpec>(dataRequest, ggRequest, useMC, useFIT, tpcRefit, strict, pushMatchable)},
AlgorithmSpec{adaptFromTask<TOFMatcherSpec>(dataRequest, ggRequest, useMC, useFIT, tpcRefit, strict, pushMatchable, nlanes)},
opts};
}

Expand Down
5 changes: 4 additions & 1 deletion Detectors/GlobalTrackingWorkflow/src/tof-matcher-workflow.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ void customize(std::vector<o2::framework::ConfigParamSpec>& workflowOptions)
// option allowing to set parameters
std::vector<o2::framework::ConfigParamSpec> options{
{"disable-mc", o2::framework::VariantType::Bool, false, {"disable MC propagation even if available"}},
{"tof-lanes", o2::framework::VariantType::Int, 3, {"number of parallel lanes up to the matcher"}},
{"disable-root-input", o2::framework::VariantType::Bool, false, {"disable root-files input reader"}},
{"disable-root-output", o2::framework::VariantType::Bool, false, {"disable root-files output writer"}},
{"track-sources", VariantType::String, std::string{GID::ALL}, {"comma-separated list of sources to use: allowed TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD (all)"}},
Expand Down Expand Up @@ -97,6 +98,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
bool writematching = 0;
bool writecalib = 0;
auto outputType = configcontext.options().get<std::string>("output-type");
auto nLanes = configcontext.options().get<int>("tof-lanes");
if (outputType.rfind("matching-info") < outputType.size()) {
writematching = 1;
}
Expand All @@ -123,6 +125,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
LOG(debug) << "TOF matching in strict mode = " << strict;
LOG(debug) << "TOF extra time tolerance for TRD tracks = " << extratolerancetrd;
LOG(debug) << "Store all matchables = " << writeMatchable;
LOG(debug) << "TOF Nlanes for matcher = " << nLanes;

//GID::mask_t alowedSources = GID::getSourcesMask("TPC,ITS-TPC");
GID::mask_t alowedSources = GID::getSourcesMask("TPC,ITS-TPC,TPC-TRD,ITS-TPC-TRD");
Expand Down Expand Up @@ -159,7 +162,7 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext)
}
}

specs.emplace_back(o2::globaltracking::getTOFMatcherSpec(src, useMC, useFIT, false, strict, extratolerancetrd, writeMatchable, lumiType)); // doTPCrefit not yet supported (need to load TPC clusters?)
specs.emplace_back(o2::globaltracking::getTOFMatcherSpec(src, useMC, useFIT, false, strict, extratolerancetrd, writeMatchable, lumiType, nLanes)); // doTPCrefit not yet supported (need to load TPC clusters?)

if (!disableRootOut) {
std::vector<DataProcessorSpec> writers;
Expand Down

0 comments on commit 61c8244

Please sign in to comment.