From cf49339e7b1c0d20d72a75ba4ee16936b98d5bf7 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Thu, 31 Oct 2024 20:33:25 -0400 Subject: [PATCH 01/14] Restore build without ROOT. --- Source/Generators/LMCArraySignalGenerator.cc | 3 ++- Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc | 2 +- Source/Kassiopeia/LMCEventHold.cc | 2 ++ Source/Kassiopeia/LMCTrackHold.cc | 4 ++++ 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index 7f55d128..e450d334 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -347,11 +347,12 @@ namespace locust bool ArraySignalGenerator::RecordRunParameters( Signal* aSignal ) { +#ifdef ROOT_FOUND fInterface->aRunParameter = new RunParameters(); fInterface->aRunParameter->fSamplingRateMHz = fAcquisitionRate; fInterface->aRunParameter->fDecimationFactor = aSignal->DecimationFactor(); fInterface->aRunParameter->fLOfrequency = fLO_Frequency; - +#endif return true; } diff --git a/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc b/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc index 3fc6db83..bc8709cc 100644 --- a/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc +++ b/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc @@ -62,11 +62,11 @@ namespace locust bool CyclotronRadiationExtractor::UpdateTrackProperties( Kassiopeia::KSParticle &aFinalParticle, unsigned index, bool bStart ) { +#ifdef ROOT_FOUND double tLOfrequency = fInterface->aRunParameter->fLOfrequency; // Hz double tSamplingRate = fInterface->aRunParameter->fSamplingRateMHz; // MHz double tOffset = -tLOfrequency + tSamplingRate * 1.e6 / 2.; // Hz double tTime = index / tSamplingRate / 1.e6 / fInterface->aRunParameter->fDecimationFactor; -#ifdef ROOT_FOUND if (bStart) { fStartingIndex = index; diff --git a/Source/Kassiopeia/LMCEventHold.cc b/Source/Kassiopeia/LMCEventHold.cc index bf343299..54bbe738 100644 --- a/Source/Kassiopeia/LMCEventHold.cc +++ b/Source/Kassiopeia/LMCEventHold.cc @@ -185,9 +185,11 @@ namespace locust fInterface->fEventInProgress = false; fInterface->fDigitizerCondition.notify_one(); // unlock LPROG( lmclog, "Kass is waking after event" ); +#ifdef ROOT_FOUND delete fInterface->anEvent; delete fInterface->aTrack; return true; +#endif } } /* namespace locust */ diff --git a/Source/Kassiopeia/LMCTrackHold.cc b/Source/Kassiopeia/LMCTrackHold.cc index 52f39c72..3e58e15f 100644 --- a/Source/Kassiopeia/LMCTrackHold.cc +++ b/Source/Kassiopeia/LMCTrackHold.cc @@ -63,7 +63,9 @@ namespace locust bool TrackHold::ExecutePreTrackModification(Kassiopeia::KSTrack &aTrack) { +#ifdef ROOT_FOUND fInterface->aTrack->Initialize(); +#endif fInterface->fNewTrackStarting = true; double tTime = aTrack.GetInitialParticle().GetTime(); @@ -76,7 +78,9 @@ namespace locust { if ( aTrack.GetTotalSteps() > 0) { +#ifdef ROOT_FOUND fInterface->anEvent->AddTrack( fInterface->aTrack ); +#endif } double tTime = aTrack.GetFinalParticle().GetTime(); From 5e3ec8ba8163c3810dd0de32c0b363f1d9348227 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Thu, 31 Oct 2024 20:35:23 -0400 Subject: [PATCH 02/14] Valgrind fixes and some indenting. --- Source/Generators/LMCCavitySignalGenerator.cc | 4 +-- .../LMCLowPassFilterFFTGenerator.cc | 30 +++++++++++-------- .../LMCDampedHarmonicOscillator.cc | 2 +- Source/Utilities/LMCCavityUtility.cc | 6 ++-- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/Source/Generators/LMCCavitySignalGenerator.cc b/Source/Generators/LMCCavitySignalGenerator.cc index 4a6317ca..6abdf7a2 100644 --- a/Source/Generators/LMCCavitySignalGenerator.cc +++ b/Source/Generators/LMCCavitySignalGenerator.cc @@ -25,12 +25,12 @@ namespace locust Generator( aName ), fDoGenerateFunc( &CavitySignalGenerator::DoGenerateTime ), fLO_Frequency( 0. ), - fDeltaT( 0. ), + fDeltaT( 0. ), gxml_filename("blank.xml"), fphiLO(0.), fNPreEventSamples( 150000 ), fRandomPreEventSamples( false ), - fTrackDelaySeed( 0 ), + fTrackDelaySeed( 0 ), fThreadCheckTime(100), fKassNeverStarted( false ), fAliasedFrequencies( false ), diff --git a/Source/Generators/LMCLowPassFilterFFTGenerator.cc b/Source/Generators/LMCLowPassFilterFFTGenerator.cc index 69c7dce7..9e12d87e 100644 --- a/Source/Generators/LMCLowPassFilterFFTGenerator.cc +++ b/Source/Generators/LMCLowPassFilterFFTGenerator.cc @@ -74,16 +74,19 @@ namespace locust // Count trailing zeroes in window. int LowPassFilterFFTGenerator::GetEndingMargin( Signal* aSignal, int windowsize, int nwin, int ch ) { - int endingMargin = 0; - for (int i=0; iLongSignalTimeComplex()[ ch*aSignal->TimeSize()*aSignal->DecimationFactor() + (nwin+1)*windowsize - i ][0]) > 0.) - { - endingMargin = i; - break; - } - } - return endingMargin; + int endingMargin = 0; + for (int i=0; iLongSignalTimeComplex()[ ch*aSignal->TimeSize()*aSignal->DecimationFactor() + (nwin+1)*windowsize - i ][0]) > 0.) + { + endingMargin = i; + break; + } + } + } + return endingMargin; } @@ -164,8 +167,10 @@ namespace locust fftw_destroy_plan(ForwardPlan); fftw_destroy_plan(ReversePlan); - delete [] SignalComplex; - delete [] FFTComplex; + fftw_free( SignalComplex ); + fftw_free( FFTComplex ); + SignalComplex = NULL; + FFTComplex = NULL; } else { @@ -175,7 +180,6 @@ namespace locust } // nwin } // NCHANNELS - return true; } diff --git a/Source/RxComponents/LMCDampedHarmonicOscillator.cc b/Source/RxComponents/LMCDampedHarmonicOscillator.cc index f4957aa1..c192f28c 100644 --- a/Source/RxComponents/LMCDampedHarmonicOscillator.cc +++ b/Source/RxComponents/LMCDampedHarmonicOscillator.cc @@ -277,7 +277,7 @@ namespace locust } } // i - delete aSignal; + aSignal->Reset(); return convolutionMag; diff --git a/Source/Utilities/LMCCavityUtility.cc b/Source/Utilities/LMCCavityUtility.cc index a202c2a9..3d323969 100644 --- a/Source/Utilities/LMCCavityUtility.cc +++ b/Source/Utilities/LMCCavityUtility.cc @@ -193,9 +193,9 @@ namespace locust #ifdef ROOT_FOUND if (fWriteOutputFile) WriteRootHisto(nSteps, freqArray, gainArray); #endif - delete aSignal; - delete freqArray; - delete gainArray; + aSignal->Reset(); + delete[] freqArray; + delete[] gainArray; LPROG( testlog, "\nSummary:"); LPROG( testlog, "dho-threshold-factor is " << dhoThresholdFactor ); From 5e0fc57d25d20f63bee20147f7454c67486b0499 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Fri, 1 Nov 2024 16:11:46 -0400 Subject: [PATCH 03/14] More valgrind and indenting fixes. --- Source/Applications/Testing/testFFT.cc | 4 ++++ .../Applications/Testing/testLMCTestSignal.cc | 5 +++++ Source/Applications/Testing/testMockEGun.cc | 3 +++ .../Applications/Testing/testMockFreeField.cc | 2 ++ Source/Core/LMCFIRFileHandler.cc | 6 +++--- Source/Core/LMCHFSSResponseFileHandler.cc | 17 +++++++++++++++++ .../RxComponents/LMCDampedHarmonicOscillator.cc | 5 ++++- Source/Utilities/LMCCavityUtility.cc | 2 ++ 8 files changed, 40 insertions(+), 4 deletions(-) diff --git a/Source/Applications/Testing/testFFT.cc b/Source/Applications/Testing/testFFT.cc index 5d45606d..7140255d 100644 --- a/Source/Applications/Testing/testFFT.cc +++ b/Source/Applications/Testing/testFFT.cc @@ -73,6 +73,10 @@ double GetPower(bool bTime) ptransform += data[j][0]*data[j][0]+data[j][1]*data[j][1]; } + fftw_destroy_plan( plan); + fftw_free( data ); + data = NULL; + if (bTime) { LPROG(testlog, "power of original data time series is: " << pdata); diff --git a/Source/Applications/Testing/testLMCTestSignal.cc b/Source/Applications/Testing/testLMCTestSignal.cc index 82966b80..14354d08 100644 --- a/Source/Applications/Testing/testLMCTestSignal.cc +++ b/Source/Applications/Testing/testLMCTestSignal.cc @@ -113,6 +113,10 @@ class testLMCTestSignal ptransform += data[j][0]*data[j][0]+data[j][1]*data[j][1]; } + fftw_destroy_plan( plan); + fftw_free( data ); + data = NULL; + if (bTime) { LPROG(testlog, "fft power of original data time series is: " << pdata); @@ -161,6 +165,7 @@ TEST_CASE( "LMCTestSignal with default parameter values (pass)", "[single-file]" double threshold = 1.e-4; REQUIRE( fabs( aTestLMCTestSignal.GetPower(aSignal, N0, 1) - aTestLMCTestSignal.GetPower(aSignal, N0, 0) ) < threshold*aTestLMCTestSignal.GetPower(aSignal, N0, 1) ); REQUIRE( fabs( aTestLMCTestSignal.GetPower(aSignal, N0, 1)/N0/50. - pow(aTestLMCTestSignal.GetAmplitude(),2.) ) < threshold*aTestLMCTestSignal.GetPower(aSignal, N0, 1)/N0/50.); + aSignal->Reset(); } diff --git a/Source/Applications/Testing/testMockEGun.cc b/Source/Applications/Testing/testMockEGun.cc index 3d657205..c8b153fb 100644 --- a/Source/Applications/Testing/testMockEGun.cc +++ b/Source/Applications/Testing/testMockEGun.cc @@ -103,7 +103,10 @@ double GetPower() } LPROG(testlog, "E-gun transformed data sum is: " << ptransform/N0); + fftw_destroy_plan(plan); + fftw_free( data ); + data = NULL; return ptransform/N0; diff --git a/Source/Applications/Testing/testMockFreeField.cc b/Source/Applications/Testing/testMockFreeField.cc index d7590bd3..5b38fb7a 100644 --- a/Source/Applications/Testing/testMockFreeField.cc +++ b/Source/Applications/Testing/testMockFreeField.cc @@ -164,6 +164,8 @@ TEST_CASE( "Mock free space Larmor power. (pass)", "[single-file]" ) fftw_destroy_plan(plan); + fftw_free( data ); + data = NULL; double threshold = 1.e-4; diff --git a/Source/Core/LMCFIRFileHandler.cc b/Source/Core/LMCFIRFileHandler.cc index e7e6fd4a..d8b1f322 100644 --- a/Source/Core/LMCFIRFileHandler.cc +++ b/Source/Core/LMCFIRFileHandler.cc @@ -24,10 +24,10 @@ namespace locust bool FIRTransmitterHandler::Configure(const scarab::param_node& aParam) { - int fNModes = 2; - if( aParam.has( "n-modes" ) ) + int fNModes = 2; + if( aParam.has( "n-modes" ) ) { - fNModes = aParam["n-modes"]().as_int(); + fNModes = aParam["n-modes"]().as_int(); } if( aParam.has( "fir-transmitter-filename" ) ) { diff --git a/Source/Core/LMCHFSSResponseFileHandler.cc b/Source/Core/LMCHFSSResponseFileHandler.cc index 03169e31..62f002aa 100644 --- a/Source/Core/LMCHFSSResponseFileHandler.cc +++ b/Source/Core/LMCHFSSResponseFileHandler.cc @@ -175,7 +175,24 @@ namespace locust fftw_free(fFIRComplex); fFIRComplex = NULL; } + + + for( int bTE=0; bTE<2; bTE++) + { + for(int l=0; lReset(); delete[] freqArray; delete[] gainArray; + delete fTFReceiverHandler; + delete fAnalyticResponseFunction; LPROG( testlog, "\nSummary:"); LPROG( testlog, "dho-threshold-factor is " << dhoThresholdFactor ); From 8a8630682c698aff37fdc5de8f54cd7cefba99b8 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Fri, 1 Nov 2024 16:16:07 -0400 Subject: [PATCH 04/14] Deprecate some unused generators. --- Source/CMakeLists.txt | 4 ---- .../LMCHighPassFilterFFTGenerator.cc | 0 .../LMCHighPassFilterFFTGenerator.hh | 0 .../{Generators => Deprecated}/LMCLocalOscillatorGenerator.cc | 0 .../{Generators => Deprecated}/LMCLocalOscillatorGenerator.hh | 0 5 files changed, 4 deletions(-) rename Source/{Generators => Deprecated}/LMCHighPassFilterFFTGenerator.cc (100%) rename Source/{Generators => Deprecated}/LMCHighPassFilterFFTGenerator.hh (100%) rename Source/{Generators => Deprecated}/LMCLocalOscillatorGenerator.cc (100%) rename Source/{Generators => Deprecated}/LMCLocalOscillatorGenerator.hh (100%) diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 6209d4b6..023984a8 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -34,8 +34,6 @@ set( LOCUST_MC_HEADER_FILES Generators/LMCTestSignalGenerator.hh Generators/LMCDecimateSignalGenerator.hh Generators/LMCLowPassFilterFFTGenerator.hh - Generators/LMCHighPassFilterFFTGenerator.hh - Generators/LMCLocalOscillatorGenerator.hh Generators/LMCButterworthLPFGenerator.hh Transmitters/LMCFieldBuffer.hh @@ -108,8 +106,6 @@ set( LOCUST_MC_SOURCE_FILES Generators/LMCTestSignalGenerator.cc Generators/LMCDecimateSignalGenerator.cc Generators/LMCLowPassFilterFFTGenerator.cc - Generators/LMCHighPassFilterFFTGenerator.cc - Generators/LMCLocalOscillatorGenerator.cc Generators/LMCButterworthLPFGenerator.cc diff --git a/Source/Generators/LMCHighPassFilterFFTGenerator.cc b/Source/Deprecated/LMCHighPassFilterFFTGenerator.cc similarity index 100% rename from Source/Generators/LMCHighPassFilterFFTGenerator.cc rename to Source/Deprecated/LMCHighPassFilterFFTGenerator.cc diff --git a/Source/Generators/LMCHighPassFilterFFTGenerator.hh b/Source/Deprecated/LMCHighPassFilterFFTGenerator.hh similarity index 100% rename from Source/Generators/LMCHighPassFilterFFTGenerator.hh rename to Source/Deprecated/LMCHighPassFilterFFTGenerator.hh diff --git a/Source/Generators/LMCLocalOscillatorGenerator.cc b/Source/Deprecated/LMCLocalOscillatorGenerator.cc similarity index 100% rename from Source/Generators/LMCLocalOscillatorGenerator.cc rename to Source/Deprecated/LMCLocalOscillatorGenerator.cc diff --git a/Source/Generators/LMCLocalOscillatorGenerator.hh b/Source/Deprecated/LMCLocalOscillatorGenerator.hh similarity index 100% rename from Source/Generators/LMCLocalOscillatorGenerator.hh rename to Source/Deprecated/LMCLocalOscillatorGenerator.hh From 156a9dc29debc45dc1d1ac122918356eb23e3d9c Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Sun, 3 Nov 2024 20:39:06 -0500 Subject: [PATCH 05/14] Delete class objects in destructors. --- Source/Generators/LMCArraySignalGenerator.cc | 12 ++++++++++++ Source/Generators/LMCCavitySignalGenerator.cc | 16 ++++++++++++++++ .../Kassiopeia/LMCCyclotronRadiationExtractor.cc | 4 ++++ Source/Kassiopeia/LMCFieldCalculator.cc | 8 ++++++++ 4 files changed, 40 insertions(+) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index e450d334..ecd3f7ac 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -55,6 +55,18 @@ namespace locust ArraySignalGenerator::~ArraySignalGenerator() { + if ( fAntennaElementPositioner != NULL ) + { + delete fAntennaElementPositioner; + } + if ( fTransmitter != NULL ) + { + delete fTransmitter; + } + if ( fPowerCombiner != NULL ) + { + delete fPowerCombiner; + } } void ArraySignalGenerator::SetParameters( const scarab::param_node& aParam ) diff --git a/Source/Generators/LMCCavitySignalGenerator.cc b/Source/Generators/LMCCavitySignalGenerator.cc index 6abdf7a2..ffde2b61 100644 --- a/Source/Generators/LMCCavitySignalGenerator.cc +++ b/Source/Generators/LMCCavitySignalGenerator.cc @@ -46,6 +46,22 @@ namespace locust CavitySignalGenerator::~CavitySignalGenerator() { + if (fPowerCombiner != NULL) + { + delete fPowerCombiner; + } + if (fFieldCalculator != NULL) + { + delete fFieldCalculator; + } + if (fTFReceiverHandler != NULL) + { + delete fTFReceiverHandler; + } + if (fAnalyticResponseFunction != NULL) + { + delete fAnalyticResponseFunction; + } } diff --git a/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc b/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc index bc8709cc..473b3737 100644 --- a/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc +++ b/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc @@ -37,6 +37,10 @@ namespace locust } CyclotronRadiationExtractor::~CyclotronRadiationExtractor() { + if (fFieldCalculator != NULL) + { + delete fFieldCalculator; + } } bool CyclotronRadiationExtractor::Configure() diff --git a/Source/Kassiopeia/LMCFieldCalculator.cc b/Source/Kassiopeia/LMCFieldCalculator.cc index cbd682cc..374cf7a6 100644 --- a/Source/Kassiopeia/LMCFieldCalculator.cc +++ b/Source/Kassiopeia/LMCFieldCalculator.cc @@ -35,6 +35,14 @@ namespace locust } FieldCalculator::~FieldCalculator() { + if (fTFReceiverHandler != NULL) + { + delete fTFReceiverHandler; + } + if (fAnalyticResponseFunction != NULL) + { + delete fAnalyticResponseFunction; + } } bool FieldCalculator::ConfigureByInterface() From c253a6428257af10cb1d4a8008722509ea9d6742 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Sun, 3 Nov 2024 21:18:17 -0500 Subject: [PATCH 06/14] Compile locust with MPI interface classes, replicated from KEMField. --- CMakeLists.txt | 6 + Source/CMakeLists.txt | 16 +- Source/Core/LMCMPIEnvironment.hh | 25 ++ Source/Core/LMCMPIInterface.cc | 428 +++++++++++++++++++++++++++++++ Source/Core/LMCMPIInterface.hh | 153 +++++++++++ 5 files changed, 625 insertions(+), 3 deletions(-) create mode 100644 Source/Core/LMCMPIEnvironment.hh create mode 100644 Source/Core/LMCMPIInterface.cc create mode 100644 Source/Core/LMCMPIInterface.hh diff --git a/CMakeLists.txt b/CMakeLists.txt index 8470f4f8..b95ae515 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -189,12 +189,18 @@ if (ROOT_FOUND) add_definitions(-DROOT_FOUND) pbuilder_add_ext_libraries (${ROOT_LIBRARIES}) message(STATUS "ROOT_LIBRARIES is ${ROOT_LIBRARIES}") + message(STATUS "ROOT_INCLUDE_DIRS is ${ROOT_INCLUDE_DIRS}") else (ROOT_FOUND) message(STATUS "Building without ROOT") remove_definitions(-DROOT_FOUND) endif (ROOT_FOUND) include_directories (${ROOT_INCLUDE_DIR}) +option (locust_mc_USE_MPI "Use MPI to accelerate calculations using multiple processors" OFF) +if (locust_mc_USE_MPI) + find_package(MPI REQUIRED) +endif (locust_mc_USE_MPI) + # Boost (1.46 required for filesystem version 3) list (APPEND Boost_COMPONENTS date_time filesystem program_options system thread) find_package (Boost 1.46.0 REQUIRED COMPONENTS ${Boost_COMPONENTS}) diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 023984a8..95753f98 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -13,6 +13,7 @@ set( LOCUST_MC_HEADER_FILES Core/LMCHFSSResponseFileHandler.hh Core/LMCFIRFileHandler.hh Core/LMCTFFileHandler.hh + Core/LMCMPIEnvironment.hh Utilities/LMCUtility.hh Utilities/LMCCavityUtility.hh @@ -142,16 +143,25 @@ set( LOCUST_MC_SOURCE_FILES ) +if (locust_mc_USE_MPI) + set( LOCUST_MC_HEADER_FILES ${LOCUST_MC_HEADER_FILES} + Core/LMCMPIInterface.hh + ) + + set( LOCUST_MC_SOURCE_FILES ${LOCUST_MC_SOURCE_FILES} + Core/LMCMPIInterface.cc + ) +endif (locust_mc_USE_MPI) + + if (locust_mc_BUILD_WITH_KASSIOPEIA) # Here we only need to add the Kassiopeia-dependent header/source files to the header/source-file lists set( LOCUST_MC_HEADER_FILES ${LOCUST_MC_HEADER_FILES} - Generators/LMCFreeFieldSignalGenerator.hh Generators/LMCKassSignalGenerator.hh Generators/LMCArraySignalGenerator.hh - Generators/LMCCavitySignalGenerator.hh - + Generators/LMCCavitySignalGenerator.hh Transmitters/LMCKassTransmitter.hh Transmitters/LMCKassCurrentTransmitter.hh diff --git a/Source/Core/LMCMPIEnvironment.hh b/Source/Core/LMCMPIEnvironment.hh new file mode 100644 index 00000000..dbaff319 --- /dev/null +++ b/Source/Core/LMCMPIEnvironment.hh @@ -0,0 +1,25 @@ +/* + * MPIEnvironment.hh + * + * Includes MPI and defines common macros. + * + * Created on: 17 Aug 2015 + * Author: wolfgang + */ + +#ifndef KMPIENVIRONMENT_HH_ +#define KMPIENVIRONMENT_HH_ + +#ifdef KEMFIELD_USE_MPI +#include "KMPIInterface.hh" +#endif + +#ifdef KEMFIELD_USE_MPI +#define MPI_SINGLE_PROCESS if (KEMField::KMPIInterface::GetInstance()->GetProcess() == 0) +#define MPI_SECOND_PROCESS if (KEMField::KMPIInterface::GetInstance()->GetProcess() == 1) +#else +#define MPI_SINGLE_PROCESS +#define MPI_SECOND_PROCESS +#endif + +#endif /* KMPIENVIRONMENT_HH_ */ diff --git a/Source/Core/LMCMPIInterface.cc b/Source/Core/LMCMPIInterface.cc new file mode 100644 index 00000000..d0be74b0 --- /dev/null +++ b/Source/Core/LMCMPIInterface.cc @@ -0,0 +1,428 @@ +#include "LMCMPIInterface.hh" + +//#include "KEMCoreMessage.hh" + +#ifdef LOCAL_RANK_MPI +//used to retrieve the host name +//should be available on on POSIX systems +#include +#include +#endif + +#define MSG_TAG 999 +#define HOST_DETERMINATION_TAG 998 +#define LOCALID_DETERMINATION_TAG 997 + +namespace locust +{ +LMCMPIInterface* LMCMPIInterface::fMPIInterface = nullptr; + +LMCMPIInterface::LMCMPIInterface() +{ + fProcess = -1; + fNProcesses = -1; + fLocalRank = -1; + fSubGroupRank = -1; + fNSubGroupProcesses = -1; + fPartnerProcessID = -1; + fSplitMode = false; +} + +LMCMPIInterface::~LMCMPIInterface() = default; + +void LMCMPIInterface::SetSplitMode(bool enable) +{ + fSplitMode = enable; + + //construct groups/communicators to evenly split up processes + SetupSubGroups(); + + //cannot make a valid split, so revert to standard mode + if (fSplitMode && !fValidSplit) { +// kem_cout(eWarning) << "Invalid MPI split mode detected - revert to standard mode with " << fNProcesses << " processes." << eom; + fSplitMode = false; + } + else if (fSplitMode) { +// kem_cout(eNormal) << "Using MPI split mode with 2 x " << fNSubGroupProcesses << " processes" << eom; + } +} + +void LMCMPIInterface::Initialize(int* argc, char*** argv, bool split_mode) +{ + /* Let the system do what it needs to start up MPI */ + int initialized = 0; + MPI_Initialized(&initialized); + + if (!initialized) + MPI_Init(argc, argv); + + /* Get my process fProcess */ + MPI_Comm_rank(MPI_COMM_WORLD, &fProcess); + + /* Find out how many processes are being used */ + MPI_Comm_size(MPI_COMM_WORLD, &fNProcesses); + + if (fProcess <= 0) { + if (fNProcesses <= 0) { +// kem_cout(eWarning) << "No MPI processes found - not running in an MPI context?" << eom; + } + else if (!initialized) { // only show this once +// kem_cout << "Running in MPI context with " << fNProcesses << " processes." << eom; + } + } + + //now determine the local rank of this process (indexed from zero) on the local host + //for example, if processes (0,2,5) are running on host A + //and processes (1,3,4) are running on host B, then the + //local rank of process 3 is 1, and the local rank of process 5 is 2 + DetermineLocalRank(); + + SetSplitMode(split_mode); +} + +void LMCMPIInterface::Finalize() +{ + /* Shut down MPI */ + int finalized = 0; + MPI_Finalized(&finalized); + + if (!finalized) + MPI_Finalize(); +} + +/** + * Interface to accessing LMCMPIInterface. + */ +LMCMPIInterface* LMCMPIInterface::GetInstance() +{ + if (fMPIInterface == nullptr) + fMPIInterface = new LMCMPIInterface(); + return fMPIInterface; +} + +/** + * Ensures that a process written between BeginSequentialProcess() and + * EndSequentialProcess() is done one processor at a time. + */ +void LMCMPIInterface::BeginSequentialProcess() +{ + int flag = 1; + + GlobalBarrier(); + + if (fProcess > 0) + MPI_Recv(&flag, 1, MPI_INT, fProcess - 1, 50, MPI_COMM_WORLD, &fStatus); +} + +/** + * @see BeginSequentialProcess() + */ +void LMCMPIInterface::EndSequentialProcess() +{ + int flag = 0; + + if (fProcess < (fNProcesses - 1)) + MPI_Send(&flag, 1, MPI_INT, fProcess + 1, 50, MPI_COMM_WORLD); + + GlobalBarrier(); +} + +//safely print messages in process order, by passing information to the +//root process for collection before calling cout +void LMCMPIInterface::PrintMessage(std::string msg) +{ + if (fNProcesses < 0 || fProcess < 0) { +// kem_cout(eError) << "MPI not initialized" << ret; +// kem_cout << "Message was: " << msg << eom; + return; + } + + unsigned int n_char = msg.size(); + + std::vector in_msg_sizes; + std::vector out_msg_sizes; + in_msg_sizes.resize(fNProcesses, 0); + out_msg_sizes.resize(fNProcesses, 0); + in_msg_sizes[fProcess] = n_char; + + //obtain the message sizes from all of the objects + MPI_Allreduce(&(in_msg_sizes[0]), &(out_msg_sizes[0]), fNProcesses, MPI_UNSIGNED, MPI_SUM, MPI_COMM_WORLD); + + //compute the total message size + unsigned int total_msg_size = 0; + std::vector msg_start_indexes; + msg_start_indexes.resize(fNProcesses); + for (int i = 0; i < fNProcesses; i++) { + total_msg_size += out_msg_sizes[i]; + }; + + for (int i = 0; i < fNProcesses; i++) { + for (int j = 0; j < i; j++) { + msg_start_indexes[i] += out_msg_sizes[j]; + } + }; + + //allocate buffers to reduce all of the messages + std::vector buf; + buf.resize(total_msg_size); + + //fill the appropriate section of the buffer + for (unsigned int i = 0; i < msg.size(); i++) { + buf[msg_start_indexes[fProcess] + i] = msg.at(i); + } + + MPI_Status status; + if (fProcess == 0) { + for (int i = 1; i < fNProcesses; i++) { + MPI_Recv(&(buf[msg_start_indexes[i]]), out_msg_sizes[i], MPI_CHAR, i, MSG_TAG, MPI_COMM_WORLD, &status); + } + } + else { + MPI_Send(&(buf[msg_start_indexes[fProcess]]), out_msg_sizes[fProcess], MPI_CHAR, 0, MSG_TAG, MPI_COMM_WORLD); + } + + if (fProcess == 0) { + //convert to string + std::stringstream final_output; + for (char c : buf) { + final_output << c; + } + std::string full_message = final_output.str(); + + //split the messages into separate lines so we can correctly use KEMField::cout + std::vector lines; + std::string::size_type pos = 0; + std::string::size_type prev = 0; + while ((pos = full_message.find('\n', prev)) != std::string::npos) { + lines.push_back(full_message.substr(prev, pos - prev)); + prev = pos + 1; + } + + if (lines.size() == 0) { + //message has no newline characters, so push the whole message into one line + lines.push_back(full_message); + } + + //print message line by line + for (auto& line : lines) { +// kem_cout() << line << eom; + } + } +} + + +void LMCMPIInterface::DetermineLocalRank() +{ +#ifdef LOCAL_RANK_MPI + //get the machine's hostname + char host_name[256]; + int ret_val = gethostname(host_name, 256); + if (ret_val != 0) { + std::cout << "Host name error!" << std::endl; + LMCMPIInterface::GetInstance()->Finalize(); + std::exit(1); + }; + + std::stringstream hostname_ss; + int count = 0; + do { + hostname_ss << host_name[count]; + count++; + } while (host_name[count] != '\0' && count < 256); + std::string hostname = hostname_ss.str(); + fHostName = hostname; + + //first we have to collect all of the hostnames that are running a process + unsigned int n_char = hostname.size(); + std::vector in_msg_sizes; + std::vector out_msg_sizes; + in_msg_sizes.resize(fNProcesses, 0); + out_msg_sizes.resize(fNProcesses, 0); + in_msg_sizes[fProcess] = n_char; + + //obtain the message sizes from all of the processes + MPI_Allreduce(&(in_msg_sizes[0]), &(out_msg_sizes[0]), fNProcesses, MPI_UNSIGNED, MPI_SUM, MPI_COMM_WORLD); + + //compute the total message size + unsigned int total_msg_size = 0; + std::vector msg_start_indexes; + msg_start_indexes.resize(fNProcesses); + for (int i = 0; i < fNProcesses; i++) { + total_msg_size += out_msg_sizes[i]; + }; + + for (int i = 0; i < fNProcesses; i++) { + for (int j = 0; j < i; j++) { + msg_start_indexes[i] += out_msg_sizes[j]; + } + }; + + //allocate buffers to reduce all of the messages + std::vector buf; + buf.resize(total_msg_size); + + //fill the appropriate section of the buffer + for (unsigned int i = 0; i < hostname.size(); i++) { + buf[msg_start_indexes[fProcess] + i] = hostname.at(i); + } + + //reduce the buffer across all processes + MPI_Status status; + if (fProcess == 0) { + for (int i = 1; i < fNProcesses; i++) { + MPI_Recv(&(buf[msg_start_indexes[i]]), + out_msg_sizes[i], + MPI_CHAR, + i, + HOST_DETERMINATION_TAG, + MPI_COMM_WORLD, + &status); + } + } + else { + unsigned int root_rank = 0; + MPI_Send(&(buf[msg_start_indexes[fProcess]]), + out_msg_sizes[fProcess], + MPI_CHAR, + root_rank, + HOST_DETERMINATION_TAG, + MPI_COMM_WORLD); + } + + //now broadcast the complete list of hostnames to all processes + MPI_Bcast(&(buf[0]), total_msg_size, MPI_CHAR, 0, MPI_COMM_WORLD); + + //now every node has a list of all host names + //we now need to figure out how many other processes are also running on + //the same host, and how many devices are available on this host + //then we can distribute the processes equitably to each device + + std::vector hostname_list; + hostname_list.resize(fNProcesses); + for (int i = 0; i < fNProcesses; i++) { + hostname_list[i] = std::string(""); + for (unsigned int j = 0; j < out_msg_sizes[i]; j++) { + hostname_list[i].push_back(buf[msg_start_indexes[i] + j]); + } + } + + //collect all the process ids of all the process running on this host + fCoHostedProcessIDs.clear(); + for (int i = 0; i < fNProcesses; i++) { + if (hostname == hostname_list[i]) { + fCoHostedProcessIDs.push_back(i); + } + } + + //determine the 'local' rank of this process + for (unsigned int i = 0; i < fCoHostedProcessIDs.size(); i++) { + if (fCoHostedProcessIDs[i] == fProcess) { + fLocalRank = i; + }; + } + +#endif +} + + +void LMCMPIInterface::SetupSubGroups() +{ +#ifdef LOCAL_RANK_MPI + //we need to retrieve the local rank from each process + //to make a associative map betweek global-rank and local-rank + std::vector local_ranks; + local_ranks.resize(fNProcesses); + local_ranks[fProcess] = fLocalRank; + + //reduce the buffer across all processes + MPI_Status status; + if (fProcess == 0) { + for (int i = 1; i < fNProcesses; i++) { + MPI_Recv(&(local_ranks[i]), 1, MPI_INT, i, LOCALID_DETERMINATION_TAG, MPI_COMM_WORLD, &status); + } + } + else { + unsigned int root_rank = 0; + MPI_Send(&(local_ranks[fProcess]), 1, MPI_INT, root_rank, LOCALID_DETERMINATION_TAG, MPI_COMM_WORLD); + } + + //now broadcast the complete list of hostnames to all processes + MPI_Bcast(&(local_ranks[0]), fNProcesses, MPI_INT, 0, MPI_COMM_WORLD); + + //now every process has a list of the local rank associated with every other process + //now we can proceed to determine which group they below to + std::vector even_members; + std::vector odd_members; + for (int i = 0; i < fNProcesses; i++) { + if (local_ranks[i] % 2 == 0) { + even_members.push_back(i); + } + else { + odd_members.push_back(i); + } + } + + fValidSplit = false; + if (even_members.size() == odd_members.size()) { + fValidSplit = true; + } + + //get the world group + MPI_Group world; + MPI_Comm_group(MPI_COMM_WORLD, &world); + + //now we go ahead and construct the groups and communicators + MPI_Group_incl(world, even_members.size(), &(even_members[0]), &fEvenGroup); + MPI_Group_incl(world, odd_members.size(), &(odd_members[0]), &fOddGroup); + + MPI_Comm_create(MPI_COMM_WORLD, fEvenGroup, &fEvenCommunicator); + MPI_Comm_create(MPI_COMM_WORLD, fOddGroup, &fOddCommunicator); + + //now we set things up for this process + if (fLocalRank % 2 == 0) { + fIsEvenGroupMember = true; + MPI_Comm_rank(fEvenCommunicator, &fSubGroupRank); + fNSubGroupProcesses = even_members.size(); + } + else { + fIsEvenGroupMember = false; + MPI_Comm_rank(fOddCommunicator, &fSubGroupRank); + fNSubGroupProcesses = odd_members.size(); + } + + //finally, if we have a valid split (equal numbers of even and odd process) + //we pair up processes so they can exchange data + if (fValidSplit) //must have even number of processes! + { + int status = 0; + int result = 0; + if (fCoHostedProcessIDs.size() % 2 == 0) { + status = 1; + }; + MPI_Allreduce(&status, &result, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + + //to make things faster we first try to pair up processes + //which share the same node/host + if (result == fNProcesses) { + //we can because each host has an even number of processes + if (fIsEvenGroupMember) { + fPartnerProcessID = fCoHostedProcessIDs[fLocalRank + 1]; + } + else { + fPartnerProcessID = fCoHostedProcessIDs[fLocalRank - 1]; + } + } + else { + //this isn't possible so we have to pair up processes across nodes + if (fIsEvenGroupMember) { + fPartnerProcessID = fProcess + 1; + } + else { + fPartnerProcessID = fProcess - 1; + } + } + } +#endif +} + +} // namespace KEMField diff --git a/Source/Core/LMCMPIInterface.hh b/Source/Core/LMCMPIInterface.hh new file mode 100644 index 00000000..e2e15b52 --- /dev/null +++ b/Source/Core/LMCMPIInterface.hh @@ -0,0 +1,153 @@ +#ifndef LMCMPIINTERFACE_DEF +#define LMCMPIINTERFACE_DEF + +#include "mpi.h" + +#include +#include + +#define LOCAL_RANK_MPI + +namespace locust +{ + +class LMCMPIInterface +{ + public: + static LMCMPIInterface* GetInstance(); + + void Initialize(int* argc, char*** argv, bool split_mode = true); + void Finalize(); + + void SetSplitMode(bool enable = true); + + bool Check() const { + return (fProcess >= 0) && (fNProcesses > 0); + } + int GetProcess() const + { + return fProcess; + } + int GetNProcesses() const + { + return fNProcesses; + } + int GetLocalRank() const + { + return fLocalRank; + } + std::string GetHostName() const + { + return fHostName; + }; + + void BeginSequentialProcess(); + void EndSequentialProcess(); + + void GlobalBarrier() const + { + MPI_Barrier(MPI_COMM_WORLD); + } + + //when called, this function must be encountered by all processes + //or the program will lock up, treat as a global barrier + void PrintMessage(std::string msg); + + //routines to be used by programs which split the processes into two + //groups bases on even/odd local process rank + bool SplitMode() + { + return fSplitMode; + }; + bool IsSplitValid() + { + return fValidSplit; + }; + bool IsEvenGroupMember() + { + return fIsEvenGroupMember; + }; + int GetNSubGroupProcesses() + { + return fNSubGroupProcesses; + } + int GetSubGroupRank() + { + return fSubGroupRank; + }; + int GetPartnerProcessID() + { + return fPartnerProcessID; + }; + + MPI_Group* GetSubGroup() + { + if (fIsEvenGroupMember) { + return &fEvenGroup; + } + else { + return &fOddGroup; + }; + } + + MPI_Comm* GetSubGroupCommunicator() + { + if (fIsEvenGroupMember) { + return &fEvenCommunicator; + } + else { + return &fOddCommunicator; + }; + } + + MPI_Group* GetEvenGroup() + { + return &fEvenGroup; + }; + MPI_Group* GetOddGroup() + { + return &fOddGroup; + }; + MPI_Comm* GetEvenCommunicator() + { + return &fEvenCommunicator; + }; + MPI_Comm* GetOddCommunicator() + { + return &fOddCommunicator; + }; + + + protected: + LMCMPIInterface(); + virtual ~LMCMPIInterface(); + + static LMCMPIInterface* fMPIInterface; + + int fProcess; + int fNProcesses; + int fLocalRank; + std::string fHostName; + std::vector fCoHostedProcessIDs; + + //groups and communicators for splitting processes into + //two sets, based on whether they have even/odd (local) ranks + bool fSplitMode; + MPI_Group fEvenGroup; //even process subgroup + MPI_Group fOddGroup; //odd process subgroup + MPI_Comm fEvenCommunicator; //comm for even group + MPI_Comm fOddCommunicator; //comm for odd group + bool fValidSplit; //true if the size of the subgroups is equal + bool fIsEvenGroupMember; //true if this process is a member of the even subgroup + int fSubGroupRank; //rank of this process in its subgroup + int fNSubGroupProcesses; //number of processes in the subgroup this process belongs to + int fPartnerProcessID; //global rank of partner process in other subgroup + + void DetermineLocalRank(); + void SetupSubGroups(); + + MPI_Status fStatus; +}; +} // namespace locust + +#endif /* LMCMPIINTERFACE_DEF */ From bec49e2a83de1fd06ff69094d36bb5b36b8454c8 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Fri, 8 Nov 2024 21:42:29 -0500 Subject: [PATCH 07/14] Continue organizing meta-data within Root structures, for convenience prior to writing. --- Source/Generators/LMCArraySignalGenerator.cc | 3 +- Source/Generators/LMCCavitySignalGenerator.cc | 12 +++++++- Source/IO/LMCRunParameters.cc | 10 ++++++- Source/IO/LMCRunParameters.hh | 3 ++ .../LMCCyclotronRadiationExtractor.cc | 2 +- Source/Kassiopeia/LMCEventHold.cc | 29 +++++++++++++++++++ Source/Kassiopeia/LMCTrackHold.cc | 4 +++ 7 files changed, 59 insertions(+), 4 deletions(-) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index 7f55d128..ad06f9c5 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -347,11 +347,12 @@ namespace locust bool ArraySignalGenerator::RecordRunParameters( Signal* aSignal ) { +#ifdef FIND_ROOT fInterface->aRunParameter = new RunParameters(); fInterface->aRunParameter->fSamplingRateMHz = fAcquisitionRate; fInterface->aRunParameter->fDecimationFactor = aSignal->DecimationFactor(); fInterface->aRunParameter->fLOfrequency = fLO_Frequency; - +#endif return true; } diff --git a/Source/Generators/LMCCavitySignalGenerator.cc b/Source/Generators/LMCCavitySignalGenerator.cc index 4a6317ca..e35d4537 100644 --- a/Source/Generators/LMCCavitySignalGenerator.cc +++ b/Source/Generators/LMCCavitySignalGenerator.cc @@ -123,7 +123,6 @@ namespace locust else if (!fInterface->fbWaveguide) // Cavity config follows { // No HFSS file is present: Cavity as damped harmonic oscillator - fAnalyticResponseFunction = new DampedHarmonicOscillator(); if ( !fAnalyticResponseFunction->Configure(tParam) || !(CrossCheckCavityConfig()) ) { @@ -384,6 +383,17 @@ namespace locust "with the unit test as in bin/testLMCCavity [-h]"); return false; } +#ifdef FIND_ROOT + fInterface->aRunParameter->fSimulationType = "cavity"; + if ( cavityFrequency > 20.e9 ) + { + fInterface->aRunParameter->fSimulationSubType = "cca"; + } + else + { + fInterface->aRunParameter->fSimulationSubType = "lfa"; + } +#endif } return true; diff --git a/Source/IO/LMCRunParameters.cc b/Source/IO/LMCRunParameters.cc index 6a389aa4..46095412 100644 --- a/Source/IO/LMCRunParameters.cc +++ b/Source/IO/LMCRunParameters.cc @@ -17,7 +17,15 @@ ClassImp(locust::RunParameters); namespace locust { - RunParameters::RunParameters() {} + RunParameters::RunParameters() : + fNoise( 0. ), + fLOfrequency( 0. ), + fSamplingRateMHz( 0. ), + fDecimationFactor( 0. ), + fDataType( "simulation" ), + fSimulationType( "rectangular-waveguide" ), + fSimulationSubType( "none" ) + {} RunParameters::~RunParameters() {} } diff --git a/Source/IO/LMCRunParameters.hh b/Source/IO/LMCRunParameters.hh index 867202de..852b6aa1 100644 --- a/Source/IO/LMCRunParameters.hh +++ b/Source/IO/LMCRunParameters.hh @@ -30,6 +30,9 @@ namespace locust double fLOfrequency; double fSamplingRateMHz; double fDecimationFactor; + std::string fDataType; + std::string fSimulationType; + std::string fSimulationSubType; ClassDef(RunParameters,1) // Root syntax. diff --git a/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc b/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc index 3fc6db83..bc8709cc 100644 --- a/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc +++ b/Source/Kassiopeia/LMCCyclotronRadiationExtractor.cc @@ -62,11 +62,11 @@ namespace locust bool CyclotronRadiationExtractor::UpdateTrackProperties( Kassiopeia::KSParticle &aFinalParticle, unsigned index, bool bStart ) { +#ifdef ROOT_FOUND double tLOfrequency = fInterface->aRunParameter->fLOfrequency; // Hz double tSamplingRate = fInterface->aRunParameter->fSamplingRateMHz; // MHz double tOffset = -tLOfrequency + tSamplingRate * 1.e6 / 2.; // Hz double tTime = index / tSamplingRate / 1.e6 / fInterface->aRunParameter->fDecimationFactor; -#ifdef ROOT_FOUND if (bStart) { fStartingIndex = index; diff --git a/Source/Kassiopeia/LMCEventHold.cc b/Source/Kassiopeia/LMCEventHold.cc index bf343299..19783590 100644 --- a/Source/Kassiopeia/LMCEventHold.cc +++ b/Source/Kassiopeia/LMCEventHold.cc @@ -8,6 +8,8 @@ #include "logger.hh" #include "LMCEventHold.hh" #include +#include + namespace locust { @@ -110,6 +112,20 @@ namespace locust { std::string tOutputPath = TOSTRING(PB_OUTPUT_DIR); std::string sFileName = tOutputPath+"/"+fTruthOutputFilename; + std::string sJsonFileName = sFileName; + const std::string ext(".root"); + if ( sFileName != ext && + sFileName.size() > ext.size() && + sFileName.substr(sFileName.size() - ext.size()) == ".root" ) + { + // Remove .root and replace it with .json: + sJsonFileName = sFileName.substr(0, sFileName.size() - ext.size()) + ".json"; + } + else + { + LERROR(lmclog,"The output file " << fTruthOutputFilename <<"doesn't end in .root"); + exit(-1); + } #ifdef ROOT_FOUND FileWriter* aRootTreeWriter = RootTreeWriter::get_instance(); aRootTreeWriter->SetFilename(sFileName); @@ -126,6 +142,17 @@ namespace locust } aRootTreeWriter->CloseFile(); #endif + + // Open the json file: + if (fAccumulateTruthInfo) + { + std::ofstream ost {sJsonFileName, std::ios_base::app}; + } + else + { + std::ofstream ost {sJsonFileName, std::ios_base::out}; + } + return true; } @@ -185,8 +212,10 @@ namespace locust fInterface->fEventInProgress = false; fInterface->fDigitizerCondition.notify_one(); // unlock LPROG( lmclog, "Kass is waking after event" ); +#ifdef FIND_ROOT delete fInterface->anEvent; delete fInterface->aTrack; +#endif return true; } diff --git a/Source/Kassiopeia/LMCTrackHold.cc b/Source/Kassiopeia/LMCTrackHold.cc index 52f39c72..1e7ac958 100644 --- a/Source/Kassiopeia/LMCTrackHold.cc +++ b/Source/Kassiopeia/LMCTrackHold.cc @@ -63,7 +63,9 @@ namespace locust bool TrackHold::ExecutePreTrackModification(Kassiopeia::KSTrack &aTrack) { +#ifdef FIND_ROOT fInterface->aTrack->Initialize(); +#endif fInterface->fNewTrackStarting = true; double tTime = aTrack.GetInitialParticle().GetTime(); @@ -76,7 +78,9 @@ namespace locust { if ( aTrack.GetTotalSteps() > 0) { +#ifdef FIND_ROOT fInterface->anEvent->AddTrack( fInterface->aTrack ); +#endif } double tTime = aTrack.GetFinalParticle().GetTime(); From 280e375195592a7fdcdb7aa9255db1cc013698f0 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Sat, 9 Nov 2024 22:32:25 -0500 Subject: [PATCH 08/14] Initialize a run ID and write it to the metadata structure. Clean up some of the Root preprocessor steps. --- Source/Generators/LMCArraySignalGenerator.cc | 2 +- Source/Generators/LMCCavitySignalGenerator.cc | 20 +++++++++++------- Source/IO/LMCRunParameters.cc | 21 +++++++++++++++++-- Source/IO/LMCRunParameters.hh | 3 +++ Source/Kassiopeia/LMCTrackHold.cc | 7 ++++--- 5 files changed, 40 insertions(+), 13 deletions(-) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index ad06f9c5..e450d334 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -347,7 +347,7 @@ namespace locust bool ArraySignalGenerator::RecordRunParameters( Signal* aSignal ) { -#ifdef FIND_ROOT +#ifdef ROOT_FOUND fInterface->aRunParameter = new RunParameters(); fInterface->aRunParameter->fSamplingRateMHz = fAcquisitionRate; fInterface->aRunParameter->fDecimationFactor = aSignal->DecimationFactor(); diff --git a/Source/Generators/LMCCavitySignalGenerator.cc b/Source/Generators/LMCCavitySignalGenerator.cc index e35d4537..1b12cc49 100644 --- a/Source/Generators/LMCCavitySignalGenerator.cc +++ b/Source/Generators/LMCCavitySignalGenerator.cc @@ -311,7 +311,6 @@ namespace locust bool CavitySignalGenerator::RecordRunParameters( Signal* aSignal ) { #ifdef ROOT_FOUND - fInterface->aRunParameter = new RunParameters(); fInterface->aRunParameter->fSamplingRateMHz = fAcquisitionRate; fInterface->aRunParameter->fDecimationFactor = aSignal->DecimationFactor(); fInterface->aRunParameter->fLOfrequency = fLO_Frequency; @@ -365,6 +364,11 @@ namespace locust LPROG(lmclog,"Running some cavity cross-checks ..."); + double timeResolution = 0.; + double thresholdFactor = 0.; + double cavityFrequency = 0.; + double qExpected = 0.; + for (int mu=0; muGetDHOTimeResolution(bTE, l, m, n); - double thresholdFactor = fAnalyticResponseFunction->GetDHOThresholdFactor(bTE, l, m, n); - double cavityFrequency = fAnalyticResponseFunction->GetCavityFrequency(bTE, l, m, n); - double qExpected = fAnalyticResponseFunction->GetCavityQ(bTE, l, m, n); + timeResolution = fAnalyticResponseFunction->GetDHOTimeResolution(bTE, l, m, n); + thresholdFactor = fAnalyticResponseFunction->GetDHOThresholdFactor(bTE, l, m, n); + cavityFrequency = fAnalyticResponseFunction->GetCavityFrequency(bTE, l, m, n); + qExpected = fAnalyticResponseFunction->GetCavityQ(bTE, l, m, n); aCavityUtility.SetOutputFile(fUnitTestRootFile); if (!aCavityUtility.CheckCavityQ( bTE, l, m, n, timeResolution, thresholdFactor, cavityFrequency, qExpected )) { @@ -383,7 +387,10 @@ namespace locust "with the unit test as in bin/testLMCCavity [-h]"); return false; } -#ifdef FIND_ROOT + } + +#ifdef ROOT_FOUND + fInterface->aRunParameter = new RunParameters(); fInterface->aRunParameter->fSimulationType = "cavity"; if ( cavityFrequency > 20.e9 ) { @@ -394,7 +401,6 @@ namespace locust fInterface->aRunParameter->fSimulationSubType = "lfa"; } #endif - } return true; } diff --git a/Source/IO/LMCRunParameters.cc b/Source/IO/LMCRunParameters.cc index 46095412..e2df78b2 100644 --- a/Source/IO/LMCRunParameters.cc +++ b/Source/IO/LMCRunParameters.cc @@ -24,8 +24,25 @@ namespace locust fDecimationFactor( 0. ), fDataType( "simulation" ), fSimulationType( "rectangular-waveguide" ), - fSimulationSubType( "none" ) - {} + fSimulationSubType( "none" ), + fRunID( 0 ) + { + time_t rawtime; + struct tm * timeInfo; + time (&rawtime); + timeInfo = localtime (&rawtime); + + struct timeval tv; + gettimeofday(&tv, NULL); + + int tDay = timeInfo->tm_mday; + int tMonth = timeInfo->tm_mon + 1; + int tYear = timeInfo->tm_year - 100; + int tMicrosec = tv.tv_usec; + + fRunID = 1e12 + tDay*1e10 + tMonth*1e8 + tYear*1e6 + tMicrosec; + + } RunParameters::~RunParameters() {} } diff --git a/Source/IO/LMCRunParameters.hh b/Source/IO/LMCRunParameters.hh index 852b6aa1..1752e09b 100644 --- a/Source/IO/LMCRunParameters.hh +++ b/Source/IO/LMCRunParameters.hh @@ -15,6 +15,8 @@ #include "TObject.h" #include "LMCRunParameters.hh" +#include "time.h" +#include namespace locust { @@ -33,6 +35,7 @@ namespace locust std::string fDataType; std::string fSimulationType; std::string fSimulationSubType; + long int fRunID; ClassDef(RunParameters,1) // Root syntax. diff --git a/Source/Kassiopeia/LMCTrackHold.cc b/Source/Kassiopeia/LMCTrackHold.cc index 1e7ac958..edd5ddf8 100644 --- a/Source/Kassiopeia/LMCTrackHold.cc +++ b/Source/Kassiopeia/LMCTrackHold.cc @@ -63,7 +63,7 @@ namespace locust bool TrackHold::ExecutePreTrackModification(Kassiopeia::KSTrack &aTrack) { -#ifdef FIND_ROOT +#ifdef ROOT_FOUND fInterface->aTrack->Initialize(); #endif fInterface->fNewTrackStarting = true; @@ -78,14 +78,15 @@ namespace locust { if ( aTrack.GetTotalSteps() > 0) { -#ifdef FIND_ROOT +#ifdef ROOT_FOUND + fInterface->aTrack->TrackID = fTrackCounter; fInterface->anEvent->AddTrack( fInterface->aTrack ); #endif + fTrackCounter += 1; } double tTime = aTrack.GetFinalParticle().GetTime(); LWARN(lmclog,"LMCTrack " << fTrackCounter << " is complete at Kass time " << tTime << " with total steps " << aTrack.GetTotalSteps() ); - fTrackCounter += 1; return true; } From 5e1d3e83f254f047527dee8b2e36c3c6419f5615 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Sat, 9 Nov 2024 22:37:35 -0500 Subject: [PATCH 09/14] First draft of an expandable meta-data file in json format. --- Source/Kassiopeia/LMCEventHold.cc | 86 ++++++++++++++++++++++++++++--- Source/Kassiopeia/LMCEventHold.hh | 2 + 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/Source/Kassiopeia/LMCEventHold.cc b/Source/Kassiopeia/LMCEventHold.cc index 19783590..5b7cb81c 100644 --- a/Source/Kassiopeia/LMCEventHold.cc +++ b/Source/Kassiopeia/LMCEventHold.cc @@ -112,14 +112,14 @@ namespace locust { std::string tOutputPath = TOSTRING(PB_OUTPUT_DIR); std::string sFileName = tOutputPath+"/"+fTruthOutputFilename; - std::string sJsonFileName = sFileName; + fJsonFileName = sFileName; const std::string ext(".root"); if ( sFileName != ext && sFileName.size() > ext.size() && sFileName.substr(sFileName.size() - ext.size()) == ".root" ) { // Remove .root and replace it with .json: - sJsonFileName = sFileName.substr(0, sFileName.size() - ext.size()) + ".json"; + fJsonFileName = sFileName.substr(0, sFileName.size() - ext.size()) + ".json"; } else { @@ -144,20 +144,88 @@ namespace locust #endif // Open the json file: - if (fAccumulateTruthInfo) + if (!fAccumulateTruthInfo) { - std::ofstream ost {sJsonFileName, std::ios_base::app}; + std::ofstream ost {fJsonFileName, std::ios_base::out}; + } + + return true; + + } + + + bool EventHold::WriteJsonFile() + { + std::ifstream jsonFile(fJsonFileName); // Open json file for inspection + bool bNewRun = true; + std::vector v; + if (jsonFile.is_open()) + { + std::string line; + while (std::getline(jsonFile, line)) + { + std::cout << line << "\n"; + bNewRun = !line.find("run-id"); + if (line != "}") // Avoid saving the last "}". It will be appended below. + { + v.push_back(line); + } + } + jsonFile.close(); } else { - std::ofstream ost {sJsonFileName, std::ios_base::out}; + LPROG( lmclog, "A json file for meta-data was not found. Opening a new one now." ); } - return true; + std::ofstream ost {fJsonFileName, std::ios_base::out}; - } + +#ifdef ROOT_FOUND + if (bNewRun) // If there are no run parameters in the json file yet, write them now: + { + ost << "{\n"; + ost << " \"run-id:\": "<< "\"" << fInterface->aRunParameter->fRunID << "\",\n"; + ost << " {\n"; + ost << " \"run-type:\": "<< "\"" << fInterface->aRunParameter->fDataType << "\",\n"; + ost << " \"simulation-type:\": "<< "\"" << fInterface->aRunParameter->fSimulationType << "\",\n"; + ost << " \"simulation-subtype:\": "<< "\"" << fInterface->aRunParameter->fSimulationSubType << "\",\n"; + ost << " \"sampling-freq-mega-hz:\": "<< "\"" << fInterface->aRunParameter->fSamplingRateMHz << "\"\n"; + ost << " }\n"; + } + else // otherwise re-write the file: + { + for (int i = 0; i < v.size(); i++) + { + ost << v[i] << "\n"; + } + } + // Write the latest event information here: + + ost << " \"event-id:\": "<< "\"" << fInterface->anEvent->fEventID << "\"\n"; + ost << " {\n"; + ost << " \"ntracks:\": "<< "\"" << fInterface->anEvent->fNTracks << "\"\n"; + for (int i=0; ianEvent->fNTracks; i++) + { + ost << " \"track-id:\": "<< "\"" << fInterface->anEvent->fTrackIDs[i] << "\"\n"; + ost << " {\n"; + ost << " \"start-time:\": "<< "\"" << fInterface->anEvent->fStartTimes[i] << "\"\n"; + ost << " \"end-time:\": "<< "\"" << fInterface->anEvent->fEndTimes[i] << "\"\n"; + ost << " \"output-avg-frequency:\": "<< "\"" << fInterface->anEvent->fOutputAvgFrequencies[i] << "\"\n"; + ost << " \"pitch-angle:\": "<< "\"" << fInterface->anEvent->fPitchAngles[i] << "\"\n"; + ost << " \"avg-axial-frequency:\": "<< "\"" << fInterface->anEvent->fAvgAxialFrequencies[i] << "\"\n"; + ost << " }\n"; + } + ost << " }\n"; + ost << "}\n"; + +#endif + + return true; + } + bool EventHold::WriteEvent() { std::string tOutputPath = TOSTRING(PB_OUTPUT_DIR); @@ -172,6 +240,7 @@ namespace locust aRootTreeWriter->WriteRunParameters(fInterface->aRunParameter); aRootTreeWriter->CloseFile(); #endif + WriteJsonFile(); return true; } @@ -212,9 +281,10 @@ namespace locust fInterface->fEventInProgress = false; fInterface->fDigitizerCondition.notify_one(); // unlock LPROG( lmclog, "Kass is waking after event" ); -#ifdef FIND_ROOT +#ifdef ROOT_FOUND delete fInterface->anEvent; delete fInterface->aTrack; + delete fInterface->aRunParameter; #endif return true; } diff --git a/Source/Kassiopeia/LMCEventHold.hh b/Source/Kassiopeia/LMCEventHold.hh index 5d490394..46bfa35d 100644 --- a/Source/Kassiopeia/LMCEventHold.hh +++ b/Source/Kassiopeia/LMCEventHold.hh @@ -31,12 +31,14 @@ namespace locust bool OpenEvent(); bool OpenFile(); bool WriteEvent(); + bool WriteJsonFile(); virtual ~EventHold(); EventHold* Clone() const; bool ConfigureByInterface(); bool Configure( const scarab::param_node& aParam ); std::string fTruthOutputFilename; + std::string fJsonFileName; bool fAccumulateTruthInfo; From 4d5279a98b0d6dbbc181d73ee2cbea488cc43803 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Sun, 10 Nov 2024 12:55:16 -0500 Subject: [PATCH 10/14] Build on push to feature branch. --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0cecfc97..775a146f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -3,7 +3,7 @@ name: Build and Test Locust on: pull_request: push: - branches: [master, develop] + branches: [master, develop, feature/metadata] tags: ['*'] workflow_dispatch: From 6ffb2446824b67f68ff0ef2d84d41b88a337613e Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Sun, 10 Nov 2024 13:08:24 -0500 Subject: [PATCH 11/14] Finish merge. --- Source/Kassiopeia/LMCEventHold.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/Kassiopeia/LMCEventHold.cc b/Source/Kassiopeia/LMCEventHold.cc index d97a1591..5b7cb81c 100644 --- a/Source/Kassiopeia/LMCEventHold.cc +++ b/Source/Kassiopeia/LMCEventHold.cc @@ -287,7 +287,6 @@ namespace locust delete fInterface->aRunParameter; #endif return true; -#endif } } /* namespace locust */ From 4aa580848652df64c7062a2e9e56789cf7c7c161 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Mon, 11 Nov 2024 10:50:07 -0500 Subject: [PATCH 12/14] Fix some json syntax. --- Source/Kassiopeia/LMCEventHold.cc | 51 +++++++++++++++++++------------ 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/Source/Kassiopeia/LMCEventHold.cc b/Source/Kassiopeia/LMCEventHold.cc index 5b7cb81c..b5308153 100644 --- a/Source/Kassiopeia/LMCEventHold.cc +++ b/Source/Kassiopeia/LMCEventHold.cc @@ -131,9 +131,6 @@ namespace locust aRootTreeWriter->SetFilename(sFileName); if (fAccumulateTruthInfo) { - // TO-DO: This option should be used when running pileup. We will need to - // figure out how to explicitly increment the event structure ID, given that the - // same (identical) simulation is being run multiple times in this case. aRootTreeWriter->OpenFile("UPDATE"); } else @@ -164,7 +161,7 @@ namespace locust std::string line; while (std::getline(jsonFile, line)) { - std::cout << line << "\n"; + LPROG( lmclog, line ); bNewRun = !line.find("run-id"); if (line != "}") // Avoid saving the last "}". It will be appended below. { @@ -185,38 +182,52 @@ namespace locust if (bNewRun) // If there are no run parameters in the json file yet, write them now: { ost << "{\n"; - ost << " \"run-id:\": "<< "\"" << fInterface->aRunParameter->fRunID << "\",\n"; + ost << " \"run-id\": "<< "\"" << fInterface->aRunParameter->fRunID << "\",\n"; ost << " {\n"; - ost << " \"run-type:\": "<< "\"" << fInterface->aRunParameter->fDataType << "\",\n"; - ost << " \"simulation-type:\": "<< "\"" << fInterface->aRunParameter->fSimulationType << "\",\n"; - ost << " \"simulation-subtype:\": "<< "\"" << fInterface->aRunParameter->fSimulationSubType << "\",\n"; - ost << " \"sampling-freq-mega-hz:\": "<< "\"" << fInterface->aRunParameter->fSamplingRateMHz << "\"\n"; - ost << " }\n"; + ost << " \"run-type\": "<< "\"" << fInterface->aRunParameter->fDataType << "\",\n"; + ost << " \"simulation-type\": "<< "\"" << fInterface->aRunParameter->fSimulationType << "\",\n"; + ost << " \"simulation-subtype\": "<< "\"" << fInterface->aRunParameter->fSimulationSubType << "\",\n"; + ost << " \"sampling-freq-mega-hz\": "<< "\"" << fInterface->aRunParameter->fSamplingRateMHz << "\"\n"; + ost << " },\n"; } else // otherwise re-write the file: { for (int i = 0; i < v.size(); i++) { - ost << v[i] << "\n"; + if (i < v.size()-1) + { + ost << v[i] << "\n"; + } + else + { + ost << v[i] << ",\n"; + } } } // Write the latest event information here: - ost << " \"event-id:\": "<< "\"" << fInterface->anEvent->fEventID << "\"\n"; + ost << " \"event-id\": "<< "\"" << fInterface->anEvent->fEventID << "\"\n"; ost << " {\n"; - ost << " \"ntracks:\": "<< "\"" << fInterface->anEvent->fNTracks << "\"\n"; + ost << " \"ntracks\": "<< "\"" << fInterface->anEvent->fNTracks << "\",\n"; for (int i=0; ianEvent->fNTracks; i++) { - ost << " \"track-id:\": "<< "\"" << fInterface->anEvent->fTrackIDs[i] << "\"\n"; + ost << " \"track-id\": "<< "\"" << fInterface->anEvent->fTrackIDs[i] << "\"\n"; ost << " {\n"; - ost << " \"start-time:\": "<< "\"" << fInterface->anEvent->fStartTimes[i] << "\"\n"; - ost << " \"end-time:\": "<< "\"" << fInterface->anEvent->fEndTimes[i] << "\"\n"; - ost << " \"output-avg-frequency:\": "<< "\"" << fInterface->anEvent->fOutputAvgFrequencies[i] << "\"\n"; - ost << " \"pitch-angle:\": "<< "\"" << fInterface->anEvent->fPitchAngles[i] << "\"\n"; - ost << " \"avg-axial-frequency:\": "<< "\"" << fInterface->anEvent->fAvgAxialFrequencies[i] << "\"\n"; - ost << " }\n"; + ost << " \"start-time\": "<< "\"" << fInterface->anEvent->fStartTimes[i] << "\",\n"; + ost << " \"end-time\": "<< "\"" << fInterface->anEvent->fEndTimes[i] << "\",\n"; + ost << " \"output-avg-frequency\": "<< "\"" << fInterface->anEvent->fOutputAvgFrequencies[i] << "\",\n"; + ost << " \"pitch-angle\": "<< "\"" << fInterface->anEvent->fPitchAngles[i] << "\",\n"; + ost << " \"avg-axial-frequency\": "<< "\"" << fInterface->anEvent->fAvgAxialFrequencies[i] << "\"\n"; + if (i < fInterface->anEvent->fNTracks-1) + { + ost << " },\n"; + } + else + { + ost << " }\n"; + } } ost << " }\n"; ost << "}\n"; From 2c4d9291c97f52899e3de922598365994a62ad33 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Tue, 12 Nov 2024 09:20:48 -0500 Subject: [PATCH 13/14] Fix json syntax again. Thank you jsoneditoronline.org! --- Source/Kassiopeia/LMCEventHold.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Source/Kassiopeia/LMCEventHold.cc b/Source/Kassiopeia/LMCEventHold.cc index b5308153..00de7638 100644 --- a/Source/Kassiopeia/LMCEventHold.cc +++ b/Source/Kassiopeia/LMCEventHold.cc @@ -183,7 +183,7 @@ namespace locust { ost << "{\n"; ost << " \"run-id\": "<< "\"" << fInterface->aRunParameter->fRunID << "\",\n"; - ost << " {\n"; + ost << " \"run-parameters\": {\n"; ost << " \"run-type\": "<< "\"" << fInterface->aRunParameter->fDataType << "\",\n"; ost << " \"simulation-type\": "<< "\"" << fInterface->aRunParameter->fSimulationType << "\",\n"; ost << " \"simulation-subtype\": "<< "\"" << fInterface->aRunParameter->fSimulationSubType << "\",\n"; @@ -208,12 +208,11 @@ namespace locust // Write the latest event information here: - ost << " \"event-id\": "<< "\"" << fInterface->anEvent->fEventID << "\"\n"; - ost << " {\n"; + ost << " \"" << fInterface->anEvent->fEventID << "\": {\n"; ost << " \"ntracks\": "<< "\"" << fInterface->anEvent->fNTracks << "\",\n"; for (int i=0; ianEvent->fNTracks; i++) { - ost << " \"track-id\": "<< "\"" << fInterface->anEvent->fTrackIDs[i] << "\"\n"; + ost << " \"" << fInterface->anEvent->fTrackIDs[i] << "\":\n"; ost << " {\n"; ost << " \"start-time\": "<< "\"" << fInterface->anEvent->fStartTimes[i] << "\",\n"; ost << " \"end-time\": "<< "\"" << fInterface->anEvent->fEndTimes[i] << "\",\n"; From 4a77aba72d5309a899d70183f2eef4f3fb9b06af Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Tue, 12 Nov 2024 09:21:40 -0500 Subject: [PATCH 14/14] Remove build on push to feature branch. --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 775a146f..0cecfc97 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -3,7 +3,7 @@ name: Build and Test Locust on: pull_request: push: - branches: [master, develop, feature/metadata] + branches: [master, develop] tags: ['*'] workflow_dispatch: