From c88abdb80d7ad79c2bb149e0797a5e47031bc611 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Wed, 10 Feb 2021 11:59:55 -0500 Subject: [PATCH 1/9] Added debug statements to find skipped samples. --- Source/Generators/LMCArraySignalGenerator.cc | 2 ++ Source/RxComponents/LMCPowerCombiner.cc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index 96f96ff9..471b3fa4 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -450,6 +450,7 @@ namespace locust else { tFieldSolution = fTransmitter->SolveKassFields(currentElement->GetPosition(), currentElement->GetPolarizationDirection(), tReceiverTime, tTotalElementIndex); + if (tFieldSolution[0] == 0.) {printf("field is zero.\n");} } tFieldSolution[0] *= currentElement->GetPatternFactor(fTransmitter->GetIncidentKVector(tTotalElementIndex), *currentElement); @@ -458,6 +459,7 @@ namespace locust FillBuffers(aSignal, tFieldSolution[1], tFieldSolution[0], fphiLO, index, channelIndex, elementIndex); double VoltageFIRSample = GetFIRSample(nfilterbins, dtfilter, channelIndex, elementIndex); + if (VoltageFIRSample == 0.) {printf("VoltageFIRSample is zero.\n");} fPowerCombiner->AddOneVoltageToStripSum(aSignal, VoltageFIRSample, fphiLO, elementIndex, IndexBuffer[channelIndex*fNElementsPerStrip+elementIndex].front()); PopBuffers(channelIndex, elementIndex); diff --git a/Source/RxComponents/LMCPowerCombiner.cc b/Source/RxComponents/LMCPowerCombiner.cc index f2383e5b..0af201bf 100644 --- a/Source/RxComponents/LMCPowerCombiner.cc +++ b/Source/RxComponents/LMCPowerCombiner.cc @@ -77,6 +77,8 @@ namespace locust aSignal->LongSignalTimeComplex()[sampleIndex][0] += 2.*VoltageFIRSample * sin(phi_LO); aSignal->LongSignalTimeComplex()[sampleIndex][1] += 2.*VoltageFIRSample * cos(phi_LO); + if (VoltageFIRSample==0.) {printf("power combining sees zero.\n");} + if ( (fvoltageCheck==true) && (sampleIndex%100 < 1) ) LWARN( lmclog, "Voltage " << z_index << " " << sampleIndex << " is <" << aSignal->LongSignalTimeComplex()[sampleIndex][1] << ">" ); return true; From fc1a524697b2b72c9eb02c30ad85242224145810 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Wed, 10 Feb 2021 13:59:39 -0500 Subject: [PATCH 2/9] Added fSkipSampleCheck and startingIndex variables to look for thread dropouts. --- Source/Generators/LMCArraySignalGenerator.cc | 14 +++++++++++--- Source/Generators/LMCArraySignalGenerator.hh | 3 ++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index 471b3fa4..46ce7883 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -43,6 +43,7 @@ namespace locust ElementFIRBuffer( 1 ), fFieldBufferSize( 50 ), fSwapFrequency( 1000 ), + fSkipSampleCheck( 1000 ), fInterface( new KassLocustInterface() ) { fRequiredSignalState = Signal::kTime; @@ -307,6 +308,10 @@ namespace locust { fSwapFrequency = aParam["swap-frequency"]().as_int(); } + if( aParam.has( "skip-sample-check" ) ) + { + fSkipSampleCheck = aParam["skip-sample-check"]().as_int(); + } if( aParam.has( "xml-filename" ) ) { gxml_filename = aParam["xml-filename"]().as_string(); @@ -419,7 +424,7 @@ namespace locust } - void ArraySignalGenerator::DriveAntenna(FILE *fp, int PreEventCounter, unsigned index, Signal* aSignal, int nfilterbins, double dtfilter) + void ArraySignalGenerator::DriveAntenna(FILE *fp, int startingIndex, unsigned index, Signal* aSignal, int nfilterbins, double dtfilter) { const int signalSize = aSignal->TimeSize(); @@ -459,7 +464,8 @@ namespace locust FillBuffers(aSignal, tFieldSolution[1], tFieldSolution[0], fphiLO, index, channelIndex, elementIndex); double VoltageFIRSample = GetFIRSample(nfilterbins, dtfilter, channelIndex, elementIndex); - if (VoltageFIRSample == 0.) {printf("VoltageFIRSample is zero.\n");} + if (VoltageFIRSample == 0.) {printf("VoltageFIRSample is zero for index %d.\n", index);} + if ((VoltageFIRSample == 0.)&&(index-startingIndex > fSkipSampleCheck)) {printf("out of spec at sample %d\n", index);} fPowerCombiner->AddOneVoltageToStripSum(aSignal, VoltageFIRSample, fphiLO, elementIndex, IndexBuffer[channelIndex*fNElementsPerStrip+elementIndex].front()); PopBuffers(channelIndex, elementIndex); @@ -601,6 +607,7 @@ namespace locust if (fTransmitter->IsKassiopeia()) { bool fTruth = false; + int startingIndex; fInterface->fKassTimeStep = 1./(fAcquisitionRate*1.e6*aSignal->DecimationFactor()); std::thread tKassiopeia (&ArraySignalGenerator::KassiopeiaInit, this, gxml_filename); // spawn new thread @@ -627,6 +634,7 @@ namespace locust { fInterface->fPreEventInProgress = false; // reset. fInterface->fEventInProgress = true; + startingIndex = index; LPROG( lmclog, "LMC about to WakeBeforeEvent()" ); WakeBeforeEvent(); // trigger Kass event. } @@ -641,7 +649,7 @@ namespace locust fInterface->fDigitizerCondition.wait( tLock ); if (fInterface->fEventInProgress) { - DriveAntenna(fp, PreEventCounter, index, aSignal, nfilterbins, dtfilter); + DriveAntenna(fp, startingIndex, index, aSignal, nfilterbins, dtfilter); PreEventCounter = 0; // reset } tLock.unlock(); diff --git a/Source/Generators/LMCArraySignalGenerator.hh b/Source/Generators/LMCArraySignalGenerator.hh index cf0b5747..eb312692 100644 --- a/Source/Generators/LMCArraySignalGenerator.hh +++ b/Source/Generators/LMCArraySignalGenerator.hh @@ -92,6 +92,7 @@ namespace locust bool fTextFileWriting; unsigned fFieldBufferSize; int fSwapFrequency; + int fSkipSampleCheck; double fphiLO; // voltage phase of LO in radians; void KassiopeiaInit(const std::string &aFile); @@ -117,7 +118,7 @@ namespace locust bool DoGenerate( Signal* aSignal ); - void DriveAntenna(FILE *fp, int PreEventCounter, unsigned index, Signal* aSignal, int nfilterbins, double dtfilter); + void DriveAntenna(FILE *fp, int startingIndex, unsigned index, Signal* aSignal, int nfilterbins, double dtfilter); bool InitializeElementArray(); AntennaElementPositioner* fAntennaElementPositioner; Transmitter* fTransmitter; // transmitter object From ead36e8e63f0bc1124d56841db90f5c17057e6be Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Wed, 10 Feb 2021 15:06:07 -0500 Subject: [PATCH 3/9] Removed extra debug statements. Defined sample range in terms of existing parameters to check for skipped samples. --- Source/Generators/LMCArraySignalGenerator.cc | 12 ++++-------- Source/Generators/LMCArraySignalGenerator.hh | 1 - Source/RxComponents/LMCPowerCombiner.cc | 2 -- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index 46ce7883..a64e5f11 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -43,7 +43,6 @@ namespace locust ElementFIRBuffer( 1 ), fFieldBufferSize( 50 ), fSwapFrequency( 1000 ), - fSkipSampleCheck( 1000 ), fInterface( new KassLocustInterface() ) { fRequiredSignalState = Signal::kTime; @@ -308,10 +307,6 @@ namespace locust { fSwapFrequency = aParam["swap-frequency"]().as_int(); } - if( aParam.has( "skip-sample-check" ) ) - { - fSkipSampleCheck = aParam["skip-sample-check"]().as_int(); - } if( aParam.has( "xml-filename" ) ) { gxml_filename = aParam["xml-filename"]().as_string(); @@ -455,7 +450,6 @@ namespace locust else { tFieldSolution = fTransmitter->SolveKassFields(currentElement->GetPosition(), currentElement->GetPolarizationDirection(), tReceiverTime, tTotalElementIndex); - if (tFieldSolution[0] == 0.) {printf("field is zero.\n");} } tFieldSolution[0] *= currentElement->GetPatternFactor(fTransmitter->GetIncidentKVector(tTotalElementIndex), *currentElement); @@ -464,8 +458,10 @@ namespace locust FillBuffers(aSignal, tFieldSolution[1], tFieldSolution[0], fphiLO, index, channelIndex, elementIndex); double VoltageFIRSample = GetFIRSample(nfilterbins, dtfilter, channelIndex, elementIndex); - if (VoltageFIRSample == 0.) {printf("VoltageFIRSample is zero for index %d.\n", index);} - if ((VoltageFIRSample == 0.)&&(index-startingIndex > fSkipSampleCheck)) {printf("out of spec at sample %d\n", index);} + if ((VoltageFIRSample == 0.)&&(index-startingIndex > fFieldBufferSize*fPowerCombiner->GetNElementsPerStrip())) + { + printf("out of spec at sample %d\n", index); + } fPowerCombiner->AddOneVoltageToStripSum(aSignal, VoltageFIRSample, fphiLO, elementIndex, IndexBuffer[channelIndex*fNElementsPerStrip+elementIndex].front()); PopBuffers(channelIndex, elementIndex); diff --git a/Source/Generators/LMCArraySignalGenerator.hh b/Source/Generators/LMCArraySignalGenerator.hh index eb312692..79e2e284 100644 --- a/Source/Generators/LMCArraySignalGenerator.hh +++ b/Source/Generators/LMCArraySignalGenerator.hh @@ -92,7 +92,6 @@ namespace locust bool fTextFileWriting; unsigned fFieldBufferSize; int fSwapFrequency; - int fSkipSampleCheck; double fphiLO; // voltage phase of LO in radians; void KassiopeiaInit(const std::string &aFile); diff --git a/Source/RxComponents/LMCPowerCombiner.cc b/Source/RxComponents/LMCPowerCombiner.cc index 0af201bf..f2383e5b 100644 --- a/Source/RxComponents/LMCPowerCombiner.cc +++ b/Source/RxComponents/LMCPowerCombiner.cc @@ -77,8 +77,6 @@ namespace locust aSignal->LongSignalTimeComplex()[sampleIndex][0] += 2.*VoltageFIRSample * sin(phi_LO); aSignal->LongSignalTimeComplex()[sampleIndex][1] += 2.*VoltageFIRSample * cos(phi_LO); - if (VoltageFIRSample==0.) {printf("power combining sees zero.\n");} - if ( (fvoltageCheck==true) && (sampleIndex%100 < 1) ) LWARN( lmclog, "Voltage " << z_index << " " << sampleIndex << " is <" << aSignal->LongSignalTimeComplex()[sampleIndex][1] << ">" ); return true; From f189033cb02a9a1008943a459789fbaf16098d27 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Wed, 10 Feb 2021 19:07:58 -0500 Subject: [PATCH 4/9] Added exit(-1) for skipped samples. Testing debug message for unresponsive Kassiopeia. --- Source/Generators/LMCArraySignalGenerator.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index a64e5f11..81775fd9 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -460,7 +460,9 @@ namespace locust double VoltageFIRSample = GetFIRSample(nfilterbins, dtfilter, channelIndex, elementIndex); if ((VoltageFIRSample == 0.)&&(index-startingIndex > fFieldBufferSize*fPowerCombiner->GetNElementsPerStrip())) { + LERROR(lmclog,"A digitizer sample was skipped due to likely unresponsive thread.\n"); printf("out of spec at sample %d\n", index); + exit(-1); } fPowerCombiner->AddOneVoltageToStripSum(aSignal, VoltageFIRSample, fphiLO, elementIndex, IndexBuffer[channelIndex*fNElementsPerStrip+elementIndex].front()); PopBuffers(channelIndex, elementIndex); @@ -661,6 +663,9 @@ namespace locust else // no Kass event ever started, unlock and break out of signal loop entirely. { tLock.unlock(); + LERROR(lmclog,"No response from Kassiopeia.\n"); + printf("no response from Kassiopeia.\n"); +// exit(-1); break; } } From abfa0c985b5516164cf65c3d21bf2bb143d360c1 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Wed, 10 Feb 2021 19:53:19 -0500 Subject: [PATCH 5/9] Unlock before exit(-1). --- Source/Generators/LMCArraySignalGenerator.cc | 28 ++++++++++++-------- Source/Generators/LMCArraySignalGenerator.hh | 2 +- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index 81775fd9..258d347c 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -419,7 +419,7 @@ namespace locust } - void ArraySignalGenerator::DriveAntenna(FILE *fp, int startingIndex, unsigned index, Signal* aSignal, int nfilterbins, double dtfilter) + bool ArraySignalGenerator::DriveAntenna(FILE *fp, int startingIndex, unsigned index, Signal* aSignal, int nfilterbins, double dtfilter) { const int signalSize = aSignal->TimeSize(); @@ -460,9 +460,8 @@ namespace locust double VoltageFIRSample = GetFIRSample(nfilterbins, dtfilter, channelIndex, elementIndex); if ((VoltageFIRSample == 0.)&&(index-startingIndex > fFieldBufferSize*fPowerCombiner->GetNElementsPerStrip())) { - LERROR(lmclog,"A digitizer sample was skipped due to likely unresponsive thread.\n"); - printf("out of spec at sample %d\n", index); - exit(-1); + LERROR(lmclog,"A digitizer sample was skipped due to likely unresponsive thread. Exiting.\n"); + return false; } fPowerCombiner->AddOneVoltageToStripSum(aSignal, VoltageFIRSample, fphiLO, elementIndex, IndexBuffer[channelIndex*fNElementsPerStrip+elementIndex].front()); PopBuffers(channelIndex, elementIndex); @@ -475,6 +474,7 @@ namespace locust fInterface->fTOld += 1./(fAcquisitionRate*1.e6*aSignal->DecimationFactor()); if ( index%fSwapFrequency == 0 ) CleanupBuffers(); // release memory + return true; } @@ -647,8 +647,16 @@ namespace locust fInterface->fDigitizerCondition.wait( tLock ); if (fInterface->fEventInProgress) { - DriveAntenna(fp, startingIndex, index, aSignal, nfilterbins, dtfilter); - PreEventCounter = 0; // reset + if (!DriveAntenna(fp, startingIndex, index, aSignal, nfilterbins, dtfilter)) + { + PreEventCounter = 0; // reset + } + else + { + tLock.unlock(); + LERROR(lmclog,"The antenna did not respond correctly. Exiting.\n"); + exit(-1); + } } tLock.unlock(); } @@ -660,13 +668,11 @@ namespace locust { tLock.unlock(); } - else // no Kass event ever started, unlock and break out of signal loop entirely. + else // no Kass event ever started, unlock and exit. { tLock.unlock(); - LERROR(lmclog,"No response from Kassiopeia.\n"); - printf("no response from Kassiopeia.\n"); -// exit(-1); - break; + LERROR(lmclog,"There was no response from Kassiopeia. Exiting.\n"); + exit(-1); } } } diff --git a/Source/Generators/LMCArraySignalGenerator.hh b/Source/Generators/LMCArraySignalGenerator.hh index 79e2e284..4cd209a2 100644 --- a/Source/Generators/LMCArraySignalGenerator.hh +++ b/Source/Generators/LMCArraySignalGenerator.hh @@ -117,7 +117,7 @@ namespace locust bool DoGenerate( Signal* aSignal ); - void DriveAntenna(FILE *fp, int startingIndex, unsigned index, Signal* aSignal, int nfilterbins, double dtfilter); + bool DriveAntenna(FILE *fp, int startingIndex, unsigned index, Signal* aSignal, int nfilterbins, double dtfilter); bool InitializeElementArray(); AntennaElementPositioner* fAntennaElementPositioner; Transmitter* fTransmitter; // transmitter object From e3a9d6e3b2d833015a36da38be7a58cce44ff0f6 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Wed, 10 Feb 2021 20:07:37 -0500 Subject: [PATCH 6/9] Exit if the code is not working, not if it is working. --- Source/Generators/LMCArraySignalGenerator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index 258d347c..7fdd45d5 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -647,7 +647,7 @@ namespace locust fInterface->fDigitizerCondition.wait( tLock ); if (fInterface->fEventInProgress) { - if (!DriveAntenna(fp, startingIndex, index, aSignal, nfilterbins, dtfilter)) + if (DriveAntenna(fp, startingIndex, index, aSignal, nfilterbins, dtfilter)) { PreEventCounter = 0; // reset } From 8728a605036a463df01a98925b3b696d6eccb227 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Wed, 10 Feb 2021 22:59:43 -0500 Subject: [PATCH 7/9] Added more log messages. Added two flags, fSkippedSamples and fKassNeverStarted to tell LMCArraySignalGenerator to return false if necessary. --- Source/Generators/LMCArraySignalGenerator.cc | 20 +++++++++++++++----- Source/Generators/LMCArraySignalGenerator.hh | 2 ++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index 7fdd45d5..4d7c1d9b 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -43,6 +43,8 @@ namespace locust ElementFIRBuffer( 1 ), fFieldBufferSize( 50 ), fSwapFrequency( 1000 ), + fKassNeverStarted( false ), + fSkippedSamples( false ), fInterface( new KassLocustInterface() ) { fRequiredSignalState = Signal::kTime; @@ -653,9 +655,10 @@ namespace locust } else { - tLock.unlock(); LERROR(lmclog,"The antenna did not respond correctly. Exiting.\n"); - exit(-1); + fSkippedSamples = true; + tLock.unlock(); + break; } } tLock.unlock(); @@ -668,11 +671,15 @@ namespace locust { tLock.unlock(); } - else // no Kass event ever started, unlock and exit. + else // Kass event has not started, unlock and exit. { + if ( index < fNPreEventSamples+1 ) + { + LERROR(lmclog,"Kass thread is unresponsive. Exiting.\n"); + fKassNeverStarted = true; + } tLock.unlock(); - LERROR(lmclog,"There was no response from Kassiopeia. Exiting.\n"); - exit(-1); + break; } } } @@ -682,8 +689,11 @@ namespace locust fInterface->fDoneWithSignalGeneration = true; if (fTextFileWriting==1) fclose(fp); LPROG( lmclog, "Finished signal loop." ); + fInterface->fWaitBeforeEvent = false; WakeBeforeEvent(); tKassiopeia.join(); // finish thread + if (fKassNeverStarted == true) return false; + if (fSkippedSamples == true) return false; } // fTransmitter->IsKassiopeia() diff --git a/Source/Generators/LMCArraySignalGenerator.hh b/Source/Generators/LMCArraySignalGenerator.hh index 4cd209a2..62457d12 100644 --- a/Source/Generators/LMCArraySignalGenerator.hh +++ b/Source/Generators/LMCArraySignalGenerator.hh @@ -92,6 +92,8 @@ namespace locust bool fTextFileWriting; unsigned fFieldBufferSize; int fSwapFrequency; + bool fKassNeverStarted; + bool fSkippedSamples; double fphiLO; // voltage phase of LO in radians; void KassiopeiaInit(const std::string &aFile); From 0b4944aa61ca84a89d5773c22fd8a46a04908079 Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Thu, 11 Feb 2021 05:06:07 -0500 Subject: [PATCH 8/9] Included LMCException in LMCArraySignalGenerator for anticipated development of more informative exits. --- Source/Generators/LMCArraySignalGenerator.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Generators/LMCArraySignalGenerator.hh b/Source/Generators/LMCArraySignalGenerator.hh index 62457d12..80d5b1b0 100644 --- a/Source/Generators/LMCArraySignalGenerator.hh +++ b/Source/Generators/LMCArraySignalGenerator.hh @@ -31,7 +31,7 @@ #include "LMCSinglePatchPositioner.hh" #include "LMCPlanarArrayPositioner.hh" #include - +#include "LMCException.hh" namespace locust From f75785455b5d425d0232a174e79cab77052884ee Mon Sep 17 00:00:00 2001 From: Penny Slocum Date: Thu, 11 Feb 2021 05:51:24 -0500 Subject: [PATCH 9/9] Updated log messages. --- Source/Generators/LMCArraySignalGenerator.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Generators/LMCArraySignalGenerator.cc b/Source/Generators/LMCArraySignalGenerator.cc index 4d7c1d9b..133f12bb 100644 --- a/Source/Generators/LMCArraySignalGenerator.cc +++ b/Source/Generators/LMCArraySignalGenerator.cc @@ -462,7 +462,7 @@ namespace locust double VoltageFIRSample = GetFIRSample(nfilterbins, dtfilter, channelIndex, elementIndex); if ((VoltageFIRSample == 0.)&&(index-startingIndex > fFieldBufferSize*fPowerCombiner->GetNElementsPerStrip())) { - LERROR(lmclog,"A digitizer sample was skipped due to likely unresponsive thread. Exiting.\n"); + LERROR(lmclog,"A digitizer sample was skipped due to likely unresponsive thread.\n"); return false; } fPowerCombiner->AddOneVoltageToStripSum(aSignal, VoltageFIRSample, fphiLO, elementIndex, IndexBuffer[channelIndex*fNElementsPerStrip+elementIndex].front()); @@ -653,9 +653,9 @@ namespace locust { PreEventCounter = 0; // reset } - else + else if (!DriveAntenna(fp, startingIndex, index, aSignal, nfilterbins, dtfilter)) { - LERROR(lmclog,"The antenna did not respond correctly. Exiting.\n"); + LERROR(lmclog,"The antenna did not respond correctly after two tries. Exiting.\n"); fSkippedSamples = true; tLock.unlock(); break;