Merge pull request #2247 from AllenInstitute/feature/2247-access-to-l…

…ogbook-from-zeromq

Allow accessing the logbook waves from ZeroMQ

Packages/MIES/MIES_ForeignFunctionInterface.ipf

@@ -84,3 +84,59 @@ Function FFI_SetCellElectrodeName(string device, variable headstage, string name
 
 	cellElectrodeNames[headstage] = name
 End
+
+/// @brief Query logbook entries from devices
+///
+/// This allows to query labnotebook/results entries from associated channels.
+///
+/// @param device          Name of the hardware device panel, @sa GetLockedDevices()
+/// @param logbookType     One of #LBT_LABNOTEBOOK or #LBT_RESULTS
+/// @param sweepNo         Sweep number
+/// @param setting         Name of the entry
+/// @param entrySourceType One of #DATA_ACQUISITION_MODE/#UNKNOWN_MODE/#TEST_PULSE_MODE
+///
+/// @return Numerical/Textual wave with #LABNOTEBOOK_LAYER_COUNT rows or a null wave reference if nothing could be found
+Function/WAVE FFI_QueryLogbook(string device, variable logbookType, variable sweepNo, string setting, variable entrySourceType)
+
+	ASSERT(logbookType != LBT_TPSTORAGE, "Invalid logbook type")
+
+	WAVE/T numericalValues = GetLogbookWaves(logbookType, LBN_NUMERICAL_VALUES, device = device)
+
+	WAVE/Z settings = GetLastSetting(numericalValues, sweepNo, setting, entrySourceType)
+
+	if(WaveExists(settings))
+		return settings
+	endif
+
+	WAVE/T textualValues = GetLogbookWaves(logbookType, LBN_TEXTUAL_VALUES, device = device)
+
+	WAVE/Z settings = GetLastSetting(textualValues, sweepNo, setting, entrySourceType)
+
+	return settings
+End
+
+/// @brief Return all unique logbook entries from devices
+///
+/// @param device      Name of the hardware device panel, @sa GetLockedDevices()
+/// @param logbookType One of #LBT_LABNOTEBOOK or #LBT_RESULTS
+/// @param setting     Name of the entry
+///
+/// @return Numerical/Textual 1D wave or a null wave reference if nothing could be found
+Function/WAVE FFI_QueryLogbookUniqueSetting(string device, variable logbookType, string setting)
+
+	ASSERT(logbookType != LBT_TPSTORAGE, "Invalid logbook type")
+
+	WAVE/T numericalValues = GetLogbookWaves(logbookType, LBN_NUMERICAL_VALUES, device = device)
+
+	WAVE/Z settings = GetUniqueSettings(numericalValues, setting)
+
+	if(WaveExists(settings))
+		return settings
+	endif
+
+	WAVE/T textualValues = GetLogbookWaves(logbookType, LBN_TEXTUAL_VALUES, device = device)
+
+	WAVE/Z settings = GetUniqueSettings(textualValues, setting)
+
+	return settings
+End

Packages/MIES/MIES_MiesUtilities_Logbook.ipf

@@ -1327,20 +1327,18 @@ threadsafe Function/WAVE GetLastSettingTextEachSCI(numericalValues, textualValue
 End
 
 /// @brief Return a wave with all labnotebook rows which have a non-empty entry for setting
-threadsafe Function/WAVE GetNonEmptyLBNRows(labnotebookValues, setting)
-	WAVE   labnotebookValues
-	string setting
+threadsafe Function [WAVE indizes, variable settingsCol] GetNonEmptyLBNRows(WAVE labnotebookValues, string setting)
 
-	variable col
+	settingsCol = GetLogbookSettingsColumn(labnotebookValues, setting)
 
-	col = FindDimLabel(labnotebookValues, COLS, setting)
-
-	if(col < 0)
-		return $""
+	if(settingsCol < 0)
+		return [$"", NaN]
 	endif
 
-	return FindIndizes(labnotebookValues, col = col, prop = PROP_EMPTY | PROP_NOT,      \
-	                   startLayer = 0, endLayer = DimSize(labnotebookValues, LAYERS) - 1)
+	WAVE/Z indizes = FindIndizes(labnotebookValues, col = settingsCol, prop = PROP_EMPTY | PROP_NOT, \
+	                             startLayer = 0, endLayer = DimSize(labnotebookValues, LAYERS) - 1)
+
+	return [indizes, settingsCol]
 End
 
 /// @brief Test helper to enforce that every query done for an INDEP_HEADSTAGE setting
@@ -1511,9 +1509,9 @@ threadsafe Function/WAVE GetSweepsWithSetting(labnotebookValues, setting)
 	WAVE   labnotebookValues
 	string setting
 
-	variable sweepCol
+	variable sweepCol, settingsCol
 
-	WAVE/Z indizes = GetNonEmptyLBNRows(labnotebookValues, setting)
+	[WAVE indizes, settingsCol] = GetNonEmptyLBNRows(labnotebookValues, setting)
 	if(!WaveExists(indizes))
 		return $""
 	endif
@@ -1536,6 +1534,50 @@ threadsafe Function/WAVE GetSweepsWithSetting(labnotebookValues, setting)
 	return GetUniqueEntries(sweeps)
 End
 
+/// @brief Return a unique list of labnotebook entries of the given setting
+///
+/// @param values  numerical logbook wave
+/// @param setting name of the value to search
+threadsafe Function/WAVE GetUniqueSettings(WAVE values, string setting)
+
+	variable numMatches, settingsCol
+
+	[WAVE indizes, settingsCol] = GetNonEmptyLBNRows(values, setting)
+	if(!WaveExists(indizes))
+		return $""
+	endif
+
+	numMatches = DimSize(indizes, ROWS)
+
+	if(IsNumericWave(values))
+		Make/D/FREE/N=(numMatches, LABNOTEBOOK_LAYER_COUNT) data
+
+		Multithread data[][] = values[indizes[p]][settingsCol][q]
+
+		Redimension/N=(numMatches * LABNOTEBOOK_LAYER_COUNT)/E=1 data
+
+		WAVE dataUnique = GetUniqueEntries(data)
+
+		return ZapNaNs(dataUnique)
+	elseif(IsTextWave(values))
+		Make/T/FREE/N=(numMatches, LABNOTEBOOK_LAYER_COUNT) dataTxt
+
+		WAVE/T valuesTxt = values
+
+		Multithread dataTxt[][] = valuesTxt[indizes[p]][settingsCol][q]
+
+		Redimension/N=(numMatches * LABNOTEBOOK_LAYER_COUNT)/E=1 dataTxt
+
+		WAVE dataUnique = GetUniqueEntries(dataTxt)
+
+		RemoveTextWaveEntry1D(dataUnique, "")
+
+		return dataUnique
+	endif
+
+	ASSERT_TS(0, "Unsupported wave type")
+End
+
 /// @brief Return the last numerical value of a setting from the labnotebook
 ///        and the sweep it was set.
 ///
@@ -1552,18 +1594,18 @@ threadsafe Function/WAVE GetLastSweepWithSetting(numericalValues, setting, sweep
 	string    setting
 	variable &sweepNo
 
-	variable idx
+	variable idx, settingsCol
 
 	sweepNo = NaN
 	ASSERT_TS(IsNumericWave(numericalValues), "Can only work with numeric waves")
 
-	WAVE/Z indizes = GetNonEmptyLBNRows(numericalValues, setting)
+	[WAVE indizes, settingsCol] = GetNonEmptyLBNRows(numericalValues, setting)
 	if(!WaveExists(indizes))
 		return $""
 	endif
 
 	idx = indizes[DimSize(indizes, ROWS) - 1]
-	Make/FREE/N=(DimSize(numericalValues, LAYERS)) data = numericalValues[idx][%$setting][p]
+	Make/FREE/N=(DimSize(numericalValues, LAYERS)) data = numericalValues[idx][settingsCol][p]
 	sweepNo = numericalValues[idx][GetSweepColumn(numericalValues)][0]
 
 	return data
@@ -1615,18 +1657,18 @@ threadsafe Function/WAVE GetLastSweepWithSettingText(textualValues, setting, swe
 	string    setting
 	variable &sweepNo
 
-	variable idx
+	variable idx, settingsCol
 
 	sweepNo = NaN
 	ASSERT_TS(IsTextWave(textualValues), "Can only work with text waves")
 
-	WAVE/Z indizes = GetNonEmptyLBNRows(textualValues, setting)
+	[WAVE indizes, settingsCol] = GetNonEmptyLBNRows(textualValues, setting)
 	if(!WaveExists(indizes))
 		return $""
 	endif
 
 	idx = indizes[DimSize(indizes, ROWS) - 1]
-	Make/FREE/T/N=(DimSize(textualValues, LAYERS)) data = textualValues[idx][%$setting][p]
+	Make/FREE/T/N=(DimSize(textualValues, LAYERS)) data = textualValues[idx][settingsCol][p]
 	sweepNo = str2num(textualValues[idx][GetSweepColumn(textualValues)][0])
 
 	return data

Packages/MIES/MIES_SweepFormula.ipf

@@ -5081,20 +5081,10 @@ Function/WAVE SF_GetAllOldCodeForGUI(string win) // parameter required for popup
 End
 
 static Function/WAVE SF_GetAllOldCode()
-	string entry
 
 	WAVE/T textualResultsValues = GetLogbookWaves(LBT_RESULTS, LBN_TEXTUAL_VALUES)
 
-	entry = "Sweep Formula code"
-	WAVE/Z indizes = GetNonEmptyLBNRows(textualResultsValues, entry)
-
-	if(!WaveExists(indizes))
-		return $""
-	endif
-
-	Make/FREE/T/N=(DimSize(indizes, ROWS)) entries = textualResultsValues[indizes[p]][%$entry][INDEP_HEADSTAGE]
-
-	return GetUniqueEntries(entries)
+	return GetUniqueSettings(textualResultsValues, "Sweep Formula code")
 End
 
 Function SF_PopMenuProc_OldCode(STRUCT WMPopupAction &pa) : PopupMenuControl

Packages/tests/Basic/UTF_Basic.ipf

@@ -14,6 +14,7 @@
 #include "UTF_DAEphyswoHardware"
 #include "UTF_Debugging"
 #include "UTF_EpochswoHardware"
+#include "UTF_ForeignFunctionInterface"
 #include "UTF_GuiUtilities"
 #include "UTF_JSONWaveNotes"
 #include "UTF_Labnotebook"
@@ -126,6 +127,7 @@ Function RunWithOpts([string testcase, string testsuite, variable allowdebug, va
 	list = AddListItem("UTF_Configuration.ipf", list, ";", Inf)
 	list = AddListItem("UTF_DAEphyswoHardware.ipf", list, ";", Inf)
 	list = AddListItem("UTF_EpochswoHardware.ipf", list, ";", Inf)
+	list = AddListItem("UTF_ForeignFunctionInterface.ipf", list, ";", Inf)
 	list = AddListItem("UTF_GuiUtilities.ipf", list, ";", Inf)
 	list = AddListItem("UTF_JSONWaveNotes.ipf", list, ";", Inf)
 	list = AddListItem("UTF_Labnotebook.ipf", list, ";", Inf)

Packages/tests/Basic/UTF_ForeignFunctionInterface.ipf

@@ -0,0 +1,43 @@
+#pragma TextEncoding="UTF-8"
+#pragma rtGlobals=3 // Use modern global access method and strict wave access.
+#pragma rtFunctionErrors=1
+#pragma ModuleName=ForeignFunctionTests
+
+static Function TestMessageFilters()
+
+	string wvNote
+
+	WAVE/T/Z filters = FFI_GetAvailableMessageFilters()
+	CHECK_WAVE(filters, FREE_WAVE | TEXT_WAVE)
+	CHECK_GT_VAR(DimSize(filters, ROWS), 0)
+
+	wvNote = note(filters)
+
+	CHECK_PROPER_STR(wvNote)
+End
+
+static Function TestLogbookQuery()
+	string key, keyTxT, device
+
+	device = "ITC16USB_0_DEV"
+	[key, keyTxt] = PrepareLBN_IGNORE(device)
+
+	WAVE/Z settings = FFI_QueryLogbook(device, LBT_LABNOTEBOOK, 0, key, DATA_ACQUISITION_MODE)
+	CHECK_WAVE(settings, NUMERIC_WAVE)
+
+	WAVE/Z settings = FFI_QueryLogbook(device, LBT_LABNOTEBOOK, 0, keyTxt, DATA_ACQUISITION_MODE)
+	CHECK_WAVE(settings, TEXT_WAVE)
+End
+
+static Function TestLogbookQueryUnique()
+	string key, keyTxT, device
+
+	device = "ITC16USB_0_DEV"
+	[key, keyTxt] = PrepareLBN_IGNORE(device)
+
+	WAVE/Z settings = FFI_QueryLogbookUniqueSetting(device, LBT_LABNOTEBOOK, key)
+	CHECK_WAVE(settings, NUMERIC_WAVE)
+
+	WAVE/Z settings = FFI_QueryLogbookUniqueSetting(device, LBT_LABNOTEBOOK, keyTxt)
+	CHECK_WAVE(settings, TEXT_WAVE)
+End

Packages/tests/Basic/UTF_Labnotebook.ipf

@@ -1312,3 +1312,30 @@ Function MultipleSameEDAdds()
 	CHECK_EQUAL_VAR(DimSize(indices, ROWS), 1)
 	CHECK_GE_VAR(indices[0], 0)
 End
+
+Function GetUniqueSettingsWorks()
+	string key, keyTxT, device
+
+	device = "ITC16USB_0_DEV"
+	[key, keyTxt] = PrepareLBN_IGNORE(device)
+
+	WAVE/T numericalValues = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_NUMERICAL_VALUES, device = device)
+	WAVE/T textualValues   = GetLogbookWaves(LBT_LABNOTEBOOK, LBN_TEXTUAL_VALUES, device = device)
+
+	// no matches
+	WAVE/Z results = GetUniqueSettings(numericalValues, "I_DONT_EXIST")
+	CHECK_WAVE(results, NULL_WAVE)
+
+	WAVE/Z resultsTxt = GetUniqueSettings(textualValues, "I_DONT_EXIST")
+	CHECK_WAVE(resultsTxt, NULL_WAVE)
+
+	// matches
+	WAVE/Z results = GetUniqueSettings(numericalValues, key)
+	CHECK_WAVE(results, NUMERIC_WAVE)
+	Make/D/FREE ref = {131415, 192021, 161718, 222324, 252627}
+	CHECK_EQUAL_WAVES(results, ref)
+
+	WAVE/Z resultsTxt = GetUniqueSettings(textualValues, keyTxt)
+	CHECK_WAVE(resultsTxt, TEXT_WAVE)
+	CHECK_EQUAL_TEXTWAVES(resultsTxt, {"131415", "192021", "161718", "222324", "252627"})
+End

Packages/tests/HardwareAnalysisFunctions/UTF_PatchSeqAccessResistanceSmoke.ipf

@@ -45,16 +45,19 @@ static Function GlobalPreAcq(string device)
 End
 
 static Function/WAVE GetResultsSingleEntry_IGNORE(string name)
+
+	variable nameCol, typeCol
+
 	WAVE/T textualResultsValues = GetTextualResultsValues()
 
-	WAVE/Z indizesName = GetNonEmptyLBNRows(textualResultsValues, name)
-	WAVE/Z indizesType = GetNonEmptyLBNRows(textualResultsValues, "EntrySourceType")
+	[WAVE indizesName, nameCol] = GetNonEmptyLBNRows(textualResultsValues, name)
+	[WAVE indizesType, typeCol] = GetNonEmptyLBNRows(textualResultsValues, "EntrySourceType")
 
 	if(!WaveExists(indizesName) || !WaveExists(indizesType))
 		return $""
 	endif
 
-	indizesType[] = (str2numSafe(textualResultsValues[indizesType[p]][%$"EntrySourceType"][INDEP_HEADSTAGE]) == SWEEP_FORMULA_RESULT) ? indizesType[p] : NaN
+	indizesType[] = (str2numSafe(textualResultsValues[indizesType[p]][typeCol][INDEP_HEADSTAGE]) == SWEEP_FORMULA_RESULT) ? indizesType[p] : NaN
 
 	WAVE/Z indizesTypeClean = ZapNaNs(indizesType)
 
@@ -68,7 +71,7 @@ static Function/WAVE GetResultsSingleEntry_IGNORE(string name)
 		return $""
 	endif
 
-	Make/FREE/T/N=(DimSize(indizes, ROWS)) entries = textualResultsValues[indizes[p]][%$name][INDEP_HEADSTAGE]
+	Make/FREE/T/N=(DimSize(indizes, ROWS)) entries = textualResultsValues[indizes[p]][nameCol][INDEP_HEADSTAGE]
 
 	return entries
 End

Packages/tests/HardwareAnalysisFunctions/UTF_PatchSeqDAScale_Adapt.ipf

@@ -66,16 +66,19 @@ static Function GlobalPreAcq(string device)
 End
 
 static Function/WAVE GetResultsSingleEntry_IGNORE(string name)
+
+	variable nameCol, typeCol
+
 	WAVE/T textualResultsValues = GetTextualResultsValues()
 
-	WAVE/Z indizesName = GetNonEmptyLBNRows(textualResultsValues, name)
-	WAVE/Z indizesType = GetNonEmptyLBNRows(textualResultsValues, "EntrySourceType")
+	[WAVE indizesName, nameCol] = GetNonEmptyLBNRows(textualResultsValues, name)
+	[WAVE indizesType, typeCol] = GetNonEmptyLBNRows(textualResultsValues, "EntrySourceType")
 
 	if(!WaveExists(indizesName) || !WaveExists(indizesType))
 		return $""
 	endif
 
-	indizesType[] = (str2numSafe(textualResultsValues[indizesType[p]][%$"EntrySourceType"][INDEP_HEADSTAGE]) == SWEEP_FORMULA_RESULT) ? indizesType[p] : NaN
+	indizesType[] = (str2numSafe(textualResultsValues[indizesType[p]][typeCol][INDEP_HEADSTAGE]) == SWEEP_FORMULA_RESULT) ? indizesType[p] : NaN
 
 	WAVE/Z indizesTypeClean = ZapNaNs(indizesType)
 
@@ -89,7 +92,7 @@ static Function/WAVE GetResultsSingleEntry_IGNORE(string name)
 		return $""
 	endif
 
-	Make/FREE/T/N=(DimSize(indizes, ROWS)) entries = textualResultsValues[indizes[p]][%$name][INDEP_HEADSTAGE]
+	Make/FREE/T/N=(DimSize(indizes, ROWS)) entries = textualResultsValues[indizes[p]][nameCol][INDEP_HEADSTAGE]
 
 	return entries
 End