From 543cacb67996f7603d8283d0685537d72fece950 Mon Sep 17 00:00:00 2001 From: Gann Date: Mon, 3 May 2021 16:01:02 +0200 Subject: [PATCH 1/3] implementation of format parser class for extended result output of bytearray answer --- o3d3xx/pcic/format_client.py | 435 +++++++++++++++++++---------------- o3d3xx/pcic/format_parser.py | 151 ++++++++++++ 2 files changed, 392 insertions(+), 194 deletions(-) create mode 100644 o3d3xx/pcic/format_parser.py diff --git a/o3d3xx/pcic/format_client.py b/o3d3xx/pcic/format_client.py index 5a645ef..6b23a41 100644 --- a/o3d3xx/pcic/format_client.py +++ b/o3d3xx/pcic/format_client.py @@ -4,206 +4,253 @@ import array import json +from o3d3xx.pcic.format_parser import * from .client import PCICV3Client class PCICFormatRecord: - def __init__(self, recordId): - self.recordMap = {} - self.recordMap["type"] = "records" - self.recordMap["id"] = str(recordId) - self.recordMap["elements"] = [] - - def addStringElement(self, id, value): - stringElement = {} - stringElement["type"] = "string" - stringElement["value"] = str(value) - stringElement["id"] = str(id) - self.recordMap["elements"].append(stringElement) - - def addBlobElement(self, id): - blobElement = {} - blobElement["type"] = "blob" - blobElement["id"] = str(id) - self.recordMap["elements"].append(blobElement) - - def toMap(self): - return self.recordMap - - def toString(self): - return json.dumps(self.recordMap) + def __init__(self, recordId): + self.recordMap = {} + self.recordMap["type"] = "records" + self.recordMap["id"] = str(recordId) + self.recordMap["elements"] = [] + + def addStringElement(self, id, value): + stringElement = {} + stringElement["type"] = "string" + stringElement["value"] = str(value) + stringElement["id"] = str(id) + self.recordMap["elements"].append(stringElement) + + def addBlobElement(self, id): + blobElement = {} + blobElement["type"] = "blob" + blobElement["id"] = str(id) + self.recordMap["elements"].append(blobElement) + + def toMap(self): + return self.recordMap + + def toString(self): + return json.dumps(self.recordMap) class PCICFormat: - def __init__(self, formatString = None): - self.formatMap = {} - if formatString != None: - self.formatMap = json.loads(formatString) - else: - self.formatMap["layouter"] = "flexible" - self.formatMap["format"] = { "dataencoding": "ascii" } - self.formatMap["elements"] = [] - - def addStringElement(self, id, value): - stringElement = {} - stringElement["type"] = "string" - stringElement["value"] = str(value) - stringElement["id"] = str(id) - self.formatMap["elements"].append(stringElement) - - def addBlobElement(self, id): - blobElement = {} - blobElement["type"] = "blob" - blobElement["id"] = str(id) - self.formatMap["elements"].append(blobElement) - - def addRecordElement(self, record): - self.formatMap["elements"].append(record.toMap()) - - def toString(self): - return json.dumps(self.formatMap) - - @staticmethod - def blobs(*blobIds): - format = PCICFormat() - for blobId in blobIds: - format.addBlobElement(blobId) - return format + def __init__(self, formatString=None): + self.formatMap = {} + if formatString != None: + self.formatMap = json.loads(formatString) + else: + self.formatMap["layouter"] = "flexible" + self.formatMap["format"] = { "dataencoding": "ascii" } + self.formatMap["elements"] = [] + + def addStringElement(self, id, value): + stringElement = {} + stringElement["type"] = "string" + stringElement["value"] = str(value) + stringElement["id"] = str(id) + self.formatMap["elements"].append(stringElement) + + def addBlobElement(self, id): + blobElement = {} + blobElement["type"] = "blob" + blobElement["id"] = str(id) + self.formatMap["elements"].append(blobElement) + + def addRecordElement(self, record): + self.formatMap["elements"].append(record.toMap()) + + def toString(self): + return json.dumps(self.formatMap) + + @staticmethod + def blobs(*blobIds): + format = PCICFormat() + for blobId in blobIds: + format.addBlobElement(blobId) + return format class PCICParser: - def __init__(self, format = None): - self.format = format - self.debug = False - - def parseBlob(self, answer, answerIndex): - # extract version independent information from chunk header - chunkType, chunkSize, headerSize, headerVersion = struct.unpack('IIII', bytes(answer[answerIndex:answerIndex+16])) - - # extract full chunk header information - if headerVersion == 1: - chunkType, chunkSize, headerSize, headerVersion, imageWidth, imageHeight, pixelFormat, timeStamp, frameCount = struct.unpack('IIIIIIIII', bytes(answer[answerIndex:answerIndex+headerSize])) - elif headerVersion == 2: - chunkType, chunkSize, headerSize, headerVersion, imageWidth, imageHeight, pixelFormat, timeStamp, frameCount, statusCode, timeStampSec, timeStampNsec = struct.unpack('IIIIIIIIIIII', bytes(answer[answerIndex:answerIndex+headerSize])) - else: - print("Unknown chunk header version %d!" % headerVersion) - return chunkType, chunkSize, None - - if self.debug == True: - print('''Data chunk: - Chunk type: %d - Chunk size: %d - Header size: %d - Header version: %d - Image width: %d - Image height: %d - Pixel format: %d - Time stamp: %d - Frame counter: %d''' % (chunkType, chunkSize, headerSize, headerVersion, imageWidth, imageHeight, pixelFormat, timeStamp, frameCount)) - - # check payload size - if len(answer) < answerIndex + chunkSize: - raise RuntimeError("data truncated ({} bytes missing)", answerIndex + chunkSize - len(answer)) - - # read chunk payload data - answerIndex += headerSize - # distinguish pixel type - if pixelFormat == 0: - image = array.array('B', bytes(answer[answerIndex:answerIndex+chunkSize-headerSize])) - elif pixelFormat == 1: - image = array.array('b', bytes(answer[answerIndex:answerIndex+chunkSize-headerSize])) - elif pixelFormat == 2: - image = array.array('H', bytes(answer[answerIndex:answerIndex+chunkSize-headerSize])) - elif pixelFormat == 3: - image = array.array('h', bytes(answer[answerIndex:answerIndex+chunkSize-headerSize])) - elif pixelFormat == 4: - image = array.array('I', bytes(answer[answerIndex:answerIndex+chunkSize-headerSize])) - elif pixelFormat == 5: - image = array.array('i', bytes(answer[answerIndex:answerIndex+chunkSize-headerSize])) - elif pixelFormat == 6: - image = array.array('f', bytes(answer[answerIndex:answerIndex+chunkSize-headerSize])) - elif pixelFormat == 8: - image = array.array('d', bytes(answer[answerIndex:answerIndex+chunkSize-headerSize])) - else: - print("Unknown pixel format %d!" % pixelFormat) - image = None - - return chunkType, chunkSize, image - - def parseElement(self, answer, answerIndex, element, result): - if element["type"] == "string": - readString = answer[answerIndex:answerIndex + len(element["value"])].decode("utf-8") - if self.debug == True: - print("String: '{}'".format(readString)) - if readString == element["value"]: - result[element["id"]] = element["value"] - return answerIndex + len(element["value"]) - else: - raise RuntimeError("read result '{}' does not match format (expected '{}')" - .format(readString, element["value"])) - elif element["type"] == "blob": - chunkType, chunkSize, blobData = self.parseBlob(answer, answerIndex) - if element["id"] in result: - if isinstance(result[element["id"]], list): - result[element["id"]].append(blobData) - else: - result[element["id"]] = [result[element["id"]], blobData] - else: - result[element["id"]] = blobData - return answerIndex + chunkSize - elif element["type"] == "records": - if self.debug == True: - print("Record: '{}'".format(element["id"])) - recordResult, answerIndex = self.parseRecord(answer, answerIndex, element["elements"]) - result[element["id"]] = recordResult - return answerIndex - raise RuntimeError("cannot handle element type {}".format(element["type"])) - - def parseRecord(self, answer, answerIndex, recordElements): - recordResult = [] - # currently only one record at the end of the answer is supported - while answerIndex < len(answer): - iterationResult = {} - for element in recordElements: - answerIndex = self.parseElement(answer, answerIndex, element, iterationResult) - recordResult.append(iterationResult) - return recordResult, answerIndex - - def parseAnswer(self, answer): - result = {} - answerIndex = 0 - # print("Parsing answer '%s' against format '%s'" % (answer, self.format.toString())) - for element in self.format.formatMap["elements"]: - answerIndex = self.parseElement(answer, answerIndex, element, result) - return result + def __init__(self, format=None): + self.format = format + self.debug = False + + def extractChunkHeaderInformation(self, answer, answerIndex): + chunkType, chunkSize, headerSize, headerVersion = struct.unpack('IIII', + bytes(answer[answerIndex:answerIndex + 16])) + # extract full chunk header information + if headerVersion == 1: + chunkType, chunkSize, headerSize, headerVersion, imageWidth, imageHeight, pixelFormat, timeStamp, frameCount = struct.unpack( + 'IIIIIIIII', bytes(answer[answerIndex:answerIndex + headerSize])) + elif headerVersion == 2: + chunkType, chunkSize, headerSize, headerVersion, imageWidth, imageHeight, pixelFormat, timeStamp, frameCount, statusCode, timeStampSec, timeStampNsec = struct.unpack( + 'IIIIIIIIIIII', bytes(answer[answerIndex:answerIndex + headerSize])) + else: + print("Unknown chunk header version %d!" % headerVersion) + return chunkType, chunkSize, None + + if self.debug == True: + print('''Data chunk: + Chunk type: %d + Chunk size: %d + Header size: %d + Header version: %d + Image width: %d + Image height: %d + Pixel format: %d + Time stamp: %d + Frame counter: %d''' % ( + chunkType, chunkSize, headerSize, headerVersion, imageWidth, imageHeight, pixelFormat, timeStamp, + frameCount)) + + return chunkType, chunkSize, headerSize, headerVersion, imageWidth, imageHeight, pixelFormat, timeStamp, frameCount + + def parseBlob(self, answer, answerIndex): + # extract version independent information from chunk header + chunkType, chunkSize, headerSize, headerVersion, imageWidth, imageHeight, pixelFormat, timeStamp, frameCount = self.extractChunkHeaderInformation(answer, answerIndex) + + # check payload size + if len(answer) < answerIndex + chunkSize: + raise RuntimeError("data truncated ({} bytes missing)", answerIndex + chunkSize - len(answer)) + + # read chunk payload data + answerIndex += headerSize + # distinguish pixel type + if pixelFormat == 0: + image = array.array('B', bytes(answer[answerIndex:answerIndex + chunkSize - headerSize])) + elif pixelFormat == 1: + image = array.array('b', bytes(answer[answerIndex:answerIndex + chunkSize - headerSize])) + elif pixelFormat == 2: + image = array.array('H', bytes(answer[answerIndex:answerIndex + chunkSize - headerSize])) + elif pixelFormat == 3: + image = array.array('h', bytes(answer[answerIndex:answerIndex + chunkSize - headerSize])) + elif pixelFormat == 4: + image = array.array('I', bytes(answer[answerIndex:answerIndex + chunkSize - headerSize])) + elif pixelFormat == 5: + image = array.array('i', bytes(answer[answerIndex:answerIndex + chunkSize - headerSize])) + elif pixelFormat == 6: + image = array.array('f', bytes(answer[answerIndex:answerIndex + chunkSize - headerSize])) + elif pixelFormat == 8: + image = array.array('d', bytes(answer[answerIndex:answerIndex + chunkSize - headerSize])) + else: + print("Unknown pixel format %d!" % pixelFormat) + image = None + + return chunkType, chunkSize, image + + def parseElement(self, answer, answerIndex, element, result): + if element["type"] == "string": + readString = answer[answerIndex:answerIndex + len(element["value"])].decode("utf-8") + if self.debug == True: + print("String: '{}'".format(readString)) + if readString == element["value"]: + result[element["id"]] = element["value"] + return answerIndex + len(element["value"]) + else: + raise RuntimeError("read result '{}' does not match format (expected '{}')" + .format(readString, element["value"])) + elif element["type"] == "blob": + chunkType, chunkSize, blobData = self.parseBlob(answer, answerIndex) + if element["id"] in result: + if isinstance(result[element["id"]], list): + result[element["id"]].append(blobData) + else: + result[element["id"]] = [result[element["id"]], blobData] + else: + result[element["id"]] = blobData + return answerIndex + chunkSize + elif element["type"] == "records": + if self.debug == True: + print("Record: '{}'".format(element["id"])) + recordResult, answerIndex = self.parseRecord(answer, answerIndex, element["elements"]) + if element["id"] in result.keys(): + result[element["id"]].append(recordResult[0]) + else: + result[element["id"]] = recordResult + return answerIndex + elif element["type"] in ["int8", "uint8", "int16", "uint16", "int32", "uint32", "float32", "float64"]: + if self.debug == True: + print("{}: '{}'".format(element["type"], element["id"])) + numericResult, answerIndex = self.parseNumeric(answer, answerIndex, element["type"]) + result[element["id"]] = numericResult + return answerIndex + raise RuntimeError("cannot handle element type {}".format(element["type"])) + + def parseNumeric(self, answer, answerIndex, elementType): + numericResult = [] + value = chr(answer[answerIndex]) + while value not in [";", "|", "*"]: + numericResult += value + answerIndex += 1 + if not answerIndex == len(answer): + value = chr(answer[answerIndex]) + # try evaluating an numeric value, else return it as a string + try: + if elementType not in ["float32", "float64"]: + value = int(''.join(numericResult)) + else: + value = eval(''.join(numericResult)) + return value, answerIndex + except SyntaxError: + value = ''.join(numericResult) + return value, answerIndex + + def parseRecord(self, answer, answerIndex, recordElements): + recordResult = [] + iterationResult = {} + for element in recordElements: + answerIndex = self.parseElement(answer, answerIndex, element, iterationResult) + recordResult.append(iterationResult) + return recordResult, answerIndex + + def parseAnswer(self, answer): + result = {} + answerIndex = 0 + # print("Parsing answer '%s' against format '%s'" % (answer, self.format.toString())) + for element in self.format.formatMap["elements"]: + answerIndex = self.parseElement(answer, answerIndex, element, result) + return result class FormatClient(PCICV3Client): - def __init__(self, address, port, format=None): - super(FormatClient, self).__init__(address, port) - # disable all result output - self.sendCommand("p0") - - self.format = format - # if format is not specified, read back format as configured by the active application - if self.format == None: - formatstring = self.sendCommand("C?").decode("utf-8")[9:] - self.format = PCICFormat(str(formatstring)) - # otherwise set provided format for this connection - else: - formatString = self.format.toString() - answer = self.sendCommand("c%09d%s" % (len(formatString), formatString)) - if str(answer, 'utf-8') != "*": - raise RuntimeError("could not change PCIC format (format string is '{}')".format(formatString)) - - self.parser = PCICParser(self.format) - - # enable result output again - self.sendCommand("p1") - - def readNextFrame(self): - result = {} - - # look for asynchronous output - ticket, answer = self.readNextAnswer() - if ticket == b"0000": - - self.parser.debug = self.debug - result = self.parser.parseAnswer(answer) - return result + def __init__(self, address, port, format=None): + self.format = format + self.answer = None + + super(FormatClient, self).__init__(address, port) + + ticket, self.answer = self.readNextAnswer() + + # disable all result output + self.sendCommand("p0") + + # if format is not specified, read back format as configured by the active application + if not format: + formatstring = self.sendCommand("C?").decode("utf-8")[9:] + self.format = PCICFormat(str(formatstring)) + # otherwise set provided format for this connection + else: + formatString = self.format.toString() + answer = self.sendCommand("c%09d%s" % (len(formatString), formatString)) + if str(answer, 'utf-8') != "*": + raise RuntimeError("could not change PCIC format (format string is '{}')".format(formatString)) + + self.parser = PCICParser(self.format) + + # parsing und unrolling the format in case of multiple or nested records + if not format: + formatParser = FormatParser(self.parser, self.answer, self.format) + formatParser.unrolledFormatMap() + + # enable result output again + self.sendCommand("p1") + + def readNextFrame(self): + result = {} + + # look for asynchronous output + ticket, self.answer = self.readNextAnswer() + + if ticket == b"0000": + + self.parser.debug = self.debug + result = self.parser.parseAnswer(self.answer) + return result diff --git a/o3d3xx/pcic/format_parser.py b/o3d3xx/pcic/format_parser.py new file mode 100644 index 0000000..fc60143 --- /dev/null +++ b/o3d3xx/pcic/format_parser.py @@ -0,0 +1,151 @@ +import copy +import re + +class FormatParser: + def __init__(self, pcicParser, answer=None, format=None): + self.pcicParser = pcicParser + self.answer = answer + self.format = format + + self.parentChildDelimiter = None + + self.nativeFormat = copy.deepcopy(self.format) + + # Parse all parent and child delimiter if no attribute is empty + self.parentChildDelimiter = self.parseParentChildDelimiter(answer) + + # self.evaluateParentChildDelimiter(answer) + # self.appendContentToFormatMap("parentChildDelimiter", self.parentChildDelimiter) + + # Unrolling whole formatMap so you can parse nested records + # self.unrolledFormatMap() + + # def appendContentToFormatMap(self, key, value): + # if "runtime" in self.format.formatMap.keys(): + # self.format.formatMap["runtime"].update({key: value}) + # else: + # self.format.formatMap["runtime"] = {key: value} + + @property + def formatMapRecord(self): + """ + Returns a copy of the next found records element in format map + :return: + """ + for i, element in enumerate(self.nativeFormat.formatMap["elements"]): + if element["type"] == "records": + return copy.deepcopy(self.nativeFormat.formatMap["elements"][i]) + + @property + def recordIndex(self): + """ + Returns the next record index in format map + :return: index as int + """ + for i, element in enumerate(self.nativeFormat.formatMap["elements"]): + if element["type"] == "records": + return i + + def unrolledFormatMap(self): + """ + Unrolling whole formatMap + :return: unrolled format map + """ + def unrollChildRecords(delimiter): + record = self.formatMapRecord + for i, element in enumerate(record["elements"]): + if element["type"] == "records": + # childElement = record["elements"][i] + amount = len(delimiter["childDelimiterIdx"]) + [record["elements"].insert(i + 1, record["elements"][i]) for _ in range(amount - 1)] + break + return record + + # Populate record elements with help of parent and child delimiter + if self.parentChildDelimiter: + unrolledRecords = [unrollChildRecords(delimiter) for delimiter in self.parentChildDelimiter] + + # Delete old record element from formatMap + del (self.format.formatMap["elements"][self.recordIndex]) + # Insert populated records in formatMap + for rec in reversed(unrolledRecords): + self.format.formatMap["elements"].insert(self.recordIndex, rec) + + return self.format + + def _parseBlobRanges(self, answer): + """ + Parsing the blob positions in answer from front and reversed order + :param answer: answer as bytearray + :return: indices which declare the positions of the blobs + """ + blobRanges = [] + answerIndex = 0 + recordsStartIndex = 0 + blobIndices = [i for i, elem in enumerate(self.format.formatMap["elements"]) if elem["type"] == "blob"] + recordsIndices = [i for i, elem in enumerate(self.format.formatMap["elements"]) if + elem["type"] == "records"] + if blobIndices: + blobAheadRecords = any([blobIndices < recordsIndices]) + blobAfterRecords = any([sorted(blobIndices, key=int, reverse=True) > recordsIndices]) + if blobAheadRecords: + for element in self.format.formatMap["elements"]: + elementType = element["type"] + if elementType == "blob": + chunkType, chunkSize, headerSize, headerVersion, imageWidth, imageHeight, pixelFormat, \ + timeStamp, frameCount = self.pcicParser.extractChunkHeaderInformation(answer, answerIndex) + blobRanges.append([answerIndex, answerIndex + chunkSize]) + answerIndex += chunkSize + elif elementType == "records": + recordsStartIndex = answerIndex + break + else: + answerIndex = self.pcicParser.parseElement(answer, answerIndex, element, {}) + + if blobAfterRecords and blobAheadRecords and recordsIndices: + blobElementAfterRecords = int(len(answer) / chunkSize) - len(blobRanges) + blobElementAfterRecordsModulo = int(len(answer) % chunkSize) + answerAheadBlob = [0, blobElementAfterRecordsModulo - blobElementAfterRecords - len("stop") - 1] + blobRanges.append([recordsStartIndex + answerAheadBlob[1], len(answer)]) + + if blobAfterRecords and not blobAheadRecords and not recordsIndices: + for element in self.format.formatMap["elements"]: + elementType = element["type"] + if elementType == "blob": + chunkType, chunkSize, headerSize, headerVersion, imageWidth, imageHeight, pixelFormat, \ + timeStamp, frameCount = self.pcicParser.extractChunkHeaderInformation(answer, answerIndex) + blobRanges.append([answerIndex, answerIndex + chunkSize]) + answerIndex += chunkSize + else: + answerIndex = self.pcicParser.parseElement(answer, answerIndex, element, {}) + + if blobAfterRecords and not blobAheadRecords and recordsIndices: + raise NotImplementedError("It is not possible to parse blob elements at the end of the bytearray " + "answer. You can place two blob elements between or ahead your records.") + + # TODO push content to meta dict + self.metaDict = 0 + # self.appendContentToFormatMap("image_resolution", {"imageWidth": imageWidth, "imageHeight": imageHeight}) + # self.appendContentToFormatMap("chunk_size", chunkSize) + return blobRanges, recordsStartIndex + + def parseParentChildDelimiter(self, answer): + """ + Parsing parent and child delimiter in answer + :param answer: answer as bytearray + :return: list + """ + parentChildDelimiter = [] + blobRanges, delimiterIndex = self._parseBlobRanges(answer) + parentDelimiter = [delimiter for delimiter in re.finditer(b"\|", answer) + if not any(True for blob in blobRanges if blob[0] <= delimiter.span()[0] <= blob[1])] + + if parentDelimiter: + for i, delimiter in enumerate(parentDelimiter): + childDelimiter = [m for m in re.finditer(b"\*", answer[delimiterIndex: delimiter.span()[0]])] + parentChildDelimiter.append({"parentDelimiterIdx": delimiter.span()[0], + "childDelimiterIdx": [d.span()[0] + delimiterIndex for d in + childDelimiter], + "parentDelimiter": "|", "childDelimiter": "*"}) + delimiterIndex = delimiter.span()[0] + return parentChildDelimiter From c6350af20cbdd179591886c095809a5bdb6d95ae Mon Sep 17 00:00:00 2001 From: Gann Date: Mon, 3 May 2021 16:01:38 +0200 Subject: [PATCH 2/3] added tests for format_parser --- tests/data/answer.txt | 1 + tests/data/formatString.txt | 71 ++++++++++++++++++++++++++++ tests/data/formatStringReference.txt | 1 + tests/test_format.py | 59 +++++++++++++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 tests/data/answer.txt create mode 100644 tests/data/formatString.txt create mode 100644 tests/data/formatStringReference.txt diff --git a/tests/data/answer.txt b/tests/data/answer.txt new file mode 100644 index 0000000..8c0db68 --- /dev/null +++ b/tests/data/answer.txt @@ -0,0 +1 @@ +star;00;6;+0.607*01;6;+0.532*02;0;+0.446*|00;6;+0.567*01;6;+0.601*02;0;+0.500*|stop \ No newline at end of file diff --git a/tests/data/formatString.txt b/tests/data/formatString.txt new file mode 100644 index 0000000..60a9312 --- /dev/null +++ b/tests/data/formatString.txt @@ -0,0 +1,71 @@ +{ + "elements": [ + { + "id": "start_string", + "type": "string", + "value": "star;" + }, + { + "elements": [ + { + "elements": [ + { + "format": { + "fill": "0", + "width": 2 + }, + "id": "id", + "type": "int32" + }, + { + "id": "delimiter", + "type": "string", + "value": ";" + }, + { + "id": "state", + "type": "int32" + }, + { + "id": "delimiter", + "type": "string", + "value": ";" + }, + { + "format": { + "fill": "+", + "precision": 3, + "width": 6 + }, + "id": "procval", + "type": "float32" + }, + { + "id": "delimiter", + "type": "string", + "value": "*" + } + ], + "id": "rois", + "type": "records" + }, + { + "id": "delimiter", + "type": "string", + "value": "|" + } + ], + "id": "models", + "type": "records" + }, + { + "id": "end_string", + "type": "string", + "value": "stop" + } + ], + "format": { + "dataencoding": "ascii" + }, + "layouter": "flexible" +} diff --git a/tests/data/formatStringReference.txt b/tests/data/formatStringReference.txt new file mode 100644 index 0000000..bcb516a --- /dev/null +++ b/tests/data/formatStringReference.txt @@ -0,0 +1 @@ +{"elements": [{"id": "start_string", "type": "string", "value": "star;"}, {"elements": [{"elements": [{"format": {"fill": "0", "width": 2}, "id": "id", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"id": "state", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"format": {"fill": "+", "precision": 3, "width": 6}, "id": "procval", "type": "float32"}, {"id": "delimiter", "type": "string", "value": "*"}], "id": "rois", "type": "records"}, {"elements": [{"format": {"fill": "0", "width": 2}, "id": "id", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"id": "state", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"format": {"fill": "+", "precision": 3, "width": 6}, "id": "procval", "type": "float32"}, {"id": "delimiter", "type": "string", "value": "*"}], "id": "rois", "type": "records"}, {"elements": [{"format": {"fill": "0", "width": 2}, "id": "id", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"id": "state", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"format": {"fill": "+", "precision": 3, "width": 6}, "id": "procval", "type": "float32"}, {"id": "delimiter", "type": "string", "value": "*"}], "id": "rois", "type": "records"}, {"id": "delimiter", "type": "string", "value": "|"}], "id": "models", "type": "records"}, {"elements": [{"elements": [{"format": {"fill": "0", "width": 2}, "id": "id", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"id": "state", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"format": {"fill": "+", "precision": 3, "width": 6}, "id": "procval", "type": "float32"}, {"id": "delimiter", "type": "string", "value": "*"}], "id": "rois", "type": "records"}, {"elements": [{"format": {"fill": "0", "width": 2}, "id": "id", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"id": "state", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"format": {"fill": "+", "precision": 3, "width": 6}, "id": "procval", "type": "float32"}, {"id": "delimiter", "type": "string", "value": "*"}], "id": "rois", "type": "records"}, {"elements": [{"format": {"fill": "0", "width": 2}, "id": "id", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"id": "state", "type": "int32"}, {"id": "delimiter", "type": "string", "value": ";"}, {"format": {"fill": "+", "precision": 3, "width": 6}, "id": "procval", "type": "float32"}, {"id": "delimiter", "type": "string", "value": "*"}], "id": "rois", "type": "records"}, {"id": "delimiter", "type": "string", "value": "|"}], "id": "models", "type": "records"}, {"id": "end_string", "type": "string", "value": "stop"}], "format": {"dataencoding": "ascii"}, "layouter": "flexible"} \ No newline at end of file diff --git a/tests/test_format.py b/tests/test_format.py index b1a9fbf..d0e300b 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -47,6 +47,7 @@ def test_records(self): images.addBlobElement("amplitude_image") format = PCICFormat() format.addRecordElement(images) + format.addRecordElement(images) parser = PCICParser(format) result = parser.parseAnswer(amplitudeImageBlob + amplitudeImageBlob) @@ -56,3 +57,61 @@ def test_records(self): {"amplitude_image": array.array('H', [1, 2])} ]} ) + + def test_formatParser(self): + with open('./data/answer.txt', 'r') as file: + string_answer = file.read() + encoded_string = string_answer.encode() + answer = bytearray(encoded_string) + with open('./data/formatString.txt', 'r') as file: + formatstring = file.read() + + format = PCICFormat(str(formatstring)) + parser = PCICParser(format) + format_parser = FormatParser(parser, answer, format) + format_unrolled = format_parser.unrolledFormatMap() + + with open('./data/formatStringReference.txt', 'r') as file: + formatstring = file.read() + + format_reference = PCICFormat(str(formatstring)) + self.assertDictEqual(format_unrolled.formatMap, format_reference.formatMap) + + def test_parseAnswer(self): + with open('./data/answer.txt', 'r') as file: + string_answer = file.read() + encoded_string = string_answer.encode() + answer = bytearray(encoded_string) + with open('./data/formatString.txt', 'r') as file: + formatstring = file.read() + + format = PCICFormat(str(formatstring)) + parser = PCICParser(format) + format_parser = FormatParser(parser, answer, format) + format_parser.unrolledFormatMap() + + result = parser.parseAnswer(answer) + + self.assertEqual(result['models'].__len__(), 2) + self.assertEqual(result['models'][0]['rois'].__len__(), 3) + self.assertEqual(result['models'][1]['rois'].__len__(), 3) + + self.assertEqual(result['models'][0]['rois'][0]['state'], 6) + self.assertEqual(result['models'][0]['rois'][1]['state'], 6) + self.assertEqual(result['models'][0]['rois'][2]['state'], 0) + self.assertEqual(result['models'][0]['rois'][0]['procval'], 0.607) + self.assertEqual(result['models'][0]['rois'][1]['procval'], 0.532) + self.assertEqual(result['models'][0]['rois'][2]['procval'], 0.446) + self.assertEqual(result['models'][0]['rois'][0]['id'], 0) + self.assertEqual(result['models'][0]['rois'][1]['id'], 1) + self.assertEqual(result['models'][0]['rois'][2]['id'], 2) + + self.assertEqual(result['models'][1]['rois'][0]['state'], 6) + self.assertEqual(result['models'][1]['rois'][1]['state'], 6) + self.assertEqual(result['models'][1]['rois'][2]['state'], 0) + self.assertEqual(result['models'][1]['rois'][0]['procval'], 0.567) + self.assertEqual(result['models'][1]['rois'][1]['procval'], 0.601) + self.assertEqual(result['models'][1]['rois'][2]['procval'], 0.5) + self.assertEqual(result['models'][1]['rois'][0]['id'], 0) + self.assertEqual(result['models'][1]['rois'][1]['id'], 1) + self.assertEqual(result['models'][1]['rois'][2]['id'], 2) From da773227021529558881dc22640121c9c90adf11 Mon Sep 17 00:00:00 2001 From: Gann Date: Mon, 3 May 2021 16:02:14 +0200 Subject: [PATCH 3/3] added description of format_parser in README.md file --- README.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.md b/README.md index cee24a4..36734fc 100644 --- a/README.md +++ b/README.md @@ -105,6 +105,30 @@ The library currently provides three basic clients: - Read back the next result (a dictionary containing all the images) with `result = pcic.readNextFrame()` +### PCIC interface (ifmVisionAssistant) +Following structure in the process interface is required for multiple and +nested records: + +* supported interface structure + - records require a special delimiter at the end like seen in below + example. default delimiters `|` and `*` can be parsed without any + changes in library. + - multiple or nested records between two blob images + - multiple models and ROIs in application which yield cascaded model + records in result. + - blob image(s) at the start or between the result + + + star ; X Image ; models Normalized amplitude image ; stop + | | + ID ; ROIs; Value of SP1 ; Value of SP2 | <-- default delimiter for models + | | + ID ; Process value ; Status * <-- default delimiter for ROIs + +* restrictions + - single blob image at the end of multiple or nested records + - positional swap of delimiters `|` and `*` in process interface + Links ----- O3D3xx related libraries for other programming languages: