From 95851a55ac59de8a76b4c5d4ae67c092f2a6e688 Mon Sep 17 00:00:00 2001 From: Marc-Christian Schulze Date: Wed, 27 Nov 2024 17:16:58 +0100 Subject: [PATCH 1/7] Improved error messages and fixed plugin not to run if no src/main/java directory existed --- .../org/structs4java/AbstractCompileMojo.java | 12 ++--- .../structs4java/StructsBatchCompiler.java | 51 ++++++++++++++----- 2 files changed, 43 insertions(+), 20 deletions(-) diff --git a/structs4java-maven-plugin/src/main/java/org/structs4java/AbstractCompileMojo.java b/structs4java-maven-plugin/src/main/java/org/structs4java/AbstractCompileMojo.java index 8e9aec9..6a4f36f 100644 --- a/structs4java-maven-plugin/src/main/java/org/structs4java/AbstractCompileMojo.java +++ b/structs4java-maven-plugin/src/main/java/org/structs4java/AbstractCompileMojo.java @@ -30,6 +30,8 @@ import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Provider; +import org.eclipse.xtext.util.Strings; +import org.eclipse.xtext.validation.Issue; public abstract class AbstractCompileMojo extends AbstractMojo { @@ -79,7 +81,7 @@ public void execute() throws MojoExecutionException { } if (!getStructsDirectory().exists()) { - getLog().info("No struct files at " + getStructsDirectory()); + getLog().info("Structs source directory does not exist: " + getStructsDirectory()); return; } @@ -166,11 +168,6 @@ protected void compile(String classPath, List sourcePaths, String struct Log log = getLog(); compiler.setResourceSet(injector.getInstance(XtextResourceSet.class)); Iterable filtered = filter(sourcePaths, FILE_EXISTS); - if (Iterables.isEmpty(filtered)) { - String dir = Iterables.toString(sourcePaths); - log.info("skip compiling sources because the configured directory '" + dir + "' does not exists."); - return; - } log.debug("Set DeleteTempDirectory: " + false); compiler.setDeleteTempDirectory(false); log.debug("Set classpath: " + classPath); @@ -190,8 +187,7 @@ protected void compile(String classPath, List sourcePaths, String struct compiler.setWriteTraceFiles(writeTraceFiles); if (!compiler.compile()) { - String dir = concat(File.pathSeparator, newArrayList(filtered)); - throw new MojoExecutionException("Error compiling struct sources in '" + dir + "'."); + throw new MojoExecutionException("Compilation of structs sources failed."); } } diff --git a/structs4java-maven-plugin/src/main/java/org/structs4java/StructsBatchCompiler.java b/structs4java-maven-plugin/src/main/java/org/structs4java/StructsBatchCompiler.java index 3ec7d2a..1218ff1 100644 --- a/structs4java-maven-plugin/src/main/java/org/structs4java/StructsBatchCompiler.java +++ b/structs4java-maven-plugin/src/main/java/org/structs4java/StructsBatchCompiler.java @@ -280,13 +280,16 @@ public boolean apply(URI input) { }); for (String src : pathes.keySet()) { URI baseDir = URI.createFileURI(src + "/"); + String identifier = Joiner.on("_").join(baseDir.segments()); URI platformResourceURI = URI.createPlatformResourceURI(identifier + "/", true); + resourceSet.getURIConverter().getURIMap().put(platformResourceURI, baseDir); + for (URI uri : pathes.get(src)) { URI uriToUse = uri.replacePrefix(baseDir, platformResourceURI); - log.info("load structs file '" + uriToUse + "'"); - resourceSet.getResource(uriToUse, true); + log.info("loading structs file '" + uri.toFileString() + "'"); + resourceSet.getResource(uri, true); } } return resourceSet; @@ -430,21 +433,45 @@ protected void reportIssues(Iterable issues) { } } + private String getFilePath(URI uri) { + if(uri.isFile()) { + return uri.toFileString(); + } + + if(uri.trimFragment().isPlatform()) { + return uri.trimFragment().toPlatformString(true); + } + + return uri.trimFragment().toString(); + } + private StringBuilder createIssueMessage(Issue issue) { - StringBuilder issueBuilder = new StringBuilder("\n"); - issueBuilder.append(issue.getSeverity()).append(": \t"); - URI uriToProblem = issue.getUriToProblem(); - if (uriToProblem != null) { - URI resourceUri = uriToProblem.trimFragment(); - issueBuilder.append(resourceUri.lastSegment()).append(" - "); - if (resourceUri.isFile()) { - issueBuilder.append(resourceUri.toFileString()); - } + StringBuilder issueBuilder = new StringBuilder(); + + String filePath = getFilePath(issue.getUriToProblem()); + + issueBuilder.append(filePath).append(":").append(issue.getLineNumber()).append(" ").append(issue.getMessage()).append("\n"); + + String sourceLineWithIssue = readLineFromResource(filePath, issue.getLineNumber()); + issueBuilder.append(sourceLineWithIssue).append("\n"); + issueBuilder.append(com.google.common.base.Strings.repeat(" ", issue.getColumn() - 1)).append("^"); + int colLength = issue.getColumnEnd() - issue.getColumn(); + if(colLength > 1) { + issueBuilder.append(com.google.common.base.Strings.repeat("-", colLength - 1)); } - issueBuilder.append("\n").append(issue.getLineNumber()).append(": ").append(issue.getMessage()); return issueBuilder; } + private String readLineFromResource(String filePath, int line) { + JavaIoFileSystemAccess fileSystemAccess = javaIoFileSystemAccessProvider.get(); + // if you don't set the output path before using you end up with below exception + // compile failed: A slot with name 'DEFAULT_OUTPUT' has not been configured. + fileSystemAccess.setOutputPath(""); + String fileContent = fileSystemAccess.readTextFile(filePath).toString(); + List lines = Strings.split(fileContent, "\n"); + return lines.get(line - 1); + } + protected void generateJavaFiles(ResourceSet resourceSet) { JavaIoFileSystemAccess javaIoFileSystemAccess = javaIoFileSystemAccessProvider.get(); javaIoFileSystemAccess.setOutputPath(outputPath); From 46fe940a21b99507d0b54587727925361d9c9eae Mon Sep 17 00:00:00 2001 From: Marc-Christian Schulze Date: Wed, 27 Nov 2024 18:09:31 +0100 Subject: [PATCH 2/7] Fixed bug when sizeof() was used to define the length of an array of structures --- .../generator/StructGenerator.xtend | 18 +++++++++++++----- .../src/main/structs/SizeOfArray.structs | 11 +++++++++++ .../example/tests/RegressionTest.java | 18 ++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 structs4java-maven-plugin-test/src/main/structs/SizeOfArray.structs diff --git a/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend b/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend index 38ac3f9..2c0ada3 100644 --- a/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend +++ b/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend @@ -599,16 +599,23 @@ class StructGenerator { } def readerMethodForArrayMember(Member m) ''' - private static java.util.ArrayList<«m.nativeTypeName().native2JavaType().box()»> «m.readerMethodName()»(java.nio.ByteBuffer buf, boolean partialRead«IF dimensionOf(m) == 0», long countof«ENDIF») throws java.io.IOException { + private static java.util.ArrayList<«m.nativeTypeName().native2JavaType().box()»> «m.readerMethodName()»(java.nio.ByteBuffer buf, boolean partialRead«IF findMemberDefiningCountOf(m) != null», long countof«ENDIF») throws java.io.IOException { java.util.ArrayList<«m.nativeTypeName().native2JavaType().box()»> lst = new java.util.ArrayList<«m.nativeTypeName().native2JavaType().box()»>(); + try { «IF dimensionOf(m) == 0» - for(long i = 0; i < countof; ++i) { - lst.add(«readerMethodName(m)»«arrayPostfix(m)»(buf, partialRead)); - } + «IF findMemberDefiningCountOf(m) != null» + for(long i = 0; i < countof; ++i) { + lst.add(«readerMethodName(m)»«arrayPostfix(m)»(buf, partialRead)); + } + «ELSE» + while(buf.hasRemaining()) { + lst.add(«readerMethodName(m)»«arrayPostfix(m)»(buf, partialRead)); + } + «ENDIF» «ELSE» «FOR i : 0 ..< dimensionOf(m)» - lst.add(«readerMethodName(m)»«arrayPostfix(m)»(buf, partialRead)); + lst.add(«readerMethodName(m)»«arrayPostfix(m)»(buf, partialRead)); «ENDFOR» «ENDIF» } catch(java.nio.BufferUnderflowException e) { @@ -616,6 +623,7 @@ class StructGenerator { throw e; } } + return lst; } diff --git a/structs4java-maven-plugin-test/src/main/structs/SizeOfArray.structs b/structs4java-maven-plugin-test/src/main/structs/SizeOfArray.structs new file mode 100644 index 0000000..f100126 --- /dev/null +++ b/structs4java-maven-plugin-test/src/main/structs/SizeOfArray.structs @@ -0,0 +1,11 @@ +package org.structs4java.example.tests.sizeofarray; + +struct Outer { + uint8_t length sizeof(fields); + Inner fields[]; +} + +struct Inner { + uint8_t length sizeof(data); + uint8_t data[]; +} diff --git a/structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/RegressionTest.java b/structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/RegressionTest.java index e678cac..f96f289 100644 --- a/structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/RegressionTest.java +++ b/structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/RegressionTest.java @@ -36,6 +36,24 @@ public class RegressionTest extends AbstractTest { + public void testSizeOfArray() throws IOException { + byte[] testData = new byte[]{ 2, 1, 0, 1, 0}; + ByteBuffer buffer = ByteBuffer.wrap(testData); + buffer.order(ByteOrder.LITTLE_ENDIAN); + org.structs4java.example.tests.sizeofarray.Outer outer = org.structs4java.example.tests.sizeofarray.Outer.read(buffer); + + Assert.assertEquals(2, outer.getLength()); + Assert.assertEquals(2, outer.getFields().size()); + Assert.assertEquals(1, outer.getFields().get(0).getLength()); + Assert.assertEquals(2, outer.getFields().get(1).getLength()); + + ByteBuffer outBuffer = ByteBuffer.allocate(testData.length); + outBuffer.order(buffer.order()); + outer.write(outBuffer); + + assertEqualBuffers(buffer, outBuffer); + } + @Test public void testSimpleStructure() throws IOException { SimpleStructure expected = createSimpleStruct(); From 62a05a70e6d4d6bdd530d6af4afb6425c06528dc Mon Sep 17 00:00:00 2001 From: Marc-Christian Schulze Date: Thu, 28 Nov 2024 11:54:38 +0100 Subject: [PATCH 3/7] Fixed bug when a sizeof member was defining the size (in bytes) of an array of structs. After the read operation the buffer position was not correctly updated as it skipped the array-member's memory --- .../generator/StructGenerator.xtend | 1 + .../example/tests/RegressionTest.java | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend b/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend index 2c0ada3..652a0e9 100644 --- a/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend +++ b/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend @@ -375,6 +375,7 @@ class StructGenerator { slice.order(buf.order()); slice.limit((int)«tempVarForMember(findMemberDefiningSizeOf(m))»); obj.«setterName(m)»(«readerMethodName(m)»(slice, true)); + buf.position(buf.position() + (int)«tempVarForMember(findMemberDefiningSizeOf(m))»); } «ELSE» obj.«setterName(m)»(«readerMethodName(m)»(buf, partialRead, (int)«tempVarForMember(findMemberDefiningSizeOf(m))»)); diff --git a/structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/RegressionTest.java b/structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/RegressionTest.java index f96f289..ee2c16f 100644 --- a/structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/RegressionTest.java +++ b/structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/RegressionTest.java @@ -36,13 +36,14 @@ public class RegressionTest extends AbstractTest { + @Test public void testSizeOfArray() throws IOException { - byte[] testData = new byte[]{ 2, 1, 0, 1, 0}; + byte[] testData = new byte[]{ 5, 1, 0, 2, 0, 0}; ByteBuffer buffer = ByteBuffer.wrap(testData); buffer.order(ByteOrder.LITTLE_ENDIAN); org.structs4java.example.tests.sizeofarray.Outer outer = org.structs4java.example.tests.sizeofarray.Outer.read(buffer); - Assert.assertEquals(2, outer.getLength()); + Assert.assertEquals(5, outer.getLength()); Assert.assertEquals(2, outer.getFields().size()); Assert.assertEquals(1, outer.getFields().get(0).getLength()); Assert.assertEquals(2, outer.getFields().get(1).getLength()); @@ -54,6 +55,18 @@ public void testSizeOfArray() throws IOException { assertEqualBuffers(buffer, outBuffer); } + @Test + public void testSizeOfArrayWithAdditionalDataBehind() throws IOException { + byte[] testData = new byte[]{ 5, 1, 0, 2, 0, 0 /* last byte of outer struct */, 4}; + ByteBuffer buffer = ByteBuffer.wrap(testData); + buffer.order(ByteOrder.LITTLE_ENDIAN); + org.structs4java.example.tests.sizeofarray.Outer outer = org.structs4java.example.tests.sizeofarray.Outer.read(buffer); + // ensure that the next read operation on the buffer actually returns the data behind the already read structure + // this is especially important for the case the the last element using a sizeof member is of type uint8_t + // and therefore actually not read but mapped to another ByteBuffer + assertEquals(4, buffer.get()); + } + @Test public void testSimpleStructure() throws IOException { SimpleStructure expected = createSimpleStruct(); From 62e82a8a15d4e8ddf821e452d18746eed331c954 Mon Sep 17 00:00:00 2001 From: Marc-Christian Schulze Date: Thu, 28 Nov 2024 11:55:19 +0100 Subject: [PATCH 4/7] Added an example how to read structures --- examples/pom.xml | 12 ++ examples/zip-file-format/pom.xml | 42 +++++++ .../src/main/structs/zip.structs | 62 ++++++++++ .../java/examples/zip/ZipFileReadingTest.java | 116 ++++++++++++++++++ .../src/test/resources/example.zip | Bin 0 -> 544 bytes .../src/test/resources/zip-content/file1.txt | 1 + .../src/test/resources/zip-content/file2.txt | 1 + pom.xml | 1 + 8 files changed, 235 insertions(+) create mode 100644 examples/pom.xml create mode 100644 examples/zip-file-format/pom.xml create mode 100644 examples/zip-file-format/src/main/structs/zip.structs create mode 100644 examples/zip-file-format/src/test/java/examples/zip/ZipFileReadingTest.java create mode 100644 examples/zip-file-format/src/test/resources/example.zip create mode 100644 examples/zip-file-format/src/test/resources/zip-content/file1.txt create mode 100644 examples/zip-file-format/src/test/resources/zip-content/file2.txt diff --git a/examples/pom.xml b/examples/pom.xml new file mode 100644 index 0000000..cb5fb42 --- /dev/null +++ b/examples/pom.xml @@ -0,0 +1,12 @@ + + 4.0.0 + com.github.marc-christian-schulze.structs4java + examples + 1.0.54-SNAPSHOT + pom + + + zip-file-format + + diff --git a/examples/zip-file-format/pom.xml b/examples/zip-file-format/pom.xml new file mode 100644 index 0000000..5359fd6 --- /dev/null +++ b/examples/zip-file-format/pom.xml @@ -0,0 +1,42 @@ + + 4.0.0 + com.github.marc-christian-schulze.structs4java + examples-zip-file-format + 1.0.54-SNAPSHOT + + + UTF-8 + 17 + 17 + + + Example project showing how to define file structures using the example of ZIP files. + + + + + com.github.marc-christian-schulze.structs4java + structs4java-maven-plugin + 1.0.54-SNAPSHOT + + + compile-structs + + compile + + + + + + + + + + junit + junit + 4.13.2 + test + + + diff --git a/examples/zip-file-format/src/main/structs/zip.structs b/examples/zip-file-format/src/main/structs/zip.structs new file mode 100644 index 0000000..989d444 --- /dev/null +++ b/examples/zip-file-format/src/main/structs/zip.structs @@ -0,0 +1,62 @@ +package examples.zip; + +struct LocalFileHeader { + uint8_t signature[4]; + uint16_t version; + bitfield uint16_t { + uint16_t generalPurposeFlags : 16; + } + uint16_t compressionMethod; + uint16_t lastModificationTime; + uint16_t lastModificationDate; + uint32_t crc32OfUncompressedData; + uint32_t compressedSizeInBytes; + uint32_t uncompressedSizeInBytes; + uint16_t fileNameLengthInBytes sizeof(fileName); + uint16_t extraFieldLengthInBytes sizeof(extraFields); + char fileName[]; + ExtraFieldRecord extraFields[]; +} + +struct ExtraFieldRecord { + uint8_t signature[2]; + uint16_t length sizeof(data); + uint8_t data[]; +} + +struct CentralDirectoryFileHeader { + uint8_t signature[4]; + uint16_t versionMadeBy; + uint16_t versionNeededToExtract; + bitfield uint16_t { + uint16_t generalPurposeFlags : 16; + } + uint16_t compressionMethod; + uint16_t fileLastModificationTime; + uint16_t fileLastModificationDate; + uint32_t crc32OfUncompressedData; + uint32_t compressedSizeInBytes; + uint32_t uncompressedSizeInBytes; + uint16_t fileNameLengthInBytes sizeof(fileName); + uint16_t extraFieldLengthInBytes sizeof(extraFields); + uint16_t fileCommentLengthInBytes sizeof(fileComment); + uint16_t diskNumber; + uint16_t internalAttributes; + uint32_t externalAttributes; + uint32_t relativeOffsetOfLocalFileHeader; + char fileName[]; + uint8_t extraFields[]; + uint8_t fileComment[]; +} + +struct EndOfCentralDirectoryRecord { + uint8_t signature[4]; + uint16_t diskNumber; + uint16_t diskWhereCentralDirectoryStarts; + uint16_t numberOfCentralDirectoryRecordsOnThisDisk; + uint16_t totalNumberOfCentralDirectoryRecords; + uint32_t sizeOfCentralDirectoryInBytes; + uint32_t offsetOfStartOfCentralDirectory; + uint16_t commentLength sizeof(comment); + uint8_t comment[]; +} diff --git a/examples/zip-file-format/src/test/java/examples/zip/ZipFileReadingTest.java b/examples/zip-file-format/src/test/java/examples/zip/ZipFileReadingTest.java new file mode 100644 index 0000000..11acdb4 --- /dev/null +++ b/examples/zip-file-format/src/test/java/examples/zip/ZipFileReadingTest.java @@ -0,0 +1,116 @@ +package examples.zip; + +import org.junit.Test; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class ZipFileReadingTest { + + @Test + public void testUsingExampleZipFile() throws IOException { + try(RandomAccessFile raf = new RandomAccessFile("src/test/resources/example.zip", "r")) { + ByteBuffer buffer = raf.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, raf.length()); + + // ZIP file specification requires little endian encoding + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // first, we need to search backwards in the ZIP file for the end of central directory + assertTrue("No valid central directory found", seekToEndOfCentralDirectory(buffer)); + + // read from the buffer using the generated Java classes from the zip.structs file + EndOfCentralDirectoryRecord eocd = EndOfCentralDirectoryRecord.read(buffer); + + assertEquals("We expect 3 entries in the ZIP file", 3, eocd.getNumberOfCentralDirectoryRecordsOnThisDisk()); + assertEquals("We expect 3 entries in the ZIP file", 3, eocd.getTotalNumberOfCentralDirectoryRecords()); + + // seek to first entry of the central directory + buffer.position((int) eocd.getOffsetOfStartOfCentralDirectory()); + + // let's retrieve the file names from the directory + CentralDirectoryFileHeader cdfhDirectory = CentralDirectoryFileHeader.read(buffer); + assertEquals(cdfhDirectory.getFileName(), "zip-content/"); + + CentralDirectoryFileHeader cdfhFile2 = CentralDirectoryFileHeader.read(buffer); + assertEquals(cdfhFile2.getFileName(), "zip-content/file2.txt"); + + CentralDirectoryFileHeader cdfhFile1 = CentralDirectoryFileHeader.read(buffer); + assertEquals(cdfhFile1.getFileName(), "zip-content/file1.txt"); + + // finally, let's read the file contents + // first, we need to seek to the location of the local file header + buffer.position((int) cdfhFile1.getRelativeOffsetOfLocalFileHeader()); + // next, we read the local file header using our generated Java class + LocalFileHeader fhFile1 = LocalFileHeader.read(buffer); + // right after the header, the actual file content is placed + byte[] contentFile1 = new byte[(int) fhFile1.getCompressedSizeInBytes()]; + buffer.get(contentFile1); + assertEquals("content file 1\n", new String(contentFile1)); + + // same for file 2 + buffer.position((int) cdfhFile2.getRelativeOffsetOfLocalFileHeader()); + LocalFileHeader fhFile2 = LocalFileHeader.read(buffer); + byte[] contentFile2 = new byte[(int) fhFile2.getCompressedSizeInBytes()]; + buffer.get(contentFile2); + assertEquals("content file 2\n", new String(contentFile2)); + } + } + + private static boolean seekToEndOfCentralDirectory(ByteBuffer buffer) throws IOException { + int value; + + // we start scanning from the end of the file + buffer.position(buffer.limit()); + + // first search for 0x06 + value = readPreviousByteFromBuffer(buffer); + while(value != 0x06) { + if(buffer.position() < 4) { + return false; + } + value = readPreviousByteFromBuffer(buffer); + } + + // now we search for 0x05 + value = readPreviousByteFromBuffer(buffer); + while(value != 0x05) { + if(buffer.position() < 3) { + return false; + } + value = readPreviousByteFromBuffer(buffer); + } + + // now we search for 0x4b + value = readPreviousByteFromBuffer(buffer); + while(value != 0x4b) { + if(buffer.position() < 2) { + return false; + } + value = readPreviousByteFromBuffer(buffer); + } + + // finally we search for 0x50 + value = readPreviousByteFromBuffer(buffer); + while(value != 0x50) { + if(buffer.position() < 1) { + return false; + } + value = readPreviousByteFromBuffer(buffer); + } + + return true; + } + + private static int readPreviousByteFromBuffer(ByteBuffer buffer) throws IOException { + buffer.position(buffer.position() - 1); + int value = buffer.get() & 0xFF; + buffer.position(buffer.position() - 1); + return value; + } +} diff --git a/examples/zip-file-format/src/test/resources/example.zip b/examples/zip-file-format/src/test/resources/example.zip new file mode 100644 index 0000000000000000000000000000000000000000..170a66fa0515d59cc5b8262fd26b91a1e3401fbc GIT binary patch literal 544 zcmWIWW@h1H0D(%onn*AMO7JkqFjQq0=qBgqm89mC=!b@IGBC3Vd!%y!acKoN10%}| zW(Ec@5rATf-;wC4{6JGcSQO2aw9K4TBfXM}61Z{vn8rb^Q~)VfFyca(o(FVe@AR^L z_)Is1oBlz_Bb|qU>4rel8JXmmafN{dG&BSl{yKt~s3F1%2@#CYKs6CHJYXg=Fl=db tLo*R6bO;-b85qb$FTr9oG-!ZEgMtRb)vRnFA2I_W2O|Rm6A&{n006)xZteg8 literal 0 HcmV?d00001 diff --git a/examples/zip-file-format/src/test/resources/zip-content/file1.txt b/examples/zip-file-format/src/test/resources/zip-content/file1.txt new file mode 100644 index 0000000..1725187 --- /dev/null +++ b/examples/zip-file-format/src/test/resources/zip-content/file1.txt @@ -0,0 +1 @@ +content file 1 diff --git a/examples/zip-file-format/src/test/resources/zip-content/file2.txt b/examples/zip-file-format/src/test/resources/zip-content/file2.txt new file mode 100644 index 0000000..d5feedb --- /dev/null +++ b/examples/zip-file-format/src/test/resources/zip-content/file2.txt @@ -0,0 +1 @@ +content file 2 diff --git a/pom.xml b/pom.xml index 619477b..22130be 100644 --- a/pom.xml +++ b/pom.xml @@ -33,5 +33,6 @@ structs4java-core structs4java-maven-plugin structs4java-maven-plugin-test + examples From 3b5c94b395ae4a68a5a22dfacbbd1fd2f0ffba23 Mon Sep 17 00:00:00 2001 From: Marc-Christian Schulze Date: Thu, 28 Nov 2024 14:29:43 +0100 Subject: [PATCH 5/7] Improved null-safety --- .../generator/StructGenerator.xtend | 133 +++++++++++++++--- .../src/main/structs/null-safety.structs | 13 ++ .../example/tests/NullSafetyTest.java | 55 ++++++++ 3 files changed, 182 insertions(+), 19 deletions(-) create mode 100644 structs4java-maven-plugin-test/src/main/structs/null-safety.structs create mode 100644 structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/NullSafetyTest.java diff --git a/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend b/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend index 652a0e9..0d1fa51 100644 --- a/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend +++ b/org.structs4java.parent/org.structs4java/src/org/structs4java/generator/StructGenerator.xtend @@ -303,13 +303,21 @@ class StructGenerator { return false; } - def isBooleanType(BitfieldEntry m) { - if(m.typename == "boolean") { - return true; - } - - return false; - } + def isBooleanType(Member m) { + if(nativeTypeName(m) == "boolean") { + return true; + } + + return false; + } + + def isBooleanType(BitfieldEntry m) { + if(m.typename == "boolean") { + return true; + } + + return false; + } def isEnumType(BitfieldEntry m) { if(m.type !== null) { @@ -982,13 +990,25 @@ class StructGenerator { private void «m.writerMethodName()»(java.nio.ByteBuffer buf) throws java.io.IOException { java.util.ArrayList<«m.nativeTypeName().native2JavaType().box()»> lst = «getterName(m)»(); «IF dimensionOf(m) == 0» + if(lst == null) { + return; + } for(«m.nativeTypeName().native2JavaType().box()» item : lst) { «writerMethodName(m)»«arrayPostfix(m)»(buf, item); } «ELSE» - «FOR i : 0 ..< dimensionOf(m)» - «writerMethodName(m)»«arrayPostfix(m)»(buf, lst.get(«i»)); - «ENDFOR» + if(lst.size() > «dimensionOf(m)») { + throw new java.io.IOException("Field '«attributeName(m)»' contains " + lst.size() + " element which can't be serialized into structure with limit of «dimensionOf(m)» elements!"); + } + + for(int i = 0; i < lst.size(); ++i) { + «writerMethodName(m)»«arrayPostfix(m)»(buf, lst.get(i)); + } + + // if there are less elements than expected, we fill with default constructed + for(int i = lst.size(); i < «dimensionOf(m)»; ++i) { + «writerMethodName(m)»«arrayPostfix(m)»(buf, «defaultConstructArrayItem(m)»); + } «ENDIF» } @@ -997,12 +1017,19 @@ class StructGenerator { def writerMethodForByteBuffer(IntegerMember m) ''' private void «m.writerMethodName()»(java.nio.ByteBuffer buf) throws java.io.IOException { + java.nio.ByteBuffer buffer = «getterName(m)»(); + + // null buffers serialize like empty buffers + if(buffer == null) { + buffer = java.nio.ByteBuffer.wrap(new byte[]{}); + } + // reset position in case someone read this buffer before - «getterName(m)»().position(0); + buffer.position(0); «IF dimensionOf(m) > 0» // we need to slice the buffer in order to limit its written size - java.nio.ByteBuffer slicedBuffer = «getterName(m)»().slice(); + java.nio.ByteBuffer slicedBuffer = buffer.slice(); if(slicedBuffer.limit() > «dimensionOf(m)») { slicedBuffer.limit(«dimensionOf(m)»); } @@ -1015,11 +1042,11 @@ class StructGenerator { } «ELSE» // buffer has unbound / dynamic size - buf.put(«getterName(m)»()); + buf.put(buffer); «ENDIF» «IF m.isPadded()» - int bytesOverlap = («getterName(m)»().limit() % «m.padding»); + int bytesOverlap = (buffer.limit() % «m.padding»); if(bytesOverlap > 0) { for(int i = 0; i < «m.padding» - bytesOverlap; ++i) { buf.put((byte)«m.getUsing()»); @@ -1413,11 +1440,7 @@ class StructGenerator { def field(Member m) ''' «printComments(m)» - «IF m instanceof StringMember» - private «attributeJavaType(m)» «attributeName(m)» = ""; - «ELSE» - private «attributeJavaType(m)» «attributeName(m)»; - «ENDIF» + private «attributeJavaType(m)» «attributeName(m)» = «defaultConstruct(m)»; ''' def field(BitfieldMember m) ''' @@ -1478,6 +1501,61 @@ class StructGenerator { «ENDFOR» ''' + def defaultConstructArrayItem(Member m) { + if(!m.isArray()) { + throw new RuntimeException("compiler-error: non-array member passed to defaultConstructArrayItem()") + } + + switch (m) { + ComplexTypeMember: { + if(m.type instanceof EnumDeclaration) { + return "null" + } + "new " + javaType(m.type) + "()" + } + IntegerMember: "0" + FloatMember: "0.0f" + default: throw new RuntimeException("Unsupported member type: " + m) + } + } + + def defaultConstruct(Member m) { + if(m.isArray()) { + + if(doesAttributeJavaTypeMapToByteBuffer(m)) { + return "java.nio.ByteBuffer.wrap(new byte[]{})" + } + + if(m instanceof StringMember) { + return "\"\"" + } + + val nativeType = nativeTypeName(m) + val javaType = native2JavaType(nativeType) + return "new java.util.ArrayList<" + box(javaType) + ">()" + } + + if(m instanceof IntegerMember) { + return "0" + } + + if(m instanceof FloatMember) { + return "0.0f" + } + + if(isBooleanType(m)) { + return "false" + } + + if(m instanceof ComplexTypeMember) { + if(m.type instanceof EnumDeclaration) { + return "null" + } + } + + return "new " + attributeJavaType(m) + "()" + } + def attributeJavaType(Member m) { val nativeType = nativeTypeName(m) val javaType = native2JavaType(nativeType) @@ -1496,6 +1574,23 @@ class StructGenerator { return javaType } } + + def doesAttributeJavaTypeMapToByteBuffer(Member m) { + val nativeType = nativeTypeName(m) + val javaType = native2JavaType(nativeType) + + if (!isArray(m)) { + return false + } + + if(m instanceof IntegerMember) { + if(m.typename.equals("uint8_t") || m.typename.equals("int8_t")) { + return true + } + } + + return false + } def attributeJavaType(BitfieldEntry m) { return native2JavaType(nativeTypeName(m)) diff --git a/structs4java-maven-plugin-test/src/main/structs/null-safety.structs b/structs4java-maven-plugin-test/src/main/structs/null-safety.structs new file mode 100644 index 0000000..116f36b --- /dev/null +++ b/structs4java-maven-plugin-test/src/main/structs/null-safety.structs @@ -0,0 +1,13 @@ +package org.structs4java.example.tests.nullsafety; + +struct StructWithByteBuffer { + uint8_t buffer[10]; +} + +struct StructWithInt32Array { + int32_t array[10]; +} + +struct StructWithStructArray { + StructWithInt32Array array[10]; +} \ No newline at end of file diff --git a/structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/NullSafetyTest.java b/structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/NullSafetyTest.java new file mode 100644 index 0000000..4e60fb5 --- /dev/null +++ b/structs4java-maven-plugin-test/src/test/java/org/structs4java/example/tests/NullSafetyTest.java @@ -0,0 +1,55 @@ +package org.structs4java.example.tests; + +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.structs4java.example.tests.nullsafety.StructWithByteBuffer; +import org.structs4java.example.tests.nullsafety.StructWithInt32Array; +import org.structs4java.example.tests.nullsafety.StructWithStructArray; + +public class NullSafetyTest { + + @Test + public void testWritingDefaultStructWithByteBuffer() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(10); + StructWithByteBuffer struct = new StructWithByteBuffer(); + struct.write(buffer); + } + + @Test + public void testWritingDefaultStructWithInt32Array() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(40); + StructWithInt32Array struct = new StructWithInt32Array(); + struct.write(buffer); + } + + @Test + public void testWritingDefaultStructWithStructArray() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(400); + StructWithStructArray struct = new StructWithStructArray(); + struct.write(buffer); + } + + @Test + public void testDefaultStructWithInt32ArrayReturnsNotNull() { + StructWithInt32Array struct = new StructWithInt32Array(); + assertNotNull(struct.getArray()); + } + + @Test + public void testDefaultStructWithStructArrayReturnsNotNull() { + StructWithStructArray struct = new StructWithStructArray(); + assertNotNull(struct.getArray()); + } + + @Test + public void testDefaultStructWithByteBufferReturnsNotNull() { + StructWithByteBuffer struct = new StructWithByteBuffer(); + assertNotNull(struct.getBuffer()); + } + +} From 7d52f891649c6d98b62e116780c8c6b3b43f60d6 Mon Sep 17 00:00:00 2001 From: Marc-Christian Schulze Date: Thu, 28 Nov 2024 14:30:03 +0100 Subject: [PATCH 6/7] Added example how to write structures --- .../java/examples/zip/ZipFileWritingTest.java | 68 +++++++++++++++++++ .../src/test/resources/README.md | 4 ++ 2 files changed, 72 insertions(+) create mode 100644 examples/zip-file-format/src/test/java/examples/zip/ZipFileWritingTest.java create mode 100644 examples/zip-file-format/src/test/resources/README.md diff --git a/examples/zip-file-format/src/test/java/examples/zip/ZipFileWritingTest.java b/examples/zip-file-format/src/test/java/examples/zip/ZipFileWritingTest.java new file mode 100644 index 0000000..af694dd --- /dev/null +++ b/examples/zip-file-format/src/test/java/examples/zip/ZipFileWritingTest.java @@ -0,0 +1,68 @@ +package examples.zip; + +import org.junit.Test; +import static org.junit.Assert.assertEquals; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.zip.CRC32; +import java.util.zip.Checksum; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class ZipFileWritingTest { + + @Test + public void testWritingZipFile() throws IOException { + ByteBuffer buffer = ByteBuffer.allocate(4096); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + // our file content to store + byte[] fileData = "example file content".getBytes("UTF-8"); + + // compute the CRC32 checksum of the file content + Checksum checksum = new CRC32(); + checksum.update(fileData, 0, fileData.length); + long crc32 = checksum.getValue(); + + // first we create and write the local file header + long positionOfLocalFileHeader = buffer.position(); + LocalFileHeader fhFile = new LocalFileHeader(); + fhFile.setSignature(ByteBuffer.wrap(new byte[]{ 0x50, 0x4b, 0x03, 0x04 })); + fhFile.setFileName("example.txt"); + fhFile.setCompressedSizeInBytes(fileData.length); + fhFile.setUncompressedSizeInBytes(fileData.length); + fhFile.setCrc32OfUncompressedData(crc32); + fhFile.write(buffer); + // right after the local file header, the actual file content needs to be stored + buffer.put(fileData); + + // now we need to create the central directory header + long positionOfFirstCentralDirectoryHeader = buffer.position(); + CentralDirectoryFileHeader cdfh = new CentralDirectoryFileHeader(); + cdfh.setSignature(ByteBuffer.wrap(new byte[]{ 0x50, 0x4b, 0x01, 0x02 })); + cdfh.setFileName("example.txt"); + cdfh.setCrc32OfUncompressedData(crc32); + cdfh.setCompressedSizeInBytes(fileData.length); + cdfh.setUncompressedSizeInBytes(fileData.length); + cdfh.setRelativeOffsetOfLocalFileHeader(positionOfLocalFileHeader); + cdfh.write(buffer); + + // finally, we need to write the closing end of central directory header + EndOfCentralDirectoryRecord eocdr = new EndOfCentralDirectoryRecord(); + eocdr.setSignature(ByteBuffer.wrap(new byte[]{ 0x50, 0x4b, 0x05, 0x06 })); + eocdr.setOffsetOfStartOfCentralDirectory(positionOfFirstCentralDirectoryHeader); + eocdr.setNumberOfCentralDirectoryRecordsOnThisDisk(1); + eocdr.setTotalNumberOfCentralDirectoryRecords(1); + eocdr.write(buffer); + + // now we read the ZIP file using Java on-board classes and verify + ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(buffer.array())); + ZipEntry fileEntry = zis.getNextEntry(); + assertEquals("example.txt", fileEntry.getName()); + byte[] fileContentRead = zis.readAllBytes(); + assertEquals("example file content", new String(fileContentRead, "UTF-8")); + } +} diff --git a/examples/zip-file-format/src/test/resources/README.md b/examples/zip-file-format/src/test/resources/README.md new file mode 100644 index 0000000..a5cff9a --- /dev/null +++ b/examples/zip-file-format/src/test/resources/README.md @@ -0,0 +1,4 @@ +The `example.zip` file was created without compression using +``` +zip -0 -r example.zip zip-content/ +``` \ No newline at end of file From 1fe0f7e81bd172273245aa7de1ecf7dca9c0a60d Mon Sep 17 00:00:00 2001 From: Marc-Christian Schulze Date: Thu, 28 Nov 2024 14:37:44 +0100 Subject: [PATCH 7/7] Link examples on landing page --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index f5c30c9..f574d8d 100644 --- a/README.md +++ b/README.md @@ -425,6 +425,11 @@ Below is a full example configuration, including the default values, of the plug ``` +# Examples + +* [examples/zip-file-format - ZipFileReadingTest](blob/master/examples/zip-file-format/src/test/java/examples/zip/ZipFileReadingTest.java) - Shows how to read a ZIP file using structs4java +* [examples/zip-file-format - ZipFileWritingTest](blob/master/examples/zip-file-format/src/test/java/examples/zip/ZipFileWritingTest.java) - Shows how to write a ZIP files using structs4java + # Comparison to Javolution If you do not want to rely on code generation you should have a look at [Javolution](http://javolution.org/) which is a plain Java implementation.