WIP: improve binary ftrace parsing speed (~10x)

* mmap the whole file for faster parsing * re-use the same mapping to read events on a laptop with ubuntu 24.04 the file is parsed an order of magnitude faster on a windows laptop I measured two orders of magnitude faster Signed-off-by: Fabrizio Iannetti <[email protected]>
eclipse-tracecompass-incubator · Jun 4, 2024 · b4230f4 · b4230f4
1 parent f230c18
commit b4230f4
Show file tree

Hide file tree

Showing 12 changed files with 313 additions and 321 deletions.
diff --git a/...se/tracecompass/incubator/ftrace/core/tests/binary/buffer/BinaryFTraceByteBufferTest.java b/...se/tracecompass/incubator/ftrace/core/tests/binary/buffer/BinaryFTraceByteBufferTest.java
@@ -45,38 +45,37 @@ public void testOffsetCounter() throws IOException {
         String traceLocation = FTraceUtils.getTraceAbsolutePath(FtraceTestTrace.TEST_2_6_MULTIPLE_CPUS);
         long currentOffset = 0;
 
-        try (BinaryFTraceByteBuffer buffer = new BinaryFTraceByteBuffer(traceLocation)) {
-            // Read the magic values
-            buffer.getNextBytes(BinaryFTraceHeaderElementSize.MAGIC_VALUE);
-            currentOffset += BinaryFTraceHeaderElementSize.MAGIC_VALUE;
-            assertEquals(buffer.getCurrentOffset(), currentOffset);
+        BinaryFTraceByteBuffer buffer = new BinaryFTraceByteBuffer(traceLocation);
+        // Read the magic values
+        buffer.getNextBytes(BinaryFTraceHeaderElementSize.MAGIC_VALUE);
+        currentOffset += BinaryFTraceHeaderElementSize.MAGIC_VALUE;
+        assertEquals(buffer.getCurrentOffset(), currentOffset);
 
-            String tracingString = buffer.getNextBytesAsString(BinaryFTraceHeaderElementSize.TRACING_STRING);
-            currentOffset += BinaryFTraceHeaderElementSize.TRACING_STRING;
-            assertEquals(tracingString, TRACING);
-            assertEquals(buffer.getCurrentOffset(), currentOffset);
+        String tracingString = buffer.getNextBytesAsString(BinaryFTraceHeaderElementSize.TRACING_STRING);
+        currentOffset += BinaryFTraceHeaderElementSize.TRACING_STRING;
+        assertEquals(tracingString, TRACING);
+        assertEquals(buffer.getCurrentOffset(), currentOffset);
 
-            String versionString = buffer.getNextString();
-            currentOffset += (BinaryFTraceHeaderElementSize.FTRACE_VERSION + BinaryFTraceHeaderElementSize.STRING_TERMINATOR);
-            assertEquals(versionString, VERSION);
-            assertEquals(buffer.getCurrentOffset(), currentOffset);
+        String versionString = buffer.getNextString();
+        currentOffset += (BinaryFTraceHeaderElementSize.FTRACE_VERSION + BinaryFTraceHeaderElementSize.STRING_TERMINATOR);
+        assertEquals(versionString, VERSION);
+        assertEquals(buffer.getCurrentOffset(), currentOffset);
 
-            // Read various data types and validate offset changes
-            buffer.getNextShort();
-            currentOffset += BinaryFTraceDataType.SHORT.getSize();
-            assertEquals(buffer.getCurrentOffset(), currentOffset);
+        // Read various data types and validate offset changes
+        buffer.getNextShort();
+        currentOffset += BinaryFTraceDataType.SHORT.getSize();
+        assertEquals(buffer.getCurrentOffset(), currentOffset);
 
-            buffer.getNextInt();
-            currentOffset += BinaryFTraceDataType.INT.getSize();
-            assertEquals(buffer.getCurrentOffset(), currentOffset);
+        buffer.getNextInt();
+        currentOffset += BinaryFTraceDataType.INT.getSize();
+        assertEquals(buffer.getCurrentOffset(), currentOffset);
 
-            buffer.getNextLong();
-            currentOffset += BinaryFTraceDataType.LONG.getSize();
-            assertEquals(buffer.getCurrentOffset(), currentOffset);
+        buffer.getNextLong();
+        currentOffset += BinaryFTraceDataType.LONG.getSize();
+        assertEquals(buffer.getCurrentOffset(), currentOffset);
 
-            buffer.movePointerToOffset(RANDOM_OFFSET);
-            currentOffset = RANDOM_OFFSET;
-            assertEquals(buffer.getCurrentOffset(), currentOffset);
-        }
+        buffer.movePointerToOffset(RANDOM_OFFSET);
+        currentOffset = RANDOM_OFFSET;
+        assertEquals(buffer.getCurrentOffset(), currentOffset);
     }
 }
diff --git a/...pse/tracecompass/incubator/internal/ftrace/core/binary/header/BinaryFTraceHeaderInfo.java b/...pse/tracecompass/incubator/internal/ftrace/core/binary/header/BinaryFTraceHeaderInfo.java
@@ -17,6 +17,9 @@
 import java.util.Map;
 import java.util.Optional;
 
+import org.eclipse.tracecompass.incubator.internal.ftrace.core.binary.parser.BinaryFTraceByteBuffer;
+import org.eclipse.tracecompass.incubator.internal.ftrace.core.binary.parser.BinaryFTraceFileMapping;
+
 /**
  * A representation of all header information required for parsing the events in
  * the FTrace file.
@@ -46,12 +49,15 @@ public class BinaryFTraceHeaderInfo {
 
     private List<BinaryFTraceFileCPU> cpus;
 
+    private BinaryFTraceFileMapping fTraceMapping;
+
     /**
      * Constructor
      *
      * @param builder The builder used to construct the trace header
+     * @param traceMapping the file memory mapping used to read data from the file
      */
-    private BinaryFTraceHeaderInfo(BinaryFTraceHeaderInfoBuilder builder) {
+    private BinaryFTraceHeaderInfo(BinaryFTraceHeaderInfoBuilder builder, BinaryFTraceFileMapping traceMapping) {
         fFilePath = builder.fBuilderFilePath;
         fVersion = builder.fBuilderVersion;
         fEndianess = builder.fBuilderEndianess; // Because Java by default is
@@ -72,25 +78,26 @@ private BinaryFTraceHeaderInfo(BinaryFTraceHeaderInfoBuilder builder) {
         fOptions = builder.fBuilderOptions;
         fEventCommonFields = builder.fBuilderEventCommonFields;
         cpus = builder.fBuilderCpus;
+        fTraceMapping = traceMapping;
+        fTraceMapping.order(fEndianess);
     }
 
     /**
-     * Get the file path to the trace file.
+     * Get the trace contents as a ByteBuffer.
      *
-     * @return The file path of the trace file.
+     * @return the trace contents as a ByteBuffer
      */
-    public String getFilePath() {
-        return fFilePath;
+    public BinaryFTraceByteBuffer getMappedBuffer() {
+        return new BinaryFTraceByteBuffer(fTraceMapping);
     }
 
     /**
-     * Set the file path to the trace file.
+     * Get the file path to the trace file.
      *
-     * @param filePath
-     *            The file path of the trace file.
+     * @return The file path of the trace file.
      */
-    public void setFilePath(String filePath) {
-        this.fFilePath = filePath;
+    public String getFilePath() {
+        return fFilePath;
     }
 
     /**
@@ -612,11 +619,12 @@ public BinaryFTraceHeaderInfoBuilder cpus(List<BinaryFTraceFileCPU> builderCpus)
          * Build an immutable {@link BinaryFTraceHeaderInfo} using the current
          * state of the builder.
          *
+         * @param traceMapping the memory mapped file buffer to be used for reading data
          * @return A {@link BinaryFTraceHeaderInfo} object.
          */
-        public BinaryFTraceHeaderInfo build() {
+        public BinaryFTraceHeaderInfo build(BinaryFTraceFileMapping traceMapping) {
             fBuilderEventCommonFields = getCommonFields();
-            return new BinaryFTraceHeaderInfo(this);
+            return new BinaryFTraceHeaderInfo(this, traceMapping);
         }
 
         private Map<String, BinaryFTraceFormatField> getCommonFields() {

diff --git a/...cecompass/incubator/internal/ftrace/core/binary/iterator/BinaryFTraceCPUPageIterator.java b/...cecompass/incubator/internal/ftrace/core/binary/iterator/BinaryFTraceCPUPageIterator.java
@@ -64,10 +64,8 @@ public class BinaryFTraceCPUPageIterator implements Closeable {
      *            The CPU data page to read
      * @param fileHeader
      *            The FTrace header information object
-     * @throws IOException
-     *             When there is an error without opening/reading from the file
      */
-    public BinaryFTraceCPUPageIterator(@NonNull BinaryFTraceCPUDataPage page, @NonNull BinaryFTraceHeaderInfo fileHeader) throws IOException {
+    public BinaryFTraceCPUPageIterator(@NonNull BinaryFTraceCPUDataPage page, @NonNull BinaryFTraceHeaderInfo fileHeader) {
         // Initialize buffer data
         fFileHeader = fileHeader;
         this.fPage = page;
@@ -103,11 +101,7 @@ public long getCurrentTimeStamp() {
      */
     @Override
     public void close() throws IOException {
-        BinaryFTraceByteBuffer buffer = fBuffer;
-        if (buffer != null) {
-            buffer.close();
-            fBuffer = null;
-        }
+        fBuffer = null;
     }
 
     /**
@@ -152,10 +146,8 @@ public BinaryFTraceResponse readNextEvent() {
     /**
      * Read the time stamp of the next event only (without reading the actual
      * event)
-     *
-     * @throws IOException
      */
-    private void readNextEventTimeStamp() throws IOException {
+    private void readNextEventTimeStamp() {
         boolean readSuccess = readNextEventHeader();
 
         /**
@@ -205,9 +197,8 @@ private void readNextEventTimeStamp() throws IOException {
      * increment the offset
      *
      * @return true if the event is read sucessfully
-     * @throws IOException
      */
-    private boolean readNextEventHeader() throws IOException {
+    private boolean readNextEventHeader() {
         if (!hasNext()) {
             return false;
         }
@@ -232,11 +223,8 @@ private boolean readNextEventHeader() throws IOException {
      *
      * @return -1 if the there is no next event, else return the event type
      *         length.
-     * @throws IOException
-     *             if there is an error reading the next event, likely because
-     *             of buffer overflow caused a logical error during reading.
      */
-    private int peekNextEventType() throws IOException {
+    private int peekNextEventType() {
         if (!hasNext()) {
             return -1;
         }
@@ -254,13 +242,8 @@ private int peekNextEventType() throws IOException {
     /**
      * Update the time stamp based on the state of the current event header and
      * increment the offset if needed
-     *
-     * @throws IOException
-     *             IOException if there is an error reading the next event,
-     *             likely because of buffer overflow caused a logical error
-     *             during reading.
      */
-    private void updateTimeStamp() throws IOException {
+    private void updateTimeStamp() {
         if (fCurrentTypeLen <= fFileHeader.getHeaderEventInfo().getDataMaxTypeLen()) {
             fCurrentTimeStamp += fCurrentTimeDelta;
         } else {
@@ -289,11 +272,8 @@ private void skip(int bytesToSkip) {
      * Get the pay load size of the event according to its type length
      *
      * @return The size of the payload of the event
-     * @throws IOException
-     *             if there is an error reading the next event, likely because
-     *             of buffer overflow caused a logical error during reading.
      */
-    private int getCurrentEventPayloadSize() throws IOException {
+    private int getCurrentEventPayloadSize() {
 
         int payloadSize = 0;
         if (fCurrentTypeLen == fFileHeader.getHeaderEventInfo().getCustomLengthEventTypeLen()) {
@@ -326,39 +306,34 @@ public BinaryFTraceCPUDataPage getPage() {
      * Lazily read the next event using the event definition information.
      *
      * @return A BinaryFTraceEvent at the current event definition.
-     * @throws IOException
-     *             if there is an error reading the next event, likely because
-     *             of buffer overflow caused a logical error during reading.
      */
-    public @Nullable BinaryFTraceEvent getCurrentEvent() throws IOException {
+    public @Nullable BinaryFTraceEvent getCurrentEvent() {
         // Make a local copy to prevent multithreading null check
         BinaryFTraceEventDefinition eventDef = fEventDef;
 
         if (eventDef != null) {
             // We use a new buffer to read the current event
-            try (BinaryFTraceByteBuffer tempBuffer = new BinaryFTraceByteBuffer(fFileHeader.getFilePath());) {
-                tempBuffer.setByteOrder(fFileHeader.getEndianess());
-                tempBuffer.movePointerToOffset(eventDef.getPayloadOffset());
-
-                byte[] data = tempBuffer.getNextBytes(eventDef.getPayloadSize());
-                BinaryFTraceEventFormat eventFormat = fDataParser.getEventFormat(data);
-                if (eventFormat == null) {
-                    return null;
-                }
+            BinaryFTraceByteBuffer tempBuffer = fFileHeader.getMappedBuffer();
+            tempBuffer.movePointerToOffset(eventDef.getPayloadOffset());
 
-                Map<String, Object> properties;
-                if (eventDef.getPayloadSize() > 0) {
-                    properties = fDataParser.parseEventData(eventFormat, data);
-                } else {
-                    properties = new HashMap<>();
-                }
+            byte[] data = tempBuffer.getNextBytes(eventDef.getPayloadSize());
+            BinaryFTraceEventFormat eventFormat = fDataParser.getEventFormat(data);
+            if (eventFormat == null) {
+                return null;
+            }
 
-                BinaryFTraceEvent event = new BinaryFTraceEvent(fCurrentTimeStamp,
-                        properties,
-                        eventFormat.getEventName(),
-                        fPage.getCpu());
-                return event;
+            Map<String, Object> properties;
+            if (eventDef.getPayloadSize() > 0) {
+                properties = fDataParser.parseEventData(eventFormat, data);
+            } else {
+                properties = new HashMap<>();
             }
+
+            BinaryFTraceEvent event = new BinaryFTraceEvent(fCurrentTimeStamp,
+                    properties,
+                    eventFormat.getEventName(),
+                    fPage.getCpu());
+            return event;
         }
 
         return null;
@@ -370,11 +345,8 @@ public BinaryFTraceCPUDataPage getPage() {
      * @param timeStampToSeekTo
      *            The timestamp to seek to
      * @return true if the seek was successful
-     * @throws IOException
-     *             if there is an error reading the next event, likely because
-     *             that the iterator has reached the end.
      */
-    public boolean seek(long timeStampToSeekTo) throws IOException {
+    public boolean seek(long timeStampToSeekTo) {
         // Reset the iterator
         initializeIterator();
 
@@ -403,12 +375,8 @@ public boolean seek(long timeStampToSeekTo) throws IOException {
      * Check if there are more stuff to read in this iterator
      *
      * @return true if there are more events to read in this page
-     *
-     * @throws IOException
-     *             if there is an error reading the next event, likely because
-     *             that the iterator has reached the end.
      */
-    public boolean hasNext() throws IOException {
+    public boolean hasNext() {
         boolean ret = true;
 
         BinaryFTraceByteBuffer buffer = fBuffer;
@@ -454,7 +422,7 @@ public boolean hasNext() throws IOException {
         return ret;
     }
 
-    private void initializeIterator() throws IOException {
+    private void initializeIterator() {
         fCurrentOffset = fPage.getDataStartingOffset();
         fCurrentTimeStamp = fPage.getTimeStamp();
         fCurrentTypeLen = -1;
@@ -466,8 +434,7 @@ private void initializeIterator() throws IOException {
             fBuffer.movePointerToOffset(fPage.getDataStartingOffset());
         } else {
             // Prepare the buffer
-            BinaryFTraceByteBuffer newBuffer = new BinaryFTraceByteBuffer(fFileHeader.getFilePath());
-            newBuffer.setByteOrder(fFileHeader.getEndianess());
+            BinaryFTraceByteBuffer newBuffer = fFileHeader.getMappedBuffer();
             newBuffer.movePointerToOffset(fPage.getDataStartingOffset());
 
             fBuffer = newBuffer;

diff --git a/...ompass/incubator/internal/ftrace/core/binary/iterator/BinaryFTraceCPUSectionIterator.java b/...ompass/incubator/internal/ftrace/core/binary/iterator/BinaryFTraceCPUSectionIterator.java
@@ -140,11 +140,8 @@ public void close() throws IOException {
      * Get the current event of this iterator lazily.
      *
      * @return The current event as a BinaryFTrace event.
-     * @throws IOException
-     *             if there is an error reading the event, likely because of
-     *             byte buffer failure.
      */
-    public @Nullable BinaryFTraceEvent getCurrentEvent() throws IOException {
+    public @Nullable BinaryFTraceEvent getCurrentEvent() {
         BinaryFTraceCPUPageIterator iter = fCurrPageIterator;
         if (iter != null) {
             return iter.getCurrentEvent();