File backed hybrid heap implementation (#101)

* File backed hybrid heap implementation

* Fix mem leak on system hybrid heap test as the heap release doesn't free the allocations. Fixing Windows build.

* Fixing test leak with system heap in hybrid heap test. This was not caught locally by the tools (odd)
Fixing Windows build issue with missing header
Fixing some warnings with unreferenced params that were only triggered by Windows VC compiler
Fixing some warnings with type cast loss of precision warnings triggered by Windows VC compiler

src/client/src/Client.c

@@ -187,6 +187,7 @@ STATUS createKinesisVideoClient(PDeviceInfo pDeviceInfo, PClientCallbacks pClien
     CHK_STATUS(heapInitialize(pKinesisVideoClient->deviceInfo.storageInfo.storageSize,
                               pKinesisVideoClient->deviceInfo.storageInfo.spillRatio,
                               heapFlags,
+                              pKinesisVideoClient->deviceInfo.storageInfo.rootDirectory,
                               &pKinesisVideoClient->pHeap));
 
     // Using content store allocator if needed

src/client/src/Stream.c

@@ -742,7 +742,7 @@ STATUS putFrame(PKinesisVideoStream pKinesisVideoStream, PFrame pFrame)
     UINT64 remainingSize = 0, remainingDuration = 0, thresholdPercent = 0, duration = 0, viewByteSize = 0, allocSize = 0;
     PBYTE pAlloc = NULL;
     UINT32 trackIndex, packagedSize = 0, packagedMetadataSize = 0, overallSize = 0,
-        itemFlags = ITEM_FLAG_NONE, clusterOverhead;
+        itemFlags = ITEM_FLAG_NONE;
     BOOL streamLocked = FALSE, clientLocked = FALSE, freeOnError = TRUE;
     EncodedFrameInfo encodedFrameInfo;
     MKV_STREAM_STATE generatorState = MKV_STATE_START_BLOCK;
@@ -2061,7 +2061,7 @@ STATUS resetCurrentViewItemStreamStart(PKinesisVideoStream pKinesisVideoStream)
     STATUS retStatus = STATUS_SUCCESS;
     PViewItem pViewItem = NULL;
     BOOL streamLocked = FALSE, clientLocked = FALSE;
-    UINT32 clusterHeaderSize, packagedSize, overallSize, dataOffset;
+    UINT32 packagedSize, overallSize, dataOffset;
     UINT64 allocSize;
     PBYTE pFrame = NULL;
     PKinesisVideoClient pKinesisVideoClient = NULL;
@@ -2329,7 +2329,7 @@ VOID deleteStreamUploadInfo(PKinesisVideoStream pKinesisVideoStream, PUploadHand
 
         if (currentTime >= pUploadHandleInfo->createTime) {
             UINT64 uptime = currentTime - pUploadHandleInfo->createTime;
-            pKinesisVideoStream->diagnostics.avgSessionDuration = EMA_ACCUMULATOR_GET_NEXT(pKinesisVideoStream->diagnostics.avgSessionDuration, uptime);
+            pKinesisVideoStream->diagnostics.avgSessionDuration = (UINT64) EMA_ACCUMULATOR_GET_NEXT(pKinesisVideoStream->diagnostics.avgSessionDuration, uptime);
         }
 
         MEMFREE(pUploadHandleInfo);

src/client/src/StreamEvent.c

@@ -966,9 +966,9 @@ STATUS calculateCallLatency(PKinesisVideoStream pKinesisVideoStream, BOOL cplApi
 
     // Exponential mean averaging
     if (cplApiCall) {
-        pKinesisVideoStream->diagnostics.cplApiCallLatency = EMA_ACCUMULATOR_GET_NEXT(pKinesisVideoStream->diagnostics.cplApiCallLatency, latency);
+        pKinesisVideoStream->diagnostics.cplApiCallLatency = (UINT64) EMA_ACCUMULATOR_GET_NEXT(pKinesisVideoStream->diagnostics.cplApiCallLatency, latency);
     } else {
-        pKinesisVideoStream->diagnostics.dataApiCallLatency = EMA_ACCUMULATOR_GET_NEXT(pKinesisVideoStream->diagnostics.dataApiCallLatency, latency);
+        pKinesisVideoStream->diagnostics.dataApiCallLatency = (UINT64) EMA_ACCUMULATOR_GET_NEXT(pKinesisVideoStream->diagnostics.dataApiCallLatency, latency);
     }
 
 CleanUp:

src/common/include/com/amazonaws/kinesis/video/common/CommonDefs.h

@@ -492,7 +492,11 @@ typedef CID*                PCID;
 #include <malloc.h>
 #endif
 
-#if defined _WIN32 || defined _WIN64 || defined __CYGWIN__
+#if defined __WINDOWS_BUILD__
+
+// Include the posix IO
+#include <fcntl.h>
+#include <io.h>
 
 #include "dlfcn_win_stub.h"
 

src/heap/include/com/amazonaws/kinesis/video/heap/Include.h

@@ -118,7 +118,7 @@ typedef struct
 /**
  * Error values
  */
-#define STATUS_HEAP_BASE                                        0x010000000
+#define STATUS_HEAP_BASE                                        0x10000000
 #define STATUS_HEAP_FLAGS_ERROR                                 STATUS_HEAP_BASE + 0x00000001
 #define STATUS_HEAP_NOT_INITIALIZED                             STATUS_HEAP_BASE + 0x00000002
 #define STATUS_HEAP_CORRUPTED                                   STATUS_HEAP_BASE + 0x00000003
@@ -141,14 +141,15 @@ typedef struct
 #define STATUS_HEAP_VRAM_UNINIT_FAILED                          STATUS_HEAP_BASE + 0x00000014
 #define STATUS_INVALID_ALLOCATION_SIZE                          STATUS_HEAP_BASE + 0x00000015
 #define STATUS_HEAP_REALLOC_ERROR                               STATUS_HEAP_BASE + 0x00000016
+#define STATUS_HEAP_FILE_HEAP_FILE_CORRUPT                      STATUS_HEAP_BASE + 0x00000017
 
 //////////////////////////////////////////////////////////////////////////
 // Public functions
 //////////////////////////////////////////////////////////////////////////
 /**
  * Creates and initializes the heap
  */
-PUBLIC_API STATUS heapInitialize(UINT64, UINT32, UINT32, PHeap*);
+PUBLIC_API STATUS heapInitialize(UINT64, UINT32, UINT32, PCHAR, PHeap*);
 
 /**
  * Releases the entire heap.

src/heap/src/Common.c

@@ -187,8 +187,8 @@ DEFINE_HEAP_SET_ALLOC_SIZE(commonHeapSetAllocSize)
     // Check if the larger size can spill over the heap limit
     if (newSize > size) {
         diff = newSize - size;
-        CHK_ERR(diff + pHeap->heapSize <= pHeap->heapLimit, STATUS_NOT_ENOUGH_MEMORY,
-                "Allocating %" PRIu64 " bytes failed due to heap limit", newSize);
+        CHK_WARN(diff + pHeap->heapSize <= pHeap->heapLimit, STATUS_NOT_ENOUGH_MEMORY,
+                 "Allocating %" PRIu64 " bytes failed due to heap limit", newSize);
         // Increment the current allocations size
         pHeap->heapSize += diff;
     } else {
@@ -234,8 +234,8 @@ DEFINE_HEAP_ALLOC(commonHeapAlloc)
     overallSize = pBaseHeap->getAllocationHeaderSizeFn() +
             pBaseHeap->getAllocationAlignedSizeFn(size) +
             pBaseHeap->getAllocationFooterSizeFn();
-    CHK_ERR(overallSize + pHeap->heapSize <= pHeap->heapLimit, STATUS_NOT_ENOUGH_MEMORY,
-            "Allocating %" PRIu64 " bytes failed due to heap limit", size);
+    CHK_WARN(overallSize + pHeap->heapSize <= pHeap->heapLimit, STATUS_NOT_ENOUGH_MEMORY,
+             "Allocating %" PRIu64 " bytes failed due to heap limit", size);
 
     // Validate the heap
     CHK_STATUS(validateHeap(pHeap));

src/heap/src/Common.h

@@ -38,6 +38,7 @@ typedef struct
     union {
         UINT32 vramHandle;
         UINT32 flags;
+        UINT32 fileHandle;
     };
 
 #ifdef HEAP_DEBUG

src/heap/src/Heap.c

@@ -33,14 +33,16 @@ STATUS heapDebugCheckAllocator(PHeap pHeap, BOOL dump)
  *      @heapLimit - The overall size of the heap
  *      @spillRatio - Spill ratio in percentage of direct allocation RAM vs. vRAM in the hybrid heap scenario
  *      @behaviorFlags - Flags controlling the behavior/type of the heap
+ *      @pRootDirectoru - Optional path to the root directory in case of the file-based heap
  *      @ppHeap - The returned pointer to the Heap object
  */
-STATUS heapInitialize(UINT64 heapLimit, UINT32 spillRatio, UINT32 behaviorFlags, PHeap* ppHeap)
+STATUS heapInitialize(UINT64 heapLimit, UINT32 spillRatio, UINT32 behaviorFlags, PCHAR pRootDirectory, PHeap* ppHeap)
 {
     ENTERS();
     STATUS retStatus = STATUS_SUCCESS;
     PHeap pHeap = NULL;
     PHybridHeap pHybridHeap = NULL;
+    PHybridFileHeap pFileHeap = NULL;
     UINT32 heapTypeFlags = (behaviorFlags & (FLAGS_USE_AIV_HEAP | FLAGS_USE_SYSTEM_HEAP));
 
     CHK(ppHeap != NULL, STATUS_NULL_ARG);
@@ -74,6 +76,12 @@ STATUS heapInitialize(UINT64 heapLimit, UINT32 spillRatio, UINT32 behaviorFlags,
 
         // Store the hybrid heap as the returned heap object
         pHeap = (PHeap) pHybridHeap;
+    } else if ((behaviorFlags & FLAGS_USE_HYBRID_FILE_HEAP) != HEAP_FLAGS_NONE) {
+        DLOGI("Creating hybrid file heap with flags: 0x%08x", behaviorFlags);
+        CHK_STATUS(hybridFileCreateHeap(pHeap, spillRatio, pRootDirectory, &pFileHeap));
+
+        // Store the file hybrid heap as the returned heap object
+        pHeap = (PHeap) pFileHeap;
     }
 
     // Just in case - validate the final heap object
@@ -92,7 +100,7 @@ STATUS heapInitialize(UINT64 heapLimit, UINT32 spillRatio, UINT32 behaviorFlags,
     // Clean up if we failed
     if (STATUS_FAILED(retStatus)) {
         DLOGE("Failed to initialize native heap.");
-        // The call is indempotent anyway
+        // The call is idempotent anyway
         heapRelease(pHeap);
     }