delete --kv-cache-storage arg.

README.md

@@ -25,7 +25,7 @@ Python 3 and C++ compiler required. The command will download the model and the
 
 ### 🛠️ Convert Model Manually
 
-Supported architectures: Llama, Mixtral, Grok
+Supported architectures: Llama, Mixtral
 
 * [How to Convert Llama 2, Llama 3, Llama 3.1](./docs/LLAMA.md)
 * [How to Convert Hugging Face Model](./docs/HUGGINGFACE.md)

src/app.cpp

@@ -43,8 +43,6 @@ AppArgs AppArgs::parse(int argc, char** argv, bool hasMode) {
     args.chatTemplateType = TEMPLATE_UNKNOWN;
     args.maxSeqLen = 0;
     args.packetAlignment = 0;
-    args.useDiscForKvCache = false;
-
     int i = 1;
     if (hasMode && argc > 1) {
         args.mode = argv[1];
@@ -103,8 +101,6 @@ AppArgs AppArgs::parse(int argc, char** argv, bool hasMode) {
             args.chatTemplateType = parseChatTemplateType(value);
         } else if (strcmp(name, "--max-seq-len") == 0) {
             args.maxSeqLen = (unsigned int)atoi(value);
-        } else if (strcmp(name, "--kv-cache-storage") == 0) {
-            args.useDiscForKvCache = strcmp(value, "disc") == 0;
         } else if (strcmp(name, "--packet-alignment") == 0) {
             args.packetAlignment = (size_t)atoi(value);
         } else {
@@ -142,7 +138,6 @@ void App::run(AppArgs* args, void (*program)(Inference* inference, SocketPool* s
     }
 
     TransformerConfig config;
-    config.useDiscForKvCache = args->useDiscForKvCache;
 
     Transformer transformer = Transformer::loadRootFromFile(args->modelPath, &spec, &config, socketPool);
     socketPool->setTurbo(true);

src/app.hpp

@@ -16,7 +16,6 @@ class AppArgs {
 public:
     char* mode;
     int nThreads;
-    bool useDiscForKvCache;
     size_t packetAlignment;
 
     // inference

src/apps/dllama/dllama.cpp

@@ -208,7 +208,6 @@ void worker(AppArgs* args) {
     }
 
     TransformerConfig config;
-    config.useDiscForKvCache = args->useDiscForKvCache;
 
     SocketPool* socketPool = SocketPool::serve(args->port);
     TransformerSpec spec;

src/transformer.cpp

@@ -310,13 +310,8 @@ TransformerBlock::TransformerBlock(TransformerSpec* spec, TransformerConfig* con
 #endif
 
     kvCacheSlice = new KvCacheSlice(spec->kvDim, spec->seqLen, spec->nSlices);
-    if (config->useDiscForKvCache) {
-        keyCache = (float*)newMmapFileBuffer(sliceIndex, kvCacheSlice->keyCacheSize);
-        valueCache = (float*)newMmapFileBuffer(sliceIndex, kvCacheSlice->valueCacheSize);
-    } else {
-        keyCache = (float*)newBuffer(kvCacheSlice->keyCacheSize);
-        valueCache = (float*)newBuffer(kvCacheSlice->valueCacheSize);
-    }
+    keyCache = (float*)newBuffer(kvCacheSlice->keyCacheSize);
+    valueCache = (float*)newBuffer(kvCacheSlice->valueCacheSize);
 
     multiHeadAttSlice = new MultiHeadAttSlice(spec->nHeads, spec->seqLen, spec->nSlices, sliceIndex);
     att = (float*)newBuffer(multiHeadAttSlice->attSize);
@@ -376,13 +371,8 @@ TransformerBlock::~TransformerBlock() {
 #endif
 
     delete kvCacheSlice;
-    if (config->useDiscForKvCache) {
-        freeMmapFileBuffer(keyCache);
-        freeMmapFileBuffer(valueCache);
-    } else {
-        freeBuffer(keyCache);
-        freeBuffer(valueCache);
-    }
+    freeBuffer(keyCache);
+    freeBuffer(valueCache);
     delete multiHeadAttSlice;
     freeBuffer(att);
 

src/transformer.hpp

@@ -90,7 +90,7 @@ struct TransformerSpec {
 };
 
 struct TransformerConfig {
-    bool useDiscForKvCache;
+    short __unused__;
 };
 
 class TransformerBlock {

src/utils.cpp

@@ -45,31 +45,6 @@ void freeBuffer(void* buffer) {
 #endif
 }
 
-unsigned int lastMmapFileBufferIndex = 0;
-
-void* newMmapFileBuffer(unsigned int appInstanceId, size_t size) {
-#ifdef _WIN32
-    throw new std::runtime_error("Mmap file buffer is not supported on Windows yet");
-#else
-    char path[256];
-    snprintf(path, 256, "mmap-buffer-%d-%d.temp", appInstanceId, lastMmapFileBufferIndex++);
-    int fd = open(path, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
-    if (fd == -1)
-        throw new std::runtime_error("Cannot create mmap buffer file");
-    if (ftruncate(fd, size) == -1)
-        throw new std::runtime_error("Cannot truncate mmap buffer file. Not enough disk space?");
-    void *addr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-    if (addr == MAP_FAILED) 
-        throw new std::runtime_error("Cannot mmap buffer file");
-    close(fd);
-    return addr;
-#endif
-}
-
-void freeMmapFileBuffer(void* addr) {
-    // TODO
-}
-
 unsigned long timeMs() {
     struct timeval te; 
     gettimeofday(&te, NULL);

src/utils.hpp

@@ -23,9 +23,6 @@
 void* newBuffer(size_t size);
 void freeBuffer(void* buffer);
 
-void* newMmapFileBuffer(unsigned int appInstanceId, size_t size);
-void freeMmapFileBuffer(void* addr);
-
 unsigned long timeMs();
 unsigned int randomU32(unsigned long long *state);
 float randomF32(unsigned long long *state);