feat: sort cuda graphs in descending order (huggingface#2104)

yuanwu2017 · Sep 24, 2024 · d930724 · d930724
1 parent f0ed8d2
commit d930724
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/server/text_generation_server/models/globals.py b/server/text_generation_server/models/globals.py
@@ -15,6 +15,13 @@
 else:
     cuda_graphs = None
 
+
+# sorting the cuda graphs in descending order helps reduce the
+# memory impact and results in less memory usage
+if cuda_graphs is not None:
+    cuda_graphs.sort(reverse=True)
+
+
 CUDA_GRAPHS = cuda_graphs
 
 # This is overridden at model loading.