fix: local generations not considering n_predict for context building

Vali-98 · Jul 30, 2024 · 01570c6 · 01570c6
1 parent cff18ad
commit 01570c6
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/app/constants/APIState/LocalAPI.ts b/app/constants/APIState/LocalAPI.ts
@@ -30,12 +30,15 @@ class LocalAPI extends APIBase {
     buildPayload = () => {
         const payloadFields = this.getSamplerFields()
         const rep_pen = payloadFields?.['penalty_repeat']
+        const n_predict =
+            (typeof payloadFields?.['n_predict'] === 'number' && payloadFields?.['n_predict']) || 0
+
         const localPreset: LlamaPreset = this.getObject(Global.LocalPreset)
         return {
             ...payloadFields,
             penalize_nl: typeof rep_pen === 'number' && rep_pen > 1,
             n_threads: localPreset.threads,
-            prompt: this.buildTextCompletionContext(localPreset.context_length),
+            prompt: this.buildTextCompletionContext(localPreset.context_length - n_predict),
             stop: this.constructStopSequence(),
             emit_partial_completion: true,
         }