feat: added bypass for context length

Vali-98 · Nov 1, 2024 · 70db45b · 70db45b
1 parent 2936cfe
commit 70db45b
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 21 deletions.
diff --git a/app/AppSettingsMenu.tsx b/app/AppSettingsMenu.tsx
@@ -65,6 +65,9 @@ const AppSettingsMenu = () => {
     const [chatOnStartup, setChatOnStartup] = useMMKVBoolean(AppSettings.ChatOnStartup)
     const [autoScroll, setAutoScroll] = useMMKVBoolean(AppSettings.AutoScroll)
     const [sendOnEnter, setSendOnEnter] = useMMKVBoolean(AppSettings.SendOnEnter)
+    const [bypassContextLength, setBypassContextLength] = useMMKVBoolean(
+        AppSettings.BypassContextLength
+    )
 
     return (
         <ScrollView style={styles.mainContainer}>
@@ -119,6 +122,13 @@ const AppSettingsMenu = () => {
                 description="Prints the generation context to logs for debugging"
             />
 
+            <SwitchWithDescription
+                title="Bypass Context Length"
+                value={printContext}
+                onValueChange={setPrintContext}
+                description="Ignores context length limits when building prompts"
+            />
+
             <SectionTitle>Character Management</SectionTitle>
             <TouchableOpacity
                 style={styles.button}

diff --git a/app/constants/APIState/BaseAPI.ts b/app/constants/APIState/BaseAPI.ts
@@ -123,6 +123,9 @@ export abstract class APIBase implements IAPIBase {
         // this is needed to check if examples should be added
         let first_message_reached = false
 
+        // check the limit bypass
+        const bypassContextLength = mmkv.getBoolean(AppSettings.BypassContextLength)
+
         // we require lengths for names if use_names is enabled
         for (const message of messages.reverse()) {
             const swipe_len = Chats.useChat.getState().getTokenCount(index)
@@ -156,7 +159,10 @@ export abstract class APIBase implements IAPIBase {
                 swipe_len + instruct_len + name_length + timestamp_length + wrap_length
 
             // check if within context window
-            if (message_acc_length + payload_length + shard_length > max_length) {
+            if (
+                message_acc_length + payload_length + shard_length > max_length &&
+                !bypassContextLength
+            ) {
                 break
             }
 
@@ -262,6 +268,9 @@ export abstract class APIBase implements IAPIBase {
         const payload = [{ role: systemRole, content: replaceMacros(initial) }]
         const messageBuffer = []
 
+        // check the limit bypass
+        const bypassContextLength = mmkv.getBoolean(AppSettings.BypassContextLength)
+
         let index = messages.length - 1
         for (const message of messages.reverse()) {
             const swipe_data = message.swipes[message.swipe_id]
@@ -277,7 +286,7 @@ export abstract class APIBase implements IAPIBase {
                 total_length +
                 name_length +
                 timestamp_length
-            if (len > max_length) break
+            if (len > max_length && !bypassContextLength) break
             messageBuffer.push({
                 role: message.is_user ? userRole : assistantRole,
                 content: replaceMacros(message.swipes[message.swipe_id].swipe),

diff --git a/app/constants/Global.ts b/app/constants/Global.ts
@@ -9,7 +9,7 @@ import { Platform } from 'react-native'
 import { API } from './API'
 import { Characters } from './Characters'
 import { Chats } from './Chat'
-import { Global, AppSettings, AppMode } from './GlobalValues'
+import { Global, AppSettings, AppMode, AppSettingsDefault } from './GlobalValues'
 import { Instructs } from './Instructs'
 import { Llama } from './LlamaLocal'
 import { Logger } from './Logger'
@@ -83,24 +83,6 @@ export const saveStringToDownload = async (
     await writeFile(`${DownloadDirectoryPath}/${filename}`, data, encoding)
 }
 
-/**
- * Default settings on first install
- */
-const AppSettingsDefault: Record<AppSettings, boolean | number> = {
-    [AppSettings.AnimateEditor]: true,
-    [AppSettings.AutoLoadLocal]: false,
-    [AppSettings.AutoScroll]: true,
-    [AppSettings.ChatOnStartup]: false,
-    [AppSettings.CreateFirstMes]: true,
-    [AppSettings.DarkMode]: true,
-    [AppSettings.DevMode]: false,
-    [AppSettings.PrimaryHue]: 270,
-    [AppSettings.SendOnEnter]: false,
-    [AppSettings.SaveLocalKV]: false,
-    [AppSettings.PrintContext]: false,
-    [AppSettings.CreateDefaultCard]: true,
-}
-
 const loadChatOnInit = async () => {
     const newestChat = await Chats.db.query.chatNewest()
     if (!newestChat) return

diff --git a/app/constants/GlobalValues.ts b/app/constants/GlobalValues.ts
@@ -131,9 +131,29 @@ export enum AppSettings {
     SaveLocalKV = 'savelocalkv',
     PrintContext = 'printcontext',
     CreateDefaultCard = 'createdefaultcard',
+    BypassContextLength = 'bypasscontextlength',
 }
 
 export enum AppMode {
     LOCAL = 'local',
     REMOTE = 'remote',
 }
+
+/**
+ * Default settings on first install
+ */
+export const AppSettingsDefault: Record<AppSettings, boolean | number> = {
+    [AppSettings.AnimateEditor]: true,
+    [AppSettings.AutoLoadLocal]: false,
+    [AppSettings.AutoScroll]: true,
+    [AppSettings.ChatOnStartup]: false,
+    [AppSettings.CreateFirstMes]: true,
+    [AppSettings.DarkMode]: true,
+    [AppSettings.DevMode]: false,
+    [AppSettings.PrimaryHue]: 240,
+    [AppSettings.SendOnEnter]: false,
+    [AppSettings.SaveLocalKV]: false,
+    [AppSettings.PrintContext]: false,
+    [AppSettings.CreateDefaultCard]: true,
+    [AppSettings.BypassContextLength]: false,
+}