- void;
+ storeMessageHistory: (messages: Message[]) => Promise;
+ }) => {
+ const { messages, initialMessages, isLoading, parseMessages, storeMessageHistory } = options;
+ parseMessages(messages, isLoading);
+
+ if (messages.length > initialMessages.length) {
+ storeMessageHistory(messages).catch((error) => toast.error(error.message));
+ }
+ },
+ 50,
+);
+
interface ChatProps {
initialMessages: Message[];
storeMessageHistory: (messages: Message[]) => Promise;
@@ -103,7 +122,7 @@ export const ChatImpl = memo(
});
const [provider, setProvider] = useState(() => {
const savedProvider = Cookies.get('selectedProvider');
- return PROVIDER_LIST.find((p) => p.name === savedProvider) || DEFAULT_PROVIDER;
+ return (PROVIDER_LIST.find((p) => p.name === savedProvider) || DEFAULT_PROVIDER) as ProviderInfo;
});
const { showChat } = useStore(chatStore);
@@ -169,11 +188,13 @@ export const ChatImpl = memo(
}, []);
useEffect(() => {
- parseMessages(messages, isLoading);
-
- if (messages.length > initialMessages.length) {
- storeMessageHistory(messages).catch((error) => toast.error(error.message));
- }
+ processSampledMessages({
+ messages,
+ initialMessages,
+ isLoading,
+ parseMessages,
+ storeMessageHistory,
+ });
}, [messages, isLoading, parseMessages]);
const scrollTextArea = () => {
diff --git a/app/components/chat/ModelSelector.tsx b/app/components/chat/ModelSelector.tsx
index 7571d636d..ec4da63ff 100644
--- a/app/components/chat/ModelSelector.tsx
+++ b/app/components/chat/ModelSelector.tsx
@@ -1,6 +1,6 @@
import type { ProviderInfo } from '~/types/model';
-import type { ModelInfo } from '~/utils/types';
import { useEffect } from 'react';
+import type { ModelInfo } from '~/lib/modules/llm/types';
interface ModelSelectorProps {
model?: string;
diff --git a/app/components/settings/SettingsWindow.tsx b/app/components/settings/SettingsWindow.tsx
index 541323f50..1fffcf4b6 100644
--- a/app/components/settings/SettingsWindow.tsx
+++ b/app/components/settings/SettingsWindow.tsx
@@ -63,7 +63,7 @@ export const SettingsWindow = ({ open, onClose }: SettingsProps) => {
variants={dialogBackdropVariants}
/>
-
+
LOCAL_PROVIDERS.includes(provider.name))
.map(async ([, provider]) => {
const envVarName =
- provider.name.toLowerCase() === 'ollama'
- ? 'OLLAMA_API_BASE_URL'
- : provider.name.toLowerCase() === 'lmstudio'
- ? 'LMSTUDIO_API_BASE_URL'
- : `REACT_APP_${provider.name.toUpperCase()}_URL`;
+ providerBaseUrlEnvKeys[provider.name].baseUrlKey || `REACT_APP_${provider.name.toUpperCase()}_URL`;
// Access environment variables through import.meta.env
- const url = import.meta.env[envVarName] || provider.settings.baseUrl || null; // Ensure baseUrl is used
+ let settingsUrl = provider.settings.baseUrl;
+
+ if (settingsUrl && settingsUrl.trim().length === 0) {
+ settingsUrl = undefined;
+ }
+
+ const url = settingsUrl || import.meta.env[envVarName] || null; // Ensure baseUrl is used
console.log(`[Debug] Using URL for ${provider.name}:`, url, `(from ${envVarName})`);
const status = await checkProviderStatus(url, provider.name);
@@ -521,7 +528,7 @@ export default function DebugTab() {
) {
- /**
- * The `cloudflareEnv` is only used when deployed or when previewing locally.
- * In development the environment variables are available through `env`.
- */
-
- // First check user-provided API keys
- if (userApiKeys?.[provider]) {
- return userApiKeys[provider];
- }
-
- // Fall back to environment variables
- switch (provider) {
- case 'Anthropic':
- return env.ANTHROPIC_API_KEY || cloudflareEnv.ANTHROPIC_API_KEY;
- case 'OpenAI':
- return env.OPENAI_API_KEY || cloudflareEnv.OPENAI_API_KEY;
- case 'Google':
- return env.GOOGLE_GENERATIVE_AI_API_KEY || cloudflareEnv.GOOGLE_GENERATIVE_AI_API_KEY;
- case 'Groq':
- return env.GROQ_API_KEY || cloudflareEnv.GROQ_API_KEY;
- case 'HuggingFace':
- return env.HuggingFace_API_KEY || cloudflareEnv.HuggingFace_API_KEY;
- case 'OpenRouter':
- return env.OPEN_ROUTER_API_KEY || cloudflareEnv.OPEN_ROUTER_API_KEY;
- case 'Deepseek':
- return env.DEEPSEEK_API_KEY || cloudflareEnv.DEEPSEEK_API_KEY;
- case 'Mistral':
- return env.MISTRAL_API_KEY || cloudflareEnv.MISTRAL_API_KEY;
- case 'OpenAILike':
- return env.OPENAI_LIKE_API_KEY || cloudflareEnv.OPENAI_LIKE_API_KEY;
- case 'Together':
- return env.TOGETHER_API_KEY || cloudflareEnv.TOGETHER_API_KEY;
- case 'xAI':
- return env.XAI_API_KEY || cloudflareEnv.XAI_API_KEY;
- case 'Perplexity':
- return env.PERPLEXITY_API_KEY || cloudflareEnv.PERPLEXITY_API_KEY;
- case 'Cohere':
- return env.COHERE_API_KEY;
- case 'AzureOpenAI':
- return env.AZURE_OPENAI_API_KEY;
- default:
- return '';
- }
-}
-
-export function getBaseURL(cloudflareEnv: Env, provider: string) {
- switch (provider) {
- case 'Together':
- return env.TOGETHER_API_BASE_URL || cloudflareEnv.TOGETHER_API_BASE_URL || 'https://api.together.xyz/v1';
- case 'OpenAILike':
- return env.OPENAI_LIKE_API_BASE_URL || cloudflareEnv.OPENAI_LIKE_API_BASE_URL;
- case 'LMStudio':
- return env.LMSTUDIO_API_BASE_URL || cloudflareEnv.LMSTUDIO_API_BASE_URL || 'http://localhost:1234';
- case 'Ollama': {
- let baseUrl = env.OLLAMA_API_BASE_URL || cloudflareEnv.OLLAMA_API_BASE_URL || 'http://localhost:11434';
-
- if (env.RUNNING_IN_DOCKER === 'true') {
- baseUrl = baseUrl.replace('localhost', 'host.docker.internal');
- }
-
- return baseUrl;
- }
- default:
- return '';
- }
-}
diff --git a/app/lib/.server/llm/model.ts b/app/lib/.server/llm/model.ts
deleted file mode 100644
index 1a5aab7eb..000000000
--- a/app/lib/.server/llm/model.ts
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * @ts-nocheck
- * Preventing TS checks with files presented in the video for a better presentation.
- */
-import { getAPIKey, getBaseURL } from '~/lib/.server/llm/api-key';
-import { createAnthropic } from '@ai-sdk/anthropic';
-import { createOpenAI } from '@ai-sdk/openai';
-import { createGoogleGenerativeAI } from '@ai-sdk/google';
-import { ollama } from 'ollama-ai-provider';
-import { createOpenRouter } from '@openrouter/ai-sdk-provider';
-import { createMistral } from '@ai-sdk/mistral';
-import { createCohere } from '@ai-sdk/cohere';
-import type { LanguageModelV1 } from 'ai';
-import type { IProviderSetting } from '~/types/model';
-
-export const DEFAULT_NUM_CTX = process.env.DEFAULT_NUM_CTX ? parseInt(process.env.DEFAULT_NUM_CTX, 10) : 32768;
-
-type OptionalApiKey = string | undefined;
-
-export function getAnthropicModel(apiKey: OptionalApiKey, model: string) {
- const anthropic = createAnthropic({
- apiKey,
- });
-
- return anthropic(model);
-}
-export function getOpenAILikeModel(baseURL: string, apiKey: OptionalApiKey, model: string) {
- const openai = createOpenAI({
- baseURL,
- apiKey,
- });
-
- return openai(model);
-}
-
-export function getCohereAIModel(apiKey: OptionalApiKey, model: string) {
- const cohere = createCohere({
- apiKey,
- });
-
- return cohere(model);
-}
-
-export function getOpenAIModel(apiKey: OptionalApiKey, model: string) {
- const openai = createOpenAI({
- apiKey,
- });
-
- return openai(model);
-}
-
-export function getMistralModel(apiKey: OptionalApiKey, model: string) {
- const mistral = createMistral({
- apiKey,
- });
-
- return mistral(model);
-}
-
-export function getGoogleModel(apiKey: OptionalApiKey, model: string) {
- const google = createGoogleGenerativeAI({
- apiKey,
- });
-
- return google(model);
-}
-
-export function getGroqModel(apiKey: OptionalApiKey, model: string) {
- const openai = createOpenAI({
- baseURL: 'https://api.groq.com/openai/v1',
- apiKey,
- });
-
- return openai(model);
-}
-
-export function getHuggingFaceModel(apiKey: OptionalApiKey, model: string) {
- const openai = createOpenAI({
- baseURL: 'https://api-inference.huggingface.co/v1/',
- apiKey,
- });
-
- return openai(model);
-}
-
-export function getOllamaModel(baseURL: string, model: string) {
- const ollamaInstance = ollama(model, {
- numCtx: DEFAULT_NUM_CTX,
- }) as LanguageModelV1 & { config: any };
-
- ollamaInstance.config.baseURL = `${baseURL}/api`;
-
- return ollamaInstance;
-}
-
-export function getDeepseekModel(apiKey: OptionalApiKey, model: string) {
- const openai = createOpenAI({
- baseURL: 'https://api.deepseek.com/beta',
- apiKey,
- });
-
- return openai(model);
-}
-
-export function getOpenRouterModel(apiKey: OptionalApiKey, model: string) {
- const openRouter = createOpenRouter({
- apiKey,
- });
-
- return openRouter.chat(model);
-}
-
-export function getLMStudioModel(baseURL: string, model: string) {
- const lmstudio = createOpenAI({
- baseUrl: `${baseURL}/v1`,
- apiKey: '',
- });
-
- return lmstudio(model);
-}
-
-export function getXAIModel(apiKey: OptionalApiKey, model: string) {
- const openai = createOpenAI({
- baseURL: 'https://api.x.ai/v1',
- apiKey,
- });
-
- return openai(model);
-}
-
-export function getPerplexityModel(apiKey: OptionalApiKey, model: string) {
- const perplexity = createOpenAI({
- baseURL: 'https://api.perplexity.ai/',
- apiKey,
- });
-
- return perplexity(model);
-}
-
-export function getModel(
- provider: string,
- model: string,
- env: Env,
- apiKeys?: Record,
- providerSettings?: Record,
-) {
- /*
- * let apiKey; // Declare first
- * let baseURL;
- */
-
- const apiKey = getAPIKey(env, provider, apiKeys); // Then assign
- const baseURL = providerSettings?.[provider].baseUrl || getBaseURL(env, provider);
-
- switch (provider) {
- case 'Anthropic':
- return getAnthropicModel(apiKey, model);
- case 'OpenAI':
- return getOpenAIModel(apiKey, model);
- case 'Groq':
- return getGroqModel(apiKey, model);
- case 'HuggingFace':
- return getHuggingFaceModel(apiKey, model);
- case 'OpenRouter':
- return getOpenRouterModel(apiKey, model);
- case 'Google':
- return getGoogleModel(apiKey, model);
- case 'OpenAILike':
- return getOpenAILikeModel(baseURL, apiKey, model);
- case 'Together':
- return getOpenAILikeModel(baseURL, apiKey, model);
- case 'Deepseek':
- return getDeepseekModel(apiKey, model);
- case 'Mistral':
- return getMistralModel(apiKey, model);
- case 'LMStudio':
- return getLMStudioModel(baseURL, model);
- case 'xAI':
- return getXAIModel(apiKey, model);
- case 'Cohere':
- return getCohereAIModel(apiKey, model);
- case 'Perplexity':
- return getPerplexityModel(apiKey, model);
- default:
- return getOllamaModel(baseURL, model);
- }
-}
diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts
index 74cdd9d4f..a71792288 100644
--- a/app/lib/.server/llm/stream-text.ts
+++ b/app/lib/.server/llm/stream-text.ts
@@ -1,5 +1,4 @@
import { convertToCoreMessages, streamText as _streamText } from 'ai';
-import { getModel } from '~/lib/.server/llm/model';
import { MAX_TOKENS } from './constants';
import { getSystemPrompt } from '~/lib/common/prompts/prompts';
import {
@@ -8,6 +7,7 @@ import {
getModelList,
MODEL_REGEX,
MODIFICATIONS_TAG_NAME,
+ PROVIDER_LIST,
PROVIDER_REGEX,
WORK_DIR,
} from '~/utils/constants';
@@ -151,10 +151,13 @@ export async function streamText(props: {
providerSettings?: Record;
promptId?: string;
}) {
- const { messages, env, options, apiKeys, files, providerSettings, promptId } = props;
+ const { messages, env: serverEnv, options, apiKeys, files, providerSettings, promptId } = props;
+
+ // console.log({serverEnv});
+
let currentModel = DEFAULT_MODEL;
let currentProvider = DEFAULT_PROVIDER.name;
- const MODEL_LIST = await getModelList(apiKeys || {}, providerSettings);
+ const MODEL_LIST = await getModelList({ apiKeys, providerSettings, serverEnv: serverEnv as any });
const processedMessages = messages.map((message) => {
if (message.role === 'user') {
const { model, provider, content } = extractPropertiesFromMessage(message);
@@ -181,6 +184,8 @@ export async function streamText(props: {
const dynamicMaxTokens = modelDetails && modelDetails.maxTokenAllowed ? modelDetails.maxTokenAllowed : MAX_TOKENS;
+ const provider = PROVIDER_LIST.find((p) => p.name === currentProvider) || DEFAULT_PROVIDER;
+
let systemPrompt =
PromptLibrary.getPropmtFromLibrary(promptId || 'default', {
cwd: WORK_DIR,
@@ -196,7 +201,12 @@ export async function streamText(props: {
}
return _streamText({
- model: getModel(currentProvider, currentModel, env, apiKeys, providerSettings) as any,
+ model: provider.getModelInstance({
+ model: currentModel,
+ serverEnv,
+ apiKeys,
+ providerSettings,
+ }),
system: systemPrompt,
maxTokens: dynamicMaxTokens,
messages: convertToCoreMessages(processedMessages as any),
diff --git a/app/lib/hooks/useEditChatDescription.ts b/app/lib/hooks/useEditChatDescription.ts
index 5230d6cf9..25147a020 100644
--- a/app/lib/hooks/useEditChatDescription.ts
+++ b/app/lib/hooks/useEditChatDescription.ts
@@ -92,6 +92,7 @@ export function useEditChatDescription({
}
const lengthValid = trimmedDesc.length > 0 && trimmedDesc.length <= 100;
+
// Allow letters, numbers, spaces, and common punctuation but exclude characters that could cause issues
const characterValid = /^[a-zA-Z0-9\s\-_.,!?()[\]{}'"]+$/.test(trimmedDesc);
diff --git a/app/lib/hooks/useSettings.tsx b/app/lib/hooks/useSettings.tsx
index cbdc1894a..3f5515102 100644
--- a/app/lib/hooks/useSettings.tsx
+++ b/app/lib/hooks/useSettings.tsx
@@ -12,14 +12,16 @@ import { useCallback, useEffect, useState } from 'react';
import Cookies from 'js-cookie';
import type { IProviderSetting, ProviderInfo } from '~/types/model';
import { logStore } from '~/lib/stores/logs'; // assuming logStore is imported from this location
-import commit from '~/commit.json';
interface CommitData {
commit: string;
version?: string;
}
-const commitJson: CommitData = commit;
+const versionData: CommitData = {
+ commit: __COMMIT_HASH,
+ version: __APP_VERSION,
+};
export function useSettings() {
const providers = useStore(providersStore);
@@ -34,7 +36,7 @@ export function useSettings() {
const checkIsStableVersion = async () => {
try {
const stableResponse = await fetch(
- `https://raw.githubusercontent.com/stackblitz-labs/bolt.diy/refs/tags/v${commitJson.version}/app/commit.json`,
+ `https://raw.githubusercontent.com/stackblitz-labs/bolt.diy/refs/tags/v${versionData.version}/app/commit.json`,
);
if (!stableResponse.ok) {
@@ -44,7 +46,7 @@ export function useSettings() {
const stableData = (await stableResponse.json()) as CommitData;
- return commit.commit === stableData.commit;
+ return versionData.commit === stableData.commit;
} catch (error) {
console.warn('Error checking stable version:', error);
return false;
@@ -58,15 +60,18 @@ export function useSettings() {
if (savedProviders) {
try {
const parsedProviders: Record = JSON.parse(savedProviders);
- Object.keys(parsedProviders).forEach((provider) => {
- const currentProvider = providers[provider];
- providersStore.setKey(provider, {
- ...currentProvider,
- settings: {
- ...parsedProviders[provider],
- enabled: parsedProviders[provider].enabled ?? true,
- },
- });
+ Object.keys(providers).forEach((provider) => {
+ const currentProviderSettings = parsedProviders[provider];
+
+ if (currentProviderSettings) {
+ providersStore.setKey(provider, {
+ ...providers[provider],
+ settings: {
+ ...currentProviderSettings,
+ enabled: currentProviderSettings.enabled ?? true,
+ },
+ });
+ }
});
} catch (error) {
console.error('Failed to parse providers from cookies:', error);
@@ -105,16 +110,16 @@ export function useSettings() {
let checkCommit = Cookies.get('commitHash');
if (checkCommit === undefined) {
- checkCommit = commit.commit;
+ checkCommit = versionData.commit;
}
- if (savedLatestBranch === undefined || checkCommit !== commit.commit) {
+ if (savedLatestBranch === undefined || checkCommit !== versionData.commit) {
// If setting hasn't been set by user, check version
checkIsStableVersion().then((isStable) => {
const shouldUseLatest = !isStable;
latestBranchStore.set(shouldUseLatest);
Cookies.set('isLatestBranch', String(shouldUseLatest));
- Cookies.set('commitHash', String(commit.commit));
+ Cookies.set('commitHash', String(versionData.commit));
});
} else {
latestBranchStore.set(savedLatestBranch === 'true');
diff --git a/app/lib/modules/llm/base-provider.ts b/app/lib/modules/llm/base-provider.ts
new file mode 100644
index 000000000..ebc2a3a53
--- /dev/null
+++ b/app/lib/modules/llm/base-provider.ts
@@ -0,0 +1,72 @@
+import type { LanguageModelV1 } from 'ai';
+import type { ProviderInfo, ProviderConfig, ModelInfo } from './types';
+import type { IProviderSetting } from '~/types/model';
+import { createOpenAI } from '@ai-sdk/openai';
+import { LLMManager } from './manager';
+
+export abstract class BaseProvider implements ProviderInfo {
+ abstract name: string;
+ abstract staticModels: ModelInfo[];
+ abstract config: ProviderConfig;
+
+ getApiKeyLink?: string;
+ labelForGetApiKey?: string;
+ icon?: string;
+
+ getProviderBaseUrlAndKey(options: {
+ apiKeys?: Record;
+ providerSettings?: IProviderSetting;
+ serverEnv?: Record;
+ defaultBaseUrlKey: string;
+ defaultApiTokenKey: string;
+ }) {
+ const { apiKeys, providerSettings, serverEnv, defaultBaseUrlKey, defaultApiTokenKey } = options;
+ let settingsBaseUrl = providerSettings?.baseUrl;
+ const manager = LLMManager.getInstance();
+
+ if (settingsBaseUrl && settingsBaseUrl.length == 0) {
+ settingsBaseUrl = undefined;
+ }
+
+ const baseUrlKey = this.config.baseUrlKey || defaultBaseUrlKey;
+ let baseUrl = settingsBaseUrl || serverEnv?.[baseUrlKey] || process?.env?.[baseUrlKey] || manager.env?.[baseUrlKey];
+
+ if (baseUrl && baseUrl.endsWith('/')) {
+ baseUrl = baseUrl.slice(0, -1);
+ }
+
+ const apiTokenKey = this.config.apiTokenKey || defaultApiTokenKey;
+ const apiKey =
+ apiKeys?.[this.name] || serverEnv?.[apiTokenKey] || process?.env?.[apiTokenKey] || manager.env?.[baseUrlKey];
+
+ return {
+ baseUrl,
+ apiKey,
+ };
+ }
+
+ // Declare the optional getDynamicModels method
+ getDynamicModels?(
+ apiKeys?: Record,
+ settings?: IProviderSetting,
+ serverEnv?: Record,
+ ): Promise;
+
+ abstract getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1;
+}
+
+type OptionalApiKey = string | undefined;
+
+export function getOpenAILikeModel(baseURL: string, apiKey: OptionalApiKey, model: string) {
+ const openai = createOpenAI({
+ baseURL,
+ apiKey,
+ });
+
+ return openai(model);
+}
diff --git a/app/lib/modules/llm/manager.ts b/app/lib/modules/llm/manager.ts
new file mode 100644
index 000000000..38dc8254d
--- /dev/null
+++ b/app/lib/modules/llm/manager.ts
@@ -0,0 +1,116 @@
+import type { IProviderSetting } from '~/types/model';
+import { BaseProvider } from './base-provider';
+import type { ModelInfo, ProviderInfo } from './types';
+import * as providers from './registry';
+
+export class LLMManager {
+ private static _instance: LLMManager;
+ private _providers: Map = new Map();
+ private _modelList: ModelInfo[] = [];
+ private readonly _env: any = {};
+
+ private constructor(_env: Record) {
+ this._registerProvidersFromDirectory();
+ this._env = _env;
+ }
+
+ static getInstance(env: Record = {}): LLMManager {
+ if (!LLMManager._instance) {
+ LLMManager._instance = new LLMManager(env);
+ }
+
+ return LLMManager._instance;
+ }
+ get env() {
+ return this._env;
+ }
+
+ private async _registerProvidersFromDirectory() {
+ try {
+ /*
+ * Dynamically import all files from the providers directory
+ * const providerModules = import.meta.glob('./providers/*.ts', { eager: true });
+ */
+
+ // Look for exported classes that extend BaseProvider
+ for (const exportedItem of Object.values(providers)) {
+ if (typeof exportedItem === 'function' && exportedItem.prototype instanceof BaseProvider) {
+ const provider = new exportedItem();
+
+ try {
+ this.registerProvider(provider);
+ } catch (error: any) {
+ console.log('Failed To Register Provider: ', provider.name, 'error:', error.message);
+ }
+ }
+ }
+ } catch (error) {
+ console.error('Error registering providers:', error);
+ }
+ }
+
+ registerProvider(provider: BaseProvider) {
+ if (this._providers.has(provider.name)) {
+ console.warn(`Provider ${provider.name} is already registered. Skipping.`);
+ return;
+ }
+
+ console.log('Registering Provider: ', provider.name);
+ this._providers.set(provider.name, provider);
+ this._modelList = [...this._modelList, ...provider.staticModels];
+ }
+
+ getProvider(name: string): BaseProvider | undefined {
+ return this._providers.get(name);
+ }
+
+ getAllProviders(): BaseProvider[] {
+ return Array.from(this._providers.values());
+ }
+
+ getModelList(): ModelInfo[] {
+ return this._modelList;
+ }
+
+ async updateModelList(options: {
+ apiKeys?: Record;
+ providerSettings?: Record;
+ serverEnv?: Record;
+ }): Promise {
+ const { apiKeys, providerSettings, serverEnv } = options;
+
+ // Get dynamic models from all providers that support them
+ const dynamicModels = await Promise.all(
+ Array.from(this._providers.values())
+ .filter(
+ (provider): provider is BaseProvider & Required> =>
+ !!provider.getDynamicModels,
+ )
+ .map((provider) =>
+ provider.getDynamicModels(apiKeys, providerSettings?.[provider.name], serverEnv).catch((err) => {
+ console.error(`Error getting dynamic models ${provider.name} :`, err);
+ return [];
+ }),
+ ),
+ );
+
+ // Combine static and dynamic models
+ const modelList = [
+ ...dynamicModels.flat(),
+ ...Array.from(this._providers.values()).flatMap((p) => p.staticModels || []),
+ ];
+ this._modelList = modelList;
+
+ return modelList;
+ }
+
+ getDefaultProvider(): BaseProvider {
+ const firstProvider = this._providers.values().next().value;
+
+ if (!firstProvider) {
+ throw new Error('No providers registered');
+ }
+
+ return firstProvider;
+ }
+}
diff --git a/app/lib/modules/llm/providers/anthropic.ts b/app/lib/modules/llm/providers/anthropic.ts
new file mode 100644
index 000000000..099f02d62
--- /dev/null
+++ b/app/lib/modules/llm/providers/anthropic.ts
@@ -0,0 +1,58 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { LanguageModelV1 } from 'ai';
+import type { IProviderSetting } from '~/types/model';
+import { createAnthropic } from '@ai-sdk/anthropic';
+
+export default class AnthropicProvider extends BaseProvider {
+ name = 'Anthropic';
+ getApiKeyLink = 'https://console.anthropic.com/settings/keys';
+
+ config = {
+ apiTokenKey: 'ANTHROPIC_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ {
+ name: 'claude-3-5-sonnet-latest',
+ label: 'Claude 3.5 Sonnet (new)',
+ provider: 'Anthropic',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: 'claude-3-5-sonnet-20240620',
+ label: 'Claude 3.5 Sonnet (old)',
+ provider: 'Anthropic',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: 'claude-3-5-haiku-latest',
+ label: 'Claude 3.5 Haiku (new)',
+ provider: 'Anthropic',
+ maxTokenAllowed: 8000,
+ },
+ { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 },
+ { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 },
+ { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 },
+ ];
+ getModelInstance: (options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }) => LanguageModelV1 = (options) => {
+ const { apiKeys, providerSettings, serverEnv, model } = options;
+ const { apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings,
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: '',
+ defaultApiTokenKey: 'ANTHROPIC_API_KEY',
+ });
+ const anthropic = createAnthropic({
+ apiKey,
+ });
+
+ return anthropic(model);
+ };
+}
diff --git a/app/lib/modules/llm/providers/cohere.ts b/app/lib/modules/llm/providers/cohere.ts
new file mode 100644
index 000000000..9233b966b
--- /dev/null
+++ b/app/lib/modules/llm/providers/cohere.ts
@@ -0,0 +1,54 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+import { createCohere } from '@ai-sdk/cohere';
+
+export default class CohereProvider extends BaseProvider {
+ name = 'Cohere';
+ getApiKeyLink = 'https://dashboard.cohere.com/api-keys';
+
+ config = {
+ apiTokenKey: 'COHERE_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere', maxTokenAllowed: 4096 },
+ { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere', maxTokenAllowed: 4096 },
+ { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere', maxTokenAllowed: 4096 },
+ { name: 'command-r', label: 'Command R', provider: 'Cohere', maxTokenAllowed: 4096 },
+ { name: 'command', label: 'Command', provider: 'Cohere', maxTokenAllowed: 4096 },
+ { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere', maxTokenAllowed: 4096 },
+ { name: 'command-light', label: 'Command Light', provider: 'Cohere', maxTokenAllowed: 4096 },
+ { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere', maxTokenAllowed: 4096 },
+ { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere', maxTokenAllowed: 4096 },
+ { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere', maxTokenAllowed: 4096 },
+ ];
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: '',
+ defaultApiTokenKey: 'COHERE_API_KEY',
+ });
+
+ if (!apiKey) {
+ throw new Error(`Missing API key for ${this.name} provider`);
+ }
+
+ const cohere = createCohere({
+ apiKey,
+ });
+
+ return cohere(model);
+ }
+}
diff --git a/app/lib/modules/llm/providers/deepseek.ts b/app/lib/modules/llm/providers/deepseek.ts
new file mode 100644
index 000000000..dac0d23f1
--- /dev/null
+++ b/app/lib/modules/llm/providers/deepseek.ts
@@ -0,0 +1,47 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+import { createOpenAI } from '@ai-sdk/openai';
+
+export default class DeepseekProvider extends BaseProvider {
+ name = 'Deepseek';
+ getApiKeyLink = 'https://platform.deepseek.com/apiKeys';
+
+ config = {
+ apiTokenKey: 'DEEPSEEK_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek', maxTokenAllowed: 8000 },
+ { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek', maxTokenAllowed: 8000 },
+ ];
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: '',
+ defaultApiTokenKey: 'DEEPSEEK_API_KEY',
+ });
+
+ if (!apiKey) {
+ throw new Error(`Missing API key for ${this.name} provider`);
+ }
+
+ const openai = createOpenAI({
+ baseURL: 'https://api.deepseek.com/beta',
+ apiKey,
+ });
+
+ return openai(model);
+ }
+}
diff --git a/app/lib/modules/llm/providers/google.ts b/app/lib/modules/llm/providers/google.ts
new file mode 100644
index 000000000..edc8e1dd3
--- /dev/null
+++ b/app/lib/modules/llm/providers/google.ts
@@ -0,0 +1,51 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+import { createGoogleGenerativeAI } from '@ai-sdk/google';
+
+export default class GoogleProvider extends BaseProvider {
+ name = 'Google';
+ getApiKeyLink = 'https://aistudio.google.com/app/apikey';
+
+ config = {
+ apiTokenKey: 'GOOGLE_GENERATIVE_AI_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 },
+ { name: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash', provider: 'Google', maxTokenAllowed: 8192 },
+ { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 },
+ { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 },
+ { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 },
+ { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 },
+ { name: 'gemini-exp-1206', label: 'Gemini exp-1206', provider: 'Google', maxTokenAllowed: 8192 },
+ ];
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: any;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: '',
+ defaultApiTokenKey: 'GOOGLE_GENERATIVE_AI_API_KEY',
+ });
+
+ if (!apiKey) {
+ throw new Error(`Missing API key for ${this.name} provider`);
+ }
+
+ const google = createGoogleGenerativeAI({
+ apiKey,
+ });
+
+ return google(model);
+ }
+}
diff --git a/app/lib/modules/llm/providers/groq.ts b/app/lib/modules/llm/providers/groq.ts
new file mode 100644
index 000000000..034dab12e
--- /dev/null
+++ b/app/lib/modules/llm/providers/groq.ts
@@ -0,0 +1,51 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+import { createOpenAI } from '@ai-sdk/openai';
+
+export default class GroqProvider extends BaseProvider {
+ name = 'Groq';
+ getApiKeyLink = 'https://console.groq.com/keys';
+
+ config = {
+ apiTokenKey: 'GROQ_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ { name: 'llama-3.2-90b-vision-preview', label: 'Llama 3.2 90b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ { name: 'llama-3.3-70b-versatile', label: 'Llama 3.3 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ ];
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: '',
+ defaultApiTokenKey: 'GROQ_API_KEY',
+ });
+
+ if (!apiKey) {
+ throw new Error(`Missing API key for ${this.name} provider`);
+ }
+
+ const openai = createOpenAI({
+ baseURL: 'https://api.groq.com/openai/v1',
+ apiKey,
+ });
+
+ return openai(model);
+ }
+}
diff --git a/app/lib/modules/llm/providers/huggingface.ts b/app/lib/modules/llm/providers/huggingface.ts
new file mode 100644
index 000000000..5ae1d1b46
--- /dev/null
+++ b/app/lib/modules/llm/providers/huggingface.ts
@@ -0,0 +1,69 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+import { createOpenAI } from '@ai-sdk/openai';
+
+export default class HuggingFaceProvider extends BaseProvider {
+ name = 'HuggingFace';
+ getApiKeyLink = 'https://huggingface.co/settings/tokens';
+
+ config = {
+ apiTokenKey: 'HuggingFace_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ {
+ name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+ label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)',
+ provider: 'HuggingFace',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: '01-ai/Yi-1.5-34B-Chat',
+ label: 'Yi-1.5-34B-Chat (HuggingFace)',
+ provider: 'HuggingFace',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: 'meta-llama/Llama-3.1-70B-Instruct',
+ label: 'Llama-3.1-70B-Instruct (HuggingFace)',
+ provider: 'HuggingFace',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: 'meta-llama/Llama-3.1-405B',
+ label: 'Llama-3.1-405B (HuggingFace)',
+ provider: 'HuggingFace',
+ maxTokenAllowed: 8000,
+ },
+ ];
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: '',
+ defaultApiTokenKey: 'HuggingFace_API_KEY',
+ });
+
+ if (!apiKey) {
+ throw new Error(`Missing API key for ${this.name} provider`);
+ }
+
+ const openai = createOpenAI({
+ baseURL: 'https://api-inference.huggingface.co/v1/',
+ apiKey,
+ });
+
+ return openai(model);
+ }
+}
diff --git a/app/lib/modules/llm/providers/lmstudio.ts b/app/lib/modules/llm/providers/lmstudio.ts
new file mode 100644
index 000000000..404045342
--- /dev/null
+++ b/app/lib/modules/llm/providers/lmstudio.ts
@@ -0,0 +1,73 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import { createOpenAI } from '@ai-sdk/openai';
+import type { LanguageModelV1 } from 'ai';
+
+export default class LMStudioProvider extends BaseProvider {
+ name = 'LMStudio';
+ getApiKeyLink = 'https://lmstudio.ai/';
+ labelForGetApiKey = 'Get LMStudio';
+ icon = 'i-ph:cloud-arrow-down';
+
+ config = {
+ baseUrlKey: 'LMSTUDIO_API_BASE_URL',
+ };
+
+ staticModels: ModelInfo[] = [];
+
+ async getDynamicModels(
+ apiKeys?: Record,
+ settings?: IProviderSetting,
+ serverEnv: Record = {},
+ ): Promise {
+ try {
+ const { baseUrl } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: settings,
+ serverEnv,
+ defaultBaseUrlKey: 'LMSTUDIO_API_BASE_URL',
+ defaultApiTokenKey: '',
+ });
+
+ if (!baseUrl) {
+ return [];
+ }
+
+ const response = await fetch(`${baseUrl}/v1/models`);
+ const data = (await response.json()) as { data: Array<{ id: string }> };
+
+ return data.data.map((model) => ({
+ name: model.id,
+ label: model.id,
+ provider: this.name,
+ maxTokenAllowed: 8000,
+ }));
+ } catch (error: any) {
+ console.log('Error getting LMStudio models:', error.message);
+
+ return [];
+ }
+ }
+ getModelInstance: (options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }) => LanguageModelV1 = (options) => {
+ const { apiKeys, providerSettings, serverEnv, model } = options;
+ const { baseUrl } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings,
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: 'OLLAMA_API_BASE_URL',
+ defaultApiTokenKey: '',
+ });
+ const lmstudio = createOpenAI({
+ baseUrl: `${baseUrl}/v1`,
+ apiKey: '',
+ });
+
+ return lmstudio(model);
+ };
+}
diff --git a/app/lib/modules/llm/providers/mistral.ts b/app/lib/modules/llm/providers/mistral.ts
new file mode 100644
index 000000000..c5becee00
--- /dev/null
+++ b/app/lib/modules/llm/providers/mistral.ts
@@ -0,0 +1,53 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+import { createMistral } from '@ai-sdk/mistral';
+
+export default class MistralProvider extends BaseProvider {
+ name = 'Mistral';
+ getApiKeyLink = 'https://console.mistral.ai/api-keys/';
+
+ config = {
+ apiTokenKey: 'MISTRAL_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral', maxTokenAllowed: 8000 },
+ { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral', maxTokenAllowed: 8000 },
+ { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral', maxTokenAllowed: 8000 },
+ { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral', maxTokenAllowed: 8000 },
+ { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral', maxTokenAllowed: 8000 },
+ { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral', maxTokenAllowed: 8000 },
+ { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral', maxTokenAllowed: 8000 },
+ { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral', maxTokenAllowed: 8000 },
+ { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral', maxTokenAllowed: 8000 },
+ ];
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: '',
+ defaultApiTokenKey: 'MISTRAL_API_KEY',
+ });
+
+ if (!apiKey) {
+ throw new Error(`Missing API key for ${this.name} provider`);
+ }
+
+ const mistral = createMistral({
+ apiKey,
+ });
+
+ return mistral(model);
+ }
+}
diff --git a/app/lib/modules/llm/providers/ollama.ts b/app/lib/modules/llm/providers/ollama.ts
new file mode 100644
index 000000000..2c99be3cf
--- /dev/null
+++ b/app/lib/modules/llm/providers/ollama.ts
@@ -0,0 +1,106 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+import { ollama } from 'ollama-ai-provider';
+
+interface OllamaModelDetails {
+ parent_model: string;
+ format: string;
+ family: string;
+ families: string[];
+ parameter_size: string;
+ quantization_level: string;
+}
+
+export interface OllamaModel {
+ name: string;
+ model: string;
+ modified_at: string;
+ size: number;
+ digest: string;
+ details: OllamaModelDetails;
+}
+
+export interface OllamaApiResponse {
+ models: OllamaModel[];
+}
+
+export const DEFAULT_NUM_CTX = process?.env?.DEFAULT_NUM_CTX ? parseInt(process.env.DEFAULT_NUM_CTX, 10) : 32768;
+
+export default class OllamaProvider extends BaseProvider {
+ name = 'Ollama';
+ getApiKeyLink = 'https://ollama.com/download';
+ labelForGetApiKey = 'Download Ollama';
+ icon = 'i-ph:cloud-arrow-down';
+
+ config = {
+ baseUrlKey: 'OLLAMA_API_BASE_URL',
+ };
+
+ staticModels: ModelInfo[] = [];
+
+ async getDynamicModels(
+ apiKeys?: Record,
+ settings?: IProviderSetting,
+ serverEnv: Record = {},
+ ): Promise {
+ try {
+ const { baseUrl } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: settings,
+ serverEnv,
+ defaultBaseUrlKey: 'OLLAMA_API_BASE_URL',
+ defaultApiTokenKey: '',
+ });
+
+ if (!baseUrl) {
+ return [];
+ }
+
+ const response = await fetch(`${baseUrl}/api/tags`);
+ const data = (await response.json()) as OllamaApiResponse;
+
+ // console.log({ ollamamodels: data.models });
+
+ return data.models.map((model: OllamaModel) => ({
+ name: model.name,
+ label: `${model.name} (${model.details.parameter_size})`,
+ provider: this.name,
+ maxTokenAllowed: 8000,
+ }));
+ } catch (e) {
+ console.error('Failed to get Ollama models:', e);
+ return [];
+ }
+ }
+ getModelInstance: (options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }) => LanguageModelV1 = (options) => {
+ const { apiKeys, providerSettings, serverEnv, model } = options;
+ let { baseUrl } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings,
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: 'OLLAMA_API_BASE_URL',
+ defaultApiTokenKey: '',
+ });
+
+ // Backend: Check if we're running in Docker
+ const isDocker = process.env.RUNNING_IN_DOCKER === 'true';
+
+ baseUrl = isDocker ? baseUrl.replace('localhost', 'host.docker.internal') : baseUrl;
+ baseUrl = isDocker ? baseUrl.replace('127.0.0.1', 'host.docker.internal') : baseUrl;
+
+ const ollamaInstance = ollama(model, {
+ numCtx: DEFAULT_NUM_CTX,
+ }) as LanguageModelV1 & { config: any };
+
+ ollamaInstance.config.baseURL = `${baseUrl}/api`;
+
+ return ollamaInstance;
+ };
+}
diff --git a/app/lib/modules/llm/providers/open-router.ts b/app/lib/modules/llm/providers/open-router.ts
new file mode 100644
index 000000000..45defb2a8
--- /dev/null
+++ b/app/lib/modules/llm/providers/open-router.ts
@@ -0,0 +1,132 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+import { createOpenRouter } from '@openrouter/ai-sdk-provider';
+
+interface OpenRouterModel {
+ name: string;
+ id: string;
+ context_length: number;
+ pricing: {
+ prompt: number;
+ completion: number;
+ };
+}
+
+interface OpenRouterModelsResponse {
+ data: OpenRouterModel[];
+}
+
+export default class OpenRouterProvider extends BaseProvider {
+ name = 'OpenRouter';
+ getApiKeyLink = 'https://openrouter.ai/settings/keys';
+
+ config = {
+ apiTokenKey: 'OPEN_ROUTER_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI', maxTokenAllowed: 8000 },
+ {
+ name: 'anthropic/claude-3.5-sonnet',
+ label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)',
+ provider: 'OpenRouter',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: 'anthropic/claude-3-haiku',
+ label: 'Anthropic: Claude 3 Haiku (OpenRouter)',
+ provider: 'OpenRouter',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: 'deepseek/deepseek-coder',
+ label: 'Deepseek-Coder V2 236B (OpenRouter)',
+ provider: 'OpenRouter',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: 'google/gemini-flash-1.5',
+ label: 'Google Gemini Flash 1.5 (OpenRouter)',
+ provider: 'OpenRouter',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: 'google/gemini-pro-1.5',
+ label: 'Google Gemini Pro 1.5 (OpenRouter)',
+ provider: 'OpenRouter',
+ maxTokenAllowed: 8000,
+ },
+ { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 },
+ {
+ name: 'mistralai/mistral-nemo',
+ label: 'OpenRouter Mistral Nemo (OpenRouter)',
+ provider: 'OpenRouter',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: 'qwen/qwen-110b-chat',
+ label: 'OpenRouter Qwen 110b Chat (OpenRouter)',
+ provider: 'OpenRouter',
+ maxTokenAllowed: 8000,
+ },
+ { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 },
+ ];
+
+ async getDynamicModels(
+ _apiKeys?: Record,
+ _settings?: IProviderSetting,
+ _serverEnv: Record = {},
+ ): Promise {
+ try {
+ const response = await fetch('https://openrouter.ai/api/v1/models', {
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ });
+
+ const data = (await response.json()) as OpenRouterModelsResponse;
+
+ return data.data
+ .sort((a, b) => a.name.localeCompare(b.name))
+ .map((m) => ({
+ name: m.id,
+ label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed(2)} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`,
+ provider: this.name,
+ maxTokenAllowed: 8000,
+ }));
+ } catch (error) {
+ console.error('Error getting OpenRouter models:', error);
+ return [];
+ }
+ }
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: '',
+ defaultApiTokenKey: 'OPEN_ROUTER_API_KEY',
+ });
+
+ if (!apiKey) {
+ throw new Error(`Missing API key for ${this.name} provider`);
+ }
+
+ const openRouter = createOpenRouter({
+ apiKey,
+ });
+ const instance = openRouter.chat(model) as LanguageModelV1;
+
+ return instance;
+ }
+}
diff --git a/app/lib/modules/llm/providers/openai-like.ts b/app/lib/modules/llm/providers/openai-like.ts
new file mode 100644
index 000000000..44fb1ab9c
--- /dev/null
+++ b/app/lib/modules/llm/providers/openai-like.ts
@@ -0,0 +1,77 @@
+import { BaseProvider, getOpenAILikeModel } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+
+export default class OpenAILikeProvider extends BaseProvider {
+ name = 'OpenAILike';
+ getApiKeyLink = undefined;
+
+ config = {
+ baseUrlKey: 'OPENAI_LIKE_API_BASE_URL',
+ apiTokenKey: 'OPENAI_LIKE_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [];
+
+ async getDynamicModels(
+ apiKeys?: Record,
+ settings?: IProviderSetting,
+ serverEnv: Record = {},
+ ): Promise {
+ try {
+ const { baseUrl, apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: settings,
+ serverEnv,
+ defaultBaseUrlKey: 'OPENAI_LIKE_API_BASE_URL',
+ defaultApiTokenKey: 'OPENAI_LIKE_API_KEY',
+ });
+
+ if (!baseUrl || !apiKey) {
+ return [];
+ }
+
+ const response = await fetch(`${baseUrl}/models`, {
+ headers: {
+ Authorization: `Bearer ${apiKey}`,
+ },
+ });
+
+ const res = (await response.json()) as any;
+
+ return res.data.map((model: any) => ({
+ name: model.id,
+ label: model.id,
+ provider: this.name,
+ maxTokenAllowed: 8000,
+ }));
+ } catch (error) {
+ console.error('Error getting OpenAILike models:', error);
+ return [];
+ }
+ }
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { baseUrl, apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: 'OPENAI_LIKE_API_BASE_URL',
+ defaultApiTokenKey: 'OPENAI_LIKE_API_KEY',
+ });
+
+ if (!baseUrl || !apiKey) {
+ throw new Error(`Missing configuration for ${this.name} provider`);
+ }
+
+ return getOpenAILikeModel(baseUrl, apiKey, model);
+ }
+}
diff --git a/app/lib/modules/llm/providers/openai.ts b/app/lib/modules/llm/providers/openai.ts
new file mode 100644
index 000000000..9a5411893
--- /dev/null
+++ b/app/lib/modules/llm/providers/openai.ts
@@ -0,0 +1,48 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+import { createOpenAI } from '@ai-sdk/openai';
+
+export default class OpenAIProvider extends BaseProvider {
+ name = 'OpenAI';
+ getApiKeyLink = 'https://platform.openai.com/api-keys';
+
+ config = {
+ apiTokenKey: 'OPENAI_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI', maxTokenAllowed: 8000 },
+ { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 },
+ { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI', maxTokenAllowed: 8000 },
+ { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 },
+ ];
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: '',
+ defaultApiTokenKey: 'OPENAI_API_KEY',
+ });
+
+ if (!apiKey) {
+ throw new Error(`Missing API key for ${this.name} provider`);
+ }
+
+ const openai = createOpenAI({
+ apiKey,
+ });
+
+ return openai(model);
+ }
+}
diff --git a/app/lib/modules/llm/providers/perplexity.ts b/app/lib/modules/llm/providers/perplexity.ts
new file mode 100644
index 000000000..eb58d7456
--- /dev/null
+++ b/app/lib/modules/llm/providers/perplexity.ts
@@ -0,0 +1,63 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+import { createOpenAI } from '@ai-sdk/openai';
+
+export default class PerplexityProvider extends BaseProvider {
+ name = 'Perplexity';
+ getApiKeyLink = 'https://www.perplexity.ai/settings/api';
+
+ config = {
+ apiTokenKey: 'PERPLEXITY_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ {
+ name: 'llama-3.1-sonar-small-128k-online',
+ label: 'Sonar Small Online',
+ provider: 'Perplexity',
+ maxTokenAllowed: 8192,
+ },
+ {
+ name: 'llama-3.1-sonar-large-128k-online',
+ label: 'Sonar Large Online',
+ provider: 'Perplexity',
+ maxTokenAllowed: 8192,
+ },
+ {
+ name: 'llama-3.1-sonar-huge-128k-online',
+ label: 'Sonar Huge Online',
+ provider: 'Perplexity',
+ maxTokenAllowed: 8192,
+ },
+ ];
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: '',
+ defaultApiTokenKey: 'PERPLEXITY_API_KEY',
+ });
+
+ if (!apiKey) {
+ throw new Error(`Missing API key for ${this.name} provider`);
+ }
+
+ const perplexity = createOpenAI({
+ baseURL: 'https://api.perplexity.ai/',
+ apiKey,
+ });
+
+ return perplexity(model);
+ }
+}
diff --git a/app/lib/modules/llm/providers/together.ts b/app/lib/modules/llm/providers/together.ts
new file mode 100644
index 000000000..1a908b849
--- /dev/null
+++ b/app/lib/modules/llm/providers/together.ts
@@ -0,0 +1,100 @@
+import { BaseProvider, getOpenAILikeModel } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+
+export default class TogetherProvider extends BaseProvider {
+ name = 'Together';
+ getApiKeyLink = 'https://api.together.xyz/settings/api-keys';
+
+ config = {
+ baseUrlKey: 'TOGETHER_API_BASE_URL',
+ apiTokenKey: 'TOGETHER_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ {
+ name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+ label: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+ provider: 'Together',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo',
+ label: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo',
+ provider: 'Together',
+ maxTokenAllowed: 8000,
+ },
+ {
+ name: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
+ label: 'Mixtral 8x7B Instruct',
+ provider: 'Together',
+ maxTokenAllowed: 8192,
+ },
+ ];
+
+ async getDynamicModels(
+ apiKeys?: Record,
+ settings?: IProviderSetting,
+ serverEnv: Record = {},
+ ): Promise {
+ try {
+ const { baseUrl: fetchBaseUrl, apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: settings,
+ serverEnv,
+ defaultBaseUrlKey: 'TOGETHER_API_BASE_URL',
+ defaultApiTokenKey: 'TOGETHER_API_KEY',
+ });
+ const baseUrl = fetchBaseUrl || 'https://api.together.xyz/v1';
+
+ if (!baseUrl || !apiKey) {
+ return [];
+ }
+
+ // console.log({ baseUrl, apiKey });
+
+ const response = await fetch(`${baseUrl}/models`, {
+ headers: {
+ Authorization: `Bearer ${apiKey}`,
+ },
+ });
+
+ const res = (await response.json()) as any;
+ const data = (res || []).filter((model: any) => model.type === 'chat');
+
+ return data.map((m: any) => ({
+ name: m.id,
+ label: `${m.display_name} - in:$${m.pricing.input.toFixed(2)} out:$${m.pricing.output.toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`,
+ provider: this.name,
+ maxTokenAllowed: 8000,
+ }));
+ } catch (error: any) {
+ console.error('Error getting Together models:', error.message);
+ return [];
+ }
+ }
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { baseUrl, apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: 'TOGETHER_API_BASE_URL',
+ defaultApiTokenKey: 'TOGETHER_API_KEY',
+ });
+
+ if (!baseUrl || !apiKey) {
+ throw new Error(`Missing configuration for ${this.name} provider`);
+ }
+
+ return getOpenAILikeModel(baseUrl, apiKey, model);
+ }
+}
diff --git a/app/lib/modules/llm/providers/xai.ts b/app/lib/modules/llm/providers/xai.ts
new file mode 100644
index 000000000..032b01bfe
--- /dev/null
+++ b/app/lib/modules/llm/providers/xai.ts
@@ -0,0 +1,47 @@
+import { BaseProvider } from '~/lib/modules/llm/base-provider';
+import type { ModelInfo } from '~/lib/modules/llm/types';
+import type { IProviderSetting } from '~/types/model';
+import type { LanguageModelV1 } from 'ai';
+import { createOpenAI } from '@ai-sdk/openai';
+
+export default class XAIProvider extends BaseProvider {
+ name = 'xAI';
+ getApiKeyLink = 'https://docs.x.ai/docs/quickstart#creating-an-api-key';
+
+ config = {
+ apiTokenKey: 'XAI_API_KEY',
+ };
+
+ staticModels: ModelInfo[] = [
+ { name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI', maxTokenAllowed: 8000 },
+ { name: 'grok-2-1212', label: 'xAI Grok2 1212', provider: 'xAI', maxTokenAllowed: 8000 },
+ ];
+
+ getModelInstance(options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }): LanguageModelV1 {
+ const { model, serverEnv, apiKeys, providerSettings } = options;
+
+ const { apiKey } = this.getProviderBaseUrlAndKey({
+ apiKeys,
+ providerSettings: providerSettings?.[this.name],
+ serverEnv: serverEnv as any,
+ defaultBaseUrlKey: '',
+ defaultApiTokenKey: 'XAI_API_KEY',
+ });
+
+ if (!apiKey) {
+ throw new Error(`Missing API key for ${this.name} provider`);
+ }
+
+ const openai = createOpenAI({
+ baseURL: 'https://api.x.ai/v1',
+ apiKey,
+ });
+
+ return openai(model);
+ }
+}
diff --git a/app/lib/modules/llm/registry.ts b/app/lib/modules/llm/registry.ts
new file mode 100644
index 000000000..fb5a31f93
--- /dev/null
+++ b/app/lib/modules/llm/registry.ts
@@ -0,0 +1,33 @@
+import AnthropicProvider from './providers/anthropic';
+import CohereProvider from './providers/cohere';
+import DeepseekProvider from './providers/deepseek';
+import GoogleProvider from './providers/google';
+import GroqProvider from './providers/groq';
+import HuggingFaceProvider from './providers/huggingface';
+import LMStudioProvider from './providers/lmstudio';
+import MistralProvider from './providers/mistral';
+import OllamaProvider from './providers/ollama';
+import OpenRouterProvider from './providers/open-router';
+import OpenAILikeProvider from './providers/openai-like';
+import OpenAIProvider from './providers/openai';
+import PerplexityProvider from './providers/perplexity';
+import TogetherProvider from './providers/together';
+import XAIProvider from './providers/xai';
+
+export {
+ AnthropicProvider,
+ CohereProvider,
+ DeepseekProvider,
+ GoogleProvider,
+ GroqProvider,
+ HuggingFaceProvider,
+ MistralProvider,
+ OllamaProvider,
+ OpenAIProvider,
+ OpenRouterProvider,
+ OpenAILikeProvider,
+ PerplexityProvider,
+ XAIProvider,
+ TogetherProvider,
+ LMStudioProvider,
+};
diff --git a/app/lib/modules/llm/types.ts b/app/lib/modules/llm/types.ts
new file mode 100644
index 000000000..337b110c7
--- /dev/null
+++ b/app/lib/modules/llm/types.ts
@@ -0,0 +1,32 @@
+import type { LanguageModelV1 } from 'ai';
+import type { IProviderSetting } from '~/types/model';
+
+export interface ModelInfo {
+ name: string;
+ label: string;
+ provider: string;
+ maxTokenAllowed: number;
+}
+
+export interface ProviderInfo {
+ name: string;
+ staticModels: ModelInfo[];
+ getDynamicModels?: (
+ apiKeys?: Record,
+ settings?: IProviderSetting,
+ serverEnv?: Record,
+ ) => Promise;
+ getModelInstance: (options: {
+ model: string;
+ serverEnv: Env;
+ apiKeys?: Record;
+ providerSettings?: Record;
+ }) => LanguageModelV1;
+ getApiKeyLink?: string;
+ labelForGetApiKey?: string;
+ icon?: string;
+}
+export interface ProviderConfig {
+ baseUrlKey?: string;
+ apiTokenKey?: string;
+}
diff --git a/app/types/model.ts b/app/types/model.ts
index 3bfbfde92..d16b10ae4 100644
--- a/app/types/model.ts
+++ b/app/types/model.ts
@@ -1,9 +1,14 @@
-import type { ModelInfo } from '~/utils/types';
+import type { ModelInfo } from '~/lib/modules/llm/types';
export type ProviderInfo = {
staticModels: ModelInfo[];
name: string;
- getDynamicModels?: (apiKeys?: Record, providerSettings?: IProviderSetting) => Promise;
+ getDynamicModels?: (
+ providerName: string,
+ apiKeys?: Record,
+ providerSettings?: IProviderSetting,
+ serverEnv?: Record,
+ ) => Promise;
getApiKeyLink?: string;
labelForGetApiKey?: string;
icon?: string;
diff --git a/app/utils/constants.ts b/app/utils/constants.ts
index 642599541..64e08a739 100644
--- a/app/utils/constants.ts
+++ b/app/utils/constants.ts
@@ -1,8 +1,7 @@
-import Cookies from 'js-cookie';
-import type { ModelInfo, OllamaApiResponse, OllamaModel } from './types';
-import type { ProviderInfo, IProviderSetting } from '~/types/model';
-import { createScopedLogger } from './logger';
-import { logStore } from '~/lib/stores/logs';
+import type { IProviderSetting } from '~/types/model';
+
+import { LLMManager } from '~/lib/modules/llm/manager';
+import type { ModelInfo } from '~/lib/modules/llm/types';
export const WORK_DIR_NAME = 'project';
export const WORK_DIR = `/home/${WORK_DIR_NAME}`;
@@ -12,533 +11,351 @@ export const PROVIDER_REGEX = /\[Provider: (.*?)\]\n\n/;
export const DEFAULT_MODEL = 'claude-3-5-sonnet-latest';
export const PROMPT_COOKIE_KEY = 'cachedPrompt';
-const logger = createScopedLogger('Constants');
-
-const PROVIDER_LIST: ProviderInfo[] = [
- {
- name: 'Anthropic',
- staticModels: [
- {
- name: 'claude-3-5-sonnet-latest',
- label: 'Claude 3.5 Sonnet (new)',
- provider: 'Anthropic',
- maxTokenAllowed: 8000,
- },
- {
- name: 'claude-3-5-sonnet-20240620',
- label: 'Claude 3.5 Sonnet (old)',
- provider: 'Anthropic',
- maxTokenAllowed: 8000,
- },
- {
- name: 'claude-3-5-haiku-latest',
- label: 'Claude 3.5 Haiku (new)',
- provider: 'Anthropic',
- maxTokenAllowed: 8000,
- },
- { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 },
- { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 },
- { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 },
- ],
- getApiKeyLink: 'https://console.anthropic.com/settings/keys',
- },
- {
- name: 'Ollama',
- staticModels: [],
- getDynamicModels: getOllamaModels,
- getApiKeyLink: 'https://ollama.com/download',
- labelForGetApiKey: 'Download Ollama',
- icon: 'i-ph:cloud-arrow-down',
- },
- {
- name: 'OpenAILike',
- staticModels: [],
- getDynamicModels: getOpenAILikeModels,
- },
- {
- name: 'Cohere',
- staticModels: [
- { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere', maxTokenAllowed: 4096 },
- { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere', maxTokenAllowed: 4096 },
- { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere', maxTokenAllowed: 4096 },
- { name: 'command-r', label: 'Command R', provider: 'Cohere', maxTokenAllowed: 4096 },
- { name: 'command', label: 'Command', provider: 'Cohere', maxTokenAllowed: 4096 },
- { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere', maxTokenAllowed: 4096 },
- { name: 'command-light', label: 'Command Light', provider: 'Cohere', maxTokenAllowed: 4096 },
- { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere', maxTokenAllowed: 4096 },
- { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere', maxTokenAllowed: 4096 },
- { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere', maxTokenAllowed: 4096 },
- ],
- getApiKeyLink: 'https://dashboard.cohere.com/api-keys',
- },
- {
- name: 'OpenRouter',
- staticModels: [
- { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI', maxTokenAllowed: 8000 },
- {
- name: 'anthropic/claude-3.5-sonnet',
- label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)',
- provider: 'OpenRouter',
- maxTokenAllowed: 8000,
- },
- {
- name: 'anthropic/claude-3-haiku',
- label: 'Anthropic: Claude 3 Haiku (OpenRouter)',
- provider: 'OpenRouter',
- maxTokenAllowed: 8000,
- },
- {
- name: 'deepseek/deepseek-coder',
- label: 'Deepseek-Coder V2 236B (OpenRouter)',
- provider: 'OpenRouter',
- maxTokenAllowed: 8000,
- },
- {
- name: 'google/gemini-flash-1.5',
- label: 'Google Gemini Flash 1.5 (OpenRouter)',
- provider: 'OpenRouter',
- maxTokenAllowed: 8000,
- },
- {
- name: 'google/gemini-pro-1.5',
- label: 'Google Gemini Pro 1.5 (OpenRouter)',
- provider: 'OpenRouter',
- maxTokenAllowed: 8000,
- },
- { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 },
- {
- name: 'mistralai/mistral-nemo',
- label: 'OpenRouter Mistral Nemo (OpenRouter)',
- provider: 'OpenRouter',
- maxTokenAllowed: 8000,
- },
- {
- name: 'qwen/qwen-110b-chat',
- label: 'OpenRouter Qwen 110b Chat (OpenRouter)',
- provider: 'OpenRouter',
- maxTokenAllowed: 8000,
- },
- { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 },
- ],
- getDynamicModels: getOpenRouterModels,
- getApiKeyLink: 'https://openrouter.ai/settings/keys',
- },
- {
- name: 'Google',
- staticModels: [
- { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 },
- { name: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash', provider: 'Google', maxTokenAllowed: 8192 },
- { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 },
- { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 },
- { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 },
- { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 },
- { name: 'gemini-exp-1206', label: 'Gemini exp-1206', provider: 'Google', maxTokenAllowed: 8192 },
- ],
- getApiKeyLink: 'https://aistudio.google.com/app/apikey',
- },
- {
- name: 'Groq',
- staticModels: [
- { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
- { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
- { name: 'llama-3.2-90b-vision-preview', label: 'Llama 3.2 90b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
- { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
- { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
- { name: 'llama-3.3-70b-versatile', label: 'Llama 3.3 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
- ],
- getApiKeyLink: 'https://console.groq.com/keys',
- },
- {
- name: 'HuggingFace',
- staticModels: [
- {
- name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
- label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)',
- provider: 'HuggingFace',
- maxTokenAllowed: 8000,
- },
- {
- name: '01-ai/Yi-1.5-34B-Chat',
- label: 'Yi-1.5-34B-Chat (HuggingFace)',
- provider: 'HuggingFace',
- maxTokenAllowed: 8000,
- },
- {
- name: 'codellama/CodeLlama-34b-Instruct-hf',
- label: 'CodeLlama-34b-Instruct (HuggingFace)',
- provider: 'HuggingFace',
- maxTokenAllowed: 8000,
- },
- {
- name: 'NousResearch/Hermes-3-Llama-3.1-8B',
- label: 'Hermes-3-Llama-3.1-8B (HuggingFace)',
- provider: 'HuggingFace',
- maxTokenAllowed: 8000,
- },
- {
- name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
- label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)',
- provider: 'HuggingFace',
- maxTokenAllowed: 8000,
- },
- {
- name: 'Qwen/Qwen2.5-72B-Instruct',
- label: 'Qwen2.5-72B-Instruct (HuggingFace)',
- provider: 'HuggingFace',
- maxTokenAllowed: 8000,
- },
- {
- name: 'meta-llama/Llama-3.1-70B-Instruct',
- label: 'Llama-3.1-70B-Instruct (HuggingFace)',
- provider: 'HuggingFace',
- maxTokenAllowed: 8000,
- },
- {
- name: 'meta-llama/Llama-3.1-405B',
- label: 'Llama-3.1-405B (HuggingFace)',
- provider: 'HuggingFace',
- maxTokenAllowed: 8000,
- },
- {
- name: '01-ai/Yi-1.5-34B-Chat',
- label: 'Yi-1.5-34B-Chat (HuggingFace)',
- provider: 'HuggingFace',
- maxTokenAllowed: 8000,
- },
- {
- name: 'codellama/CodeLlama-34b-Instruct-hf',
- label: 'CodeLlama-34b-Instruct (HuggingFace)',
- provider: 'HuggingFace',
- maxTokenAllowed: 8000,
- },
- {
- name: 'NousResearch/Hermes-3-Llama-3.1-8B',
- label: 'Hermes-3-Llama-3.1-8B (HuggingFace)',
- provider: 'HuggingFace',
- maxTokenAllowed: 8000,
- },
- ],
- getApiKeyLink: 'https://huggingface.co/settings/tokens',
- },
-
- {
- name: 'OpenAI',
- staticModels: [
- { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI', maxTokenAllowed: 8000 },
- { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 },
- { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI', maxTokenAllowed: 8000 },
- { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 },
- ],
- getApiKeyLink: 'https://platform.openai.com/api-keys',
- },
- {
- name: 'xAI',
- staticModels: [{ name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI', maxTokenAllowed: 8000 }],
- getApiKeyLink: 'https://docs.x.ai/docs/quickstart#creating-an-api-key',
- },
- {
- name: 'Deepseek',
- staticModels: [
- { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek', maxTokenAllowed: 8000 },
- { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek', maxTokenAllowed: 8000 },
- ],
- getApiKeyLink: 'https://platform.deepseek.com/apiKeys',
- },
- {
- name: 'Mistral',
- staticModels: [
- { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral', maxTokenAllowed: 8000 },
- { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral', maxTokenAllowed: 8000 },
- { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral', maxTokenAllowed: 8000 },
- { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral', maxTokenAllowed: 8000 },
- { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral', maxTokenAllowed: 8000 },
- { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral', maxTokenAllowed: 8000 },
- { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral', maxTokenAllowed: 8000 },
- { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral', maxTokenAllowed: 8000 },
- { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral', maxTokenAllowed: 8000 },
- ],
- getApiKeyLink: 'https://console.mistral.ai/api-keys/',
- },
- {
- name: 'LMStudio',
- staticModels: [],
- getDynamicModels: getLMStudioModels,
- getApiKeyLink: 'https://lmstudio.ai/',
- labelForGetApiKey: 'Get LMStudio',
- icon: 'i-ph:cloud-arrow-down',
- },
- {
- name: 'Together',
- getDynamicModels: getTogetherModels,
- staticModels: [
- {
- name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
- label: 'Qwen/Qwen2.5-Coder-32B-Instruct',
- provider: 'Together',
- maxTokenAllowed: 8000,
- },
- {
- name: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo',
- label: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo',
- provider: 'Together',
- maxTokenAllowed: 8000,
- },
-
- {
- name: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
- label: 'Mixtral 8x7B Instruct',
- provider: 'Together',
- maxTokenAllowed: 8192,
- },
- ],
- getApiKeyLink: 'https://api.together.xyz/settings/api-keys',
- },
- {
- name: 'Perplexity',
- staticModels: [
- {
- name: 'llama-3.1-sonar-small-128k-online',
- label: 'Sonar Small Online',
- provider: 'Perplexity',
- maxTokenAllowed: 8192,
- },
- {
- name: 'llama-3.1-sonar-large-128k-online',
- label: 'Sonar Large Online',
- provider: 'Perplexity',
- maxTokenAllowed: 8192,
- },
- {
- name: 'llama-3.1-sonar-huge-128k-online',
- label: 'Sonar Huge Online',
- provider: 'Perplexity',
- maxTokenAllowed: 8192,
- },
- ],
- getApiKeyLink: 'https://www.perplexity.ai/settings/api',
- },
-];
-
-export const DEFAULT_PROVIDER = PROVIDER_LIST[0];
-
-const staticModels: ModelInfo[] = PROVIDER_LIST.map((p) => p.staticModels).flat();
-
-export let MODEL_LIST: ModelInfo[] = [...staticModels];
-
-export async function getModelList(
- apiKeys: Record,
- providerSettings?: Record,
-) {
- MODEL_LIST = [
- ...(
- await Promise.all(
- PROVIDER_LIST.filter(
- (p): p is ProviderInfo & { getDynamicModels: () => Promise } => !!p.getDynamicModels,
- ).map((p) => p.getDynamicModels(apiKeys, providerSettings?.[p.name])),
- )
- ).flat(),
- ...staticModels,
- ];
- return MODEL_LIST;
-}
-
-async function getTogetherModels(apiKeys?: Record, settings?: IProviderSetting): Promise {
- try {
- const baseUrl = settings?.baseUrl || import.meta.env.TOGETHER_API_BASE_URL || '';
- const provider = 'Together';
-
- if (!baseUrl) {
- return [];
- }
-
- let apiKey = import.meta.env.OPENAI_LIKE_API_KEY ?? '';
-
- if (apiKeys && apiKeys[provider]) {
- apiKey = apiKeys[provider];
- }
-
- if (!apiKey) {
- return [];
- }
-
- const response = await fetch(`${baseUrl}/models`, {
- headers: {
- Authorization: `Bearer ${apiKey}`,
- },
- });
- const res = (await response.json()) as any;
- const data: any[] = (res || []).filter((model: any) => model.type == 'chat');
-
- return data.map((m: any) => ({
- name: m.id,
- label: `${m.display_name} - in:$${m.pricing.input.toFixed(
- 2,
- )} out:$${m.pricing.output.toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`,
- provider,
- maxTokenAllowed: 8000,
- }));
- } catch (e) {
- console.error('Error getting OpenAILike models:', e);
- return [];
- }
-}
-
-const getOllamaBaseUrl = (settings?: IProviderSetting) => {
- const defaultBaseUrl = settings?.baseUrl || import.meta.env.OLLAMA_API_BASE_URL || 'http://localhost:11434';
-
- // Check if we're in the browser
- if (typeof window !== 'undefined') {
- // Frontend always uses localhost
- return defaultBaseUrl;
- }
-
- // Backend: Check if we're running in Docker
- const isDocker = process.env.RUNNING_IN_DOCKER === 'true';
-
- return isDocker ? defaultBaseUrl.replace('localhost', 'host.docker.internal') : defaultBaseUrl;
-};
-
-async function getOllamaModels(apiKeys?: Record, settings?: IProviderSetting): Promise {
- try {
- const baseUrl = getOllamaBaseUrl(settings);
- const response = await fetch(`${baseUrl}/api/tags`);
- const data = (await response.json()) as OllamaApiResponse;
-
- return data.models.map((model: OllamaModel) => ({
- name: model.name,
- label: `${model.name} (${model.details.parameter_size})`,
- provider: 'Ollama',
- maxTokenAllowed: 8000,
- }));
- } catch (e: any) {
- logStore.logError('Failed to get Ollama models', e, { baseUrl: settings?.baseUrl });
- logger.warn('Failed to get Ollama models: ', e.message || '');
-
- return [];
- }
+const llmManager = LLMManager.getInstance(import.meta.env);
+
+export const PROVIDER_LIST = llmManager.getAllProviders();
+export const DEFAULT_PROVIDER = llmManager.getDefaultProvider();
+
+let MODEL_LIST = llmManager.getModelList();
+
+/*
+ *const PROVIDER_LIST_OLD: ProviderInfo[] = [
+ * {
+ * name: 'Anthropic',
+ * staticModels: [
+ * {
+ * name: 'claude-3-5-sonnet-latest',
+ * label: 'Claude 3.5 Sonnet (new)',
+ * provider: 'Anthropic',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'claude-3-5-sonnet-20240620',
+ * label: 'Claude 3.5 Sonnet (old)',
+ * provider: 'Anthropic',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'claude-3-5-haiku-latest',
+ * label: 'Claude 3.5 Haiku (new)',
+ * provider: 'Anthropic',
+ * maxTokenAllowed: 8000,
+ * },
+ * { name: 'claude-3-opus-latest', label: 'Claude 3 Opus', provider: 'Anthropic', maxTokenAllowed: 8000 },
+ * { name: 'claude-3-sonnet-20240229', label: 'Claude 3 Sonnet', provider: 'Anthropic', maxTokenAllowed: 8000 },
+ * { name: 'claude-3-haiku-20240307', label: 'Claude 3 Haiku', provider: 'Anthropic', maxTokenAllowed: 8000 },
+ * ],
+ * getApiKeyLink: 'https://console.anthropic.com/settings/keys',
+ * },
+ * {
+ * name: 'Ollama',
+ * staticModels: [],
+ * getDynamicModels: getOllamaModels,
+ * getApiKeyLink: 'https://ollama.com/download',
+ * labelForGetApiKey: 'Download Ollama',
+ * icon: 'i-ph:cloud-arrow-down',
+ * },
+ * {
+ * name: 'OpenAILike',
+ * staticModels: [],
+ * getDynamicModels: getOpenAILikeModels,
+ * },
+ * {
+ * name: 'Cohere',
+ * staticModels: [
+ * { name: 'command-r-plus-08-2024', label: 'Command R plus Latest', provider: 'Cohere', maxTokenAllowed: 4096 },
+ * { name: 'command-r-08-2024', label: 'Command R Latest', provider: 'Cohere', maxTokenAllowed: 4096 },
+ * { name: 'command-r-plus', label: 'Command R plus', provider: 'Cohere', maxTokenAllowed: 4096 },
+ * { name: 'command-r', label: 'Command R', provider: 'Cohere', maxTokenAllowed: 4096 },
+ * { name: 'command', label: 'Command', provider: 'Cohere', maxTokenAllowed: 4096 },
+ * { name: 'command-nightly', label: 'Command Nightly', provider: 'Cohere', maxTokenAllowed: 4096 },
+ * { name: 'command-light', label: 'Command Light', provider: 'Cohere', maxTokenAllowed: 4096 },
+ * { name: 'command-light-nightly', label: 'Command Light Nightly', provider: 'Cohere', maxTokenAllowed: 4096 },
+ * { name: 'c4ai-aya-expanse-8b', label: 'c4AI Aya Expanse 8b', provider: 'Cohere', maxTokenAllowed: 4096 },
+ * { name: 'c4ai-aya-expanse-32b', label: 'c4AI Aya Expanse 32b', provider: 'Cohere', maxTokenAllowed: 4096 },
+ * ],
+ * getApiKeyLink: 'https://dashboard.cohere.com/api-keys',
+ * },
+ * {
+ * name: 'OpenRouter',
+ * staticModels: [
+ * { name: 'gpt-4o', label: 'GPT-4o', provider: 'OpenAI', maxTokenAllowed: 8000 },
+ * {
+ * name: 'anthropic/claude-3.5-sonnet',
+ * label: 'Anthropic: Claude 3.5 Sonnet (OpenRouter)',
+ * provider: 'OpenRouter',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'anthropic/claude-3-haiku',
+ * label: 'Anthropic: Claude 3 Haiku (OpenRouter)',
+ * provider: 'OpenRouter',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'deepseek/deepseek-coder',
+ * label: 'Deepseek-Coder V2 236B (OpenRouter)',
+ * provider: 'OpenRouter',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'google/gemini-flash-1.5',
+ * label: 'Google Gemini Flash 1.5 (OpenRouter)',
+ * provider: 'OpenRouter',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'google/gemini-pro-1.5',
+ * label: 'Google Gemini Pro 1.5 (OpenRouter)',
+ * provider: 'OpenRouter',
+ * maxTokenAllowed: 8000,
+ * },
+ * { name: 'x-ai/grok-beta', label: 'xAI Grok Beta (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 8000 },
+ * {
+ * name: 'mistralai/mistral-nemo',
+ * label: 'OpenRouter Mistral Nemo (OpenRouter)',
+ * provider: 'OpenRouter',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'qwen/qwen-110b-chat',
+ * label: 'OpenRouter Qwen 110b Chat (OpenRouter)',
+ * provider: 'OpenRouter',
+ * maxTokenAllowed: 8000,
+ * },
+ * { name: 'cohere/command', label: 'Cohere Command (OpenRouter)', provider: 'OpenRouter', maxTokenAllowed: 4096 },
+ * ],
+ * getDynamicModels: getOpenRouterModels,
+ * getApiKeyLink: 'https://openrouter.ai/settings/keys',
+ * },
+ * {
+ * name: 'Google',
+ * staticModels: [
+ * { name: 'gemini-1.5-flash-latest', label: 'Gemini 1.5 Flash', provider: 'Google', maxTokenAllowed: 8192 },
+ * { name: 'gemini-2.0-flash-exp', label: 'Gemini 2.0 Flash', provider: 'Google', maxTokenAllowed: 8192 },
+ * { name: 'gemini-1.5-flash-002', label: 'Gemini 1.5 Flash-002', provider: 'Google', maxTokenAllowed: 8192 },
+ * { name: 'gemini-1.5-flash-8b', label: 'Gemini 1.5 Flash-8b', provider: 'Google', maxTokenAllowed: 8192 },
+ * { name: 'gemini-1.5-pro-latest', label: 'Gemini 1.5 Pro', provider: 'Google', maxTokenAllowed: 8192 },
+ * { name: 'gemini-1.5-pro-002', label: 'Gemini 1.5 Pro-002', provider: 'Google', maxTokenAllowed: 8192 },
+ * { name: 'gemini-exp-1206', label: 'Gemini exp-1206', provider: 'Google', maxTokenAllowed: 8192 },
+ * ],
+ * getApiKeyLink: 'https://aistudio.google.com/app/apikey',
+ * },
+ * {
+ * name: 'Groq',
+ * staticModels: [
+ * { name: 'llama-3.1-8b-instant', label: 'Llama 3.1 8b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ * { name: 'llama-3.2-11b-vision-preview', label: 'Llama 3.2 11b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ * { name: 'llama-3.2-90b-vision-preview', label: 'Llama 3.2 90b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ * { name: 'llama-3.2-3b-preview', label: 'Llama 3.2 3b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ * { name: 'llama-3.2-1b-preview', label: 'Llama 3.2 1b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ * { name: 'llama-3.3-70b-versatile', label: 'Llama 3.3 70b (Groq)', provider: 'Groq', maxTokenAllowed: 8000 },
+ * ],
+ * getApiKeyLink: 'https://console.groq.com/keys',
+ * },
+ * {
+ * name: 'HuggingFace',
+ * staticModels: [
+ * {
+ * name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+ * label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)',
+ * provider: 'HuggingFace',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: '01-ai/Yi-1.5-34B-Chat',
+ * label: 'Yi-1.5-34B-Chat (HuggingFace)',
+ * provider: 'HuggingFace',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'codellama/CodeLlama-34b-Instruct-hf',
+ * label: 'CodeLlama-34b-Instruct (HuggingFace)',
+ * provider: 'HuggingFace',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'NousResearch/Hermes-3-Llama-3.1-8B',
+ * label: 'Hermes-3-Llama-3.1-8B (HuggingFace)',
+ * provider: 'HuggingFace',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+ * label: 'Qwen2.5-Coder-32B-Instruct (HuggingFace)',
+ * provider: 'HuggingFace',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'Qwen/Qwen2.5-72B-Instruct',
+ * label: 'Qwen2.5-72B-Instruct (HuggingFace)',
+ * provider: 'HuggingFace',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'meta-llama/Llama-3.1-70B-Instruct',
+ * label: 'Llama-3.1-70B-Instruct (HuggingFace)',
+ * provider: 'HuggingFace',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'meta-llama/Llama-3.1-405B',
+ * label: 'Llama-3.1-405B (HuggingFace)',
+ * provider: 'HuggingFace',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: '01-ai/Yi-1.5-34B-Chat',
+ * label: 'Yi-1.5-34B-Chat (HuggingFace)',
+ * provider: 'HuggingFace',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'codellama/CodeLlama-34b-Instruct-hf',
+ * label: 'CodeLlama-34b-Instruct (HuggingFace)',
+ * provider: 'HuggingFace',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'NousResearch/Hermes-3-Llama-3.1-8B',
+ * label: 'Hermes-3-Llama-3.1-8B (HuggingFace)',
+ * provider: 'HuggingFace',
+ * maxTokenAllowed: 8000,
+ * },
+ * ],
+ * getApiKeyLink: 'https://huggingface.co/settings/tokens',
+ * },
+ * {
+ * name: 'OpenAI',
+ * staticModels: [
+ * { name: 'gpt-4o-mini', label: 'GPT-4o Mini', provider: 'OpenAI', maxTokenAllowed: 8000 },
+ * { name: 'gpt-4-turbo', label: 'GPT-4 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 },
+ * { name: 'gpt-4', label: 'GPT-4', provider: 'OpenAI', maxTokenAllowed: 8000 },
+ * { name: 'gpt-3.5-turbo', label: 'GPT-3.5 Turbo', provider: 'OpenAI', maxTokenAllowed: 8000 },
+ * ],
+ * getApiKeyLink: 'https://platform.openai.com/api-keys',
+ * },
+ * {
+ * name: 'xAI',
+ * staticModels: [{ name: 'grok-beta', label: 'xAI Grok Beta', provider: 'xAI', maxTokenAllowed: 8000 }],
+ * getApiKeyLink: 'https://docs.x.ai/docs/quickstart#creating-an-api-key',
+ * },
+ * {
+ * name: 'Deepseek',
+ * staticModels: [
+ * { name: 'deepseek-coder', label: 'Deepseek-Coder', provider: 'Deepseek', maxTokenAllowed: 8000 },
+ * { name: 'deepseek-chat', label: 'Deepseek-Chat', provider: 'Deepseek', maxTokenAllowed: 8000 },
+ * ],
+ * getApiKeyLink: 'https://platform.deepseek.com/apiKeys',
+ * },
+ * {
+ * name: 'Mistral',
+ * staticModels: [
+ * { name: 'open-mistral-7b', label: 'Mistral 7B', provider: 'Mistral', maxTokenAllowed: 8000 },
+ * { name: 'open-mixtral-8x7b', label: 'Mistral 8x7B', provider: 'Mistral', maxTokenAllowed: 8000 },
+ * { name: 'open-mixtral-8x22b', label: 'Mistral 8x22B', provider: 'Mistral', maxTokenAllowed: 8000 },
+ * { name: 'open-codestral-mamba', label: 'Codestral Mamba', provider: 'Mistral', maxTokenAllowed: 8000 },
+ * { name: 'open-mistral-nemo', label: 'Mistral Nemo', provider: 'Mistral', maxTokenAllowed: 8000 },
+ * { name: 'ministral-8b-latest', label: 'Mistral 8B', provider: 'Mistral', maxTokenAllowed: 8000 },
+ * { name: 'mistral-small-latest', label: 'Mistral Small', provider: 'Mistral', maxTokenAllowed: 8000 },
+ * { name: 'codestral-latest', label: 'Codestral', provider: 'Mistral', maxTokenAllowed: 8000 },
+ * { name: 'mistral-large-latest', label: 'Mistral Large Latest', provider: 'Mistral', maxTokenAllowed: 8000 },
+ * ],
+ * getApiKeyLink: 'https://console.mistral.ai/api-keys/',
+ * },
+ * {
+ * name: 'LMStudio',
+ * staticModels: [],
+ * getDynamicModels: getLMStudioModels,
+ * getApiKeyLink: 'https://lmstudio.ai/',
+ * labelForGetApiKey: 'Get LMStudio',
+ * icon: 'i-ph:cloud-arrow-down',
+ * },
+ * {
+ * name: 'Together',
+ * getDynamicModels: getTogetherModels,
+ * staticModels: [
+ * {
+ * name: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+ * label: 'Qwen/Qwen2.5-Coder-32B-Instruct',
+ * provider: 'Together',
+ * maxTokenAllowed: 8000,
+ * },
+ * {
+ * name: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo',
+ * label: 'meta-llama/Llama-3.2-90B-Vision-Instruct-Turbo',
+ * provider: 'Together',
+ * maxTokenAllowed: 8000,
+ * },
+ *
+ * {
+ * name: 'mistralai/Mixtral-8x7B-Instruct-v0.1',
+ * label: 'Mixtral 8x7B Instruct',
+ * provider: 'Together',
+ * maxTokenAllowed: 8192,
+ * },
+ * ],
+ * getApiKeyLink: 'https://api.together.xyz/settings/api-keys',
+ * },
+ * {
+ * name: 'Perplexity',
+ * staticModels: [
+ * {
+ * name: 'llama-3.1-sonar-small-128k-online',
+ * label: 'Sonar Small Online',
+ * provider: 'Perplexity',
+ * maxTokenAllowed: 8192,
+ * },
+ * {
+ * name: 'llama-3.1-sonar-large-128k-online',
+ * label: 'Sonar Large Online',
+ * provider: 'Perplexity',
+ * maxTokenAllowed: 8192,
+ * },
+ * {
+ * name: 'llama-3.1-sonar-huge-128k-online',
+ * label: 'Sonar Huge Online',
+ * provider: 'Perplexity',
+ * maxTokenAllowed: 8192,
+ * },
+ * ],
+ * getApiKeyLink: 'https://www.perplexity.ai/settings/api',
+ * },
+ *];
+ */
+
+const providerBaseUrlEnvKeys: Record = {};
+PROVIDER_LIST.forEach((provider) => {
+ providerBaseUrlEnvKeys[provider.name] = {
+ baseUrlKey: provider.config.baseUrlKey,
+ apiTokenKey: provider.config.apiTokenKey,
+ };
+});
+
+// Export the getModelList function using the manager
+export async function getModelList(options: {
+ apiKeys?: Record;
+ providerSettings?: Record;
+ serverEnv?: Record;
+}) {
+ return await llmManager.updateModelList(options);
}
-async function getOpenAILikeModels(
- apiKeys?: Record,
- settings?: IProviderSetting,
-): Promise {
- try {
- const baseUrl = settings?.baseUrl || import.meta.env.OPENAI_LIKE_API_BASE_URL || '';
-
- if (!baseUrl) {
- return [];
- }
-
- let apiKey = '';
-
- if (apiKeys && apiKeys.OpenAILike) {
- apiKey = apiKeys.OpenAILike;
- }
-
- const response = await fetch(`${baseUrl}/models`, {
- headers: {
- Authorization: `Bearer ${apiKey}`,
- },
- });
- const res = (await response.json()) as any;
-
- return res.data.map((model: any) => ({
- name: model.id,
- label: model.id,
- provider: 'OpenAILike',
- }));
- } catch (e) {
- console.error('Error getting OpenAILike models:', e);
- return [];
- }
-}
-
-type OpenRouterModelsResponse = {
- data: {
- name: string;
- id: string;
- context_length: number;
- pricing: {
- prompt: number;
- completion: number;
- };
- }[];
-};
-
-async function getOpenRouterModels(): Promise {
- const data: OpenRouterModelsResponse = await (
- await fetch('https://openrouter.ai/api/v1/models', {
- headers: {
- 'Content-Type': 'application/json',
- },
- })
- ).json();
-
- return data.data
- .sort((a, b) => a.name.localeCompare(b.name))
- .map((m) => ({
- name: m.id,
- label: `${m.name} - in:$${(m.pricing.prompt * 1_000_000).toFixed(
- 2,
- )} out:$${(m.pricing.completion * 1_000_000).toFixed(2)} - context ${Math.floor(m.context_length / 1000)}k`,
- provider: 'OpenRouter',
- maxTokenAllowed: 8000,
- }));
-}
-
-async function getLMStudioModels(_apiKeys?: Record, settings?: IProviderSetting): Promise {
- try {
- const baseUrl = settings?.baseUrl || import.meta.env.LMSTUDIO_API_BASE_URL || 'http://localhost:1234';
- const response = await fetch(`${baseUrl}/v1/models`);
- const data = (await response.json()) as any;
-
- return data.data.map((model: any) => ({
- name: model.id,
- label: model.id,
- provider: 'LMStudio',
- }));
- } catch (e: any) {
- logStore.logError('Failed to get LMStudio models', e, { baseUrl: settings?.baseUrl });
- return [];
- }
-}
-
-async function initializeModelList(providerSettings?: Record): Promise {
- let apiKeys: Record = {};
-
- try {
- const storedApiKeys = Cookies.get('apiKeys');
-
- if (storedApiKeys) {
- const parsedKeys = JSON.parse(storedApiKeys);
-
- if (typeof parsedKeys === 'object' && parsedKeys !== null) {
- apiKeys = parsedKeys;
- }
- }
- } catch (error: any) {
- logStore.logError('Failed to fetch API keys from cookies', error);
- logger.warn(`Failed to fetch apikeys from cookies: ${error?.message}`);
- }
- MODEL_LIST = [
- ...(
- await Promise.all(
- PROVIDER_LIST.filter(
- (p): p is ProviderInfo & { getDynamicModels: () => Promise } => !!p.getDynamicModels,
- ).map((p) => p.getDynamicModels(apiKeys, providerSettings?.[p.name])),
- )
- ).flat(),
- ...staticModels,
- ];
-
- return MODEL_LIST;
+async function initializeModelList(options: {
+ env?: Record;
+ providerSettings?: Record;
+ apiKeys?: Record;
+}): Promise {
+ const { providerSettings, apiKeys, env } = options;
+ const list = await getModelList({
+ apiKeys,
+ providerSettings,
+ serverEnv: env,
+ });
+ MODEL_LIST = list || MODEL_LIST;
+
+ return list;
}
-export {
- getOllamaModels,
- getOpenAILikeModels,
- getLMStudioModels,
- initializeModelList,
- getOpenRouterModels,
- PROVIDER_LIST,
-};
+// initializeModelList({})
+export { initializeModelList, providerBaseUrlEnvKeys, MODEL_LIST };
diff --git a/app/utils/shell.ts b/app/utils/shell.ts
index 53b450f4c..0ffea4225 100644
--- a/app/utils/shell.ts
+++ b/app/utils/shell.ts
@@ -105,6 +105,7 @@ export class BoltShell {
* this.#shellInputStream?.write('\x03');
*/
this.terminal.input('\x03');
+ await this.waitTillOscCode('prompt');
if (state && state.executionPrms) {
await state.executionPrms;
diff --git a/app/utils/types.ts b/app/utils/types.ts
index 1fa253fa4..5f8bc392b 100644
--- a/app/utils/types.ts
+++ b/app/utils/types.ts
@@ -19,10 +19,3 @@ export interface OllamaModel {
export interface OllamaApiResponse {
models: OllamaModel[];
}
-
-export interface ModelInfo {
- name: string;
- label: string;
- provider: string;
- maxTokenAllowed: number;
-}
diff --git a/app/vite-env.d.ts b/app/vite-env.d.ts
new file mode 100644
index 000000000..ab9286583
--- /dev/null
+++ b/app/vite-env.d.ts
@@ -0,0 +1,2 @@
+declare const __COMMIT_HASH: string;
+declare const __APP_VERSION: string;
diff --git a/docs/docs/CONTRIBUTING.md b/docs/docs/CONTRIBUTING.md
index 7b18010da..3a8d5be8f 100644
--- a/docs/docs/CONTRIBUTING.md
+++ b/docs/docs/CONTRIBUTING.md
@@ -1,246 +1,219 @@
# Contribution Guidelines
-## π Table of Contents
-- [Code of Conduct](#code-of-conduct)
-- [How Can I Contribute?](#how-can-i-contribute)
-- [Pull Request Guidelines](#pull-request-guidelines)
-- [Coding Standards](#coding-standards)
-- [Development Setup](#development-setup)
-- [Deploymnt with Docker](#docker-deployment-documentation)
+Welcome! This guide provides all the details you need to contribute effectively to the project. Thank you for helping us make **bolt.diy** a better tool for developers worldwide. π‘
---
-## Code of Conduct
+## π Table of Contents
-This project and everyone participating in it is governed by our Code of Conduct. By participating, you are expected to uphold this code. Please report unacceptable behavior to the project maintainers.
+1. [Code of Conduct](#code-of-conduct)
+2. [How Can I Contribute?](#how-can-i-contribute)
+3. [Pull Request Guidelines](#pull-request-guidelines)
+4. [Coding Standards](#coding-standards)
+5. [Development Setup](#development-setup)
+6. [Testing](#testing)
+7. [Deployment](#deployment)
+8. [Docker Deployment](#docker-deployment)
+9. [VS Code Dev Containers Integration](#vs-code-dev-containers-integration)
---
-## How Can I Contribute?
-
-### π Reporting Bugs and Feature Requests
-- Check the issue tracker to avoid duplicates
-- Use the issue templates when available
-- Include as much relevant information as possible
-- For bugs, add steps to reproduce the issue
-
-### π§ Code Contributions
-1. Fork the repository
-2. Create a new branch for your feature/fix
-3. Write your code
-4. Submit a pull request
+## π‘οΈ Code of Conduct
-### β¨ Becoming a Core Contributor
-We're looking for dedicated contributors to help maintain and grow this project. If you're interested in becoming a core contributor, please fill out our [Contributor Application Form](https://forms.gle/TBSteXSDCtBDwr5m7).
+This project is governed by our **Code of Conduct**. By participating, you agree to uphold this code. Report unacceptable behavior to the project maintainers.
---
-## Pull Request Guidelines
+## π οΈ How Can I Contribute?
-### π PR Checklist
-- [ ] Branch from the main branch
-- [ ] Update documentation if needed
-- [ ] Manually verify all new functionality works as expected
-- [ ] Keep PRs focused and atomic
+### 1οΈβ£ Reporting Bugs or Feature Requests
+- Check the [issue tracker](#) to avoid duplicates.
+- Use issue templates (if available).
+- Provide detailed, relevant information and steps to reproduce bugs.
-### π Review Process
-1. Manually test the changes
-2. At least one maintainer review required
-3. Address all review comments
-4. Maintain clean commit history
+### 2οΈβ£ Code Contributions
+1. Fork the repository.
+2. Create a feature or fix branch.
+3. Write and test your code.
+4. Submit a pull request (PR).
----
-
-## Coding Standards
-
-### π» General Guidelines
-- Follow existing code style
-- Comment complex logic
-- Keep functions focused and small
-- Use meaningful variable names
+### 3οΈβ£ Join as a Core Contributor
+Interested in maintaining and growing the project? Fill out our [Contributor Application Form](https://forms.gle/TBSteXSDCtBDwr5m7).
---
-## Development Setup
+## β
Pull Request Guidelines
-### π Initial Setup
-1. Clone the repository:
-```bash
-git clone https://github.com/stackblitz-labs/bolt.diy.git
-```
+### PR Checklist
+- Branch from the **main** branch.
+- Update documentation, if needed.
+- Test all functionality manually.
+- Focus on one feature/bug per PR.
-2. Install dependencies:
-```bash
-pnpm install
-```
+### Review Process
+1. Manual testing by reviewers.
+2. At least one maintainer review required.
+3. Address review comments.
+4. Maintain a clean commit history.
-3. Set up environment variables:
- - Rename `.env.example` to `.env.local`
- - Add your LLM API keys (only set the ones you plan to use):
-```bash
-GROQ_API_KEY=XXX
-HuggingFace_API_KEY=XXX
-OPENAI_API_KEY=XXX
-ANTHROPIC_API_KEY=XXX
-...
-```
- - Optionally set debug level:
-```bash
-VITE_LOG_LEVEL=debug
-```
+---
- - Optionally set context size:
-```bash
-DEFAULT_NUM_CTX=32768
-```
+## π Coding Standards
-Some Example Context Values for the qwen2.5-coder:32b models are.
-
-* DEFAULT_NUM_CTX=32768 - Consumes 36GB of VRAM
-* DEFAULT_NUM_CTX=24576 - Consumes 32GB of VRAM
-* DEFAULT_NUM_CTX=12288 - Consumes 26GB of VRAM
-* DEFAULT_NUM_CTX=6144 - Consumes 24GB of VRAM
+### General Guidelines
+- Follow existing code style.
+- Comment complex logic.
+- Keep functions small and focused.
+- Use meaningful variable names.
-**Important**: Never commit your `.env.local` file to version control. It's already included in .gitignore.
+---
-### π Running the Development Server
+## π₯οΈ Development Setup
+
+### 1οΈβ£ Initial Setup
+- Clone the repository:
+ ```bash
+ git clone https://github.com/stackblitz-labs/bolt.diy.git
+ ```
+- Install dependencies:
+ ```bash
+ pnpm install
+ ```
+- Set up environment variables:
+ 1. Rename `.env.example` to `.env.local`.
+ 2. Add your API keys:
+ ```bash
+ GROQ_API_KEY=XXX
+ HuggingFace_API_KEY=XXX
+ OPENAI_API_KEY=XXX
+ ...
+ ```
+ 3. Optionally set:
+ - Debug level: `VITE_LOG_LEVEL=debug`
+ - Context size: `DEFAULT_NUM_CTX=32768`
+
+**Note**: Never commit your `.env.local` file to version control. Itβs already in `.gitignore`.
+
+### 2οΈβ£ Run Development Server
```bash
pnpm run dev
```
-
-**Note**: You will need Google Chrome Canary to run this locally if you use Chrome! It's an easy install and a good browser for web development anyway.
+**Tip**: Use **Google Chrome Canary** for local testing.
---
-## Testing
-
-Run the test suite with:
+## π§ͺ Testing
+Run the test suite with:
```bash
pnpm test
```
---
-## Deployment
-
-To deploy the application to Cloudflare Pages:
+## π Deployment
+### Deploy to Cloudflare Pages
```bash
pnpm run deploy
```
-
-Make sure you have the necessary permissions and Wrangler is correctly configured for your Cloudflare account.
+Ensure you have required permissions and that Wrangler is configured.
---
-# Docker Deployment Documentation
+## π³ Docker Deployment
-This guide outlines various methods for building and deploying the application using Docker.
+This section outlines the methods for deploying the application using Docker. The processes for **Development** and **Production** are provided separately for clarity.
-## Build Methods
+---
-### 1. Using Helper Scripts
+### π§βπ» Development Environment
-NPM scripts are provided for convenient building:
+#### Build Options
+**Option 1: Helper Scripts**
```bash
# Development build
npm run dockerbuild
-
-# Production build
-npm run dockerbuild:prod
```
-### 2. Direct Docker Build Commands
-
-You can use Docker's target feature to specify the build environment:
-
+**Option 2: Direct Docker Build Command**
```bash
-# Development build
docker build . --target bolt-ai-development
-
-# Production build
-docker build . --target bolt-ai-production
```
-### 3. Docker Compose with Profiles
-
-Use Docker Compose profiles to manage different environments:
-
+**Option 3: Docker Compose Profile**
```bash
-# Development environment
docker-compose --profile development up
+```
-# Production environment
-docker-compose --profile production up
+#### Running the Development Container
+```bash
+docker run -p 5173:5173 --env-file .env.local bolt-ai:development
```
---
-## Running the Application
+### π Production Environment
-After building using any of the methods above, run the container with:
+#### Build Options
+**Option 1: Helper Scripts**
```bash
-# Development
-docker run -p 5173:5173 --env-file .env.local bolt-ai:development
+# Production build
+npm run dockerbuild:prod
+```
+
+**Option 2: Direct Docker Build Command**
+```bash
+docker build . --target bolt-ai-production
+```
+
+**Option 3: Docker Compose Profile**
+```bash
+docker-compose --profile production up
+```
-# Production
+#### Running the Production Container
+```bash
docker run -p 5173:5173 --env-file .env.local bolt-ai:production
```
---
-## Deployment with Coolify
+### Coolify Deployment
-[Coolify](https://github.com/coollabsio/coolify) provides a straightforward deployment process:
-
-1. Import your Git repository as a new project
-2. Select your target environment (development/production)
-3. Choose "Docker Compose" as the Build Pack
-4. Configure deployment domains
-5. Set the custom start command:
+For an easy deployment process, use [Coolify](https://github.com/coollabsio/coolify):
+1. Import your Git repository into Coolify.
+2. Choose **Docker Compose** as the build pack.
+3. Configure environment variables (e.g., API keys).
+4. Set the start command:
```bash
docker compose --profile production up
```
-6. Configure environment variables
- - Add necessary AI API keys
- - Adjust other environment variables as needed
-7. Deploy the application
---
-## VS Code Integration
+## π οΈ VS Code Dev Containers Integration
-The `docker-compose.yaml` configuration is compatible with VS Code dev containers:
+The `docker-compose.yaml` configuration is compatible with **VS Code Dev Containers**, making it easy to set up a development environment directly in Visual Studio Code.
-1. Open the command palette in VS Code
-2. Select the dev container configuration
-3. Choose the "development" profile from the context menu
+### Steps to Use Dev Containers
----
-
-## Environment Files
-
-Ensure you have the appropriate `.env.local` file configured before running the containers. This file should contain:
-- API keys
-- Environment-specific configurations
-- Other required environment variables
+1. Open the command palette in VS Code (`Ctrl+Shift+P` or `Cmd+Shift+P` on macOS).
+2. Select **Dev Containers: Reopen in Container**.
+3. Choose the **development** profile when prompted.
+4. VS Code will rebuild the container and open it with the pre-configured environment.
---
-## DEFAULT_NUM_CTX
+## π Environment Variables
-The `DEFAULT_NUM_CTX` environment variable can be used to limit the maximum number of context values used by the qwen2.5-coder model. For example, to limit the context to 24576 values (which uses 32GB of VRAM), set `DEFAULT_NUM_CTX=24576` in your `.env.local` file.
+Ensure `.env.local` is configured correctly with:
+- API keys.
+- Context-specific configurations.
-First off, thank you for considering contributing to bolt.diy! This fork aims to expand the capabilities of the original project by integrating multiple LLM providers and enhancing functionality. Every contribution helps make bolt.diy a better tool for developers worldwide.
-
----
-
-## Notes
-
-- Port 5173 is exposed and mapped for both development and production environments
-- Environment variables are loaded from `.env.local`
-- Different profiles (development/production) can be used for different deployment scenarios
-- The configuration supports both local development and production deployment
+Example for the `DEFAULT_NUM_CTX` variable:
+```bash
+DEFAULT_NUM_CTX=24576 # Uses 32GB VRAM
+```
\ No newline at end of file
diff --git a/docs/docs/FAQ.md b/docs/docs/FAQ.md
index 9f18a88ac..a09fae885 100644
--- a/docs/docs/FAQ.md
+++ b/docs/docs/FAQ.md
@@ -1,6 +1,21 @@
# Frequently Asked Questions (FAQ)
-## How do I get the best results with bolt.diy?
+
Version
- {versionHash.slice(0, 7)} + {connitJson.commit.slice(0, 7)} (v{versionTag || '0.0.1'}) - {isLatestBranch ? 'nightly' : 'stable'} diff --git a/app/components/settings/providers/ProvidersTab.tsx b/app/components/settings/providers/ProvidersTab.tsx index 281b4c80d..e03731f43 100644 --- a/app/components/settings/providers/ProvidersTab.tsx +++ b/app/components/settings/providers/ProvidersTab.tsx @@ -7,6 +7,7 @@ import { logStore } from '~/lib/stores/logs'; // Import a default fallback icon import DefaultIcon from '/icons/Default.svg'; // Adjust the path as necessary +import { providerBaseUrlEnvKeys } from '~/utils/constants'; export default function ProvidersTab() { const { providers, updateProviderSettings, isLocalModel } = useSettings(); @@ -33,9 +34,87 @@ export default function ProvidersTab() { newFilteredProviders.sort((a, b) => a.name.localeCompare(b.name)); - setFilteredProviders(newFilteredProviders); + // Split providers into regular and URL-configurable + const regular = newFilteredProviders.filter((p) => !URL_CONFIGURABLE_PROVIDERS.includes(p.name)); + const urlConfigurable = newFilteredProviders.filter((p) => URL_CONFIGURABLE_PROVIDERS.includes(p.name)); + + setFilteredProviders([...regular, ...urlConfigurable]); }, [providers, searchTerm, isLocalModel]); + const renderProviderCard = (provider: IProviderConfig) => { + const envBaseUrlKey = providerBaseUrlEnvKeys[provider.name].baseUrlKey; + const envBaseUrl = envBaseUrlKey ? import.meta.env[envBaseUrlKey] : undefined; + const isUrlConfigurable = URL_CONFIGURABLE_PROVIDERS.includes(provider.name); + + return ( +
+
+ );
+ };
+
+ const regularProviders = filteredProviders.filter((p) => !URL_CONFIGURABLE_PROVIDERS.includes(p.name));
+ const urlConfigurableProviders = filteredProviders.filter((p) => URL_CONFIGURABLE_PROVIDERS.includes(p.name));
+
return (
+ {
+ updateProviderSettings(provider.name, { ...provider.settings, enabled });
+
+ if (enabled) {
+ logStore.logProvider(`Provider ${provider.name} enabled`, { provider: provider.name });
+ } else {
+ logStore.logProvider(`Provider ${provider.name} disabled`, { provider: provider.name });
+ }
+ }}
+ />
+
+ {isUrlConfigurable && provider.settings.enabled && (
+
+ 
{
+ e.currentTarget.src = DefaultIcon;
+ }}
+ alt={`${provider.name} icon`}
+ className="w-6 h-6 dark:invert"
+ />
+ {provider.name}
+
+ {
+ e.currentTarget.src = DefaultIcon;
+ }}
+ alt={`${provider.name} icon`}
+ className="w-6 h-6 dark:invert"
+ />
+ {provider.name}
+
+ {envBaseUrl && (
+
+ )}
+
+ {
+ let newBaseUrl: string | undefined = e.target.value;
+
+ if (newBaseUrl && newBaseUrl.trim().length === 0) {
+ newBaseUrl = undefined;
+ }
+
+ updateProviderSettings(provider.name, { ...provider.settings, baseUrl: newBaseUrl });
+ logStore.logProvider(`Base URL updated for ${provider.name}`, {
+ provider: provider.name,
+ baseUrl: newBaseUrl,
+ });
+ }}
+ placeholder={`Enter ${provider.name} base URL`}
+ className="w-full bg-white dark:bg-bolt-elements-background-depth-4 relative px-2 py-1.5 rounded-md focus:outline-none placeholder-bolt-elements-textTertiary text-bolt-elements-textPrimary dark:text-bolt-elements-textPrimary border border-bolt-elements-borderColor"
+ />
+
+ )}
+
@@ -47,60 +126,21 @@ export default function ProvidersTab() {
className="w-full bg-white dark:bg-bolt-elements-background-depth-4 relative px-2 py-1.5 rounded-md focus:outline-none placeholder-bolt-elements-textTertiary text-bolt-elements-textPrimary dark:text-bolt-elements-textPrimary border border-bolt-elements-borderColor"
/>
- {filteredProviders.map((provider) => (
-
-
);
}
diff --git a/app/entry.server.tsx b/app/entry.server.tsx
index a44917f02..5e92d21ec 100644
--- a/app/entry.server.tsx
+++ b/app/entry.server.tsx
@@ -14,7 +14,7 @@ export default async function handleRequest(
remixContext: EntryContext,
_loadContext: AppLoadContext,
) {
- await initializeModelList();
+ await initializeModelList({});
const readable = await renderToReadableStream(
- {
- updateProviderSettings(provider.name, { ...provider.settings, enabled });
-
- if (enabled) {
- logStore.logProvider(`Provider ${provider.name} enabled`, { provider: provider.name });
- } else {
- logStore.logProvider(`Provider ${provider.name} disabled`, { provider: provider.name });
- }
- }}
- />
-
- {/* Base URL input for configurable providers */}
- {URL_CONFIGURABLE_PROVIDERS.includes(provider.name) && provider.settings.enabled && (
-
- {
- // Fallback to default icon on error
- e.currentTarget.src = DefaultIcon;
- }}
- alt={`${provider.name} icon`}
- className="w-6 h-6 dark:invert"
- />
- {provider.name}
-
- {
- // Fallback to default icon on error
- e.currentTarget.src = DefaultIcon;
- }}
- alt={`${provider.name} icon`}
- className="w-6 h-6 dark:invert"
- />
- {provider.name}
-
-
- {
- const newBaseUrl = e.target.value;
- updateProviderSettings(provider.name, { ...provider.settings, baseUrl: newBaseUrl });
- logStore.logProvider(`Base URL updated for ${provider.name}`, {
- provider: provider.name,
- baseUrl: newBaseUrl,
- });
- }}
- placeholder={`Enter ${provider.name} base URL`}
- className="w-full bg-white dark:bg-bolt-elements-background-depth-4 relative px-2 py-1.5 rounded-md focus:outline-none placeholder-bolt-elements-textTertiary text-bolt-elements-textPrimary dark:text-bolt-elements-textPrimary border border-bolt-elements-borderColor"
- />
-
- )}
+
+ {/* Regular Providers Grid */}
+ {regularProviders.map(renderProviderCard)}
+
+ {/* URL Configurable Providers Section */}
+ {urlConfigurableProviders.length > 0 && (
+
+
- ))}
+ )}
Experimental Providers
++ These providers are experimental and allow you to run AI models locally or connect to your own + infrastructure. They require additional setup but offer more flexibility. +
+{urlConfigurableProviders.map(renderProviderCard)}
+
+
+What are the best models for bolt.diy?
+ +For the best experience with bolt.diy, we recommend using the following models: + +- **Claude 3.5 Sonnet (old)**: Best overall coder, providing excellent results across all use cases +- **Gemini 2.0 Flash**: Exceptional speed while maintaining good performance +- **GPT-4o**: Strong alternative to Claude 3.5 Sonnet with comparable capabilities +- **DeepSeekCoder V2 236b**: Best open source model (available through OpenRouter, DeepSeek API, or self-hosted) +- **Qwen 2.5 Coder 32b**: Best model for self-hosting with reasonable hardware requirements + +**Note**: Models with less than 7b parameters typically lack the capability to properly interact with bolt! +
+
----
-
-## How do I contribute to bolt.diy?
+How do I get the best results with bolt.diy?
- **Be specific about your stack**: Mention the frameworks or libraries you want to use (e.g., Astro, Tailwind, ShadCN) in your initial prompt. This ensures that bolt.diy scaffolds the project according to your preferences. @@ -14,66 +29,62 @@ - **Batch simple instructions**: Combine simple tasks into a single prompt to save time and reduce API credit consumption. For example: *"Change the color scheme, add mobile responsiveness, and restart the dev server."* +
+
----
-
-## What are the future plans for bolt.diy?
+How do I contribute to bolt.diy?
Check out our [Contribution Guide](CONTRIBUTING.md) for more details on how to get involved! +
+
----
-
-## Why are there so many open issues/pull requests?
+What are the future plans for bolt.diy?
Visit our [Roadmap](https://roadmap.sh/r/ottodev-roadmap-2ovzo) for the latest updates. New features and improvements are on the way! +
+
----
-
-## How do local LLMs compare to larger models like Claude 3.5 Sonnet for bolt.diy?
+Why are there so many open issues/pull requests?
bolt.diy began as a small showcase project on @ColeMedin's YouTube channel to explore editing open-source projects with local LLMs. However, it quickly grew into a massive community effort! -Weβre forming a team of maintainers to manage demand and streamline issue resolution. The maintainers are rockstars, and weβre also exploring partnerships to help the project thrive. +We're forming a team of maintainers to manage demand and streamline issue resolution. The maintainers are rockstars, and we're also exploring partnerships to help the project thrive. +
+
----
-
-## Common Errors and Troubleshooting
+How do local LLMs compare to larger models like Claude 3.5 Sonnet for bolt.diy?
While local LLMs are improving rapidly, larger models like GPT-4o, Claude 3.5 Sonnet, and DeepSeek Coder V2 236b still offer the best results for complex applications. Our ongoing focus is to improve prompts, agents, and the platform to better support smaller local LLMs. +
+
---
diff --git a/docs/docs/index.md b/docs/docs/index.md
index 389e74ffa..641d45a72 100644
--- a/docs/docs/index.md
+++ b/docs/docs/index.md
@@ -1,6 +1,24 @@
# Welcome to bolt diy
bolt.diy allows you to choose the LLM that you use for each prompt! Currently, you can use OpenAI, Anthropic, Ollama, OpenRouter, Gemini, LMStudio, Mistral, xAI, HuggingFace, DeepSeek, or Groq models - and it is easily extended to use any other model supported by the Vercel AI SDK! See the instructions below for running this locally and extending it to include more models.
+## Table of Contents
+- [Join the community!](#join-the-community)
+- [Features](#features)
+- [Setup](#setup)
+ - [Prerequisites](#prerequisites)
+ - [Clone the Repository](#clone-the-repository)
+ - [Entering API Keys](#entering-api-keys)
+ - [1. Set API Keys in the `.env.local` File](#1-set-api-keys-in-the-envlocal-file)
+ - [2. Configure API Keys Directly in the Application](#2-configure-api-keys-directly-in-the-application)
+- [Run the Application](#run-the-application)
+ - [Option 1: Without Docker](#option-1-without-docker)
+ - [Option 2: With Docker](#option-2-with-docker)
+- [Update Your Local Version to the Latest](#update-your-local-version-to-the-latest)
+- [Adding New LLMs](#adding-new-llms)
+- [Available Scripts](#available-scripts)
+- [Development](#development)
+- [Tips and Tricks](#tips-and-tricks)
+
---
## Join the community!
@@ -9,72 +27,65 @@ bolt.diy allows you to choose the LLM that you use for each prompt! Currently, y
---
-## Whats bolt.diy
+## Features
-bolt.diy is an AI-powered web development agent that allows you to prompt, run, edit, and deploy full-stack applications directly from your browserβno local setup required. If you're here to build your own AI-powered web dev agent using the Bolt open source codebase, [click here to get started!](./CONTRIBUTING.md)
+- **AI-powered full-stack web development** directly in your browser.
+- **Support for multiple LLMs** with an extensible architecture to integrate additional models.
+- **Attach images to prompts** for better contextual understanding.
+- **Integrated terminal** to view output of LLM-run commands.
+- **Revert code to earlier versions** for easier debugging and quicker changes.
+- **Download projects as ZIP** for easy portability.
+- **Integration-ready Docker support** for a hassle-free setup.
---
-## What Makes bolt.diy Different
-
-Claude, v0, etc are incredible- but you can't install packages, run backends, or edit code. Thatβs where bolt.diy stands out:
-
-- **Full-Stack in the Browser**: bolt.diy integrates cutting-edge AI models with an in-browser development environment powered by **StackBlitzβs WebContainers**. This allows you to:
- - Install and run npm tools and libraries (like Vite, Next.js, and more)
- - Run Node.js servers
- - Interact with third-party APIs
- - Deploy to production from chat
- - Share your work via a URL
-
-- **AI with Environment Control**: Unlike traditional dev environments where the AI can only assist in code generation, bolt.diy gives AI models **complete control** over the entire environment including the filesystem, node server, package manager, terminal, and browser console. This empowers AI agents to handle the whole app lifecycleβfrom creation to deployment.
-
-Whether youβre an experienced developer, a PM, or a designer, bolt.diy allows you to easily build production-grade full-stack applications.
+## Setup
-For developers interested in building their own AI-powered development tools with WebContainers, check out the open-source Bolt codebase in this repo!
+If you're new to installing software from GitHub, don't worry! If you encounter any issues, feel free to submit an "issue" using the provided links or improve this documentation by forking the repository, editing the instructions, and submitting a pull request. The following instruction will help you get the stable branch up and running on your local machine in no time.
----
-
-## Setup
+### Prerequisites
-Many of you are new users to installing software from Github. If you have any installation troubles reach out and submit an "issue" using the links above, or feel free to enhance this documentation by forking, editing the instructions, and doing a pull request.
+1. **Install Git**: [Download Git](https://git-scm.com/downloads)
+2. **Install Node.js**: [Download Node.js](https://nodejs.org/en/download/)
-1. [Install Git from](https://git-scm.com/downloads)
+ - After installation, the Node.js path is usually added to your system automatically. To verify:
+ - **Windows**: Search for "Edit the system environment variables," click "Environment Variables," and check if `Node.js` is in the `Path` variable.
+ - **Mac/Linux**: Open a terminal and run:
+ ```bash
+ echo $PATH
+ ```
+ Look for `/usr/local/bin` in the output.
-2. [Install Node.js from](https://nodejs.org/en/download/)
+### Clone the Repository
-Pay attention to the installer notes after completion.
+Alternatively, you can download the latest version of the project directly from the [Releases Page](https://github.com/stackblitz-labs/bolt.diy/releases/latest). Simply download the .zip file, extract it, and proceed with the setup instructions below. If you are comfertiable using git then run the command below.
-On all operating systems, the path to Node.js should automatically be added to your system path. But you can check your path if you want to be sure. On Windows, you can search for "edit the system environment variables" in your system, select "Environment Variables..." once you are in the system properties, and then check for a path to Node in your "Path" system variable. On a Mac or Linux machine, it will tell you to check if /usr/local/bin is in your $PATH. To determine if usr/local/bin is included in $PATHΒ open your Terminal and run:
+Clone the repository using Git:
-```
-echo $PATHΒ .
+```bash
+git clone -b stable https://github.com/stackblitz-labs/bolt.diy
```
-If you see usr/local/bin in the output then you're good to go.
+---
-3. Clone the repository (if you haven't already) by opening a Terminal window (or CMD with admin permissions) and then typing in this:
+### Entering API Keys
-```
-git clone https://github.com/stackblitz-labs/bolt.diy.git
-```
+There are two ways to configure your API keys in bolt.diy:
-3. Rename .env.example to .env.local and add your LLM API keys. You will find this file on a Mac at "[your name]/bolt.diy/.env.example". For Windows and Linux the path will be similar.
+#### 1. Set API Keys in the `.env.local` File
-
+When setting up the application, you will need to add your API keys for the LLMs you wish to use. You can do this by renaming the `.env.example` file to `.env.local` and adding your API keys there.
-If you can't see the file indicated above, its likely you can't view hidden files. On Mac, open a Terminal window and enter this command below. On Windows, you will see the hidden files option in File Explorer Settings. A quick Google search will help you if you are stuck here.
+- On **Mac**, you can find the file at `[your name]/bolt.diy/.env.example`.
+- On **Windows/Linux**, the path will be similar.
-```
+If you can't see the file, it's likely because hidden files are not being shown. On **Mac**, open a Terminal window and enter the following command to show hidden files:
+
+```bash
defaults write com.apple.finder AppleShowAllFiles YES
```
-**NOTE**: you only have to set the ones you want to use and Ollama doesn't need an API key because it runs locally on your computer:
-
-[Get your GROQ API Key here](https://console.groq.com/keys)
-
-[Get your Open AI API Key by following these instructions](https://help.openai.com/en/articles/4936850-where-do-i-find-my-openai-api-key)
-
-Get your Anthropic API Key in your [account settings](https://console.anthropic.com/settings/keys)
+Make sure to add your API keys for each provider you want to use, for example:
```
GROQ_API_KEY=XXX
@@ -82,81 +93,108 @@ OPENAI_API_KEY=XXX
ANTHROPIC_API_KEY=XXX
```
-Optionally, you can set the debug level:
+Once you've set your keys, you can proceed with running the app. You will set these keys up during the initial setup, and you can revisit and update them later after the app is running.
-```
-VITE_LOG_LEVEL=debug
-```
+**Note**: Never commit your `.env.local` file to version control. Itβs already included in the `.gitignore`.
-**Important**: Never commit your `.env.local` file to version control. It's already included in .gitignore.
+#### 2. Configure API Keys Directly in the Application
-## Run with Docker
+Alternatively, you can configure your API keys directly in the application once it's running. To do this:
-Prerequisites:
+1. Launch the application and navigate to the provider selection dropdown.
+2. Select the provider you wish to configure.
+3. Click the pencil icon next to the selected provider.
+4. Enter your API key in the provided field.
-Git and Node.js as mentioned above, as well as Docker: https://www.docker.com/
+This method allows you to easily add or update your keys without needing to modify files directly.
-### 1a. Using Helper Scripts
+Once you've configured your keys, the application will be ready to use the selected LLMs.
-NPM scripts are provided for convenient building:
-```bash
-# Development build
-npm run dockerbuild
+---
-# Production build
-npm run dockerbuild:prod
-```
+## Run the Application
-### 1b. Direct Docker Build Commands (alternative to using NPM scripts)
+### Option 1: Without Docker
-You can use Docker's target feature to specify the build environment instead of using NPM scripts if you wish:
+1. **Install Dependencies**:
+ ```bash
+ pnpm install
+ ```
+ If `pnpm` is not installed, install it using:
+ ```bash
+ sudo npm install -g pnpm
+ ```
-```bash
-# Development build
-docker build . --target bolt-ai-development
+2. **Start the Application**:
+ ```bash
+ pnpm run dev
+ ```
+ This will start the Remix Vite development server. You will need Google Chrome Canary to run this locally if you use Chrome! It's an easy install and a good browser for web development anyway.
-# Production build
-docker build . --target bolt-ai-production
-```
+### Option 2: With Docker
-### 2. Docker Compose with Profiles to Run the Container
+#### Prerequisites
+- Ensure Git, Node.js, and Docker are installed: [Download Docker](https://www.docker.com/)
-Use Docker Compose profiles to manage different environments:
+#### Steps
-```bash
-# Development environment
-docker-compose --profile development up
+1. **Build the Docker Image**:
-# Production environment
-docker-compose --profile production up
-```
+ Use the provided NPM scripts:
+ ```bash
+ npm run dockerbuild
+ ```
+
+ Alternatively, use Docker commands directly:
+ ```bash
+ docker build . --target bolt-ai-development
+ ```
+
+2. **Run the Container**:
+ Use Docker Compose profiles to manage environments:
+ ```bash
+ docker-compose --profile development up
+ ```
-When you run the Docker Compose command with the development profile, any changes you
-make on your machine to the code will automatically be reflected in the site running
-on the container (i.e. hot reloading still applies!).
+ - With the development profile, changes to your code will automatically reflect in the running container (hot reloading).
---
-## Run Without Docker
+### Update Your Local Version to the Latest
-1. Install dependencies using Terminal (or CMD in Windows with admin permissions):
+To keep your local version of bolt.diy up to date with the latest changes, follow these steps for your operating system:
-```
-pnpm install
-```
+#### 1. **Navigate to your project folder**
+ Navigate to the directory where you cloned the repository and open a terminal:
-If you get an error saying "command not found: pnpm" or similar, then that means pnpm isn't installed. You can install it via this:
+#### 2. **Fetch the Latest Changes**
+ Use Git to pull the latest changes from the main repository:
-```
-sudo npm install -g pnpm
-```
+ ```bash
+ git pull origin main
+ ```
-2. Start the application with the command:
+#### 3. **Update Dependencies**
+ After pulling the latest changes, update the project dependencies by running the following command:
-```bash
-pnpm run dev
-```
+ ```bash
+ pnpm install
+ ```
+
+#### 4. **Rebuild and Start the Application**
+
+ - **If using Docker**, ensure you rebuild the Docker image to avoid using a cached version:
+ ```bash
+ docker-compose --profile development up --build
+ ```
+
+ - **If not using Docker**, you can start the application as usual with:
+ ```bash
+ pnpm run dev
+ ```
+
+This ensures that you're running the latest version of bolt.diy and can take advantage of all the newest features and bug fixes.
---
diff --git a/docs/images/api-key-ui-section.png b/docs/images/api-key-ui-section.png
new file mode 100644
index 000000000..4dc4a7b56
Binary files /dev/null and b/docs/images/api-key-ui-section.png differ
diff --git a/docs/images/bolt-settings-button.png b/docs/images/bolt-settings-button.png
new file mode 100644
index 000000000..2a3c55bae
Binary files /dev/null and b/docs/images/bolt-settings-button.png differ
diff --git a/docs/images/provider-base-url.png b/docs/images/provider-base-url.png
new file mode 100644
index 000000000..49e59d48d
Binary files /dev/null and b/docs/images/provider-base-url.png differ
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index efec89864..da73295af 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -11957,4 +11957,4 @@ snapshots:
zod@3.23.8: {}
- zwitch@2.0.4: {}
+ zwitch@2.0.4: {}
\ No newline at end of file
diff --git a/pre-start.cjs b/pre-start.cjs
index e6b7001ef..cd24d93c4 100644
--- a/pre-start.cjs
+++ b/pre-start.cjs
@@ -1,4 +1,18 @@
-const { commit } = require('./app/commit.json');
+const { execSync } =require('child_process');
+
+// Get git hash with fallback
+const getGitHash = () => {
+ try {
+ return execSync('git rev-parse --short HEAD').toString().trim();
+ } catch {
+ return 'no-git-info';
+ }
+};
+
+let commitJson = {
+ hash: JSON.stringify(getGitHash()),
+ version: JSON.stringify(process.env.npm_package_version),
+};
console.log(`
β
ββββββββββββββββββββββββββββββββββββββββ
@@ -6,5 +20,7 @@ console.log(`
β‘οΈ Welcome β‘οΈ
β
ββββββββββββββββββββββββββββββββββββββββ
`);
-console.log('π Current Commit Version:', commit);
-console.log('β
ββββββββββββββββββββββββββββββββββββββββ
');
\ No newline at end of file
+console.log('π Current Version Tag:', `v${commitJson.version}`);
+console.log('π Current Commit Version:', commitJson.hash);
+console.log(' Please wait until the URL appears here');
+console.log('β
ββββββββββββββββββββββββββββββββββββββββ
');
diff --git a/vite.config.ts b/vite.config.ts
index f18b8b934..312230a0a 100644
--- a/vite.config.ts
+++ b/vite.config.ts
@@ -5,8 +5,24 @@ import { nodePolyfills } from 'vite-plugin-node-polyfills';
import { optimizeCssModules } from 'vite-plugin-optimize-css-modules';
import tsconfigPaths from 'vite-tsconfig-paths';
+import { execSync } from 'child_process';
+
+// Get git hash with fallback
+const getGitHash = () => {
+ try {
+ return execSync('git rev-parse --short HEAD').toString().trim();
+ } catch {
+ return 'no-git-info';
+ }
+};
+
+
export default defineConfig((config) => {
return {
+ define: {
+ __COMMIT_HASH: JSON.stringify(getGitHash()),
+ __APP_VERSION: JSON.stringify(process.env.npm_package_version),
+ },
build: {
target: 'esnext',
},
@@ -28,7 +44,7 @@ export default defineConfig((config) => {
chrome129IssuePlugin(),
config.mode === 'production' && optimizeCssModules({ apply: 'build' }),
],
- envPrefix: ["VITE_", "OPENAI_LIKE_API_", "OLLAMA_API_BASE_URL", "LMSTUDIO_API_BASE_URL","TOGETHER_API_BASE_URL"],
+ envPrefix: ["VITE_","OPENAI_LIKE_API_BASE_URL", "OLLAMA_API_BASE_URL", "LMSTUDIO_API_BASE_URL","TOGETHER_API_BASE_URL"],
css: {
preprocessorOptions: {
scss: {
diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts
index ed2afcac3..fb5157da6 100644
--- a/worker-configuration.d.ts
+++ b/worker-configuration.d.ts
@@ -1,4 +1,5 @@
interface Env {
+ DEFAULT_NUM_CTX:Settings;
ANTHROPIC_API_KEY: string;
OPENAI_API_KEY: string;
GROQ_API_KEY: string;