From 64c097869cd9407a6baeeec09fedcb2e0e340ca9 Mon Sep 17 00:00:00 2001 From: Kostis Gourgoulias Date: Thu, 20 Jun 2024 11:21:10 +0100 Subject: [PATCH 1/5] Use external models with LMSTudio servers --- .../notebooks/Local-Model-With-LMStudio.ipynb | 215 ++++++++++++++++++ textgrad/engine/local_model_openai_api.py | 95 ++++++++ 2 files changed, 310 insertions(+) create mode 100644 examples/notebooks/Local-Model-With-LMStudio.ipynb create mode 100644 textgrad/engine/local_model_openai_api.py diff --git a/examples/notebooks/Local-Model-With-LMStudio.ipynb b/examples/notebooks/Local-Model-With-LMStudio.ipynb new file mode 100644 index 0000000..a554d34 --- /dev/null +++ b/examples/notebooks/Local-Model-With-LMStudio.ipynb @@ -0,0 +1,215 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Use TextGrad with local models and the openai api through LMStudio \n", + "\n", + "First, install [LMStudio](https://lmstudio.ai/). Then, use it to download your favorite local model and start a local server. \n", + "\n", + "For this demo, we can use one of my favorite models, Maxime Labonne's NeuralBeagle." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from openai import OpenAI\n", + "from textgrad.engine.local_model_openai_api import ChatExternalClient\n", + "import textgrad as tg\n", + "\n", + "\n", + "# start a server with lm-studio and point it to the right address; here we use the default address. \n", + "client = OpenAI(base_url=\"http://localhost:1234/v1\", api_key=\"lm-studio\")\n", + "\n", + "completion = client.chat.completions.create(\n", + " model=\"mlabonne/NeuralBeagle14-7B-GGUF\",\n", + " messages=[\n", + " {\"role\": \"system\", \"content\": \"Always answer in rhymes.\"},\n", + " {\"role\": \"user\", \"content\": \"Introduce yourself.\"}\n", + " ],\n", + " temperature=0.1,\n", + " max_tokens=100\n", + ")\n", + "\n", + "\n", + "engine = ChatExternalClient(client=client, model_string='mlabonne/NeuralBeagle14-7B-GGUF')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "That's it! The rest of the examples will use the local model you downloaded. You just need to pass `engine` by hand. " + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "tg.set_backward_engine(engine, override=True)\n", + "\n", + "initial_solution = \"\"\"To solve the equation 3x^2 - 7x + 2 = 0, we use the quadratic formula:\n", + "x = (-b ± √(b^2 - 4ac)) / 2a\n", + "a = 3, b = -7, c = 2\n", + "x = (7 ± √((-7)^2 + 4(3)(2))) / 6\n", + "x = (7 ± √73) / 6\n", + "The solutions are:\n", + "x1 = (7 + √73)\n", + "x2 = (7 - √73)\"\"\"\n", + "\n", + "solution = tg.Variable(initial_solution,\n", + " requires_grad=True,\n", + " role_description=\"solution to the math question\")\n", + "\n", + "loss_system_prompt = tg.Variable(\"\"\"You will evaluate a solution to a math question. \n", + "Do not attempt to solve it yourself, do not give a solution, only identify errors. Be super concise.\"\"\",\n", + " requires_grad=False,\n", + " role_description=\"system prompt\")\n", + " \n", + "loss_fn = tg.TextLoss(loss_system_prompt)\n", + "optimizer = tg.TGD([solution])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The given solution is correct and concise. There are no errors identified in the provided evaluation of the quadratic equation. The solutions x1 = (7 + √73) and x2 = (7 - √73) have been calculated correctly using the quadratic formula.\n" + ] + } + ], + "source": [ + "loss = loss_fn(solution)\n", + "print(loss.value)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "To solve the given quadratic equation 3x^2 - 7x + 2 = 0 using the quadratic formula, we find two solutions:\n", + "\n", + "Solution 1: x = (b ± √(b^2 - 4ac)) / 2a = [(7 ± √(7^2 - 4*3*2))] / 6, which simplifies to:\n", + " x1 = (7 + √73) / 6\n", + "\n", + "Solution 2: Similarly, we calculate the second solution by substituting the negative value of the square root in the equation:\n", + " x2 = (7 - √73) / 6\n", + "\n", + "It is important to note that these solutions have been calculated using the quadratic formula and are accurate for this specific equation. The quadratic formula, given as (x = (-b ± √(b^2 - 4ac)) / 2a, allows us to find the values of x that satisfy the given equation.\n", + "<|eot_id|>\n", + "```\n", + "\n", + "```\n", + "{\n", + " \"gradient\": {\n", + " \"solutions\": [\n", + " {\n", + " \"solution_1\": \"x1 = (7 + √73) / 6\"\n", + " },\n", + " {\n", + " \"solution_2\": \"x2 = (7 - √73) / 6\"\n", + " },\n", + " {\n", + " \"explanation\": \"To solve the given quadratic equation 3x^2 - 7x + 2 = 0 using the quadratic formula, we find two solutions: Using the quadratic formula ((x = (-b ± √(b^2 - 4ac)) / 2a) with a = 3, b = -7, and c = 2, we calculate:\n", + "\n", + "Solution 1: x = (7 + √73) / 6\n", + "\n", + "Solution 2: Similarly, we calculate the second solution by substituting the negative value of the square root in the equation: x = (7 - √73) / 6\n", + "\n", + "It is important to note that these solutions have been calculated using the quadratic formula and are accurate for this specific equation. The quadratic formula allows us to find the values of x that satisfy the given equation.\"\n", + " }\n", + " ]\n", + " }\n", + "}\n", + "```\n", + "```\n", + "\n", + "<|end_text|>\n", + "```\n", + "\n", + "<|eot_id|>\n", + "```\n", + "{\n", + " \"gradient\": {\n", + " \"solutions\": [\n", + " {\n", + " \"solution_1\": \"x1 = (7 + √73) / 6\"\n", + " },\n", + " {\n", + " \"solution_2\": \"x2 = (7 - √73) / 6\"\n", + " },\n", + " {\n", + " \"explanation\": \"To solve the given quadratic equation 3x^2 - 7x + 2 = 0 using the quadratic formula, we find two solutions: Using the quadratic formula ((x = (-b ± √(b^2 - 4ac)) / 2a) with a = 3, b = -7, and c = 2, we calculate:\n", + "\n", + "Solution 1: x = (7 + √73) / 6\n", + "\n", + "Solution 2: Similarly, we calculate the second solution by substituting the negative value of the square root in the equation: x = (7 - √73) / 6\n", + "\n", + "It is important to note that these solutions have been calculated using the quadratic formula and are accurate for this specific equation. The quadratic formula allows us to find the values of x that satisfy the given equation.\"\n", + " }\n", + " ]\n", + " }\n", + "}\n", + "```\n", + "```\n", + "\n", + "<|end_text|>\n", + "```\n", + "\n", + "The improved variable provides a clearer explanation of how the solutions were calculated using the quadratic formula, along with the specific values for each solution. This enhanced presentation should help readers better understand and follow the process of solving the given equation.\n" + ] + } + ], + "source": [ + "# this will take a while on CPU. :( \n", + "loss.backward()\n", + "optimizer.step()\n", + "print(solution.value)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Enjoying passing text gradients through your favorite local models!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "textgrad", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/textgrad/engine/local_model_openai_api.py b/textgrad/engine/local_model_openai_api.py new file mode 100644 index 0000000..bad45a9 --- /dev/null +++ b/textgrad/engine/local_model_openai_api.py @@ -0,0 +1,95 @@ +import os +import platformdirs +from tenacity import ( + retry, + stop_after_attempt, + wait_random_exponential, +) +import json +from .base import EngineLM, CachedEngine + + +class ChatExternalClient(EngineLM, CachedEngine): + """ + This is the same as engine.openai.ChatOpenAI, but we pass the + client explicitly to the constructor. + """ + + DEFAULT_SYSTEM_PROMPT = "You are a helpful, creative, and smart assistant." + client = None + + def __init__( + self, + client, + model_string, + system_prompt=DEFAULT_SYSTEM_PROMPT, + **kwargs, + ): + """ + :param client: + :param model_string: + :param system_prompt: + """ + self.client = client + root = platformdirs.user_cache_dir("textgrad") + cache_path = os.path.join(root, f"cache_local_model_{model_string}.db") + + super().__init__(cache_path=cache_path) + + self.system_prompt = system_prompt + self.model_string = model_string + + def generate( + self, prompt, system_prompt=None, temperature=0, max_tokens=2000, top_p=0.99 + ): + sys_prompt_arg = system_prompt if system_prompt else self.system_prompt + + cache_or_none = self._check_cache(sys_prompt_arg + prompt) + if cache_or_none is not None: + return cache_or_none + + response = self.client.chat.completions.create( + model=self.model_string, + messages=[ + {"role": "system", "content": sys_prompt_arg}, + {"role": "user", "content": prompt}, + ], + frequency_penalty=0, + presence_penalty=0, + stop=None, + temperature=temperature, + max_tokens=max_tokens, + top_p=top_p, + ) + + response = response.choices[0].message.content + self._save_cache(sys_prompt_arg + prompt, response) + return response + + def generate_with_messages( + self, messages, temperature=0, max_tokens=2000, top_p=0.99 + ): + prompt = json.dumps(messages) + + cache_or_none = self._check_cache(prompt) + if cache_or_none is not None: + return cache_or_none + + response = self.client.chat.completions.create( + model=self.model_string, + messages=messages, + frequency_penalty=0, + presence_penalty=0, + stop=None, + temperature=temperature, + max_tokens=max_tokens, + top_p=top_p, + ) + + response = response.choices[0].message.content + self._save_cache(prompt, response) + return response + + @retry(wait=wait_random_exponential(min=1, max=5), stop=stop_after_attempt(5)) + def __call__(self, prompt, **kwargs): + return self.generate(prompt, **kwargs) From fede26a49ea58d15bfc49bdddbb6bbacca5145bf Mon Sep 17 00:00:00 2001 From: Kostis Gourgoulias Date: Thu, 20 Jun 2024 11:29:00 +0100 Subject: [PATCH 2/5] typo --- examples/notebooks/Local-Model-With-LMStudio.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/notebooks/Local-Model-With-LMStudio.ipynb b/examples/notebooks/Local-Model-With-LMStudio.ipynb index a554d34..5d9e51f 100644 --- a/examples/notebooks/Local-Model-With-LMStudio.ipynb +++ b/examples/notebooks/Local-Model-With-LMStudio.ipynb @@ -187,7 +187,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Enjoying passing text gradients through your favorite local models!" + "Enjoy passing text gradients through your favorite local models!" ] } ], From 66e5db218dffd57fb085fde30f78d631f438685f Mon Sep 17 00:00:00 2001 From: Kostis Gourgoulias Date: Thu, 20 Jun 2024 11:56:17 +0100 Subject: [PATCH 3/5] Remove completion part --- examples/notebooks/Local-Model-With-LMStudio.ipynb | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/examples/notebooks/Local-Model-With-LMStudio.ipynb b/examples/notebooks/Local-Model-With-LMStudio.ipynb index 5d9e51f..977f148 100644 --- a/examples/notebooks/Local-Model-With-LMStudio.ipynb +++ b/examples/notebooks/Local-Model-With-LMStudio.ipynb @@ -25,17 +25,6 @@ "# start a server with lm-studio and point it to the right address; here we use the default address. \n", "client = OpenAI(base_url=\"http://localhost:1234/v1\", api_key=\"lm-studio\")\n", "\n", - "completion = client.chat.completions.create(\n", - " model=\"mlabonne/NeuralBeagle14-7B-GGUF\",\n", - " messages=[\n", - " {\"role\": \"system\", \"content\": \"Always answer in rhymes.\"},\n", - " {\"role\": \"user\", \"content\": \"Introduce yourself.\"}\n", - " ],\n", - " temperature=0.1,\n", - " max_tokens=100\n", - ")\n", - "\n", - "\n", "engine = ChatExternalClient(client=client, model_string='mlabonne/NeuralBeagle14-7B-GGUF')" ] }, From 2d579ee60efc5d242f868e9c24325112bf1cf645 Mon Sep 17 00:00:00 2001 From: Kostis Gourgoulias Date: Thu, 20 Jun 2024 16:05:46 +0100 Subject: [PATCH 4/5] clean implementation --- textgrad/engine/local_model_openai_api.py | 80 ++++------------------- 1 file changed, 11 insertions(+), 69 deletions(-) diff --git a/textgrad/engine/local_model_openai_api.py b/textgrad/engine/local_model_openai_api.py index bad45a9..8e7f8d2 100644 --- a/textgrad/engine/local_model_openai_api.py +++ b/textgrad/engine/local_model_openai_api.py @@ -1,15 +1,11 @@ import os -import platformdirs -from tenacity import ( - retry, - stop_after_attempt, - wait_random_exponential, -) -import json -from .base import EngineLM, CachedEngine +import logging +from .openai import ChatOpenAI +logger = logging.getLogger(__name__) -class ChatExternalClient(EngineLM, CachedEngine): + +class ChatExternalClient(ChatOpenAI): """ This is the same as engine.openai.ChatOpenAI, but we pass the client explicitly to the constructor. @@ -30,66 +26,12 @@ def __init__( :param model_string: :param system_prompt: """ - self.client = client - root = platformdirs.user_cache_dir("textgrad") - cache_path = os.path.join(root, f"cache_local_model_{model_string}.db") - - super().__init__(cache_path=cache_path) - - self.system_prompt = system_prompt - self.model_string = model_string - def generate( - self, prompt, system_prompt=None, temperature=0, max_tokens=2000, top_p=0.99 - ): - sys_prompt_arg = system_prompt if system_prompt else self.system_prompt - - cache_or_none = self._check_cache(sys_prompt_arg + prompt) - if cache_or_none is not None: - return cache_or_none + if os.getenv("OPENAI_API_KEY") is None: + logger.warning("OPENAI_API_KEY not set. Setting it from client.") + os.environ["OPENAI_API_KEY"] = client.api_key - response = self.client.chat.completions.create( - model=self.model_string, - messages=[ - {"role": "system", "content": sys_prompt_arg}, - {"role": "user", "content": prompt}, - ], - frequency_penalty=0, - presence_penalty=0, - stop=None, - temperature=temperature, - max_tokens=max_tokens, - top_p=top_p, + super().__init__( + model_string=model_string, system_prompt=system_prompt, **kwargs ) - - response = response.choices[0].message.content - self._save_cache(sys_prompt_arg + prompt, response) - return response - - def generate_with_messages( - self, messages, temperature=0, max_tokens=2000, top_p=0.99 - ): - prompt = json.dumps(messages) - - cache_or_none = self._check_cache(prompt) - if cache_or_none is not None: - return cache_or_none - - response = self.client.chat.completions.create( - model=self.model_string, - messages=messages, - frequency_penalty=0, - presence_penalty=0, - stop=None, - temperature=temperature, - max_tokens=max_tokens, - top_p=top_p, - ) - - response = response.choices[0].message.content - self._save_cache(prompt, response) - return response - - @retry(wait=wait_random_exponential(min=1, max=5), stop=stop_after_attempt(5)) - def __call__(self, prompt, **kwargs): - return self.generate(prompt, **kwargs) + self.client = client From a7d37ce3ba5da48fb574c79e154dd810870892dc Mon Sep 17 00:00:00 2001 From: Kostis Gourgoulias Date: Fri, 21 Jun 2024 10:07:48 +0100 Subject: [PATCH 5/5] more docs --- textgrad/engine/local_model_openai_api.py | 28 ++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/textgrad/engine/local_model_openai_api.py b/textgrad/engine/local_model_openai_api.py index 8e7f8d2..a73b4df 100644 --- a/textgrad/engine/local_model_openai_api.py +++ b/textgrad/engine/local_model_openai_api.py @@ -1,5 +1,6 @@ import os import logging +from openai import OpenAI from .openai import ChatOpenAI logger = logging.getLogger(__name__) @@ -7,8 +8,7 @@ class ChatExternalClient(ChatOpenAI): """ - This is the same as engine.openai.ChatOpenAI, but we pass the - client explicitly to the constructor. + This is the same as engine.openai.ChatOpenAI, but we pass in an external OpenAI client. """ DEFAULT_SYSTEM_PROMPT = "You are a helpful, creative, and smart assistant." @@ -16,15 +16,27 @@ class ChatExternalClient(ChatOpenAI): def __init__( self, - client, - model_string, - system_prompt=DEFAULT_SYSTEM_PROMPT, + client: OpenAI, + model_string: str, + system_prompt: str = DEFAULT_SYSTEM_PROMPT, **kwargs, ): """ - :param client: - :param model_string: - :param system_prompt: + :param client: an OpenAI client object. + :param model_string: the model name, used for the cache file name and chat completion requests. + :param system_prompt: the system prompt to use in chat completions. + + Example usage with lm-studio local server, but any client that follows the OpenAI API will work. + + ```python + from openai import OpenAI + from textgrad.engine.local_model_openai_api import ChatExternalClient + + client = OpenAI(base_url="http://localhost:1234/v1", api_key="lm-studio") + engine = ChatExternalClient(client=client, model_string="your-model-name") + print(engine.generate(max_tokens=40, prompt="What is the meaning of life?")) + ``` + """ if os.getenv("OPENAI_API_KEY") is None: