diff --git a/cookbooks/Multi-LLM-Consistency/Multi-LLM Consistency Prompting & LLM-based Quorum_aiconfig.json b/cookbooks/Multi-LLM-Consistency/Multi-LLM Consistency.json similarity index 100% rename from cookbooks/Multi-LLM-Consistency/Multi-LLM Consistency Prompting & LLM-based Quorum_aiconfig.json rename to cookbooks/Multi-LLM-Consistency/Multi-LLM Consistency.json diff --git a/cookbooks/Multi-LLM-Consistency/multi_llm_consistency.json b/cookbooks/Multi-LLM-Consistency/multi_llm_consistency.json index 83826bf5e..5f71f260f 100644 --- a/cookbooks/Multi-LLM-Consistency/multi_llm_consistency.json +++ b/cookbooks/Multi-LLM-Consistency/multi_llm_consistency.json @@ -12,7 +12,7 @@ "presence_penalty": 0, "frequency_penalty": 0 }, - "ChatGPT": { + "gpt-3.5-turbo": { "model": "gpt-3.5-turbo", "top_p": 1, "max_tokens": 4096, @@ -53,7 +53,7 @@ "input": "{{input_question}}", "metadata": { "model": { - "name": "ChatGPT", + "name": "gpt-3.5-turbo", "settings": {} }, "parameters": {}, @@ -65,7 +65,7 @@ "input": "{{ few_shot_prompting_examples }}\n\n{{input_question}}", "metadata": { "model": { - "name": "PaLM", + "name": "models/text-bison-001", "settings": {} }, "parameters": {}, diff --git a/cookbooks/Multi-LLM-Consistency/multi_llm_consistency_prompting.ipynb b/cookbooks/Multi-LLM-Consistency/multi_llm_consistency_prompting.ipynb index f4c82d9e8..65efe34b9 100644 --- a/cookbooks/Multi-LLM-Consistency/multi_llm_consistency_prompting.ipynb +++ b/cookbooks/Multi-LLM-Consistency/multi_llm_consistency_prompting.ipynb @@ -1,66 +1,155 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, "cells": [ { "cell_type": "code", - "source": [ - "# Install libraries\n", - "!pip install openai\n", - "!pip install python-aiconfig" - ], + "execution_count": 1, "metadata": { "id": "RDkePRKCS8rU" }, - "execution_count": null, - "outputs": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: python-aiconfig in /opt/homebrew/lib/python3.11/site-packages (1.1.3)\n", + "Requirement already satisfied: requests==2.30.0 in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (2.30.0)\n", + "Requirement already satisfied: black in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (23.11.0)\n", + "Requirement already satisfied: flake8 in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (6.1.0)\n", + "Requirement already satisfied: pytest in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (7.4.3)\n", + "Requirement already satisfied: pydantic>=2.1 in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (2.5.2)\n", + "Requirement already satisfied: pybars3 in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (0.9.7)\n", + "Requirement already satisfied: google-generativeai in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (0.2.2)\n", + "Requirement already satisfied: openai>=1.0.0 in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (1.3.6)\n", + "Requirement already satisfied: python-dotenv in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (1.0.0)\n", + "Requirement already satisfied: huggingface-hub in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (0.19.4)\n", + "Requirement already satisfied: result in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (0.14.0)\n", + "Requirement already satisfied: nest-asyncio in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (1.5.8)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.11/site-packages (from requests==2.30.0->python-aiconfig) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.11/site-packages (from requests==2.30.0->python-aiconfig) (3.6)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/homebrew/lib/python3.11/site-packages (from requests==2.30.0->python-aiconfig) (2.1.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.11/site-packages (from requests==2.30.0->python-aiconfig) (2023.11.17)\n", + "Requirement already satisfied: anyio<4,>=3.5.0 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (3.7.1)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (1.8.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (0.25.2)\n", + "Requirement already satisfied: sniffio in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (1.3.0)\n", + "Requirement already satisfied: tqdm>4 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (4.66.1)\n", + "Requirement already satisfied: typing-extensions<5,>=4.5 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (4.8.0)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /opt/homebrew/lib/python3.11/site-packages (from pydantic>=2.1->python-aiconfig) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.14.5 in /opt/homebrew/lib/python3.11/site-packages (from pydantic>=2.1->python-aiconfig) (2.14.5)\n", + "Requirement already satisfied: click>=8.0.0 in /opt/homebrew/lib/python3.11/site-packages (from black->python-aiconfig) (8.1.7)\n", + "Requirement already satisfied: mypy-extensions>=0.4.3 in /opt/homebrew/lib/python3.11/site-packages (from black->python-aiconfig) (1.0.0)\n", + "Requirement already satisfied: packaging>=22.0 in /opt/homebrew/lib/python3.11/site-packages (from black->python-aiconfig) (23.2)\n", + "Requirement already satisfied: pathspec>=0.9.0 in /opt/homebrew/lib/python3.11/site-packages (from black->python-aiconfig) (0.11.2)\n", + "Requirement already satisfied: platformdirs>=2 in /opt/homebrew/lib/python3.11/site-packages (from black->python-aiconfig) (4.0.0)\n", + "Requirement already satisfied: mccabe<0.8.0,>=0.7.0 in /opt/homebrew/lib/python3.11/site-packages (from flake8->python-aiconfig) (0.7.0)\n", + "Requirement already satisfied: pycodestyle<2.12.0,>=2.11.0 in /opt/homebrew/lib/python3.11/site-packages (from flake8->python-aiconfig) (2.11.1)\n", + "Requirement already satisfied: pyflakes<3.2.0,>=3.1.0 in /opt/homebrew/lib/python3.11/site-packages (from flake8->python-aiconfig) (3.1.0)\n", + "Requirement already satisfied: google-ai-generativelanguage==0.3.3 in /opt/homebrew/lib/python3.11/site-packages (from google-generativeai->python-aiconfig) (0.3.3)\n", + "Requirement already satisfied: google-auth in /opt/homebrew/lib/python3.11/site-packages (from google-generativeai->python-aiconfig) (2.23.4)\n", + "Requirement already satisfied: google-api-core in /opt/homebrew/lib/python3.11/site-packages (from google-generativeai->python-aiconfig) (2.14.0)\n", + "Requirement already satisfied: protobuf in /opt/homebrew/lib/python3.11/site-packages (from google-generativeai->python-aiconfig) (4.25.1)\n", + "Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.0 in /opt/homebrew/lib/python3.11/site-packages (from google-ai-generativelanguage==0.3.3->google-generativeai->python-aiconfig) (1.22.3)\n", + "Requirement already satisfied: filelock in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub->python-aiconfig) (3.13.1)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub->python-aiconfig) (2023.10.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub->python-aiconfig) (6.0.1)\n", + "Requirement already satisfied: PyMeta3>=0.5.1 in /opt/homebrew/lib/python3.11/site-packages (from pybars3->python-aiconfig) (0.5.1)\n", + "Requirement already satisfied: iniconfig in /opt/homebrew/lib/python3.11/site-packages (from pytest->python-aiconfig) (2.0.0)\n", + "Requirement already satisfied: pluggy<2.0,>=0.12 in /opt/homebrew/lib/python3.11/site-packages (from pytest->python-aiconfig) (1.3.0)\n", + "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/homebrew/lib/python3.11/site-packages (from google-api-core->google-generativeai->python-aiconfig) (1.61.0)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/homebrew/lib/python3.11/site-packages (from google-auth->google-generativeai->python-aiconfig) (5.3.2)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/homebrew/lib/python3.11/site-packages (from google-auth->google-generativeai->python-aiconfig) (0.3.0)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/homebrew/lib/python3.11/site-packages (from google-auth->google-generativeai->python-aiconfig) (4.9)\n", + "Requirement already satisfied: httpcore==1.* in /opt/homebrew/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->python-aiconfig) (1.0.2)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /opt/homebrew/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.0.0->python-aiconfig) (0.14.0)\n", + "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /opt/homebrew/lib/python3.11/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-ai-generativelanguage==0.3.3->google-generativeai->python-aiconfig) (1.59.3)\n", + "Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /opt/homebrew/lib/python3.11/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-ai-generativelanguage==0.3.3->google-generativeai->python-aiconfig) (1.59.3)\n", + "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /opt/homebrew/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth->google-generativeai->python-aiconfig) (0.5.1)\n" + ] + } + ], + "source": [ + "# Install libraries\n", + "!pip install python-aiconfig" + ] }, { "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "nat03JkI9Dw2", + "outputId": "90207ded-030a-4f4d-cbd1-17e1e1e23a50" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_fields.py:149: UserWarning: Field \"model_parsers\" has conflict with protected namespace \"model_\".\n", + "\n", + "You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n", + " warnings.warn(\n" + ] + } + ], "source": [ "#Import libraries\n", "import openai\n", + "import google.generativeai as palm\n", "import os\n", "from aiconfig.Config import AIConfigRuntime\n", "\n", - "#Set the OpenAI key. The following code is specific for Google Colab\n", - "#We recommend using environment variables for setting the open ai key\n", - "from google.colab import userdata\n", - "openai.api_key = userdata.get('OPENAI_API_KEY')" - ], - "metadata": { - "id": "nat03JkI9Dw2" - }, - "execution_count": null, - "outputs": [] + "#Set the OpenAI key\n", + "#We recommend using environment variables for api keys\n", + "try:\n", + " # The following code is specific for Google Colab\n", + " from google.colab import userdata\n", + " openai.api_key = userdata.get('OPENAI_API_KEY')\n", + " palm.configure(api_key=os.getenv(\"PALM_KEY\"))\n", + "except:\n", + " import dotenv\n", + " dotenv.load_dotenv()\n", + "\n", + " # Set API Keys\n", + " openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n", + " palm.configure(api_key=os.getenv(\"PALM_KEY\")) " + ] }, { "cell_type": "code", - "source": [ - "# Load AI Config\n", - "config_file_path = \"multi-llm-consistency.json\"\n", - "config = AIConfigRuntime.load(config_file_path)" - ], + "execution_count": 3, "metadata": { "id": "VDZ4rhDr9E9E" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "# Load AI Config\n", + "config_file_path = \"Multi-LLM Consistency.json\"\n", + "config = AIConfigRuntime.load(config_file_path)" + ] }, { "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 508 + }, + "id": "Z_BqhhxstmBV", + "outputId": "817eecf1-4bf7-48fe-dcbf-fbc088f0f402" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "When you were 6, your sister was half your age, which means she was 3. So she is 3 years younger than you. Now that you're 70, she is 70 - 3 = 67 years old. The answer is 67.The majority across the outputs is 67." + ] + } + ], "source": [ "# Run the AI Config\n", "from aiconfig.default_parsers.parameterized_model_parser import InferenceOptions\n", @@ -68,34 +157,64 @@ "params = {}\n", "inference_options = InferenceOptions()\n", "\n", - "response_completion = await config.run(\"majority-evaluator\", params, inference_options)\n", + "response_completion = await config.run(\"majority-evaluator\", params, inference_options, run_with_dependencies=True)\n", "response = config.get_output_text(\"majority-evaluator\")" - ], - "metadata": { - "id": "Z_BqhhxstmBV" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", - "source": [ - "print(response)" - ], + "execution_count": 5, "metadata": { - "id": "SUDF3Jj_tqw4" + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "SUDF3Jj_tqw4", + "outputId": "103ecfc6-2c88-4874-8291-bb7489145f85" }, - "execution_count": null, - "outputs": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The majority across the outputs is 67.\n" + ] + } + ], + "source": [ + "print(response)" + ] }, { "cell_type": "code", - "source": [], + "execution_count": null, "metadata": { "id": "ZhyBEWGcM6zB" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.5" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}