Skip to content

Commit

Permalink
[cookbook] multi llm consistency update
Browse files Browse the repository at this point in the history 
- update aiconfig json, repulled it from the LastMile workbook linked in the readme

- update notebook with run with dependencies

- use PaLM api key
  • Loading branch information
Ankush Pala [email protected] committed Nov 30, 2023
1 parent 8f97d73 commit 2574e66
Show file tree
Hide file tree
Showing 3 changed files with 178 additions and 59 deletions.
0 ...rompting & LLM-based Quorum_aiconfig.json → ...LM-Consistency/Multi-LLM Consistency.json
View file
File renamed without changes.
6 changes: 3 additions & 3 deletions cookbooks/Multi-LLM-Consistency/multi_llm_consistency.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"presence_penalty": 0,
"frequency_penalty": 0
},
"ChatGPT": {
"gpt-3.5-turbo": {
"model": "gpt-3.5-turbo",
"top_p": 1,
"max_tokens": 4096,
Expand Down Expand Up @@ -53,7 +53,7 @@
"input": "{{input_question}}",
"metadata": {
"model": {
"name": "ChatGPT",
"name": "gpt-3.5-turbo",
"settings": {}
},
"parameters": {},
Expand All @@ -65,7 +65,7 @@
"input": "{{ few_shot_prompting_examples }}\n\n{{input_question}}",
"metadata": {
"model": {
"name": "PaLM",
"name": "models/text-bison-001",
"settings": {}
},
"parameters": {},
Expand Down
231 changes: 175 additions & 56 deletions cookbooks/Multi-LLM-Consistency/multi_llm_consistency_prompting.ipynb
View file
Original file line number Diff line number Diff line change
@@ -1,101 +1,220 @@
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "code",
"source": [
"# Install libraries\n",
"!pip install openai\n",
"!pip install python-aiconfig"
],
"execution_count": 1,
"metadata": {
"id": "RDkePRKCS8rU"
},
"execution_count": null,
"outputs": []
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Requirement already satisfied: python-aiconfig in /opt/homebrew/lib/python3.11/site-packages (1.1.3)\n",
"Requirement already satisfied: requests==2.30.0 in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (2.30.0)\n",
"Requirement already satisfied: black in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (23.11.0)\n",
"Requirement already satisfied: flake8 in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (6.1.0)\n",
"Requirement already satisfied: pytest in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (7.4.3)\n",
"Requirement already satisfied: pydantic>=2.1 in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (2.5.2)\n",
"Requirement already satisfied: pybars3 in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (0.9.7)\n",
"Requirement already satisfied: google-generativeai in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (0.2.2)\n",
"Requirement already satisfied: openai>=1.0.0 in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (1.3.6)\n",
"Requirement already satisfied: python-dotenv in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (1.0.0)\n",
"Requirement already satisfied: huggingface-hub in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (0.19.4)\n",
"Requirement already satisfied: result in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (0.14.0)\n",
"Requirement already satisfied: nest-asyncio in /opt/homebrew/lib/python3.11/site-packages (from python-aiconfig) (1.5.8)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.11/site-packages (from requests==2.30.0->python-aiconfig) (3.3.2)\n",
"Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.11/site-packages (from requests==2.30.0->python-aiconfig) (3.6)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/homebrew/lib/python3.11/site-packages (from requests==2.30.0->python-aiconfig) (2.1.0)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.11/site-packages (from requests==2.30.0->python-aiconfig) (2023.11.17)\n",
"Requirement already satisfied: anyio<4,>=3.5.0 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (3.7.1)\n",
"Requirement already satisfied: distro<2,>=1.7.0 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (1.8.0)\n",
"Requirement already satisfied: httpx<1,>=0.23.0 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (0.25.2)\n",
"Requirement already satisfied: sniffio in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (1.3.0)\n",
"Requirement already satisfied: tqdm>4 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (4.66.1)\n",
"Requirement already satisfied: typing-extensions<5,>=4.5 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->python-aiconfig) (4.8.0)\n",
"Requirement already satisfied: annotated-types>=0.4.0 in /opt/homebrew/lib/python3.11/site-packages (from pydantic>=2.1->python-aiconfig) (0.6.0)\n",
"Requirement already satisfied: pydantic-core==2.14.5 in /opt/homebrew/lib/python3.11/site-packages (from pydantic>=2.1->python-aiconfig) (2.14.5)\n",
"Requirement already satisfied: click>=8.0.0 in /opt/homebrew/lib/python3.11/site-packages (from black->python-aiconfig) (8.1.7)\n",
"Requirement already satisfied: mypy-extensions>=0.4.3 in /opt/homebrew/lib/python3.11/site-packages (from black->python-aiconfig) (1.0.0)\n",
"Requirement already satisfied: packaging>=22.0 in /opt/homebrew/lib/python3.11/site-packages (from black->python-aiconfig) (23.2)\n",
"Requirement already satisfied: pathspec>=0.9.0 in /opt/homebrew/lib/python3.11/site-packages (from black->python-aiconfig) (0.11.2)\n",
"Requirement already satisfied: platformdirs>=2 in /opt/homebrew/lib/python3.11/site-packages (from black->python-aiconfig) (4.0.0)\n",
"Requirement already satisfied: mccabe<0.8.0,>=0.7.0 in /opt/homebrew/lib/python3.11/site-packages (from flake8->python-aiconfig) (0.7.0)\n",
"Requirement already satisfied: pycodestyle<2.12.0,>=2.11.0 in /opt/homebrew/lib/python3.11/site-packages (from flake8->python-aiconfig) (2.11.1)\n",
"Requirement already satisfied: pyflakes<3.2.0,>=3.1.0 in /opt/homebrew/lib/python3.11/site-packages (from flake8->python-aiconfig) (3.1.0)\n",
"Requirement already satisfied: google-ai-generativelanguage==0.3.3 in /opt/homebrew/lib/python3.11/site-packages (from google-generativeai->python-aiconfig) (0.3.3)\n",
"Requirement already satisfied: google-auth in /opt/homebrew/lib/python3.11/site-packages (from google-generativeai->python-aiconfig) (2.23.4)\n",
"Requirement already satisfied: google-api-core in /opt/homebrew/lib/python3.11/site-packages (from google-generativeai->python-aiconfig) (2.14.0)\n",
"Requirement already satisfied: protobuf in /opt/homebrew/lib/python3.11/site-packages (from google-generativeai->python-aiconfig) (4.25.1)\n",
"Requirement already satisfied: proto-plus<2.0.0dev,>=1.22.0 in /opt/homebrew/lib/python3.11/site-packages (from google-ai-generativelanguage==0.3.3->google-generativeai->python-aiconfig) (1.22.3)\n",
"Requirement already satisfied: filelock in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub->python-aiconfig) (3.13.1)\n",
"Requirement already satisfied: fsspec>=2023.5.0 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub->python-aiconfig) (2023.10.0)\n",
"Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub->python-aiconfig) (6.0.1)\n",
"Requirement already satisfied: PyMeta3>=0.5.1 in /opt/homebrew/lib/python3.11/site-packages (from pybars3->python-aiconfig) (0.5.1)\n",
"Requirement already satisfied: iniconfig in /opt/homebrew/lib/python3.11/site-packages (from pytest->python-aiconfig) (2.0.0)\n",
"Requirement already satisfied: pluggy<2.0,>=0.12 in /opt/homebrew/lib/python3.11/site-packages (from pytest->python-aiconfig) (1.3.0)\n",
"Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /opt/homebrew/lib/python3.11/site-packages (from google-api-core->google-generativeai->python-aiconfig) (1.61.0)\n",
"Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/homebrew/lib/python3.11/site-packages (from google-auth->google-generativeai->python-aiconfig) (5.3.2)\n",
"Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/homebrew/lib/python3.11/site-packages (from google-auth->google-generativeai->python-aiconfig) (0.3.0)\n",
"Requirement already satisfied: rsa<5,>=3.1.4 in /opt/homebrew/lib/python3.11/site-packages (from google-auth->google-generativeai->python-aiconfig) (4.9)\n",
"Requirement already satisfied: httpcore==1.* in /opt/homebrew/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->python-aiconfig) (1.0.2)\n",
"Requirement already satisfied: h11<0.15,>=0.13 in /opt/homebrew/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.0.0->python-aiconfig) (0.14.0)\n",
"Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /opt/homebrew/lib/python3.11/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-ai-generativelanguage==0.3.3->google-generativeai->python-aiconfig) (1.59.3)\n",
"Requirement already satisfied: grpcio-status<2.0.dev0,>=1.33.2 in /opt/homebrew/lib/python3.11/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.10.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,<3.0.0dev,>=1.34.0->google-ai-generativelanguage==0.3.3->google-generativeai->python-aiconfig) (1.59.3)\n",
"Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /opt/homebrew/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth->google-generativeai->python-aiconfig) (0.5.1)\n"
]
}
],
"source": [
"# Install libraries\n",
"!pip install python-aiconfig"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "nat03JkI9Dw2",
"outputId": "90207ded-030a-4f4d-cbd1-17e1e1e23a50"
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/opt/homebrew/lib/python3.11/site-packages/pydantic/_internal/_fields.py:149: UserWarning: Field \"model_parsers\" has conflict with protected namespace \"model_\".\n",
"\n",
"You may be able to resolve this warning by setting `model_config['protected_namespaces'] = ()`.\n",
" warnings.warn(\n"
]
}
],
"source": [
"#Import libraries\n",
"import openai\n",
"import google.generativeai as palm\n",
"import os\n",
"from aiconfig.Config import AIConfigRuntime\n",
"\n",
"#Set the OpenAI key. The following code is specific for Google Colab\n",
"#We recommend using environment variables for setting the open ai key\n",
"from google.colab import userdata\n",
"openai.api_key = userdata.get('OPENAI_API_KEY')"
],
"metadata": {
"id": "nat03JkI9Dw2"
},
"execution_count": null,
"outputs": []
"#Set the OpenAI key\n",
"#We recommend using environment variables for api keys\n",
"try:\n",
" # The following code is specific for Google Colab\n",
" from google.colab import userdata\n",
" openai.api_key = userdata.get('OPENAI_API_KEY')\n",
" palm.configure(api_key=os.getenv(\"PALM_KEY\"))\n",
"except:\n",
" import dotenv\n",
" dotenv.load_dotenv()\n",
"\n",
" # Set API Keys\n",
" openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
" palm.configure(api_key=os.getenv(\"PALM_KEY\")) "
]
},
{
"cell_type": "code",
"source": [
"# Load AI Config\n",
"config_file_path = \"multi-llm-consistency.json\"\n",
"config = AIConfigRuntime.load(config_file_path)"
],
"execution_count": 3,
"metadata": {
"id": "VDZ4rhDr9E9E"
},
"execution_count": null,
"outputs": []
"outputs": [],
"source": [
"# Load AI Config\n",
"config_file_path = \"Multi-LLM Consistency.json\"\n",
"config = AIConfigRuntime.load(config_file_path)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 508
},
"id": "Z_BqhhxstmBV",
"outputId": "817eecf1-4bf7-48fe-dcbf-fbc088f0f402"
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"When you were 6, your sister was half your age, which means she was 3. So she is 3 years younger than you. Now that you're 70, she is 70 - 3 = 67 years old. The answer is 67.The majority across the outputs is 67."
]
}
],
"source": [
"# Run the AI Config\n",
"from aiconfig.default_parsers.parameterized_model_parser import InferenceOptions\n",
"\n",
"params = {}\n",
"inference_options = InferenceOptions()\n",
"\n",
"response_completion = await config.run(\"majority-evaluator\", params, inference_options)\n",
"response_completion = await config.run(\"majority-evaluator\", params, inference_options, run_with_dependencies=True)\n",
"response = config.get_output_text(\"majority-evaluator\")"
],
"metadata": {
"id": "Z_BqhhxstmBV"
},
"execution_count": null,
"outputs": []
]
},
{
"cell_type": "code",
"source": [
"print(response)"
],
"execution_count": 5,
"metadata": {
"id": "SUDF3Jj_tqw4"
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "SUDF3Jj_tqw4",
"outputId": "103ecfc6-2c88-4874-8291-bb7489145f85"
},
"execution_count": null,
"outputs": []
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The majority across the outputs is 67.\n"
]
}
],
"source": [
"print(response)"
]
},
{
"cell_type": "code",
"source": [],
"execution_count": null,
"metadata": {
"id": "ZhyBEWGcM6zB"
},
"execution_count": null,
"outputs": []
"outputs": [],
"source": []
}
]
}
],
"metadata": {
"colab": {
"provenance": []
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.5"
}
},
"nbformat": 4,
"nbformat_minor": 0
}

0 comments on commit 2574e66

Please sign in to comment.