diff --git a/docs/docs/reference/gen_notebooks/feedback_prod.md b/docs/docs/reference/gen_notebooks/feedback_prod.md index 9a55d833fe9..3b3ce2d29eb 100644 --- a/docs/docs/reference/gen_notebooks/feedback_prod.md +++ b/docs/docs/reference/gen_notebooks/feedback_prod.md @@ -18,22 +18,24 @@ title: Log Feedback from Production It is often hard to automatically evaluate a generated LLM response so, depending on your risk tolerance, you can gather direct user feedback to find areas to improve. -In this tutorial, we'll use a custom RAG chatbot as an example app with which the users can interact and which allows us to collect user feedback. +In this tutorial, we'll use a custom chatbot as an example app from which to collect user feedback. We'll use Streamlit to build the interface and we'll capture the LLM interactions and feedback in Weave. ## Setup ```python -!pip install weave openai streamlit +!pip install weave openai streamlit wandb +!pip install set-env-colab-kaggle-dotenv -q # for env var ``` -First, create a file called `secrets.toml` and add an OpenAI key so it works with [st.secrets](https://docs.streamlit.io/develop/api-reference/connections/st.secrets). You can [sign up](https://platform.openai.com/signup) on the OpenAI platform to get your own API key. - ```python -# secrets.toml -OPENAI_API_KEY = "your OpenAI key" +# Add a .env file with your OpenAI and WandB API keys +from set_env import set_env + +_ = set_env("OPENAI_API_KEY") +_ = set_env("WANDB_API_KEY") ``` Next, create a file called `chatbot.py` with the following contents: @@ -43,17 +45,28 @@ Next, create a file called `chatbot.py` with the following contents: # chatbot.py import streamlit as st +import wandb from openai import OpenAI +from set_env import set_env import weave -st.title("Add feedback") +_ = set_env("OPENAI_API_KEY") +_ = set_env("WANDB_API_KEY") + +# highlight-next-line +wandb.login() + +# highlight-next-line +weave_client = weave.init("feedback-example") + +oai_client = OpenAI() # highlight-next-line @weave.op def chat_response(prompt): - stream = client.chat.completions.create( + stream = oai_client.chat.completions.create( model="gpt-4o", messages=[ {"role": "user", "content": prompt}, @@ -68,60 +81,103 @@ def chat_response(prompt): return {"response": response} -client = OpenAI(api_key=st.secrets["OPENAI_API_KEY"]) -# highlight-next-line -weave_client = weave.init("feedback-example") - - def display_chat_messages(): - for message in st.session_state.messages: + for idx, message in enumerate(st.session_state.messages): with st.chat_message(message["role"]): st.markdown(message["content"]) + # Only show feedback options for assistant messages + if message["role"] == "assistant": + # Get index of this call in the session state + call_idx = ( + sum( + m["role"] == "assistant" + for m in st.session_state.messages[: idx + 1] + ) + - 1 + ) -def get_and_process_prompt(): + # Create a container for feedback options + feedback_container = st.container() + with feedback_container: + col1, col2, col3 = st.columns([1, 1, 4]) + + # Thumbs up button + with col1: + # highlight-next-line + if st.button("👍", key=f"thumbs_up_{idx}"): + if "calls" in st.session_state and call_idx < len( + st.session_state.calls + ): + # highlight-next-line + st.session_state.calls[call_idx].feedback.add_reaction( + "👍" + ) + st.success("Thanks for the feedback!") + + # Thumbs down button + with col2: + # highlight-next-line + if st.button("👎", key=f"thumbs_down_{idx}"): + if "calls" in st.session_state and call_idx < len( + st.session_state.calls + ): + # highlight-next-line + st.session_state.calls[call_idx].feedback.add_reaction( + "👎" + ) + st.success("Thanks for the feedback!") + + # Text feedback + with col3: + feedback_text = st.text_input( + "Feedback", key=f"feedback_input_{idx}" + ) + if st.button("Submit Feedback", key=f"submit_feedback_{idx}"): + if feedback_text and call_idx < len(st.session_state.calls): + # highlight-next-line + st.session_state.calls[call_idx].feedback.add_note( + feedback_text + ) + st.success("Feedback submitted!") + + +def show_chat_prompt(): if prompt := st.chat_input("What is up?"): st.session_state.messages.append({"role": "user", "content": prompt}) - with st.chat_message("user"): - st.markdown(prompt) - with st.chat_message("assistant"): - # highlight-next-line with weave.attributes( {"session": st.session_state["session_id"], "env": "prod"} ): - # This could also be weave model.predict.call if you're using a weave.Model subclass - result, call = chat_response.call( - prompt - ) # call the function with `.call`, this returns a tuple with a new Call object - # highlight-next-line - st.button( - ":thumbsup:", - on_click=lambda: call.feedback.add_reaction("👍"), - key="up", - ) - # highlight-next-line - st.button( - ":thumbsdown:", - on_click=lambda: call.feedback.add_reaction("👎"), - key="down", - ) + result, call = chat_response.call(prompt) + st.write(result["response"]) st.session_state.messages.append( {"role": "assistant", "content": result["response"]} ) + # highlight-next-line + st.session_state.calls.append(call) + +def init_session_state(): + if "session_id" not in st.session_state: + st.session_state["session_id"] = "123abc" -def init_chat_history(): if "messages" not in st.session_state: - st.session_state.messages = st.session_state.messages = [] + st.session_state.messages = [] + + if "calls" not in st.session_state: + st.session_state.calls = [] def main(): - st.session_state["session_id"] = "123abc" - init_chat_history() + st.title("Add feedback") + + init_session_state() display_chat_messages() - get_and_process_prompt() + show_chat_prompt() + + st.rerun() if __name__ == "__main__": diff --git a/docs/notebooks/feedback_prod.ipynb b/docs/notebooks/feedback_prod.ipynb index fad44ddd310..fa48fc8ea70 100644 --- a/docs/notebooks/feedback_prod.ipynb +++ b/docs/notebooks/feedback_prod.ipynb @@ -10,7 +10,6 @@ "---\n", "docusaurus_head_meta::end -->\n", "\n", - "" ] }, @@ -25,7 +24,7 @@ "\n", "It is often hard to automatically evaluate a generated LLM response so, depending on your risk tolerance, you can gather direct user feedback to find areas to improve.\n", "\n", - "In this tutorial, we'll use a custom RAG chatbot as an example app with which the users can interact and which allows us to collect user feedback.\n", + "In this tutorial, we'll use a custom chatbot as an example app from which to collect user feedback.\n", "We'll use Streamlit to build the interface and we'll capture the LLM interactions and feedback in Weave." ] }, @@ -46,24 +45,21 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install weave openai streamlit" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, create a file called `secrets.toml` and add an OpenAI key so it works with [st.secrets](https://docs.streamlit.io/develop/api-reference/connections/st.secrets). You can [sign up](https://platform.openai.com/signup) on the OpenAI platform to get your own API key. " + "!pip install weave openai streamlit wandb\n", + "!pip install set-env-colab-kaggle-dotenv -q # for env var" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "# secrets.toml\n", - "OPENAI_API_KEY = \"your OpenAI key\"" + "# Add a .env file with your OpenAI and WandB API keys\n", + "from set_env import set_env\n", + "\n", + "_ = set_env(\"OPENAI_API_KEY\")\n", + "_ = set_env(\"WANDB_API_KEY\")" ] }, { @@ -82,17 +78,28 @@ "# chatbot.py\n", "\n", "import streamlit as st\n", + "import wandb\n", "from openai import OpenAI\n", + "from set_env import set_env\n", "\n", "import weave\n", "\n", - "st.title(\"Add feedback\")\n", + "_ = set_env(\"OPENAI_API_KEY\")\n", + "_ = set_env(\"WANDB_API_KEY\")\n", + "\n", + "# highlight-next-line\n", + "wandb.login()\n", + "\n", + "# highlight-next-line\n", + "weave_client = weave.init(\"feedback-example\")\n", + "\n", + "oai_client = OpenAI()\n", "\n", "\n", "# highlight-next-line\n", "@weave.op\n", "def chat_response(prompt):\n", - " stream = client.chat.completions.create(\n", + " stream = oai_client.chat.completions.create(\n", " model=\"gpt-4o\",\n", " messages=[\n", " {\"role\": \"user\", \"content\": prompt},\n", @@ -107,60 +114,103 @@ " return {\"response\": response}\n", "\n", "\n", - "client = OpenAI(api_key=st.secrets[\"OPENAI_API_KEY\"])\n", - "# highlight-next-line\n", - "weave_client = weave.init(\"feedback-example\")\n", - "\n", - "\n", "def display_chat_messages():\n", - " for message in st.session_state.messages:\n", + " for idx, message in enumerate(st.session_state.messages):\n", " with st.chat_message(message[\"role\"]):\n", " st.markdown(message[\"content\"])\n", "\n", + " # Only show feedback options for assistant messages\n", + " if message[\"role\"] == \"assistant\":\n", + " # Get index of this call in the session state\n", + " call_idx = (\n", + " sum(\n", + " m[\"role\"] == \"assistant\"\n", + " for m in st.session_state.messages[: idx + 1]\n", + " )\n", + " - 1\n", + " )\n", "\n", - "def get_and_process_prompt():\n", + " # Create a container for feedback options\n", + " feedback_container = st.container()\n", + " with feedback_container:\n", + " col1, col2, col3 = st.columns([1, 1, 4])\n", + "\n", + " # Thumbs up button\n", + " with col1:\n", + " # highlight-next-line\n", + " if st.button(\"👍\", key=f\"thumbs_up_{idx}\"):\n", + " if \"calls\" in st.session_state and call_idx < len(\n", + " st.session_state.calls\n", + " ):\n", + " # highlight-next-line\n", + " st.session_state.calls[call_idx].feedback.add_reaction(\n", + " \"👍\"\n", + " )\n", + " st.success(\"Thanks for the feedback!\")\n", + "\n", + " # Thumbs down button\n", + " with col2:\n", + " # highlight-next-line\n", + " if st.button(\"👎\", key=f\"thumbs_down_{idx}\"):\n", + " if \"calls\" in st.session_state and call_idx < len(\n", + " st.session_state.calls\n", + " ):\n", + " # highlight-next-line\n", + " st.session_state.calls[call_idx].feedback.add_reaction(\n", + " \"👎\"\n", + " )\n", + " st.success(\"Thanks for the feedback!\")\n", + "\n", + " # Text feedback\n", + " with col3:\n", + " feedback_text = st.text_input(\n", + " \"Feedback\", key=f\"feedback_input_{idx}\"\n", + " )\n", + " if st.button(\"Submit Feedback\", key=f\"submit_feedback_{idx}\"):\n", + " if feedback_text and call_idx < len(st.session_state.calls):\n", + " # highlight-next-line\n", + " st.session_state.calls[call_idx].feedback.add_note(\n", + " feedback_text\n", + " )\n", + " st.success(\"Feedback submitted!\")\n", + "\n", + "\n", + "def show_chat_prompt():\n", " if prompt := st.chat_input(\"What is up?\"):\n", " st.session_state.messages.append({\"role\": \"user\", \"content\": prompt})\n", "\n", - " with st.chat_message(\"user\"):\n", - " st.markdown(prompt)\n", - "\n", " with st.chat_message(\"assistant\"):\n", - " # highlight-next-line\n", " with weave.attributes(\n", " {\"session\": st.session_state[\"session_id\"], \"env\": \"prod\"}\n", " ):\n", - " # This could also be weave model.predict.call if you're using a weave.Model subclass\n", - " result, call = chat_response.call(\n", - " prompt\n", - " ) # call the function with `.call`, this returns a tuple with a new Call object\n", - " # highlight-next-line\n", - " st.button(\n", - " \":thumbsup:\",\n", - " on_click=lambda: call.feedback.add_reaction(\"👍\"),\n", - " key=\"up\",\n", - " )\n", - " # highlight-next-line\n", - " st.button(\n", - " \":thumbsdown:\",\n", - " on_click=lambda: call.feedback.add_reaction(\"👎\"),\n", - " key=\"down\",\n", - " )\n", + " result, call = chat_response.call(prompt)\n", + " st.write(result[\"response\"])\n", " st.session_state.messages.append(\n", " {\"role\": \"assistant\", \"content\": result[\"response\"]}\n", " )\n", + " # highlight-next-line\n", + " st.session_state.calls.append(call)\n", + "\n", "\n", + "def init_session_state():\n", + " if \"session_id\" not in st.session_state:\n", + " st.session_state[\"session_id\"] = \"123abc\"\n", "\n", - "def init_chat_history():\n", " if \"messages\" not in st.session_state:\n", - " st.session_state.messages = st.session_state.messages = []\n", + " st.session_state.messages = []\n", + "\n", + " if \"calls\" not in st.session_state:\n", + " st.session_state.calls = []\n", "\n", "\n", "def main():\n", - " st.session_state[\"session_id\"] = \"123abc\"\n", - " init_chat_history()\n", + " st.title(\"Add feedback\")\n", + "\n", + " init_session_state()\n", " display_chat_messages()\n", - " get_and_process_prompt()\n", + " show_chat_prompt()\n", + "\n", + " st.rerun()\n", "\n", "\n", "if __name__ == \"__main__\":\n", @@ -187,21 +237,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'weave'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mweave\u001b[39;00m\n\u001b[1;32m 3\u001b[0m weave\u001b[38;5;241m.\u001b[39minit(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfeedback-example\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 6\u001b[0m \u001b[38;5;66;03m# highlight-next-line\u001b[39;00m\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'weave'" - ] - } - ], + "outputs": [], "source": [ "import weave\n", "\n", @@ -294,7 +332,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.7" + "version": "3.11.11" } }, "nbformat": 4,