Novelty.AI.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d27a0d22",
   "metadata": {},
   "source": [
    "Set everything up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b522c64",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import json\n",
    "import csv\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "from collections import OrderedDict\n",
    "import re\n",
    "\n",
    "def open_file(filepath):\n",
    "    with open(filepath, 'r', encoding='utf-8') as infile:\n",
    "        return infile.read()\n",
    "\n",
    "\n",
    "openai.api_key = open_file('openaiapikey.txt')\n",
    "\n",
    "def gpt3_embedding(content, engine='text-similarity-ada-001'):\n",
    "    response = openai.Embedding.create(input=content,engine=engine)\n",
    "    vector = response['data'][0]['embedding']  # this is a normal list\n",
    "    return vector"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5baa9636",
   "metadata": {},
   "source": [
    "If movie synopsis embeddings data doesn't exist yet, load movie synopsis data and compute embeddings for the movie synopses\n",
    "\n",
    "Note: Try the synopsis by itself, and the synopsis with a prefix of \"Movie synopsis: \" (to provide more context to the LLM).\n",
    "My theory is that providing the prefix will produce a more tightly clustered set of resulting embeddings, as the model will have more understanding of the context of the paragraphs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77bb38f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "json_file = 'index.json'\n",
    "\n",
    "# Check if the JSON file does NOT exist\n",
    "if not os.path.exists(json_file):\n",
    "    # File doesn't exist - load synopses and generate embeddings\n",
    "    synopses = []\n",
    "\n",
    "    # Open the CSV data file\n",
    "    with open('imdb_top_1000.csv', 'r', encoding='utf-8', errors='replace') as csvfile:\n",
    "        # Create a CSV reader object\n",
    "        csvreader = csv.reader(csvfile)\n",
    "        \n",
    "        # Skip the header if your CSV has one\n",
    "        next(csvreader, None)\n",
    "        \n",
    "        # Loop through each row in the CSV\n",
    "        for row in csvreader:\n",
    "            # Extract the description \n",
    "            # synopsis = \"Movie synopsis: \" + row[7]\n",
    "            synopsis = row[7]\n",
    "            \n",
    "            # Append the 8th field to the list\n",
    "            synopses.append(synopsis)\n",
    "    \n",
    "    # Print the first synopsis\n",
    "    # print(synopses[0])\n",
    "    \n",
    "    result = list()\n",
    "    \n",
    "    for synopsis in synopses:\n",
    "            embedding = gpt3_embedding(synopsis.encode(encoding='ASCII',errors='ignore').decode())\n",
    "            info = {'content': synopsis, 'vector': embedding}\n",
    "    #       print(info, '\\n\\n\\n')\n",
    "            result.append(info)\n",
    "    \n",
    "    with open('index.json', 'w') as outfile:\n",
    "            json.dump(result, outfile, indent=2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71c2543c",
   "metadata": {},
   "source": [
    "Iterate through the embeddings to create lower and upper bound vectors (for each vector parameter / dimension), and the size of the range of values at each parameter position / dimension.\n",
    "\n",
    "This establishes a bounded subset of the latent space, within which all of the synopses exist.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4cda6a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('index.json', 'r') as infile:\n",
    "    data = json.load(infile)\n",
    "\n",
    "min_vector = []\n",
    "max_vector = []\n",
    "range_vector = []\n",
    "\n",
    "dimensionality = len(data[0]['vector'])\n",
    "#print(dimensionality)\n",
    "\n",
    "for j in range(dimensionality):\n",
    "    # Initialize variable to store the minimum and maximum values for the current dimension\n",
    "    min_value = 1\n",
    "    max_value = -1\n",
    "\n",
    "    for i in data:\n",
    "        if i['vector'][j] < min_value:\n",
    "            min_value = i['vector'][j]\n",
    "        if i['vector'][j] > max_value:\n",
    "            max_value = i['vector'][j]\n",
    "            \n",
    "    range_value = max_value - min_value\n",
    "    min_vector.append(min_value)\n",
    "    max_vector.append(max_value)\n",
    "    range_vector.append(range_value)\n",
    "\n",
    "    \n",
    "print(\"Min vector\")\n",
    "print(min_vector)\n",
    "print(\"Max vector\")\n",
    "print(max_vector)\n",
    "print(\"Range vector\")\n",
    "print(range_vector)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35999b2d",
   "metadata": {},
   "source": [
    "This is debug.\n",
    "\n",
    "This step looks into the sizes of the ranges for each vector dimension. Plot range for each vector dimension. Perhaps sorted from highest range to lowest. It will be interesting to see what this looks like; could be one of...\n",
    "a) range on all dimensions is high (bad)\n",
    "b) range on some dimensions is high, but low for the (majority of?) others. This is my guess.\n",
    "c) range on all dimensions is low."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e1b82c2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Sort the vector in decreasing order\n",
    "range_sorted_vector = sorted(range_vector, reverse=True)\n",
    "\n",
    "# Create the plot\n",
    "#plt.figure(figsize=(10, 5))  # Optional: Set the figure size\n",
    "plt.plot(range_sorted_vector, marker=',', linestyle='-')\n",
    "#plt.plot(range_vector, marker=',', linestyle='-')\n",
    "\n",
    "# Add labels and title\n",
    "plt.xlabel('Index')\n",
    "plt.ylabel('Value')\n",
    "plt.title('Plot of Range for each vector dimension')\n",
    "\n",
    "# Show grid\n",
    "plt.grid(True)\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7f4b099",
   "metadata": {},
   "source": [
    "This is debug.\n",
    "\n",
    "Turns out there is a small number of embedding dimensions that have a larger range. Drill into the dimensions where the range is greater than 0.15."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "266e790e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find indices of values above 0.15\n",
    "indices = [index for index, value in enumerate(range_vector) if value > 0.15]\n",
    "\n",
    "# Print the indices\n",
    "print(\"Indices of values above 0.15:\", indices)\n",
    "for i in range(len(indices)):\n",
    "    print(f\"Index: {indices[i]}  Range: {range_vector[indices[i]]}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e0217c4",
   "metadata": {},
   "source": [
    "This is debug.\n",
    "\n",
    "Side-track.\n",
    "Plot the values for embedding dimension 822, across all of data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3e80fa2",
   "metadata": {},
   "outputs": [],
   "source": [
    "sidetrack_vector = []\n",
    "for i in range(len(data)):\n",
    "    sidetrack_vector.append(data[i]['vector'][822])\n",
    "\n",
    "print(f\"Min: {min(sidetrack_vector)}, Max: {max(sidetrack_vector)}, Range: {max(sidetrack_vector) - min(sidetrack_vector)}\")\n",
    "\n",
    "# Create the plot\n",
    "#plt.figure(figsize=(10, 5))  # Optional: Set the figure size\n",
    "plt.plot(sidetrack_vector, marker=',', linestyle='-')\n",
    "\n",
    "# Add labels and title\n",
    "plt.xlabel('Index')\n",
    "plt.ylabel('Value')\n",
    "plt.title('Plot of emdedding index 822 values')\n",
    "\n",
    "# Show grid\n",
    "plt.grid(True)\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d7f33a6",
   "metadata": {},
   "source": [
    "Create 1,000 random embeddings (potential novel outliers), with all elements of each vector between the lower and upper bounds\n",
    "\n",
    "There is an option here to extend the possible range for the random numbers to some percentage below the lower bound, and above the upper bound to increase the possible candidate space."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6e3e142",
   "metadata": {},
   "outputs": [],
   "source": [
    "randoms = []\n",
    "\n",
    "for i in range(1000):\n",
    "    random_vector = []\n",
    "    \n",
    "    for j in range(dimensionality):\n",
    "        random_vector.append(random.uniform(min_vector[j],max_vector[j]))\n",
    "\n",
    "    randoms.append(random_vector)\n",
    "    \n",
    "# print(randoms[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afec41f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot to check that the random vector is actually between and min and max values in each dimension\n",
    "\n",
    "# Plot the vectors\n",
    "plt.plot(max_vector, marker=',', linestyle='-', label='Max Vector')\n",
    "plt.plot(randoms[0], marker='x', linestyle='--', label='First Random Vector')\n",
    "plt.plot(min_vector, marker=',', linestyle='-.', label='Min Vector')\n",
    "\n",
    "# Add labels, title, and legend\n",
    "plt.xlabel('Index')\n",
    "plt.ylabel('Value')\n",
    "plt.title('Plot of Three Vectors')\n",
    "plt.legend()\n",
    "\n",
    "# Show grid\n",
    "plt.grid(True)\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4757c8f",
   "metadata": {},
   "source": [
    "For each random embedding, compute pairwise distances against all of the synopsis embeddings (the cosine similarity distance of the random embedding against every synopsis embedding). \n",
    "\n",
    "Initially I looked at the average cosine similarity for each random embedding against all the movie synopses. However, a better measure of novelty is the lowest maximum cosine similarity.\n",
    "\n",
    "This step is computationally expensive, and takes a while.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afb2978d",
   "metadata": {},
   "outputs": [],
   "source": [
    "max_CS_vector = []\n",
    "\n",
    "# For each of the random vectors...\n",
    "for i in range(len(randoms)):\n",
    "    \n",
    "    max_cosine_similarity = -1\n",
    "    \n",
    "    # For each movie synopsis...\n",
    "    for j in range(len(data)):\n",
    "        \n",
    "        # Convert lists to NumPy arrays\n",
    "        np_vector_a = np.array(data[j]['vector']) # Movie synopsis embedding\n",
    "        np_vector_b = np.array(randoms[i]) # The random embedding\n",
    "        \n",
    "        # Calculate the dot product\n",
    "        dot_product = np.dot(np_vector_a, np_vector_b)\n",
    "        \n",
    "        # Calculate the magnitudes (Euclidean norms)\n",
    "        magnitude_a = np.linalg.norm(np_vector_a)\n",
    "        magnitude_b = np.linalg.norm(np_vector_b)\n",
    "        \n",
    "        # Calculate the cosine similarity\n",
    "        cosine_similarity = dot_product / (magnitude_a * magnitude_b)\n",
    "\n",
    "        # Check if this cosine similarity is the maximum. If so, set the maximum to this.\n",
    "        if cosine_similarity > max_cosine_similarity:\n",
    "            max_cosine_similarity = cosine_similarity\n",
    "        # print(f\"randoms[{i}], data[{j}], Cosine Similarity: {cosine_similarity}\")\n",
    "\n",
    "    max_CS_vector.append(max_cosine_similarity)\n",
    "    # print(f\"randoms[{i}] average cosine similarity: {avg_cosine_similarity}\")\n",
    "    print(f\"randoms[{i}] max cosine similarity: {max_cosine_similarity}\")\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2624931",
   "metadata": {},
   "source": [
    "Pick the random embedding with the lowest maximum cosine similarity from all the movie synopsis embeddings.\n",
    "(1 is identical, 0 is orthogonal, -1 is diametrically opposed)\n",
    "Max cosine similarity is the movie synopsis that is closest to the random embedding.\n",
    "Random embedding with the lowest maximum cosine similarity is the one that is furthest from the closest established synopsis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3153d5a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Find the lowest max CS from the randoms\n",
    "lowest_max_CS = min(max_CS_vector)\n",
    "\n",
    "# Find the index of the best candidate random embedding\n",
    "lowest_max_CS_index = max_CS_vector.index(lowest_max_CS)\n",
    "\n",
    "outlier = []\n",
    "outlier = randoms[lowest_max_CS_index]\n",
    "print(f\"The lowest maximum cosine similarity is {lowest_max_CS} (index {lowest_max_CS_index})\")\n",
    "#print(f\"The corresponding outlier embedding is {outlier}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d61c7161",
   "metadata": {},
   "source": [
    "Write the outlier to a file, for future reference."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "da00fba6",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "result = list()\n",
    "info = {'vector': outlier, 'max CS': lowest_max_CS}\n",
    "# print(info, '\\n\\n\\n')\n",
    "result.append(info)\n",
    "    \n",
    "with open('outlier.json', 'w') as outfile:\n",
    "            json.dump(result, outfile, indent=2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00fcd434",
   "metadata": {},
   "source": [
    "Invert the outlier embedding to text\n",
    "This is the hard bit.\n",
    "\n",
    "\"Embedding inversion\"\n",
    "\n",
    "https://www.sisap.org/2023/accepted.html - Fabio Carrara, Claudio Gennaro, Lucia Vadicamo and Giuseppe Amato. Vec2Doc: Transforming Dense Vectors into Sparse Representations for Efficient Information Retrieval\n",
    "\n",
    "https://www.reddit.com/r/LanguageTechnology/comments/itb55x/vec2doc_reverse_document_embeddings_does_this/\n",
    "\n",
    "https://arxiv.org/pdf/2004.00053.pdf#:~:text=First%2C%20embedding%20vectors%20can%20be,some%20of%20the%20input%20data.\n",
    "\n",
    "https://arxiv.org/abs/2305.03010 - \"Given the black-box access to a language model, we treat sentence embeddings as initial tokens' representations and train or fine-tune a powerful decoder model to decode the whole sequences directly.\"\n",
    "\n",
    "https://arxiv.org/abs/2004.00053?ref=hackernoon.com\n",
    "\n",
    "https://stats.stackexchange.com/questions/422430/inverse-word-embedding-vector-to-word\n",
    "\n",
    "https://community.openai.com/t/embeddings-converting-a-embedded-vector-back-to-natural-language/202320\n",
    "\n",
    "https://hackernoon.com/embeddings-arent-human-readable-and-other-nonsense - good\n",
    "\n",
    "https://arxiv.org/pdf/2205.05124.pdf - may be of use\n",
    "\n",
    "Might need to brute-force a solution. \n",
    "Generate embeddings for a limited vocabulary of key words. \n",
    "See which one is closest to the outlier embedding\n",
    "Repeat for next word.\n",
    "...\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "47115aef",
   "metadata": {},
   "source": [
    "This is debug.\n",
    "\n",
    "Calculate cosine similarity of the outlier against a test synopsis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7070dd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_string = \"In a post-apocalyptic California, a gripping saga chronicles the high-stakes relationships between cold-blooded killers and desperate survivors, who make a perilous journey to Arkansas, where sacrifice and relationship struggles become the ultimate test of humanity.\"\n",
    "embedding = gpt3_embedding(test_string.encode(encoding='ASCII',errors='ignore').decode())\n",
    "\n",
    "# Convert lists to NumPy arrays\n",
    "np_vector_a = np.array(embedding)\n",
    "np_vector_b = np.array(outlier)\n",
    "        \n",
    "# Calculate the dot product\n",
    "dot_product = np.dot(np_vector_a, np_vector_b)\n",
    "       \n",
    "# Calculate the magnitudes (Euclidean norms)\n",
    "magnitude_a = np.linalg.norm(np_vector_a)\n",
    "magnitude_b = np.linalg.norm(np_vector_b)\n",
    "        \n",
    "# Calculate the cosine similarity\n",
    "cosine_similarity = dot_product / (magnitude_a * magnitude_b)\n",
    "print(f\"Cosine similarity: {cosine_similarity}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "423319a0",
   "metadata": {},
   "source": [
    "So we are going to try brute forcing the inversion of the outlier embedding.\n",
    "\n",
    "First create a vocabulary. Start with the all the unique words from the original dateset. Then remove stop words.\n",
    "Then create corresponding embeddings for each remaining word in the vocabulary.\n",
    "\n",
    "There's some cost associated with this step.\n",
    "\n",
    "It also takes a long time to run.\n",
    "\n",
    "We write the results into a file, so you should only need to run this once, since the vocabulary won't change (unless you change the data set) and the embeddings won't change (unless you change the model)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eea0e21f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize a Counter object to hold the vocabulary and frequencies\n",
    "vocab_counter = Counter()\n",
    "\n",
    "# Open the CSV file\n",
    "with open('imdb_top_1000.csv', 'r', encoding='utf-8', errors='replace') as csvfile:\n",
    "    csvreader = csv.reader(csvfile)\n",
    "    \n",
    "    # Skip the header row if your CSV has one\n",
    "    next(csvreader, None)\n",
    "    \n",
    "    # Loop through each row in the CSV\n",
    "    for row in csvreader:\n",
    "        # Extract the sentence from the 8th field (index 7)\n",
    "        sentence = row[7]\n",
    "        \n",
    "        # Tokenize the sentence into words\n",
    "        # Using regex to split by non-alphabetic characters for simplicity\n",
    "        words = re.split(r'\\W+', sentence.lower())\n",
    "        \n",
    "        # Update the vocabulary and frequencies\n",
    "        vocab_counter.update(words)\n",
    "\n",
    "# Remove empty string if exists\n",
    "if '' in vocab_counter:\n",
    "    del vocab_counter['']\n",
    "\n",
    "# Read stop words from the text file into a set\n",
    "with open('stop words.txt', 'r', encoding='utf-8') as f:\n",
    "    stop_words = set(line.strip() for line in f)\n",
    "\n",
    "# Remove stop words from the vocabulary\n",
    "filtered_vocab = {word: freq for word, freq in vocab_counter.items() if word not in stop_words}\n",
    "\n",
    "# Sort the filtered vocabulary by frequency (most common words first)\n",
    "sorted_vocab = OrderedDict(sorted(filtered_vocab.items(), key=lambda x: x[0], reverse=False))\n",
    "\n",
    "# Now sorted_vocab holds the sorted vocabulary\n",
    "#print(\"Sorted Vocabulary:\", sorted_vocab)\n",
    "\n",
    "vocab = list(sorted_vocab.keys())\n",
    "print(vocab)\n",
    "\n",
    "# Check if the JSON file does NOT exist\n",
    "if not os.path.exists('dictionary.json'):\n",
    "    # File doesn't exist - generate embeddings for first word dictionary\n",
    "    result = []\n",
    "\n",
    "    for word in vocab:\n",
    "        embedding = gpt3_embedding(word.encode(encoding='ASCII',errors='ignore').decode())\n",
    "        info = {'word': word, 'vector': embedding}\n",
    "        print(info, '\\n\\n\\n')\n",
    "        result.append(info)\n",
    "    \n",
    "    with open('dictionary.json', 'w') as outfile:\n",
    "        json.dump(result, outfile, indent=2)\n",
    "\n",
    "\n",
    "\n",
    "#print(len(data))\n",
    "#print(data[0])\n",
    "#print(len(sorted_vocab))\n",
    "#print(len(list(sorted_vocab.keys())))\n",
    "#print(list(sorted_vocab.keys())[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b87bc11",
   "metadata": {},
   "source": [
    "Next, pairwise compare the vocabulary vectors to the outlier vector to find the words that match best.\n",
    "Store the 20(?) best match words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdf8d6af",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dictionary.json', 'r') as infile:\n",
    "    data = json.load(infile)\n",
    "\n",
    "# Create a vector that will store the cosine similarities of each word in the dictionary against the outlier embedding.\n",
    "similarities_vector = []\n",
    "\n",
    "# For each word in the dictionary...\n",
    "for i in range(len(data)):\n",
    "    # Convert lists to NumPy arrays\n",
    "    np_vector_a = np.array(data[i]['vector']) # The embedding of the word\n",
    "    np_vector_b = np.array(outlier) # The outlier embedding\n",
    "        \n",
    "    # Calculate the dot product\n",
    "    dot_product = np.dot(np_vector_a, np_vector_b)\n",
    "       \n",
    "    # Calculate the magnitudes (Euclidean norms)\n",
    "    magnitude_a = np.linalg.norm(np_vector_a)\n",
    "    magnitude_b = np.linalg.norm(np_vector_b)\n",
    "        \n",
    "    # Calculate the cosine similarity\n",
    "    cosine_similarity = dot_product / (magnitude_a * magnitude_b)\n",
    "    similarities_vector.append(cosine_similarity)\n",
    "    #print(f\"Word: {data[i]['word']},  Cosine similarity: {cosine_similarity}\")\n",
    "    \n",
    "#print(len(similarities_vector))\n",
    "#print(max(similarities_vector))\n",
    "#print(data[similarities_vector.index(max(similarities_vector))]['word'])\n",
    "\n",
    "# Sort the list in descending order\n",
    "sorted_similarities_vector = sorted(similarities_vector, reverse=True)\n",
    "\n",
    "# Take the first N elements\n",
    "# top_10_values = sorted_similarities_vector[:10]\n",
    "top_20_values = sorted_similarities_vector[:20]\n",
    "\n",
    "# Print the N highest values\n",
    "\n",
    "best_words = [] \n",
    "\n",
    "#for i in top_10_values:\n",
    "for i in top_20_values:\n",
    "    print(data[similarities_vector.index(i)]['word'])\n",
    "    best_words.append(data[similarities_vector.index(i)]['word'])\n",
    "    \n",
    "for i in range(20):\n",
    "    print(best_words[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "556cb416",
   "metadata": {},
   "source": [
    "Call the openAI API to generate permutations of synopses based on the best match words. Use the best available model for this. No issue that I can see with this being a different model to what I've been using for embedding generation.\n",
    "\n",
    "Get the 5 permutations out of the response from the LLM.\n",
    "\n",
    "Calculate embeddings for each of the 5 permutations / candidates.\n",
    "\n",
    "Finally, calculate the cosine similarity for each permutation / candidate against the outlier that we generated earlier. Higher value equals a better match.\n",
    "\n",
    "This stage is annoying, because the format of the output from the LLM is somewhat non-deterministic. Sometimes you end up with name / value pairs where the name is \"synopsis\" (good). Other times the model will name the synopses as \"name\", or \"synopsis1\", \"synopsis2\", etc. My python isn't good enough to handle this. I suggest re-running this cell if it breaks. Not too expensive, since it's only one prompt per time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a2aa621",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "best_words_string = \" \".join(best_words)\n",
    "print(best_words_string)\n",
    "\n",
    "response = openai.ChatCompletion.create(\n",
    "    model=\"gpt-4\",\n",
    "    messages=[\n",
    "        {\"role\": \"system\", \"content\": \"You are a helpful special-purpose assistant. The user is working on an embedding inversion project. The user will provide you with a list of words that have some semantic similarity to the embedding vector that we are trying to invert. You will use combinations of these words to create 5 candidate narratives in the form of movie synopses. You will return the candidate synopses in JSON format. Don't prefix the JSON response with anything - just provide the bare JSON. In the JSON, don't provide a title - just provide the synopses. In the name / value pairs in the JSON, use the name 'synopsis' for all values.\"},\n",
    "        {\"role\": \"user\", \"content\": best_words_string},\n",
    "    ]\n",
    ")\n",
    "\n",
    "#print(response)\n",
    "#print(type(response))\n",
    "\n",
    "content = response['choices'][0]['message']['content']\n",
    "print(content)\n",
    "#print(type(content))\n",
    "\n",
    "# Parse the JSON data\n",
    "synopses_data = json.loads(content)\n",
    "\n",
    "# Extract the values into a Python array\n",
    "synopses_list = [item[\"synopsis\"] for item in synopses_data]\n",
    "\n",
    "# Print the resulting array\n",
    "print(synopses_list)\n",
    "\n",
    "result = []\n",
    "for i in range(5):\n",
    "    embedding = gpt3_embedding(synopses_list[i].encode(encoding='ASCII',errors='ignore').decode())\n",
    "    info = {'synopsis': synopses_list[i], 'vector': embedding}\n",
    "    print(info, '\\n\\n\\n')\n",
    "    result.append(info)\n",
    "\n",
    "for i in range(5):\n",
    "    # Convert lists to NumPy arrays\n",
    "    np_vector_a = np.array(result[i]['vector']) # The embedding of the candidate synopsis\n",
    "    np_vector_b = np.array(outlier) # The outlier embedding\n",
    "        \n",
    "    # Calculate the dot product\n",
    "    dot_product = np.dot(np_vector_a, np_vector_b)\n",
    "       \n",
    "    # Calculate the magnitudes (Euclidean norms)\n",
    "    magnitude_a = np.linalg.norm(np_vector_a)\n",
    "    magnitude_b = np.linalg.norm(np_vector_b)\n",
    "        \n",
    "    # Calculate the cosine similarity\n",
    "    cosine_similarity = dot_product / (magnitude_a * magnitude_b)\n",
    "    print(f\"Synopsis: {result[i]['synopsis']},  Cosine similarity: {cosine_similarity}\")\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1636b773",
   "metadata": {},
   "source": [
    "Thank you for following to the end."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c1a0519b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}