value_iteration_algoritm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 718,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 719,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createtransactiondata(shape, number):\n",
    "    \"\"\"\n",
    "    Artificial Transactions Data Creator.\n",
    "\n",
    "    It creates (number, shape, shape + 1) numpy array. \n",
    "    Number index means individual persons, shape index means states, also (shape + 1) index means states but includes end state.\n",
    "    Last element of each row represents probability of going to end state.\n",
    "    All rows represents probability, so sum of individual row is equals one.\n",
    "    Those probabilities represents of going from one state to another.\n",
    "    For example data[0][0][5] means probability of going state 1 to state 6.\n",
    "\n",
    "\n",
    "    Args:\n",
    "        shape: Number of states. Must be higher than 5.\n",
    "        number: Number of person.\n",
    "    \n",
    "    Return:\n",
    "        numpy array (number, shape, shape + 1)\n",
    "    \"\"\"\n",
    "    transaction_data = np.array([])\n",
    "    for u in range(number):\n",
    "        human = np.zeros((shape,shape+1))\n",
    "        for k in range(shape):\n",
    "            X = np.array([])\n",
    "\n",
    "            if k == 0:\n",
    "                for i,j in [(75,94),(16,25),(9,15)]:\n",
    "                    X = np.hstack((X,np.random.randint(i,j) + np.random.rand()))\n",
    "            elif k == 1:\n",
    "                for i,j in [(14,22),(60,90),(14,22),(2,10)]:\n",
    "                    X = np.hstack((X,np.random.randint(i,j) + np.random.rand()))\n",
    "            elif k == shape-2:\n",
    "                for i,j in [(1,10),(9,13),(16,19),(35,50)]:\n",
    "                    X = np.hstack((X,np.random.randint(i,j) + np.random.rand()))\n",
    "            elif k == shape-1:\n",
    "                for i,j in [(4,13),(13,19),(30,40)]:\n",
    "                    X = np.hstack((X,np.random.randint(i,j) + np.random.rand()))\n",
    "            else:\n",
    "                for i,j in [(1,7),(7,23),(60,94),(7,23),(1,7)]:\n",
    "                    X = np.hstack((X,np.random.randint(i,j) + np.random.rand()))\n",
    "\n",
    "            X /= 100\n",
    "\n",
    "            if X.sum() > 1:\n",
    "                surplus = X.sum() - 1\n",
    "                total = X.sum()\n",
    "                for i,j in enumerate(X):\n",
    "                    X[i] = j - j * surplus / total\n",
    "\n",
    "            elif X.sum() < 1:\n",
    "                human[k][shape] = (1 - X.sum())\n",
    "\n",
    "            if (k == 0) or (k == 1):\n",
    "                for l in range(len(X)):\n",
    "                    human[k][l] = X[l] \n",
    "            else:\n",
    "                for l in range(len(X)):\n",
    "                    human[k][k+l-2] = X[l] \n",
    "        if transaction_data.ndim == 3:\n",
    "            transaction_data = np.vstack((transaction_data,human.reshape(1,shape,shape+1)))\n",
    "        else:\n",
    "            transaction_data = human.reshape(1,shape,shape+1)\n",
    "    return transaction_data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For illustrative purpose data numpy array inserted in a pandas dataframe. \n",
    "Example data is created for 7 playing states and 1 end states for one person.\n",
    "Rows and columns represents states. Last columns represents end state so it's not exists in index."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 720,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.707881</td>\n",
       "      <td>0.204227</td>\n",
       "      <td>0.087892</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.130960</td>\n",
       "      <td>0.641314</td>\n",
       "      <td>0.172268</td>\n",
       "      <td>0.055457</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.043212</td>\n",
       "      <td>0.071591</td>\n",
       "      <td>0.705403</td>\n",
       "      <td>0.144592</td>\n",
       "      <td>0.035202</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.030337</td>\n",
       "      <td>0.134779</td>\n",
       "      <td>0.631895</td>\n",
       "      <td>0.174579</td>\n",
       "      <td>0.028410</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.045924</td>\n",
       "      <td>0.098599</td>\n",
       "      <td>0.661705</td>\n",
       "      <td>0.130133</td>\n",
       "      <td>0.019656</td>\n",
       "      <td>0.043982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.023854</td>\n",
       "      <td>0.100977</td>\n",
       "      <td>0.186833</td>\n",
       "      <td>0.464952</td>\n",
       "      <td>0.223385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.069093</td>\n",
       "      <td>0.161822</td>\n",
       "      <td>0.333117</td>\n",
       "      <td>0.435967</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          1         2         3         4         5         6         7  \\\n",
       "1  0.707881  0.204227  0.087892  0.000000  0.000000  0.000000  0.000000   \n",
       "2  0.130960  0.641314  0.172268  0.055457  0.000000  0.000000  0.000000   \n",
       "3  0.043212  0.071591  0.705403  0.144592  0.035202  0.000000  0.000000   \n",
       "4  0.000000  0.030337  0.134779  0.631895  0.174579  0.028410  0.000000   \n",
       "5  0.000000  0.000000  0.045924  0.098599  0.661705  0.130133  0.019656   \n",
       "6  0.000000  0.000000  0.000000  0.023854  0.100977  0.186833  0.464952   \n",
       "7  0.000000  0.000000  0.000000  0.000000  0.069093  0.161822  0.333117   \n",
       "\n",
       "          8  \n",
       "1  0.000000  \n",
       "2  0.000000  \n",
       "3  0.000000  \n",
       "4  0.000000  \n",
       "5  0.043982  \n",
       "6  0.223385  \n",
       "7  0.435967  "
      ]
     },
     "execution_count": 720,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shape = 7\n",
    "number = 1\n",
    "pd.DataFrame(createtransactiondata(shape=shape, number=number)[0],\n",
    "        columns=[state for state in range(1,shape+2)],\n",
    "        index=[state for state in range(1,shape+1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 710,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createrewarddata(shape,option,number,initial):\n",
    "    \"\"\"\n",
    "    Artificial Rewards Data Creator.\n",
    "\n",
    "    It creates (number, shape, option) numpy array. \n",
    "    \n",
    "    Each reward array desingned to decresing from top left to buttom right. That means\n",
    "    first states rewards higher than the last states rewards.\n",
    "\n",
    "    Args:\n",
    "        shape: Number of states. Must be higher than 5.\n",
    "        option: Number of choice. Using purpose is creating more reward columns.\n",
    "        number: Number of person.\n",
    "        initial: Base of maximum reward. Result could be +- (initial / 10).\n",
    "    \n",
    "    Return:\n",
    "        numpy array (number, shape, shape + 1)\n",
    "    \"\"\"\n",
    "    reward_data = np.array([])\n",
    "    for _ in range(number):\n",
    "        reward = np.zeros((shape,option))\n",
    "        for i in range(shape):\n",
    "            for j in range(option):\n",
    "                if j == 0 and i == 0:\n",
    "                    reward[i][j] = initial + np.random.randint(-initial/10,initial/10)\n",
    "                elif i != 0 and j == 0:\n",
    "                    reward[i][j] = reward[i-1][j] - np.random.randint(initial/50,initial/20)\n",
    "                elif i == 0 and j != 0:\n",
    "                    reward[i][j] = reward[i][j-1] - np.random.randint(initial/50,initial/20)\n",
    "                else:\n",
    "                    value_1 = reward[i][j-1] - np.random.randint(initial/50,initial/20)\n",
    "                    value_2 = reward[i-1][j] - np.random.randint(initial/50,initial/20)\n",
    "                    reward[i][j] = min(value_1,value_2)\n",
    "        if reward_data.ndim == 3:\n",
    "            reward_data = np.vstack((reward_data,reward.reshape(1,shape,option)))\n",
    "        else:\n",
    "            reward_data = reward.reshape(1,shape,option)\n",
    "    return reward_data\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For illustrative purpose data numpy array inserted in a pandas dataframe. Rows represents rewards, columns represents options.\n",
    "Example data is created for 10 playing states so there is no reward for end state. Also there 3 option and each option's policy differ than other. Finally this data created for one person.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 711,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>option_1</th>\n",
       "      <th>option_2</th>\n",
       "      <th>option_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>95.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>89.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>92.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>85.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>89.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>81.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>85.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>78.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>83.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>74.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>80.0</td>\n",
       "      <td>76.0</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>76.0</td>\n",
       "      <td>74.0</td>\n",
       "      <td>68.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>74.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>64.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>72.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>60.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>70.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>58.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    option_1  option_2  option_3\n",
       "1       95.0      92.0      89.0\n",
       "2       92.0      88.0      85.0\n",
       "3       89.0      85.0      81.0\n",
       "4       85.0      81.0      78.0\n",
       "5       83.0      78.0      74.0\n",
       "6       80.0      76.0      72.0\n",
       "7       76.0      74.0      68.0\n",
       "8       74.0      70.0      64.0\n",
       "9       72.0      66.0      60.0\n",
       "10      70.0      62.0      58.0"
      ]
     },
     "execution_count": 711,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shape=10\n",
    "option=3\n",
    "number=1\n",
    "initial=100\n",
    "pd.DataFrame(createrewarddata(shape=shape,option=option,number=number,initial=initial)[0],  \n",
    "                                columns=[\"option_\"+str(i+1) for i in range(option)],\n",
    "                                index = [index for index in range(1,shape+1)]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 712,
   "metadata": {},
   "outputs": [],
   "source": [
    "class user():\n",
    "    def __init__(self, total_state, rewards, transaction):\n",
    "        self.N = (total_state + 1) #plus one for end state\n",
    "        self.transaction = transaction #transaction array\n",
    "        self.rewards = rewards #reward array\n",
    "    def isEnd(self,state):\n",
    "        \"\"\"\n",
    "        Checks the given state is end state.\n",
    "        \"\"\"\n",
    "        return state == self.N\n",
    "    def actions(self,state):\n",
    "        \"\"\"\n",
    "        For this problem, our actions are stay or go.\n",
    "        \"\"\"\n",
    "        result = []\n",
    "        if state <= self.N:\n",
    "            result.append(\"stay\")\n",
    "        if state <= self.N:\n",
    "            result.append(\"go\")\n",
    "        return result\n",
    "    def succProbReward(self, action, state):\n",
    "        \"\"\"\n",
    "        If we pick the stay action, our reward is 2.5 and we continue playing with probability of going another state\n",
    "        or we pick go and take reward but game going to end.\n",
    "\n",
    "        return:\n",
    "            tuple(state,probability,reward)\n",
    "\n",
    "        \"\"\"\n",
    "        result = []\n",
    "        if action == \"stay\":\n",
    "            for i in range(self.N):\n",
    "                result.append((i+1, self.transaction[state-1][i],2.5))\n",
    "        elif action == \"go\":\n",
    "            result.append((self.N, 1, self.rewards[state-1]))\n",
    "        return result\n",
    "    def discount(self):\n",
    "        \"\"\"\n",
    "        Discount rate for another round.\n",
    "        \"\"\"\n",
    "        return 0.97**(1/365)\n",
    "    def states(self):\n",
    "        \"\"\"\n",
    "        Range list of total states.\n",
    "        \"\"\"\n",
    "        return range(1,self.N + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 713,
   "metadata": {},
   "outputs": [],
   "source": [
    "def valueIteration(mdp):\n",
    "    # initialize all states to 0\n",
    "    V = {}\n",
    "    for state in mdp.states():\n",
    "        V[state] = 0\n",
    "\n",
    "    def Q(state,action):\n",
    "        # returns right hand side of the algorithm.\n",
    "        return sum(prob*(reward+mdp.discount()*V[newState]) \\\n",
    "            for newState,prob,reward in mdp.succProbReward(action,state))\n",
    "    while True:\n",
    "        newV = {}\n",
    "        for state in mdp.states():\n",
    "            if mdp.isEnd(state):\n",
    "                newV[state] = 0 #if end state policy is 0\n",
    "            else:\n",
    "                newV[state] = max(Q(state,action) for action in mdp.actions(state)) #gets max policy of given state wrt probability of going another state.\n",
    "        if max(abs(V[state]-newV[state]) for state in mdp.states()) < 1e-4: #checks convergence\n",
    "            break\n",
    "        V = newV #update v after convergence\n",
    "        pi = {}\n",
    "        for state in mdp.states():\n",
    "            if mdp.isEnd(state):\n",
    "                pi[state] = 'none'\n",
    "            else: \n",
    "                pi[state] = max((Q(state,action),action) for action in mdp.actions(state))[1] #gets action of max policy\n",
    "    outcome = []\n",
    "    for state in mdp.states():\n",
    "        outcome.append((state,V[state],pi[state]))\n",
    "    return outcome\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 714,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(shape,number,option,initial):\n",
    "    transaction_data = createtransactiondata(shape=shape, number=number)\n",
    "    reward_data = createrewarddata(shape=shape,option=option,number=number,initial=initial)\n",
    "    columns = [\"option_\"+str(i+1) for i in range(option)] + [\"user_id\",\"state\"] + [\"option_\"+str(i+1)+\"_exp_life\" for i in range(option)]\n",
    "    outcome = pd.DataFrame(columns=columns) \n",
    "    for num_user in range(number):\n",
    "        option_list = {}\n",
    "        option_list[\"user_id\"] = num_user\n",
    "        for num_state in range(shape):\n",
    "            option_list[\"state\"] = num_state+1\n",
    "            for num_option in range(option):\n",
    "                mdp = user(total_state=shape,rewards=reward_data[num_user].T[num_option], transaction = transaction_data[num_user])\n",
    "                vit = valueIteration(mdp)\n",
    "                option_list[\"option_\"+str(num_option+1)] = vit[num_state][2]\n",
    "                option_list[\"option_\"+str(num_option+1)+\"_exp_life\"] = vit[num_state][1]\n",
    "            outcome = outcome.append(option_list,ignore_index=True)\n",
    "    return outcome.set_index([\"user_id\",\"state\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Result of all of these, in given conditions, person whom user_id is zero is if in state 2 and in option 3 must stay for getting maximum policy or if he/she is in option_4, must go."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 717,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>option_1</th>\n",
       "      <th>option_2</th>\n",
       "      <th>option_3</th>\n",
       "      <th>option_4</th>\n",
       "      <th>option_1_exp_life</th>\n",
       "      <th>option_2_exp_life</th>\n",
       "      <th>option_3_exp_life</th>\n",
       "      <th>option_4_exp_life</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_id</th>\n",
       "      <th>state</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"10\" valign=\"top\">0</th>\n",
       "      <th>1</th>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>3226.000000</td>\n",
       "      <td>3071.000000</td>\n",
       "      <td>2961.000000</td>\n",
       "      <td>2831.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>stay</td>\n",
       "      <td>go</td>\n",
       "      <td>3103.000000</td>\n",
       "      <td>2960.000000</td>\n",
       "      <td>2825.144675</td>\n",
       "      <td>2736.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>stay</td>\n",
       "      <td>go</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>3002.672174</td>\n",
       "      <td>2870.000000</td>\n",
       "      <td>2730.782042</td>\n",
       "      <td>2619.526051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>2881.106368</td>\n",
       "      <td>2733.269126</td>\n",
       "      <td>2615.000000</td>\n",
       "      <td>2494.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>2745.000000</td>\n",
       "      <td>2589.000000</td>\n",
       "      <td>2471.000000</td>\n",
       "      <td>2332.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>2622.000000</td>\n",
       "      <td>2439.000000</td>\n",
       "      <td>2321.000000</td>\n",
       "      <td>2231.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>2489.000000</td>\n",
       "      <td>2364.000000</td>\n",
       "      <td>2209.000000</td>\n",
       "      <td>2110.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>stay</td>\n",
       "      <td>go</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>2370.957521</td>\n",
       "      <td>2278.000000</td>\n",
       "      <td>2110.196821</td>\n",
       "      <td>1992.380458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>2253.000000</td>\n",
       "      <td>2153.000000</td>\n",
       "      <td>2015.000000</td>\n",
       "      <td>1875.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>2141.000000</td>\n",
       "      <td>2008.000000</td>\n",
       "      <td>1883.000000</td>\n",
       "      <td>1781.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"10\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>3285.000000</td>\n",
       "      <td>3197.000000</td>\n",
       "      <td>3100.000000</td>\n",
       "      <td>3010.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>3223.117333</td>\n",
       "      <td>3111.654527</td>\n",
       "      <td>3007.755153</td>\n",
       "      <td>2914.947228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>3186.716174</td>\n",
       "      <td>3063.938620</td>\n",
       "      <td>2956.690366</td>\n",
       "      <td>2862.504170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>3129.446187</td>\n",
       "      <td>2991.468092</td>\n",
       "      <td>2879.717809</td>\n",
       "      <td>2783.676176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>3055.461346</td>\n",
       "      <td>2900.863331</td>\n",
       "      <td>2784.049358</td>\n",
       "      <td>2685.823569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>2946.198518</td>\n",
       "      <td>2767.203622</td>\n",
       "      <td>2643.472158</td>\n",
       "      <td>2542.605179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>stay</td>\n",
       "      <td>2780.436982</td>\n",
       "      <td>2570.452032</td>\n",
       "      <td>2436.014030</td>\n",
       "      <td>2329.687056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>2608.000000</td>\n",
       "      <td>2355.000000</td>\n",
       "      <td>2212.000000</td>\n",
       "      <td>2105.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>2465.000000</td>\n",
       "      <td>2234.000000</td>\n",
       "      <td>2075.000000</td>\n",
       "      <td>1946.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>go</td>\n",
       "      <td>2367.000000</td>\n",
       "      <td>2081.000000</td>\n",
       "      <td>1989.000000</td>\n",
       "      <td>1856.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              option_1 option_2 option_3 option_4  option_1_exp_life  \\\n",
       "user_id state                                                          \n",
       "0       1           go       go       go       go        3226.000000   \n",
       "        2           go       go     stay       go        3103.000000   \n",
       "        3         stay       go     stay     stay        3002.672174   \n",
       "        4         stay     stay       go       go        2881.106368   \n",
       "        5           go       go       go       go        2745.000000   \n",
       "        6           go       go       go       go        2622.000000   \n",
       "        7           go       go       go       go        2489.000000   \n",
       "        8         stay       go     stay     stay        2370.957521   \n",
       "        9           go       go       go       go        2253.000000   \n",
       "        10          go       go       go       go        2141.000000   \n",
       "1       1           go       go       go       go        3285.000000   \n",
       "        2         stay     stay     stay     stay        3223.117333   \n",
       "        3         stay     stay     stay     stay        3186.716174   \n",
       "        4         stay     stay     stay     stay        3129.446187   \n",
       "        5         stay     stay     stay     stay        3055.461346   \n",
       "        6         stay     stay     stay     stay        2946.198518   \n",
       "        7         stay     stay     stay     stay        2780.436982   \n",
       "        8           go       go       go       go        2608.000000   \n",
       "        9           go       go       go       go        2465.000000   \n",
       "        10          go       go       go       go        2367.000000   \n",
       "\n",
       "               option_2_exp_life  option_3_exp_life  option_4_exp_life  \n",
       "user_id state                                                           \n",
       "0       1            3071.000000        2961.000000        2831.000000  \n",
       "        2            2960.000000        2825.144675        2736.000000  \n",
       "        3            2870.000000        2730.782042        2619.526051  \n",
       "        4            2733.269126        2615.000000        2494.000000  \n",
       "        5            2589.000000        2471.000000        2332.000000  \n",
       "        6            2439.000000        2321.000000        2231.000000  \n",
       "        7            2364.000000        2209.000000        2110.000000  \n",
       "        8            2278.000000        2110.196821        1992.380458  \n",
       "        9            2153.000000        2015.000000        1875.000000  \n",
       "        10           2008.000000        1883.000000        1781.000000  \n",
       "1       1            3197.000000        3100.000000        3010.000000  \n",
       "        2            3111.654527        3007.755153        2914.947228  \n",
       "        3            3063.938620        2956.690366        2862.504170  \n",
       "        4            2991.468092        2879.717809        2783.676176  \n",
       "        5            2900.863331        2784.049358        2685.823569  \n",
       "        6            2767.203622        2643.472158        2542.605179  \n",
       "        7            2570.452032        2436.014030        2329.687056  \n",
       "        8            2355.000000        2212.000000        2105.000000  \n",
       "        9            2234.000000        2075.000000        1946.000000  \n",
       "        10           2081.000000        1989.000000        1856.000000  "
      ]
     },
     "execution_count": 717,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "main(shape = 10, number = 2, option = 4, initial = 3120)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "590e32d2faa652b006e1dae1f0a2840dbe5275083b71e4653943457b2bebcd97"
  },
  "kernelspec": {
   "display_name": "Python 3.10.1 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.1"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}