Evacuation_RL_0930_One_Class_126_60.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Evacuation_RL_0928_One_Class_126_60.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMAt9BHfgnIGJVuficKcfIG",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bravoila/MachineLearning/blob/master/Evacuation_RL_0930_One_Class_126_60.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BXfVWLMxxEI0",
        "outputId": "7edaf2ba-7cf5-4d59-b995-06fb1ef1c3c5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "pip install rlcard"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: rlcard in /usr/local/lib/python3.6/dist-packages (0.2.5)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.6/dist-packages (from rlcard) (1.1.0)\n",
            "Requirement already satisfied: matplotlib>=3.0 in /usr/local/lib/python3.6/dist-packages (from rlcard) (3.2.2)\n",
            "Requirement already satisfied: numpy>=1.16.3 in /usr/local/lib/python3.6/dist-packages (from rlcard) (1.18.5)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from rlcard) (20.4)\n",
            "Requirement already satisfied: pillow>=5.2.0 in /usr/local/lib/python3.6/dist-packages (from rlcard) (7.0.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard) (1.2.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard) (0.10.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard) (2.8.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->rlcard) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jhGcGdwdxICT",
        "outputId": "bdcf7410-c903-469f-b906-d21bfdac569a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 564
        }
      },
      "source": [
        "pip install rlcard[tensorflow]"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: rlcard[tensorflow] in /usr/local/lib/python3.6/dist-packages (0.2.5)\n",
            "Requirement already satisfied: pillow>=5.2.0 in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (7.0.0)\n",
            "Requirement already satisfied: matplotlib>=3.0 in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (3.2.2)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (20.4)\n",
            "Requirement already satisfied: numpy>=1.16.3 in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (1.18.5)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (1.1.0)\n",
            "Requirement already satisfied: tensorflow<2.0,>=1.14; extra == \"tensorflow\" in /usr/local/lib/python3.6/dist-packages (from rlcard[tensorflow]) (1.15.4)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard[tensorflow]) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard[tensorflow]) (1.2.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard[tensorflow]) (2.8.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0->rlcard[tensorflow]) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->rlcard[tensorflow]) (1.15.0)\n",
            "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (0.35.1)\n",
            "Requirement already satisfied: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (0.2.2)\n",
            "Requirement already satisfied: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.0.8)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.1.2)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (0.8.1)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.32.0)\n",
            "Requirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (0.10.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.12.1)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (3.12.4)\n",
            "Requirement already satisfied: tensorflow-estimator==1.15.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.15.1)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (0.2.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (3.3.0)\n",
            "Requirement already satisfied: tensorboard<1.16.0,>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.15.0)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (2.10.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (50.3.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.16.0,>=1.15.0->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (3.2.2)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<1.16.0,>=1.15.0->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (2.0.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.16.0,>=1.15.0->tensorflow<2.0,>=1.14; extra == \"tensorflow\"->rlcard[tensorflow]) (3.2.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rCm-mXNVJt4U",
        "outputId": "f27b3107-e830-4fde-bcc3-2ae3617562e9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 530
        }
      },
      "source": [
        "# Stable Baselines only supports tensorflow 1.x for now\n",
        "%tensorflow_version 1.x\n",
        "!apt install swig cmake libopenmpi-dev zlib1g-dev\n",
        "!pip install stable-baselines[mpi]==2.10.0"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 1.x selected.\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "zlib1g-dev is already the newest version (1:1.2.11.dfsg-0ubuntu2).\n",
            "libopenmpi-dev is already the newest version (2.1.1-8).\n",
            "swig is already the newest version (3.0.12-1).\n",
            "cmake is already the newest version (3.10.2-1ubuntu2.18.04.1).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 21 not upgraded.\n",
            "Requirement already satisfied: stable-baselines[mpi]==2.10.0 in /usr/local/lib/python3.6/dist-packages (2.10.0)\n",
            "Requirement already satisfied: gym[atari,classic_control]>=0.11 in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (0.17.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (0.16.0)\n",
            "Requirement already satisfied: cloudpickle>=0.5.5 in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (1.3.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (1.18.5)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (4.1.2.30)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (1.1.2)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (3.2.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from stable-baselines[mpi]==2.10.0) (1.4.1)\n",
            "Requirement already satisfied: mpi4py; extra == \"mpi\" in /tensorflow-1.15.2/python3.6 (from stable-baselines[mpi]==2.10.0) (3.0.3)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.11->stable-baselines[mpi]==2.10.0) (1.5.0)\n",
            "Requirement already satisfied: Pillow; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.11->stable-baselines[mpi]==2.10.0) (7.0.0)\n",
            "Requirement already satisfied: atari-py~=0.2.0; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.11->stable-baselines[mpi]==2.10.0) (0.2.6)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas->stable-baselines[mpi]==2.10.0) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->stable-baselines[mpi]==2.10.0) (2018.9)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines[mpi]==2.10.0) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines[mpi]==2.10.0) (2.4.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines[mpi]==2.10.0) (1.2.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym[atari,classic_control]>=0.11->stable-baselines[mpi]==2.10.0) (0.16.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from atari-py~=0.2.0; extra == \"atari\"->gym[atari,classic_control]>=0.11->stable-baselines[mpi]==2.10.0) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Grxcnf0WJ7VX"
      },
      "source": [
        "Code for environment"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fj7EwdXCJ4Hs"
      },
      "source": [
        "#from google.colab import files\n",
        "#import io\n",
        "#uploaded = files.upload()"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P1BceU4TKZ2O",
        "cellView": "both"
      },
      "source": [
        "#import io\n",
        "#io.BytesIO(uploaded['patient_generattion2.xlsx'])\n",
        "#@title Code for Environment"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZUxRL33-4CWG"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QBIND4ZhJ6f7",
        "cellView": "form"
      },
      "source": [
        "#@title Code for Environment\n",
        "#!/usr/bin/env python3\n",
        "# -*- coding: utf-8 -*-\n",
        "\"\"\"\n",
        "Created on Tue Dec 17 11:46:46 2020\n",
        "\n",
        "@author: machaolun\n",
        "\"\"\"\n",
        "\"\"\"\n",
        "Unit: Minute\n",
        "\"\"\"\n",
        "import gym\n",
        "from gym import spaces\n",
        "\n",
        "# import tkinter as tk\n",
        "import random\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "# from collections import deque\n",
        "from heapq import heappush,heappop\n",
        "import csv\n",
        "import copy\n",
        "from collections import deque\n",
        "\n",
        "random.seed(2020)\n",
        "# This class is to generate patients\n",
        "total_simulation_time = 10000\n",
        "\n",
        "\n",
        "def SURV_PROB_CAL(my_facility, my_patype, my_total_time, surv_num):\n",
        "        # survival probablity\n",
        "    # Resource-Based Patient Prioritization in Mass-Casualty Incidents\n",
        "    # first is the worst case and the fifth is the optimistic case    \n",
        "    # Immediate         Delayed\n",
        "    # β0,I β1,I β2,I    β0,D β1,D β2,D\n",
        "    # 0.09,17  ,1.01    0.57,61   ,2.03\n",
        "    # 0.15,28  ,1.38    0.65,86   ,2.11\n",
        "    # 0.24,47  ,1.30    0.76,138  ,2.17\n",
        "    # 0.40,59  ,1.47    0.77,140  ,2.29\n",
        "    # 0.56,91  ,1.58    0.81,160  ,1.41\n",
        "    if surv_num == 1:\n",
        "        # set para\n",
        "        # Senerio 1, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.09,17  ,1.01]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.57,61 ,2.03]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.09,17  ,1.01]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.57,61   ,2.03]\n",
        "    elif surv_num == 2:       \n",
        "        # # Senerio 2, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.15,28  ,1.38]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.65,86   ,2.11]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.15,28  ,1.38]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.65,86   ,2.11]\n",
        "            \n",
        "    elif surv_num == 3:\n",
        "        \n",
        "        # # Senerio 3, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.24,47  ,1.30]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.76,138  ,2.17]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.24,47  ,1.30]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.76,138  ,2.17]\n",
        "    elif surv_num == 4:\n",
        "            \n",
        "        # # Senerio 4, assume no different between ACF and hospital\n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.40,59  ,1.47]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.77,140  ,2.29]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.40,59  ,1.47]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.77,140  ,2.29]            \n",
        "    elif surv_num == 5:\n",
        "        # Senerio 5, assume no different between ACF and hospital            \n",
        "        if my_facility == 1 and my_patype == 0:\n",
        "            \"\"\"Hospital, immediate\"\"\"\n",
        "            Beta = [0.56,91  ,1.58]\n",
        "        elif my_facility == 1 and my_patype == 1:\n",
        "            \"\"\"Hospital, delayed\"\"\"\n",
        "            Beta = [0.81,160  ,1.41]\n",
        "        elif my_facility == 2 and my_patype == 0:\n",
        "            \"\"\"ACF, immediate\"\"\"\n",
        "            Beta = [0.56,91  ,1.58]\n",
        "        elif my_facility == 2 and my_patype == 1:\n",
        "            \"\"\"ACF, delayed\"\"\"\n",
        "            Beta = [0.81,160  ,1.41]        \n",
        "    \n",
        "    # 0.9 is ACF discount factor\n",
        "    if my_facility == 1:\n",
        "        SP = Beta[0]/((my_total_time/Beta[1]) ** Beta[2] + 1.0)\n",
        "    else:\n",
        "        SP = 0.8 * Beta[0]/((my_total_time/Beta[1]) ** Beta[2] + 1.0)\n",
        "    return SP\n",
        "\n",
        "\n",
        "def eva_perform(env, num):\n",
        "    ''' Evaluate he performance of the agents in the environment\n",
        "    Args:\n",
        "        env (Env class): The environment to be evaluated.\n",
        "        num (int): The number of games to play.\n",
        "    Returns:\n",
        "        A list of avrage payoffs for each player\n",
        "    '''\n",
        "    payoffs = 0\n",
        "    counter = 0\n",
        "\n",
        "    while counter < num:\n",
        "        tj = env.run(1, is_training=False)\n",
        "        return_ = pd.DataFrame(tj)\n",
        "        payoffs += sum(return_[2])\n",
        "        counter += 1\n",
        "\n",
        "    payoffs /= counter\n",
        "    \n",
        "    return payoffs\n",
        "\n",
        "class Event(object):\n",
        "    # num_of_events = 0\n",
        "    def __init__(self,generate_time, ptype, surv_num):\n",
        "        self.generate_time = generate_time\n",
        "        self.trans_time = 0.2\n",
        "        self.arrival_time = 0.3\n",
        "        self.start_service_time = 0.4\n",
        "        self.service_time = 0.5\n",
        "        self.departure_time = 0.6\n",
        "        self.survival_prob_res = 0\n",
        "        self.facility = 0.7\n",
        "        self.patype = ptype\n",
        "        self.status = \"default\"\n",
        "        self.surv_num = surv_num\n",
        "        # severe , minor\n",
        "        # Event.num_of_events += 1\n",
        "    def total_time(self):\n",
        "        return self.departure_time - self.generate_time\n",
        "    \n",
        "    def sojourn_time(self):\n",
        "        return self.departure_time - self.arrival_time\n",
        "\n",
        "    def waiting_time(self):\n",
        "        # waiting time in queue\n",
        "        return self.sojourn_time() - self.service_time\n",
        "    \n",
        "    def survival_prob(self):\n",
        "        my_total_time = self.total_time()\n",
        "        SP = SURV_PROB_CAL(self.facility, self.patype, my_total_time, self.surv_num)\n",
        "        self.survival_prob_res = SP\n",
        "        return SP\n",
        "\n",
        "    def __repr__(self):\n",
        "        return f\"{self.generate_time}, {self.trans_time},{self.arrival_time}, {self.start_service_time}, {self.service_time}, {self.departure_time}, {self.survival_prob_res}, {self.facility}, {self.patype} \\n\"\n",
        "    def get_info(self):\n",
        "        #print([self.generate_time, self.trans_time, self.arrival_time, self.start_service_time, self.service_time, self.departure_time, self.facility, self.patype, self.status])\n",
        "        return [self.generate_time, self.trans_time, self.arrival_time, self.start_service_time, self.service_time, self.departure_time, self.survival_prob_res, self.facility, self.patype, self.status]\n",
        "    \n",
        "\"\"\"########################################################################\"\"\"\n",
        "\n",
        "class Server(object):\n",
        "    def __init__(self, c, status, mu0, mu1, tt):\n",
        "        # self.jobs = jobs\n",
        "        self.c = c\n",
        "        self.status = status\n",
        "        self.mu0 = mu0\n",
        "        self.mu1 = mu1\n",
        "        self.tt = tt\n",
        "        self.server_tnext = 0\n",
        "\n",
        "        self.num_busy = 0  # number of busy servers\n",
        "        self.stack = []    # store the patient info in the server\n",
        "        self.queue = []\n",
        "        self.served_jobs = [] \n",
        "\n",
        "    def handle_arrival(self, time, job):\n",
        "        # time here refers to arrival time\n",
        "        # print(\"arrival_job\", job)\n",
        "        # input(\"arrival\")\n",
        "    \n",
        "        if self.num_busy < self.c:\n",
        "            if len(self.queue) == 0:\n",
        "                # print(\"handle_arrival1 \")\n",
        "                self.start_service(time, job)\n",
        "        else:\n",
        "            job.status = \"queue\"\n",
        "            heappush(self.queue, ((job.patype, job.arrival_time), job))\n",
        "            \n",
        "        # print(\"handle_arrival_done \")\n",
        "\n",
        "    def start_service(self, time, job):\n",
        "        # time here refers to start service time\n",
        "        self.num_busy += 1  # server becomes busy.\n",
        "        job.start_service_time = time\n",
        "        job.departure_time = time + job.service_time\n",
        "        \n",
        "        job.status = \"stack\"\n",
        "        heappush(self.stack, (job.departure_time, job))\n",
        "        \n",
        "        # print(\"service_job \", job)\n",
        "        # input(\"service\")\n",
        "        \n",
        "        if job.departure_time <= self.server_tnext:\n",
        "            # print(\"start_service1 \")\n",
        "            self.handle_departure()\n",
        "            \n",
        "        \n",
        "    def handle_departure(self):\n",
        "        # time here refers to departure time \n",
        "        # self.stack.sort(reverse=True)\n",
        "        try:\n",
        "            while self.stack[0][0] <= self.server_tnext:\n",
        "                # print(\"while \", self.stack[0][0], self.server_tnext)\n",
        "                prev_depart = copy.deepcopy(self.stack[0][0])\n",
        "                self.num_busy -= 1\n",
        "                self.stack[0][1].status = \"released\"\n",
        "                next_done = heappop(self.stack)\n",
        "                next_done[1].survival_prob()\n",
        "                self.served_jobs.append(next_done)\n",
        "                \n",
        "                # print(\"next_done \",next_done)\n",
        "                # input(\"depature\")\n",
        "                \n",
        "                if self.queue and self.num_busy < self.c:\n",
        "                    if self.queue[0][1].arrival_time < self.server_tnext:\n",
        "                        # print(\"queue\")\n",
        "                        # print(\"self.server_tnext \", self.server_tnext)\n",
        "                        # print(\"prev_depart \", prev_depart)\n",
        "                        next_job = heappop(self.queue)\n",
        "                        # print(\"next_job! \", next_job)\n",
        "                        \n",
        "                        # input(\"here1\")\n",
        "                        # print(\"arr!\", next_job[1].arrival_time)\n",
        "                        # print(\"prev!\",prev_depart)\n",
        "                        # input(\"here2\")\n",
        "                        self.start_service(max(next_job[1].arrival_time, prev_depart),next_job[1])\n",
        "        except:\n",
        "            pass\n",
        "        \n",
        "    def travel_time(self):\n",
        "        return self.tt\n",
        "        # return random.expovariate(self.tt)\n",
        "        \n",
        "        # return self.tt\n",
        "        \n",
        "    def service_time(self, patient_type):\n",
        "        if patient_type == 0:\n",
        "            return random.expovariate(self.mu0)\n",
        "        else:\n",
        "            return random.expovariate(self.mu1)\n",
        "            \n",
        "    def getSize(self, tnow):\n",
        "        try:\n",
        "            count_queue0 = 0\n",
        "            count_queue1 = 0\n",
        "            copy_queue = copy.deepcopy(self.queue)\n",
        "            for item in copy_queue:\n",
        "                if item[1].arrival_time <= tnow: \n",
        "                    if item[1].patype == 0:\n",
        "                        count_queue0 += 1\n",
        "                    else:\n",
        "                        count_queue1 += 1      \n",
        "                    \n",
        "            return [count_queue0, count_queue1]\n",
        "        \n",
        "        except:\n",
        "            return [0,0]\n",
        "    \n",
        "    \n",
        "    \"\"\"need to optimize code \"\"\"\n",
        "    def getBusyServer(self, tnow):\n",
        "        count_busyserver = 0\n",
        "        copy_stack = copy.deepcopy(self.stack) + copy.deepcopy(self.served_jobs) \n",
        "        # print(\"copy_stack \", copy_stack)\n",
        "        # print(\"stack \", self.stack\n",
        "        \n",
        "        try:\n",
        "            for item in copy_stack:\n",
        "                # print(\"item \",item[1])\n",
        "                if item[1].departure_time >= tnow and item[1].start_service_time <= tnow and count_busyserver < self.c:\n",
        "                    count_busyserver += 1\n",
        "                # print(\"count_busyserver\", count_busyserver)\n",
        "                \n",
        "            return [count_busyserver]\n",
        "        \n",
        "        except:\n",
        "            return [0]\n",
        "    \n",
        "    \n",
        "\"\"\"########################################################################\"\"\"\n",
        "\n",
        "def AssignType():\n",
        "    # set para\n",
        "    p = 0\n",
        "    myassign = np.random.binomial(1,p)\n",
        "    if myassign == 1:\n",
        "        patype = 0\n",
        "    else:\n",
        "        patype = 1\n",
        "    \n",
        "    return patype\n",
        "\n",
        "\"\"\"########################################################################\"\"\"\n",
        "\n",
        "class Env(gym.Env):\n",
        "    def __init__(self, tnow):\n",
        "        self.tnow = tnow\n",
        "        self.gen_patient_time_list = []\n",
        "        self.tgen = 0\n",
        "        self.tgen_next = 0\n",
        "        self.arrival_list1 = []\n",
        "        self.arrival_list2 = []\n",
        "        self.arrival_time = self.tnow\n",
        "        self.surv_num = 5\n",
        "        # Define constants for clearer code\n",
        "        self.S1 = 0\n",
        "        self.S2 = 1\n",
        "        self.action_num = 2\n",
        "        self.action_space = spaces.Discrete(self.action_num)\n",
        "\n",
        "        \n",
        "        \"\"\"['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\"\"\"    \n",
        "        \"\"\" obs space\"\"\"\n",
        "        \"\"\" ['num_to_hos', 'num_to_acf', 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2', \n",
        "        'waiting time at hos', ' service at hos', 'waiting time at acf', ' service at acf'])\"\"\"\n",
        "        \n",
        "        self.obs_low = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])\n",
        "        self.obs_high = np.array([15, 15, 20, 20, 20, 20, 8, 4, 1000, 1000, 1000, 1000])\n",
        "        # self.observation_space = spaces.Box(self.obs_low, self.obs_high, dtype=np.int32)\n",
        "        \n",
        "        self.tgen_list = []\n",
        "        # pd.DataFrame(columns = ['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        # 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\n",
        "        self.rewards_records = {}\n",
        "        self.obs_records = {}\n",
        "        self.action_records = {}\n",
        "        self.wait_records1 = {}  # queuing_time\n",
        "        self.service_records1 = {} #service_time\n",
        "        self.wait_records2 = {}  # queuing_time\n",
        "        self.service_records2 = {}\n",
        "\n",
        "        \"\"\"Server Settings\"\"\"\n",
        "        \"\"\" mu_tt = log(x) \"\"\"\n",
        "        \"\"\"pdf = (np.exp(-(np.log(x) - mu)**2 / (2 * sigma**2)) / (x * sigma * np.sqrt(2 * np.pi)))\"\"\"\n",
        "\n",
        "        # set para server\n",
        "\n",
        "        self.server1_rate0 = 1/90  \n",
        "        self.server1_rate1 = 1/30\n",
        "        \"\"\" mean value, not mu\"\"\"\n",
        "        self.server1_tt = 60\n",
        "        \n",
        "        self.server2_rate0 = 1/90\n",
        "        self.server2_rate1 = 1/30\n",
        "        \n",
        "        \"\"\" mean value, not mu\"\"\"\n",
        "        self.server2_tt = 10\n",
        "        \n",
        "        # __init__(self, c, status, mu0, mu1, tt):\n",
        "        self.server1 = Server(12, 'open',self.server1_rate0,self.server1_rate1,self.server1_tt)\n",
        "        self.server2 = Server(6, 'open', self.server2_rate0, self.server2_rate1,self.server2_tt)\n",
        "        self.s2_isopen = True   \n",
        "        self.simulation_time = total_simulation_time\n",
        "        \n",
        "        \n",
        "    #     \"\"\" use pre-generated data\"\"\"\n",
        "    # def Get_Time(self, myi):\n",
        "    #     nlength = 50\n",
        "    #     a = pd.read_excel(\"/Users/machaolun/Documents/GitHub/EvacuationMDP/Evacuation_code/RL/patient_generattion2.xlsx\",\n",
        "    #               index_col=0)\n",
        "    #     b = list(a.iloc[myi,:nlength])\n",
        "    #     self.gen_patient_time_list = deque(b)\n",
        "        \n",
        "        \n",
        "        \"\"\" use generated data\"\"\"\n",
        "    def Get_Time(self, myi):\n",
        "        total_num_patient = 50\n",
        "        arrive_lambda = 1/5.0\n",
        "        mylist = []\n",
        "        t= 0\n",
        "        \"\"\"Generate Casulties\"\"\"\n",
        "        for j in range(total_num_patient):\n",
        "            t+= random.expovariate(arrive_lambda)\n",
        "            mylist.append(t)   \n",
        "            \n",
        "        self.gen_patient_time_list = deque(mylist)   \n",
        "    # def GenerateTime(self):\n",
        "    #     \"\"\"\n",
        "    #     Arrival rate per min\n",
        "    #     Set Arrival Rate (time inhomogeneous) Here\n",
        "    #     \"\"\"\n",
        "    #     # ref: https://zhuanlan.zhihu.com/p/28785318\n",
        "    #     t = 0\n",
        "    #     I = 0\n",
        "    #     s = []\n",
        "        \n",
        "    #     max_simulate_time = 1000\n",
        "        \n",
        "    #     while t <= max_simulate_time:\n",
        "    #         u = np.random.uniform(1e-8,1)\n",
        "    #         t = t - np.log(u)\n",
        "    #         if t <= max_simulate_time:\n",
        "    #             u1 = np.random.uniform(1e-8,1)\n",
        "    #             if u1 <= inhop(t):\n",
        "    #                 I = I + 1\n",
        "    #                 s.append(t)\n",
        "                    \n",
        "    #     # time when event happens\n",
        "    #     self.gen_patient_time_list = deque(s)        \n",
        "    \n",
        "    \"\"\" Open new ACF, the resource comes from hospital\"\"\"        \n",
        "    # def take_action(self, Action):\n",
        "    #     if (not self.s2_isopen) and Action == \"open\":\n",
        "    #         original_num_server = copy.deepcopy(self.server1.c)\n",
        "    #         self.s2_isopen = True\n",
        "            \n",
        "    #         # set para\n",
        "    #         precent_move = 0.2\n",
        "    #         self.server1.c = round((1 - precent_move)*original_num_server)\n",
        "    #         self.server2.c = int(precent_move * original_num_server) + 2 \n",
        "    #         self.server2.status = 'open'\n",
        "            \n",
        "    #         print(\"!!!!!!!!!!!!!!!\\n\",self.server1.c, self.server2.c)\n",
        "    #     else:\n",
        "    #         pass \n",
        "        \n",
        "    def step(self, Action):\n",
        "        \n",
        "        try:\n",
        "            self.tgen = self.gen_patient_time_list.popleft()\n",
        "            try:\n",
        "                self.tgen_next = self.gen_patient_time_list[0]\n",
        "            except:\n",
        "                self.tgen_next = 100000\n",
        "            # action records\n",
        "            self.action_records[self.tgen] = Action\n",
        "            \n",
        "            patient = Event(self.tgen, AssignType(), self.surv_num)\n",
        "\n",
        "            #  self.s2_isopen and         \n",
        "            if Action == self.S2:    \n",
        "                patient.trans_time = self.server2.travel_time()\n",
        "                # self.server2.tnext = min(self.tgen + patient.travel_time , self.simulation_time)\n",
        "                patient.facility = 2\n",
        "                patient.arrival_time = patient.generate_time + patient.trans_time\n",
        "                patient.service_time = self.server2.service_time(patient.patype)\n",
        "                heappush(self.arrival_list2, ((patient.arrival_time), patient))\n",
        "                \n",
        "            elif Action == self.S1:\n",
        "                patient.trans_time = self.server1.travel_time()\n",
        "                # self.server1.tnext = min(self.tgen + patient.travel_time, self.simulation_time)\n",
        "                patient.facility = 1\n",
        "                patient.arrival_time = patient.generate_time + patient.trans_time\n",
        "                patient.service_time = self.server1.service_time(patient.patype) \n",
        "                heappush(self.arrival_list1, ((patient.arrival_time), patient))\n",
        "                                \n",
        "            else:\n",
        "                raise ValueError(\"Received invalid action={} which is not part of the action space\".format(Action))\n",
        "        \n",
        "        except:\n",
        "            self.tgen_next = 100000\n",
        "            \n",
        "        try:\n",
        "            while self.arrival_list1[0][0] <= self.tgen_next:\n",
        "                try:\n",
        "                    self.server1.server_tnext = min(self.arrival_list1[0][0], self.tgen_next)\n",
        "                except:\n",
        "                    self.server1.server_tnext = self.tgen_next\n",
        "                    \n",
        "                self.server1.handle_departure()\n",
        "                mypatient_s1 = heappop(self.arrival_list1)\n",
        "                self.server1.handle_arrival(mypatient_s1[1].arrival_time, mypatient_s1[1])\n",
        "                \n",
        "        except:\n",
        "            pass\n",
        "        \n",
        "        self.server1.server_tnext = self.tgen_next    \n",
        "        self.server1.handle_departure()\n",
        "        \n",
        "        if self.s2_isopen:\n",
        "            try:\n",
        "                while self.arrival_list2[0][0] <= self.tgen_next:\n",
        "    \n",
        "                    try:\n",
        "                        self.server2.server_tnext = min(self.arrival_list2[0][0], self.tgen_next)\n",
        "                    except:\n",
        "                        self.server2.server_tnext = self.tgen_next\n",
        "                        \n",
        "                    self.server2.handle_departure()\n",
        "                    \n",
        "                    mypatient_s2 = heappop(self.arrival_list2)\n",
        "                    self.server2.handle_arrival(mypatient_s2[1].arrival_time, mypatient_s2[1])\n",
        "                    \n",
        "            except:\n",
        "                pass\n",
        "            \n",
        "            self.server2.server_tnext = self.tgen_next\n",
        "            self.server2.handle_departure()\n",
        "                \n",
        "        \"\"\" gen_time, num to hos, num to acf, \n",
        "        [ql10,ql11], [ql20, ql21],\n",
        "        busy server1, busy 2 \"\"\"        \n",
        "        \"\"\"['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\"\"\"    \n",
        "        \n",
        "        # print(\"self.tgen_next\", self.tgen_next)\n",
        "        if self.s2_isopen:\n",
        "            Legal_Action = [0,1]\n",
        "        else:\n",
        "            Legal_Action = [0,1]\n",
        "\n",
        "        # for the first one\n",
        "        if len(self.gen_patient_time_list) == 49:\n",
        "            self.tgen_list.append([self.tgen, Legal_Action])\n",
        "            self.obs_records[self.tgen] = np.array([self.tgen, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,0 ]) \n",
        "\n",
        "            #action record is above \n",
        "            \n",
        "        all_served = self.server1.served_jobs + self.server2.served_jobs\n",
        "        \n",
        "       \n",
        "        for jobs in all_served:\n",
        "            self.rewards_records[jobs[1].generate_time] = jobs[1].survival_prob_res\n",
        "  \n",
        "        # add obs to records\n",
        "        if self.tgen_next < 100000:\n",
        "            self.tgen_list.append([self.tgen_next, Legal_Action])\n",
        "            i = 0\n",
        "            j = 0\n",
        "            w_record1 = 0\n",
        "            s_record1 = 0\n",
        "            w_record2 = 0\n",
        "            s_record2 = 0\n",
        "            \n",
        "            for jobs2 in all_served:\n",
        "\n",
        "                # time window -- future consideration\n",
        "                if jobs2[1].departure_time < self.tgen_next and jobs2[1].facility  == 1: # 1hos 2acf\n",
        "                    i += 1\n",
        "                    w_record1 += jobs2[1].waiting_time()            \n",
        "                    s_record1 += jobs2[1].service_time\n",
        "                    \n",
        "                if jobs2[1].departure_time < self.tgen_next and jobs2[1].facility  == 2: # 1hos 2acf\n",
        "                    j += 1\n",
        "                    w_record2 += jobs2[1].waiting_time()            \n",
        "                    s_record2 += jobs2[1].service_time                    \n",
        "\n",
        "            self.wait_records1[self.tgen_next] = 0 if i == 0 else int(round(w_record1/i))   # queuing_time\n",
        "            self.service_records1[self.tgen_next] = 0 if i == 0 else int(round(s_record1/i)) #service_time\n",
        "\n",
        "            self.wait_records2[self.tgen_next] = 0 if j == 0 else int(round(w_record2/j))   # queuing_time\n",
        "            self.service_records2[self.tgen_next] = 0 if j == 0 else int(round(s_record2/j))  #service_time\n",
        "                   \n",
        "            # observation records\n",
        "            self.obs_records[self.tgen_next] = np.array([self.tgen_next, len(self.arrival_list1), len(self.arrival_list2)] \n",
        "                        + self.server1.getSize(self.tgen_next)\n",
        "                        + self.server2.getSize(self.tgen_next)\n",
        "                        + self.server1.getBusyServer(self.tgen_next)\n",
        "                        + self.server2.getBusyServer(self.tgen_next)\n",
        "                        + [self.wait_records1[self.tgen_next],\n",
        "                           self.service_records1[self.tgen_next],\n",
        "                           self.wait_records2[self.tgen_next],\n",
        "                           self.service_records2[self.tgen_next]])\n",
        "                                                        \n",
        "                        # + [len(self.server1.served_jobs)]\n",
        "                        # + [len(self.server2.served_jobs)])\n",
        "            states = {'obs': np.delete(self.obs_records[self.tgen_next],0), \n",
        "                      'legal_actions': Legal_Action}\n",
        "        else:\n",
        "            states = {'obs': self.obs_low,\n",
        "                      'legal_actions': [0,1]}\n",
        "        \n",
        "\n",
        "            \n",
        "             \n",
        "        returns = []\n",
        "\n",
        "        for everyone in self.tgen_list[:]:\n",
        "            try:\n",
        "                rewards = self.rewards_records[everyone[0]]\n",
        "                \n",
        "                obs = self.obs_records[everyone[0]]\n",
        "                \n",
        "                actions = self.action_records[everyone[0]]\n",
        "                \n",
        "                returns.append({'obs': obs,'legal_actions': everyone[1],'action': actions ,'reward': rewards})\n",
        "                self.tgen_list.remove(everyone)\n",
        "            except:\n",
        "                continue\n",
        "\n",
        "        done = bool(self.tgen_next == 100000)\n",
        "        # obs = np.array([1,1,1,1,1,1,1,1])\n",
        "        # reward = 1\n",
        "        info = {}\n",
        "        \n",
        "        return states, returns, done, info\n",
        "        \n",
        "    def set_agents(self, agents):\n",
        "        ''' Set the agents that will interact with the environment\n",
        "        Args:\n",
        "            agents (list): List of Agent classes\n",
        "        '''\n",
        "        self.agents = agents\n",
        "\n",
        "\n",
        "    def run(self, myj, is_training=False):\n",
        "        '''\n",
        "        Run a complete game, either for evaluation or training RL agent.\n",
        "        Args:\n",
        "            is_training (boolean): True if for training purpose.\n",
        "        Returns:\n",
        "            (tuple) Tuple containing:\n",
        "                (list): A list of trajectories generated from the environment.\n",
        "                (list): A list payoffs. Each entry corresponds to one player.\n",
        "        Note: The trajectories are 3-dimension list. The first dimension is for different players.\n",
        "              The second dimension is for different transitions. The third dimension is for the contents of each transiton\n",
        "        '''\n",
        "        return_summary = []\n",
        "        done = False\n",
        "        \n",
        "        states, returns, done, info = self.reset()\n",
        "        # read arrival data \n",
        "        self.Get_Time(myj)\n",
        "        \n",
        "        while not done:\n",
        "            # action, _states = model.predict(states)\n",
        "            if not is_training:\n",
        "                action, probs_ = self.agents.eval_step(states)\n",
        "                # print(\"best action \", action)\n",
        "                # print(\"probs \", probs_)\n",
        "            else:\n",
        "                action = self.agents.step(states)\n",
        "            \n",
        "            # if random.random() > 0.9:\n",
        "            #     action = 1   \n",
        "            # else:\n",
        "            #     action = 0\n",
        "            # action = 0\n",
        "            \n",
        "            states, returns, done, info = self.step(action)\n",
        "            \n",
        "            return_summary = np.append(return_summary, returns)\n",
        "        \n",
        "        # served_result = self.get_stat()\n",
        "        \n",
        "        # print(served_result)\n",
        "        \n",
        "        list2 = [x for x in return_summary if x != []]\n",
        "        sorted_ts = sorted(list2, key = lambda i: i['obs'][0]) \n",
        "        \n",
        "        trajectories = []\n",
        "\n",
        "        for i in range(len(sorted_ts) - 1):\n",
        "            #np.delete(sorted_ts[i]['obs'],0)\n",
        "            trajectories.append([{'obs': np.delete(sorted_ts[i]['obs'],0), 'legal_actions': sorted_ts[i]['legal_actions']}, sorted_ts[i]['action'],sorted_ts[i]['reward'],\n",
        "          {'obs': np.delete(sorted_ts[i+1]['obs'],0), 'legal_actions': sorted_ts[i+1]['legal_actions']}, bool(i == len(sorted_ts) - 2)])\n",
        "        \n",
        "        return trajectories\n",
        "    \n",
        "    def get_Reward(self):\n",
        "    # Playing Atari with Deep Reinforcement Learning    \n",
        "    # receives a reward rt representing the change in game score.\n",
        "    # survival probablity\n",
        "    # Resource-Based Patient Prioritization in Mass-Casualty Incidents\n",
        "        all_served = self.server1.served_jobs + self.server2.served_jobs\n",
        "        \n",
        "        for jobs in all_served:\n",
        "            # if jobs[1].generate_time not in list(self.rewards_records.keys()):\n",
        "            self.rewards_records[jobs[1].generate_time] = jobs[1].survival_prob_res\n",
        "\n",
        "\n",
        "    def reset(self):\n",
        "        # return observation\n",
        "        self.tnow = 0\n",
        "        self.gen_patient_time_list = []\n",
        "        self.tgen = 0\n",
        "        self.tnext = 0\n",
        "        self.arrival_list1 = []\n",
        "        self.arrival_list2 = []\n",
        "        self.arrival_time = self.tnow\n",
        "        self.surv_num = 5\n",
        "\n",
        "        \n",
        "        \"\"\"['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\"\"\"    \n",
        "        \"\"\" obs space\"\"\"\n",
        "        \"\"\" ['num_to_hos', 'num_to_acf', 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2', 'waiting time ', ' service'])\"\"\"\n",
        "        \n",
        "        # self.observation_space = spaces.Box(self.obs_low, self.obs_high, dtype=np.int32)\n",
        "        \n",
        "        self.tgen_list = []\n",
        "        # pd.DataFrame(columns = ['gen_time', 'num_to_hos', 'num_to_acf', \n",
        "        # 'ql10' ,'ql11', 'ql20', 'ql21', 'busy_1', 'busy_2','served_1', 'served_2', 'rewards'])\n",
        "        self.rewards_records = {}\n",
        "        self.obs_records = {}\n",
        "        self.action_records = {}\n",
        "        self.wait_records1 = {}  # queuing_time\n",
        "        self.service_records1 = {} #service_time\n",
        "        self.wait_records2 = {}  # queuing_time\n",
        "        self.service_records2 = {}\n",
        "        \n",
        "        \n",
        "        self.server1 = Server(12, 'open',self.server1_rate0,self.server1_rate1,self.server1_tt)\n",
        "        self.server2 = Server(6, 'not_open', self.server2_rate0, self.server2_rate1,self.server2_tt)\n",
        "        self.s2_isopen = True\n",
        "        self.simulation_time = 100000\n",
        "        \n",
        "        done = False\n",
        "        info = {}\n",
        "        returns = []\n",
        "        states = {'obs': self.obs_low,'legal_actions': [0,1]}\n",
        "        \n",
        "        return states, returns, done, info\n",
        "\n",
        "    \n",
        "    def get_stat(self):\n",
        "        # Avg_SojournTime, Avg_WaitingTime, Avg_QueueLength\n",
        "        SojournTime1 = 0.0\n",
        "        WaitingTime1 = 0.0\n",
        "        SojournTime2 = 0.0\n",
        "        WaitingTime2 = 0.0\n",
        "        \n",
        "        mydata = []\n",
        "        \n",
        "        for jobs in self.server1.served_jobs:\n",
        "            SojournTime1 += jobs[1].sojourn_time()\n",
        "            WaitingTime1 += jobs[1].waiting_time()\n",
        "            \"\"\" f\"{self.arrival_time}, {self.start_service_time}, {self.service_time}, \n",
        "            {self.departure_time}, {self.facility}, {self.patype} \\n\" \"\"\"\n",
        "\n",
        "            # heappush(mydata, (jobs[1].arrival_time, jobs[1].get_info()))\n",
        "            mydata.append(jobs[1].get_info())\n",
        "            \n",
        "            \"\"\"also attach patient in queue and not release\"\"\"\n",
        "            \n",
        "        for not_complete_job in self.server1.stack:\n",
        "            mydata.append(not_complete_job[1].get_info())\n",
        "            \n",
        "        for not_complete_job in self.server1.queue:\n",
        "            mydata.append(not_complete_job[1].get_info())\n",
        "\n",
        "        with open('Server1.csv', 'w', newline='') as file:\n",
        "            writer = csv.writer(file, delimiter=',')\n",
        "            writer.writerows(mydata)        \n",
        "        \n",
        "        mydata = []\n",
        "        \n",
        "        for jobs in self.server2.served_jobs:\n",
        "            SojournTime2 += jobs[1].sojourn_time()\n",
        "            WaitingTime2 += jobs[1].waiting_time()\n",
        "            mydata.append(jobs[1].get_info())\n",
        "            # mydata.append([jobs[1].arrival_time,jobs[1].start_service_time,jobs[1].service_time, jobs[1].departure_time])\n",
        "            # print(\"{0:.8f}\".format(jobs[1].arrival_time), \"{0:.8f}\".format(jobs[1].start_service_time),\\\n",
        "            #       \"{0:.8f}\".format(jobs[1].service_time), \"{0:.8f}\".format(jobs[1].departure_time))\n",
        "            # print(jobs[1].get_info())\n",
        "            \n",
        "        for not_complete_job in self.server2.stack:\n",
        "            mydata.append(not_complete_job[1].get_info())\n",
        "            \n",
        "        for not_complete_job in self.server2.queue:\n",
        "            mydata.append(not_complete_job[1].get_info())            \n",
        "        \n",
        "        with open('Server2.csv', 'w', newline='') as file:\n",
        "            writer = csv.writer(file, delimiter=',')\n",
        "            writer.writerows(mydata) \n",
        "            \n",
        "            \n",
        "        if len(self.server1.served_jobs) != 0:\n",
        "            Avg_SojournTime1 = SojournTime1/len(self.server1.served_jobs)\n",
        "            Avg_WaitingTime1 = WaitingTime1/len(self.server1.served_jobs)\n",
        "        else:    \n",
        "            Avg_SojournTime1 = None\n",
        "            Avg_WaitingTime1 = None\n",
        "\n",
        "        if len(self.server2.served_jobs) != 0:\n",
        "            Avg_SojournTime2 = SojournTime2/len(self.server2.served_jobs)             \n",
        "            Avg_WaitingTime2 = WaitingTime2/len(self.server2.served_jobs)\n",
        "        else:\n",
        "            Avg_SojournTime2 = None\n",
        "            Avg_WaitingTime2 = None\n",
        "            \n",
        "        \n",
        "        return  Avg_SojournTime1, Avg_WaitingTime1, \\\n",
        "            Avg_SojournTime2, Avg_WaitingTime2\n",
        "                # Avg_QueueLength1, Avg_QueueLength2\n",
        "        \n",
        "    def debug(self):\n",
        "        print(\"==============\")\n",
        "        # self.server1.queue,\n",
        "        # print(\"server1  \", self.server1.c, self.server1.action, \\\n",
        "              # self.server1.getSize(),  self.server1.stack)\n",
        "        # print(\"served job \", self.server1.served_jobs)\n",
        "        # print(\"server2  \", self.server2.c, self.server2.action, \\\n",
        "        # self.server2.getSize(), self.server2.stack)\n",
        "\n",
        "\n",
        " \n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8AkpLa7UxOYj"
      },
      "source": [
        "import tensorflow as tf\n",
        "import os\n",
        "import pandas as pd\n",
        "import rlcard\n",
        "\n",
        "from rlcard.utils import set_global_seed\n",
        "from rlcard.utils import Logger"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "au65qNEZ1MMZ",
        "cellView": "form"
      },
      "source": [
        "#@title Code for Agent\n",
        "''' DQN agent\n",
        "The code is derived from https://github.com/dennybritz/reinforcement-learning/blob/master/DQN/dqn.py\n",
        "Copyright (c) 2019 DATA Lab at Texas A&M University\n",
        "Copyright (c) 2016 Denny Britz\n",
        "Permission is hereby granted, free of charge, to any person obtaining a copy\n",
        "of this software and associated documentation files (the \"Software\"), to deal\n",
        "in the Software without restriction, including without limitation the rights\n",
        "to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
        "copies of the Software, and to permit persons to whom the Software is\n",
        "furnished to do so, subject to the following conditions:\n",
        "The above copyright notice and this permission notice shall be included in all\n",
        "copies or substantial portions of the Software.\n",
        "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
        "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
        "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n",
        "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
        "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n",
        "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n",
        "SOFTWARE.\n",
        "'''\n",
        "\n",
        "import random\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from collections import namedtuple\n",
        "\n",
        "from rlcard.utils.utils import remove_illegal\n",
        "\n",
        "Transition = namedtuple('Transition', ['state', 'action', 'reward', 'next_state', 'done'])\n",
        "\n",
        "\n",
        "class DQNAgent(object):\n",
        "\n",
        "    def __init__(self,\n",
        "                 sess,\n",
        "                 scope,\n",
        "                 replay_memory_size=10000,\n",
        "                 replay_memory_init_size=200,\n",
        "                 update_target_estimator_every=1000,\n",
        "                 discount_factor=0.99,\n",
        "                 epsilon_start=1.0,\n",
        "                 epsilon_end=0.01,\n",
        "                 epsilon_decay_steps=10000,\n",
        "                 batch_size=32,\n",
        "                 action_num=2,\n",
        "                 state_shape=None,\n",
        "                 train_every=1,\n",
        "                 mlp_layers=None,\n",
        "                 learning_rate=0.00005):\n",
        "\n",
        "        '''\n",
        "        Q-Learning algorithm for off-policy TD control using Function Approximation.\n",
        "        Finds the optimal greedy policy while following an epsilon-greedy policy.\n",
        "        Args:\n",
        "            sess (tf.Session): Tensorflow Session object.\n",
        "            scope (string): The name scope of the DQN agent.\n",
        "            replay_memory_size (int): Size of the replay memory\n",
        "            replay_memory_init_size (int): Number of random experiences to sampel when initializing\n",
        "              the reply memory.\n",
        "            train_every (int): Train the agent every X steps.\n",
        "            update_target_estimator_every (int): Copy parameters from the Q estimator to the\n",
        "              target estimator every N steps\n",
        "            discount_factor (float): Gamma discount factor\n",
        "            epsilon_start (int): Chance to sample a random action when taking an action.\n",
        "              Epsilon is decayed over time and this is the start value\n",
        "            epsilon_end (int): The final minimum value of epsilon after decaying is done\n",
        "            epsilon_decay_steps (int): Number of steps to decay epsilon over\n",
        "            batch_size (int): Size of batches to sample from the replay memory\n",
        "            evaluate_every (int): Evaluate every N steps\n",
        "            action_num (int): The number of the actions\n",
        "            state_space (list): The space of the state vector\n",
        "            train_every (int): Train the network every X steps.\n",
        "            mlp_layers (list): The layer number and the dimension of each layer in MLP\n",
        "            learning_rate (float): The learning rate of the DQN agent.\n",
        "        '''\n",
        "        self.use_raw = False\n",
        "        self.sess = sess\n",
        "        self.scope = scope\n",
        "        self.replay_memory_init_size = replay_memory_init_size\n",
        "        self.update_target_estimator_every = update_target_estimator_every\n",
        "        self.discount_factor = discount_factor\n",
        "        self.epsilon_decay_steps = epsilon_decay_steps\n",
        "        self.batch_size = batch_size\n",
        "        self.action_num = action_num\n",
        "        self.train_every = train_every\n",
        "\n",
        "        # Total timesteps\n",
        "        self.total_t = 0\n",
        "\n",
        "        # Total training step\n",
        "        self.train_t = 0\n",
        "\n",
        "        # The epsilon decay scheduler\n",
        "        self.epsilons = np.linspace(epsilon_start, epsilon_end, epsilon_decay_steps)\n",
        "\n",
        "        # Create estimators\n",
        "        self.q_estimator = Estimator(scope=self.scope+\"_q\", action_num=action_num, learning_rate=learning_rate, state_shape=state_shape, mlp_layers=mlp_layers)\n",
        "        self.target_estimator = Estimator(scope=self.scope+\"_target_q\", action_num=action_num, learning_rate=learning_rate, state_shape=state_shape, mlp_layers=mlp_layers)\n",
        "\n",
        "        # Create replay memory\n",
        "        self.memory = Memory(replay_memory_size, batch_size)\n",
        "\n",
        "    def feed(self, ts):\n",
        "        ''' Store data in to replay buffer and train the agent. There are two stages.\n",
        "            In stage 1, populate the memory without training\n",
        "            In stage 2, train the agent every several timesteps\n",
        "        Args:\n",
        "            ts (list): a list of 5 elements that represent the transition\n",
        "        '''\n",
        "        (state, action, reward, next_state, done) = tuple(ts)\n",
        "        self.feed_memory(state['obs'], action, reward, next_state['obs'], done)\n",
        "        self.total_t += 1\n",
        "        tmp = self.total_t - self.replay_memory_init_size\n",
        "        if tmp>=0 and tmp%self.train_every == 0:\n",
        "            self.train()\n",
        "\n",
        "    def step(self, state):\n",
        "        ''' Predict the action for generating training data\n",
        "        Args:\n",
        "            state (numpy.array): current state\n",
        "        Returns:\n",
        "            action (int): an action id\n",
        "        '''\n",
        "        A = self.predict(state['obs'])\n",
        "        A = remove_illegal(A, state['legal_actions'])\n",
        "        action = np.random.choice(np.arange(len(A)), p=A)\n",
        "        return action\n",
        "\n",
        "    def eval_step(self, state):\n",
        "        ''' Predict the action for evaluation purpose.\n",
        "        Args:\n",
        "            state (numpy.array): current state\n",
        "        Returns:\n",
        "            action (int): an action id\n",
        "            probs (list): a list of probabilies\n",
        "        '''\n",
        "        q_values = self.q_estimator.predict(self.sess, np.expand_dims(state['obs'], 0))[0]\n",
        "        probs = remove_illegal(np.exp(q_values), state['legal_actions'])\n",
        "        best_action = np.argmax(probs)\n",
        "\n",
        "        # print(\"q_values \", q_values)\n",
        "        # print(\"eval_step \", state, best_action, probs)\n",
        "        return best_action, probs\n",
        "\n",
        "    def predict(self, state):\n",
        "        ''' Predict the action probabilities\n",
        "        Args:\n",
        "            state (numpy.array): current state\n",
        "        Returns:\n",
        "            q_values (numpy.array): a 1-d array where each entry represents a Q value\n",
        "        '''\n",
        "        epsilon = self.epsilons[min(self.total_t, self.epsilon_decay_steps-1)]\n",
        "        A = np.ones(self.action_num, dtype=float) * epsilon / self.action_num\n",
        "        q_values = self.q_estimator.predict(self.sess, np.expand_dims(state, 0))[0]\n",
        "        best_action = np.argmax(q_values)\n",
        "        A[best_action] += (1.0 - epsilon)\n",
        "        return A\n",
        "\n",
        "    def train(self):\n",
        "        ''' Train the network\n",
        "        Returns:\n",
        "            loss (float): The loss of the current batch.\n",
        "        '''\n",
        "        state_batch, action_batch, reward_batch, next_state_batch, done_batch = self.memory.sample()\n",
        "        # Calculate q values and targets (Double DQN)\n",
        "        q_values_next = self.q_estimator.predict(self.sess, next_state_batch)\n",
        "        best_actions = np.argmax(q_values_next, axis=1)\n",
        "        q_values_next_target = self.target_estimator.predict(self.sess, next_state_batch)\n",
        "        target_batch = reward_batch + np.invert(done_batch).astype(np.float32) * \\\n",
        "            self.discount_factor * q_values_next_target[np.arange(self.batch_size), best_actions]\n",
        "\n",
        "        # Perform gradient descent update\n",
        "        state_batch = np.array(state_batch)\n",
        "        loss = self.q_estimator.update(self.sess, state_batch, action_batch, target_batch)\n",
        "        print('\\rINFO - Agent {}, step {}, rl-loss: {}'.format(self.scope, self.total_t, loss), end='')\n",
        "\n",
        "\n",
        "        # Update the target estimator\n",
        "        if self.train_t % self.update_target_estimator_every == 0:\n",
        "            copy_model_parameters(self.sess, self.q_estimator, self.target_estimator)\n",
        "            print(\"\\nINFO - Copied model parameters to target network.\")\n",
        "\n",
        "        self.train_t += 1\n",
        "\n",
        "    def feed_memory(self, state, action, reward, next_state, done):\n",
        "        ''' Feed transition to memory\n",
        "        Args:\n",
        "            state (numpy.array): the current state\n",
        "            action (int): the performed action ID\n",
        "            reward (float): the reward received\n",
        "            next_state (numpy.array): the next state after performing the action\n",
        "            done (boolean): whether the episode is finished\n",
        "        '''\n",
        "        self.memory.save(state, action, reward, next_state, done)\n",
        "\n",
        "    def copy_params_op(self, global_vars):\n",
        "        ''' Copys the variables of two estimator to others.\n",
        "        Args:\n",
        "            global_vars (list): A list of tensor\n",
        "        '''\n",
        "        self_vars = tf.contrib.slim.get_variables(scope=self.scope, collection=tf.GraphKeys.TRAINABLE_VARIABLES)\n",
        "        update_ops = []\n",
        "        for v1, v2 in zip(global_vars, self_vars):\n",
        "            op = v2.assign(v1)\n",
        "            update_ops.append(op)\n",
        "        self.sess.run(update_ops)\n",
        "\n",
        "class Estimator():\n",
        "    ''' Q-Value Estimator neural network.\n",
        "        This network is used for both the Q-Network and the Target Network.\n",
        "    '''\n",
        "\n",
        "    def __init__(self, scope=\"estimator\", action_num=2, learning_rate=0.001, state_shape=None, mlp_layers=None):\n",
        "        ''' Initilalize an Estimator object.\n",
        "        Args:\n",
        "            action_num (int): the number output actions\n",
        "            state_shap (list): the shape of the state space\n",
        "        '''\n",
        "        self.scope = scope\n",
        "        self.action_num = action_num\n",
        "        self.learning_rate=learning_rate\n",
        "        self.state_shape = state_shape if isinstance(state_shape, list) else [state_shape]\n",
        "        self.mlp_layers = map(int, mlp_layers)\n",
        "\n",
        "        with tf.variable_scope(scope):\n",
        "            # Build the graph\n",
        "            self._build_model()\n",
        "            update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope=tf.get_variable_scope().name)\n",
        "        # Optimizer Parameters from original paper\n",
        "        self.optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate, name='dqn_adam')\n",
        "\n",
        "        with tf.control_dependencies(update_ops):\n",
        "            self.train_op = self.optimizer.minimize(self.loss, global_step=tf.contrib.framework.get_global_step())\n",
        "\n",
        "    def _build_model(self):\n",
        "        ''' Build an MLP model.\n",
        "        '''\n",
        "        # Placeholders for our input\n",
        "        # Our input are 4 RGB frames of shape 160, 160 each\n",
        "        input_shape = [None]\n",
        "        input_shape.extend(self.state_shape)\n",
        "        self.X_pl = tf.placeholder(shape=input_shape, dtype=tf.float32, name=\"X\")\n",
        "        # The TD target value\n",
        "        self.y_pl = tf.placeholder(shape=[None], dtype=tf.float32, name=\"y\")\n",
        "        # Integer id of which action was selected\n",
        "        self.actions_pl = tf.placeholder(shape=[None], dtype=tf.int32, name=\"actions\")\n",
        "        # Boolean to indicate whether is training or not\n",
        "        self.is_train = tf.placeholder(tf.bool, name=\"is_train\")\n",
        "\n",
        "        batch_size = tf.shape(self.X_pl)[0]\n",
        "\n",
        "        # Batch Normalization\n",
        "        X = tf.layers.batch_normalization(self.X_pl, training=self.is_train)\n",
        "\n",
        "        # Fully connected layers\n",
        "        fc = tf.contrib.layers.flatten(X)\n",
        "        for dim in self.mlp_layers:\n",
        "            fc = tf.contrib.layers.fully_connected(fc, dim, activation_fn=tf.tanh)\n",
        "        self.predictions = tf.contrib.layers.fully_connected(fc, self.action_num, activation_fn=None)\n",
        "\n",
        "        # Get the predictions for the chosen actions only\n",
        "        gather_indices = tf.range(batch_size) * tf.shape(self.predictions)[1] + self.actions_pl\n",
        "        self.action_predictions = tf.gather(tf.reshape(self.predictions, [-1]), gather_indices)\n",
        "\n",
        "        # Calculate the loss\n",
        "        self.losses = tf.squared_difference(self.y_pl, self.action_predictions)\n",
        "        self.loss = tf.reduce_mean(self.losses)\n",
        "\n",
        "    def predict(self, sess, s):\n",
        "        ''' Predicts action values.\n",
        "        Args:\n",
        "          sess (tf.Session): Tensorflow Session object\n",
        "          s (numpy.array): State input of shape [batch_size, 4, 160, 160, 3]\n",
        "          is_train (boolean): True if is training\n",
        "        Returns:\n",
        "          Tensor of shape [batch_size, NUM_VALID_ACTIONS] containing the estimated\n",
        "          action values.\n",
        "        '''\n",
        "        return sess.run(self.predictions, { self.X_pl: s, self.is_train:False})\n",
        "\n",
        "    def update(self, sess, s, a, y):\n",
        "        ''' Updates the estimator towards the given targets.\n",
        "        Args:\n",
        "          sess (tf.Session): Tensorflow Session object\n",
        "          s (list): State input of shape [batch_size, 4, 160, 160, 3]\n",
        "          a (list): Chosen actions of shape [batch_size]\n",
        "          y (list): Targets of shape [batch_size]\n",
        "        Returns:\n",
        "          The calculated loss on the batch.\n",
        "        '''\n",
        "        feed_dict = { self.X_pl: s, self.y_pl: y, self.actions_pl: a, self.is_train: True}\n",
        "        _, _, loss = sess.run(\n",
        "                [tf.contrib.framework.get_global_step(), self.train_op, self.loss],\n",
        "                feed_dict)\n",
        "        return loss\n",
        "\n",
        "class Memory(object):\n",
        "    ''' Memory for saving transitions\n",
        "    '''\n",
        "\n",
        "    def __init__(self, memory_size, batch_size):\n",
        "        ''' Initialize\n",
        "        Args:\n",
        "            memory_size (int): the size of the memroy buffer\n",
        "        '''\n",
        "        self.memory_size = memory_size\n",
        "        self.batch_size = batch_size\n",
        "        self.memory = []\n",
        "\n",
        "    def save(self, state, action, reward, next_state, done):\n",
        "        ''' Save transition into memory\n",
        "        Args:\n",
        "            state (numpy.array): the current state\n",
        "            action (int): the performed action ID\n",
        "            reward (float): the reward received\n",
        "            next_state (numpy.array): the next state after performing the action\n",
        "            done (boolean): whether the episode is finished\n",
        "        '''\n",
        "        if len(self.memory) == self.memory_size:\n",
        "            self.memory.pop(0)\n",
        "        transition = Transition(state, action, reward, next_state, done)\n",
        "        self.memory.append(transition)\n",
        "\n",
        "    def sample(self):\n",
        "        ''' Sample a minibatch from the replay memory\n",
        "        Returns:\n",
        "            state_batch (list): a batch of states\n",
        "            action_batch (list): a batch of actions\n",
        "            reward_batch (list): a batch of rewards\n",
        "            next_state_batch (list): a batch of states\n",
        "            done_batch (list): a batch of dones\n",
        "        '''\n",
        "        samples = random.sample(self.memory, self.batch_size)\n",
        "        return map(np.array, zip(*samples))\n",
        "\n",
        "def copy_model_parameters(sess, estimator1, estimator2):\n",
        "    ''' Copys the model parameters of one estimator to another.\n",
        "    Args:\n",
        "        sess (tf.Session): Tensorflow Session object\n",
        "        estimator1 (Estimator): Estimator to copy the paramters from\n",
        "        estimator2 (Estimator): Estimator to copy the parameters to\n",
        "    '''\n",
        "    e1_params = [t for t in tf.trainable_variables() if t.name.startswith(estimator1.scope)]\n",
        "    e1_params = sorted(e1_params, key=lambda v: v.name)\n",
        "    e2_params = [t for t in tf.trainable_variables() if t.name.startswith(estimator2.scope)]\n",
        "    e2_params = sorted(e2_params, key=lambda v: v.name)\n",
        "\n",
        "    update_ops = []\n",
        "    for e1_v, e2_v in zip(e1_params, e2_params):\n",
        "        op = e2_v.assign(e1_v)\n",
        "        update_ops.append(op)\n",
        "\n",
        "    sess.run(update_ops)\n",
        "\n",
        "#if __name__ == \"__main__\":\n",
        "#    with tf.Session() as sess:\n",
        "#        agent = DQNAgent(sess,\n",
        "#                         scope='dqn',\n",
        "#                         action_num=4,\n",
        "#                         replay_memory_init_size=100,\n",
        "#                         norm_step=100,\n",
        "#                         state_shape=[2],\n",
        "#                         mlp_layers=[10,10])\n",
        "#\n",
        "#        for a in tf.global_variables():\n",
        "#            print(a)\n",
        "\n"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JdXItFuWahIb"
      },
      "source": [
        "# Make environment\n",
        "env = Env(0)\n",
        "eval_env = Env(0)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WagAZhdAYcmG"
      },
      "source": [
        "# Set the iterations numbers and how frequently we evaluate/save plot\n",
        "evaluate_every = 200\n",
        "evaluate_num = 1000\n",
        "episode_num = 10000\n",
        "\n",
        "# The intial memory size\n",
        "memory_init_size = 200\n",
        "\n",
        "# Train the agent every X steps\n",
        "train_every = 1\n",
        "\n",
        "# The paths for saving the logs and learning curves\n",
        "log_dir = './experiments/blackjack_dqn_result/'\n",
        "\n",
        "# Set a global seed\n",
        "tf.compat.v1.set_random_seed(0)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j2W2O1hOaL6P",
        "outputId": "aed4002a-ce4c-4281-e463-096fd1272c06",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "with tf.Session() as sess:\n",
        "\n",
        "    # Initialize a global step\n",
        "    global_step = tf.Variable(0, name='global_step', trainable=False)\n",
        "\n",
        "    # Set up the agents\n",
        "    agent = DQNAgent(sess,\n",
        "                     scope='dqn',\n",
        "                     action_num=env.action_num,\n",
        "                     replay_memory_init_size=memory_init_size,\n",
        "                     train_every=train_every,\n",
        "                     state_shape=[12],\n",
        "                     mlp_layers=[8,8])\n",
        "    env.set_agents(agent)\n",
        "    eval_env.set_agents(agent)\n",
        "\n",
        "    # Initialize global variables\n",
        "    sess.run(tf.global_variables_initializer())\n",
        "\n",
        "    # Init a Logger to plot the learning curve\n",
        "    logger = Logger(log_dir)\n",
        "\n",
        "    for episode in range(episode_num):\n",
        "        # print('\\n episode1 ', episode)\n",
        "        # Generate data from the environment\n",
        "        trajectories = env.run(myj = 1, is_training=True)\n",
        "        # print(\"trajectories \", trajectories)\n",
        "\n",
        "        # Feed transitions into agent memory, and train the agent\n",
        "        for ts in trajectories:\n",
        "            # print(\"ts \", ts)\n",
        "            agent.feed(ts)\n",
        "\n",
        "        # Evaluate the performance. Play with random agents.\n",
        "        #if episode % evaluate_every == 0:\n",
        "        #    logger.log_performance(env.timestep, tournament(eval_env, evaluate_num)[0])\n",
        "\n",
        "        # Evaluate the performance. Play with random agents.\n",
        "        if episode % evaluate_every == 0:\n",
        "          # print('episode2 ', episode)\n",
        "          # print(\"logger \", eva_perform(eval_env, evaluate_num))\n",
        "          logger.log_performance(episode, eva_perform(eval_env, evaluate_num))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From <ipython-input-8-a178179b8696>:253: batch_normalization (from tensorflow.python.layers.normalization) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.BatchNormalization instead.  In particular, `tf.control_dependencies(tf.GraphKeys.UPDATE_OPS)` should not be used (consult the `tf.keras.layers.batch_normalization` documentation).\n",
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/python/layers/normalization.py:327: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `layer.__call__` method instead.\n",
            "WARNING:tensorflow:\n",
            "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
            "For more information, please see:\n",
            "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
            "  * https://github.com/tensorflow/addons\n",
            "  * https://github.com/tensorflow/io (for I/O related ops)\n",
            "If you depend on functionality not listed there, please file an issue.\n",
            "\n",
            "WARNING:tensorflow:From /tensorflow-1.15.2/python3.6/tensorflow_core/contrib/layers/python/layers/layers.py:1634: flatten (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use keras.layers.flatten instead.\n",
            "WARNING:tensorflow:From <ipython-input-8-a178179b8696>:233: get_global_step (from tensorflow.contrib.framework.python.ops.variables) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please switch to tf.train.get_global_step\n",
            "\n",
            "----------------------------------------\n",
            "  timestep     |  0\n",
            "  reward       |  27.75909882489102\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 200, rl-loss: 0.8428958654403687\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 1200, rl-loss: 0.4448859691619873\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 2200, rl-loss: 0.12676510214805603\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 3200, rl-loss: 0.1844816505908966\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 4200, rl-loss: 0.08974098414182663\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 5200, rl-loss: 0.06868554651737213\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 6200, rl-loss: 0.11190006881952286\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 7200, rl-loss: 0.13760852813720703\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 8200, rl-loss: 0.18903282284736633\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 9200, rl-loss: 0.1487952470779419\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 9849, rl-loss: 0.14822842180728912\n",
            "----------------------------------------\n",
            "  timestep     |  200\n",
            "  reward       |  27.597799981832864\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 10200, rl-loss: 0.17686371505260468\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 11200, rl-loss: 0.25688961148262024\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 12200, rl-loss: 0.15193027257919312\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 13200, rl-loss: 0.11923876404762268\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 14200, rl-loss: 0.2153567224740982\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 15200, rl-loss: 0.30788862705230713\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 16200, rl-loss: 0.18254008889198303\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 17200, rl-loss: 0.306637704372406\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 18200, rl-loss: 0.33020251989364624\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 19200, rl-loss: 0.2512132525444031\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 19649, rl-loss: 0.4434109926223755\n",
            "----------------------------------------\n",
            "  timestep     |  400\n",
            "  reward       |  27.61495877420221\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 20200, rl-loss: 0.4901871979236603\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 21200, rl-loss: 0.4647088050842285\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 22200, rl-loss: 0.7315391302108765\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 23200, rl-loss: 0.8609292507171631\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 24200, rl-loss: 0.45044344663619995\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 25200, rl-loss: 0.6219924092292786\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 26200, rl-loss: 0.7111080288887024\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 27200, rl-loss: 0.7431315183639526\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 28200, rl-loss: 1.2413042783737183\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 29200, rl-loss: 1.2545561790466309\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 29449, rl-loss: 1.7330682277679443\n",
            "----------------------------------------\n",
            "  timestep     |  600\n",
            "  reward       |  27.47373179555407\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 30200, rl-loss: 1.4930200576782227\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 31200, rl-loss: 2.1272971630096436\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 32200, rl-loss: 1.2271528244018555\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 33200, rl-loss: 3.7093496322631836\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 34200, rl-loss: 1.5404233932495117\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 35200, rl-loss: 1.0143415927886963\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 36200, rl-loss: 0.8340678811073303\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 37200, rl-loss: 1.106632113456726\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 38200, rl-loss: 1.599010944366455\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 39200, rl-loss: 2.014986038208008\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 39249, rl-loss: 1.2851841449737549\n",
            "----------------------------------------\n",
            "  timestep     |  800\n",
            "  reward       |  27.72623818255872\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 40200, rl-loss: 2.229548454284668\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 41200, rl-loss: 1.8696023225784302\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 42200, rl-loss: 1.5425114631652832\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 43200, rl-loss: 1.7424284219741821\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 44200, rl-loss: 1.0314428806304932\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 45200, rl-loss: 1.0731425285339355\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 46200, rl-loss: 1.4529364109039307\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 47200, rl-loss: 3.4211578369140625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 48200, rl-loss: 1.950479507446289\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 49049, rl-loss: 1.370656132698059\n",
            "----------------------------------------\n",
            "  timestep     |  1000\n",
            "  reward       |  27.73983517877068\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 49200, rl-loss: 2.074476718902588\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 50200, rl-loss: 1.8520230054855347\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 51200, rl-loss: 1.0628242492675781\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 52200, rl-loss: 1.6870925426483154\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 53200, rl-loss: 2.6085286140441895\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 54200, rl-loss: 1.079882264137268\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 55200, rl-loss: 0.8664929866790771\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 56200, rl-loss: 0.8082568645477295\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 57200, rl-loss: 2.0389769077301025\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 58200, rl-loss: 1.0826632976531982\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 58849, rl-loss: 1.2851612567901611\n",
            "----------------------------------------\n",
            "  timestep     |  1200\n",
            "  reward       |  27.776147081884886\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 59200, rl-loss: 0.8015350103378296\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 60200, rl-loss: 0.9034667015075684\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 61200, rl-loss: 0.922744870185852\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 62200, rl-loss: 0.838853120803833\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 63200, rl-loss: 0.6976416110992432\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 64200, rl-loss: 0.6713100075721741\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 65200, rl-loss: 1.688267469406128\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 66200, rl-loss: 0.9947656393051147\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 67200, rl-loss: 1.2506322860717773\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 68200, rl-loss: 2.148130416870117\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 68649, rl-loss: 1.426446795463562\n",
            "----------------------------------------\n",
            "  timestep     |  1400\n",
            "  reward       |  27.761341588012307\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 69200, rl-loss: 1.136505126953125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 70200, rl-loss: 1.575850009918213\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 71200, rl-loss: 1.125760793685913\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 72200, rl-loss: 1.2099446058273315\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 73200, rl-loss: 0.554507315158844\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 74200, rl-loss: 1.2844327688217163\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 75200, rl-loss: 0.34259623289108276\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 76200, rl-loss: 0.647181510925293\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 77200, rl-loss: 0.5812184810638428\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 78200, rl-loss: 1.1911683082580566\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 78449, rl-loss: 1.0456358194351196\n",
            "----------------------------------------\n",
            "  timestep     |  1600\n",
            "  reward       |  27.762525361829862\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 79200, rl-loss: 0.7454583644866943\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 80200, rl-loss: 0.9301834106445312\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 81200, rl-loss: 0.5284606218338013\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 82200, rl-loss: 0.9884846210479736\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 83200, rl-loss: 0.9699673652648926\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 84200, rl-loss: 0.3198287785053253\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 85200, rl-loss: 1.2281734943389893\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 86200, rl-loss: 0.8273256421089172\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 87200, rl-loss: 0.7160711884498596\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 88200, rl-loss: 0.3434021472930908\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 88249, rl-loss: 1.4900500774383545\n",
            "----------------------------------------\n",
            "  timestep     |  1800\n",
            "  reward       |  27.717127229652483\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 89200, rl-loss: 1.0089378356933594\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 90200, rl-loss: 0.37047624588012695\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 91200, rl-loss: 1.0021865367889404\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 92200, rl-loss: 0.2011537253856659\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 93200, rl-loss: 0.6099070310592651\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 94200, rl-loss: 0.9437923431396484\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 95200, rl-loss: 0.8173500895500183\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 96200, rl-loss: 0.5152428150177002\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 97200, rl-loss: 1.1881909370422363\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 98049, rl-loss: 0.9953979253768921\n",
            "----------------------------------------\n",
            "  timestep     |  2000\n",
            "  reward       |  27.724982151646742\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 98200, rl-loss: 0.5304352045059204\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 99200, rl-loss: 0.5275249481201172\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 100200, rl-loss: 1.0738996267318726\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 101200, rl-loss: 0.8990194797515869\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 102200, rl-loss: 0.9547061324119568\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 103200, rl-loss: 0.6464481949806213\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 104200, rl-loss: 0.36707645654678345\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 105200, rl-loss: 1.024090051651001\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 106200, rl-loss: 0.8673983812332153\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 107200, rl-loss: 0.8726427555084229\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 107849, rl-loss: 1.0381661653518677\n",
            "----------------------------------------\n",
            "  timestep     |  2200\n",
            "  reward       |  27.70276884077139\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 108200, rl-loss: 0.7379590272903442\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 109200, rl-loss: 0.8357305526733398\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 110200, rl-loss: 0.7468651533126831\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 111200, rl-loss: 0.7214497327804565\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 112200, rl-loss: 0.7306482791900635\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 113200, rl-loss: 0.2740190029144287\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 114200, rl-loss: 0.8362966179847717\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 115200, rl-loss: 0.6887431740760803\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 116200, rl-loss: 1.09175705909729\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 117200, rl-loss: 1.3797918558120728\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 117649, rl-loss: 0.7039934992790222\n",
            "----------------------------------------\n",
            "  timestep     |  2400\n",
            "  reward       |  27.716486075691645\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 118200, rl-loss: 1.498814344406128\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 119200, rl-loss: 1.2849392890930176\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 120200, rl-loss: 1.2792572975158691\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 121200, rl-loss: 0.5919762253761292\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 122200, rl-loss: 0.5515418648719788\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 123200, rl-loss: 0.9905586838722229\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 124200, rl-loss: 1.2845935821533203\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 125200, rl-loss: 0.5677615404129028\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 126200, rl-loss: 0.2801835238933563\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 127200, rl-loss: 0.6836562752723694\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 127449, rl-loss: 0.7629616260528564\n",
            "----------------------------------------\n",
            "  timestep     |  2600\n",
            "  reward       |  27.754288184165656\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 128200, rl-loss: 0.677984356880188\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 129200, rl-loss: 0.9874642491340637\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 130200, rl-loss: 0.73426353931427\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 131200, rl-loss: 0.9879897832870483\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 132200, rl-loss: 1.1891248226165771\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 133200, rl-loss: 1.2727582454681396\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 134200, rl-loss: 0.748960018157959\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 135200, rl-loss: 0.8863283395767212\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 136200, rl-loss: 1.5782527923583984\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 137200, rl-loss: 1.497298240661621\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 137249, rl-loss: 0.5587266683578491\n",
            "----------------------------------------\n",
            "  timestep     |  2800\n",
            "  reward       |  27.737528862475564\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 138200, rl-loss: 2.025233268737793\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 139200, rl-loss: 1.114209532737732\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 140200, rl-loss: 1.2249058485031128\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 141200, rl-loss: 2.487685441970825\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 142200, rl-loss: 1.6795388460159302\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 143200, rl-loss: 0.6984187364578247\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 144200, rl-loss: 1.8610107898712158\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 145200, rl-loss: 1.2568535804748535\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 146200, rl-loss: 0.7893291711807251\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 147049, rl-loss: 1.0907299518585205\n",
            "----------------------------------------\n",
            "  timestep     |  3000\n",
            "  reward       |  27.751889679119117\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 147200, rl-loss: 1.0589122772216797\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 148200, rl-loss: 1.4042115211486816\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 149200, rl-loss: 0.6545634269714355\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 150200, rl-loss: 0.6314008831977844\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 151200, rl-loss: 1.068169355392456\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 152200, rl-loss: 0.6798356771469116\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 153200, rl-loss: 1.5497732162475586\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 154200, rl-loss: 1.0255777835845947\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 155200, rl-loss: 0.4404224157333374\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 156200, rl-loss: 0.3669818639755249\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 156849, rl-loss: 0.377300500869751\n",
            "----------------------------------------\n",
            "  timestep     |  3200\n",
            "  reward       |  27.71638969565681\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 157200, rl-loss: 1.0109220743179321\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 158200, rl-loss: 1.4908595085144043\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 159200, rl-loss: 0.8071169853210449\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 160200, rl-loss: 0.9834407567977905\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 161200, rl-loss: 0.39245545864105225\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 162200, rl-loss: 0.6705456972122192\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 163200, rl-loss: 0.5733063817024231\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 164200, rl-loss: 0.7839306592941284\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 165200, rl-loss: 2.4580299854278564\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 166200, rl-loss: 2.766341209411621\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 166649, rl-loss: 3.2560524940490723\n",
            "----------------------------------------\n",
            "  timestep     |  3400\n",
            "  reward       |  27.756156952453246\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 167200, rl-loss: 1.4546911716461182\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 168200, rl-loss: 1.4817496538162231\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 169200, rl-loss: 2.0482444763183594\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 170200, rl-loss: 2.485135555267334\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 171200, rl-loss: 4.387177467346191\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 172200, rl-loss: 5.170493125915527\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 173200, rl-loss: 1.3009655475616455\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 174200, rl-loss: 1.6184828281402588\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 175200, rl-loss: 5.493759632110596\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 176200, rl-loss: 3.171201229095459\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 176449, rl-loss: 4.686903953552246\n",
            "----------------------------------------\n",
            "  timestep     |  3600\n",
            "  reward       |  27.770778058841536\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 177200, rl-loss: 2.089963912963867\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 178200, rl-loss: 1.9735267162322998\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 179200, rl-loss: 4.233149528503418\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 180200, rl-loss: 1.131903886795044\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 181200, rl-loss: 3.619431257247925\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 182200, rl-loss: 3.942586898803711\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 183200, rl-loss: 2.019695281982422\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 184200, rl-loss: 2.6239566802978516\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 185200, rl-loss: 7.758338928222656\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 186200, rl-loss: 8.655580520629883\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 186249, rl-loss: 5.556006908416748\n",
            "----------------------------------------\n",
            "  timestep     |  3800\n",
            "  reward       |  27.781790750462278\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 187200, rl-loss: 4.97157096862793\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 188200, rl-loss: 9.34809684753418\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 189200, rl-loss: 6.723016262054443\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 190200, rl-loss: 9.894144058227539\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 191200, rl-loss: 6.403730869293213\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 192200, rl-loss: 8.719905853271484\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 193200, rl-loss: 8.942564010620117\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 194200, rl-loss: 9.368691444396973\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 195200, rl-loss: 11.625746726989746\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 196049, rl-loss: 18.83758544921875\n",
            "----------------------------------------\n",
            "  timestep     |  4000\n",
            "  reward       |  27.772991545359872\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 196200, rl-loss: 5.192886829376221\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 197200, rl-loss: 7.2768964767456055\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 198200, rl-loss: 2.5116796493530273\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 199200, rl-loss: 4.391143321990967\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 200200, rl-loss: 7.078960418701172\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 201200, rl-loss: 9.195428848266602\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 202200, rl-loss: 8.86390209197998\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 203200, rl-loss: 14.879081726074219\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 204200, rl-loss: 34.034820556640625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 205200, rl-loss: 15.422128677368164\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 205849, rl-loss: 18.945568084716797\n",
            "----------------------------------------\n",
            "  timestep     |  4200\n",
            "  reward       |  27.765612491306324\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 206200, rl-loss: 13.323934555053711\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 207200, rl-loss: 24.939186096191406\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 208200, rl-loss: 19.02816390991211\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 209200, rl-loss: 11.07590389251709\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 210200, rl-loss: 9.186118125915527\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 211200, rl-loss: 7.170732498168945\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 212200, rl-loss: 24.188676834106445\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 213200, rl-loss: 14.266999244689941\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 214200, rl-loss: 8.086508750915527\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 215200, rl-loss: 11.987798690795898\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 215649, rl-loss: 7.479057312011719\n",
            "----------------------------------------\n",
            "  timestep     |  4400\n",
            "  reward       |  27.767987071017405\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 216200, rl-loss: 12.25473403930664\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 217200, rl-loss: 6.956241607666016\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 218200, rl-loss: 17.946521759033203\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 219200, rl-loss: 21.49799346923828\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 220200, rl-loss: 3.1012954711914062\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 221200, rl-loss: 21.79113006591797\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 222200, rl-loss: 3.2436656951904297\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 223200, rl-loss: 8.624958992004395\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 224200, rl-loss: 2.9207794666290283\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 225200, rl-loss: 12.870585441589355\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 225449, rl-loss: 6.6468825340271\n",
            "----------------------------------------\n",
            "  timestep     |  4600\n",
            "  reward       |  27.75381934807081\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 226200, rl-loss: 21.134620666503906\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 227200, rl-loss: 32.54197311401367\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 228200, rl-loss: 20.165203094482422\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 229200, rl-loss: 14.447210311889648\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 230200, rl-loss: 25.820274353027344\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 231200, rl-loss: 19.984535217285156\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 232200, rl-loss: 7.349827289581299\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 233200, rl-loss: 48.34603500366211\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 234200, rl-loss: 39.89399337768555\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 235200, rl-loss: 15.415410041809082\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 235249, rl-loss: 7.58293342590332\n",
            "----------------------------------------\n",
            "  timestep     |  4800\n",
            "  reward       |  27.770905336317046\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 236200, rl-loss: 15.460674285888672\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 237200, rl-loss: 17.167743682861328\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 238200, rl-loss: 15.532097816467285\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 239200, rl-loss: 13.103415489196777\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 240200, rl-loss: 5.507604598999023\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 241200, rl-loss: 21.975875854492188\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 242200, rl-loss: 6.682589054107666\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 243200, rl-loss: 21.94074249267578\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 244200, rl-loss: 17.767982482910156\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 245049, rl-loss: 26.13346290588379\n",
            "----------------------------------------\n",
            "  timestep     |  5000\n",
            "  reward       |  27.794091470486293\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 245200, rl-loss: 24.03084373474121\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 246200, rl-loss: 24.071632385253906\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 247200, rl-loss: 22.566978454589844\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 248200, rl-loss: 14.995637893676758\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 249200, rl-loss: 4.289474964141846\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 250200, rl-loss: 16.573911666870117\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 251200, rl-loss: 11.877581596374512\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 252200, rl-loss: 3.0121359825134277\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 253200, rl-loss: 16.601917266845703\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 254200, rl-loss: 8.88155746459961\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 254849, rl-loss: 22.830759048461914\n",
            "----------------------------------------\n",
            "  timestep     |  5200\n",
            "  reward       |  27.757879094435086\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 255200, rl-loss: 12.308279037475586\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 256200, rl-loss: 9.399585723876953\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 257200, rl-loss: 21.935304641723633\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 258200, rl-loss: 15.333915710449219\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 259200, rl-loss: 53.073246002197266\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 260200, rl-loss: 43.92958068847656\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 261200, rl-loss: 20.156896591186523\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 262200, rl-loss: 4.347764015197754\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 263200, rl-loss: 35.89318084716797\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 264200, rl-loss: 41.336822509765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 264649, rl-loss: 15.695995330810547\n",
            "----------------------------------------\n",
            "  timestep     |  5400\n",
            "  reward       |  27.78820930408436\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 265200, rl-loss: 21.347185134887695\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 266200, rl-loss: 21.510374069213867\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 267200, rl-loss: 14.639189720153809\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 268200, rl-loss: 16.923681259155273\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 269200, rl-loss: 32.751426696777344\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 270200, rl-loss: 8.765783309936523\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 271200, rl-loss: 33.53172302246094\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 272200, rl-loss: 27.708396911621094\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 273200, rl-loss: 14.551055908203125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 274200, rl-loss: 16.187013626098633\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 274449, rl-loss: 7.858327865600586\n",
            "----------------------------------------\n",
            "  timestep     |  5600\n",
            "  reward       |  27.785651022879257\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 275200, rl-loss: 56.370086669921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 276200, rl-loss: 11.738935470581055\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 277200, rl-loss: 7.053065299987793\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 278200, rl-loss: 23.77726936340332\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 279200, rl-loss: 20.525489807128906\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 280200, rl-loss: 36.82968521118164\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 281200, rl-loss: 29.120065689086914\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 282200, rl-loss: 12.104499816894531\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 283200, rl-loss: 26.0667724609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 284200, rl-loss: 3.2131810188293457\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 284249, rl-loss: 13.7831449508667\n",
            "----------------------------------------\n",
            "  timestep     |  5800\n",
            "  reward       |  27.798654380219137\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 285200, rl-loss: 19.216659545898438\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 286200, rl-loss: 17.660015106201172\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 287200, rl-loss: 29.654666900634766\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 288200, rl-loss: 28.385032653808594\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 289200, rl-loss: 8.194643020629883\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 290200, rl-loss: 31.99637222290039\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 291200, rl-loss: 36.340396881103516\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 292200, rl-loss: 25.519758224487305\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 293200, rl-loss: 9.815801620483398\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 294049, rl-loss: 16.07252311706543\n",
            "----------------------------------------\n",
            "  timestep     |  6000\n",
            "  reward       |  27.752295873290056\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 294200, rl-loss: 9.508451461791992\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 295200, rl-loss: 17.52602767944336\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 296200, rl-loss: 20.21781349182129\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 297200, rl-loss: 21.615266799926758\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 298200, rl-loss: 37.4766845703125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 299200, rl-loss: 13.392343521118164\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 300200, rl-loss: 25.012832641601562\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 301200, rl-loss: 20.068218231201172\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 302200, rl-loss: 33.71237564086914\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 303200, rl-loss: 17.026723861694336\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 303849, rl-loss: 20.917484283447266\n",
            "----------------------------------------\n",
            "  timestep     |  6200\n",
            "  reward       |  27.774003742102746\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 304200, rl-loss: 14.13888931274414\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 305200, rl-loss: 41.67106628417969\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 306200, rl-loss: 14.835787773132324\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 307200, rl-loss: 38.02416229248047\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 308200, rl-loss: 75.20437622070312\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 309200, rl-loss: 11.683454513549805\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 310200, rl-loss: 23.84881019592285\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 311200, rl-loss: 35.93586730957031\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 312200, rl-loss: 38.038055419921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 313200, rl-loss: 17.175552368164062\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 313649, rl-loss: 56.75410461425781\n",
            "----------------------------------------\n",
            "  timestep     |  6400\n",
            "  reward       |  27.74471100617428\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 314200, rl-loss: 30.183883666992188\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 315200, rl-loss: 41.05679702758789\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 316200, rl-loss: 41.680850982666016\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 317200, rl-loss: 26.37974739074707\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 318200, rl-loss: 32.057960510253906\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 319200, rl-loss: 41.83982849121094\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 320200, rl-loss: 25.61248779296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 321200, rl-loss: 35.359561920166016\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 322200, rl-loss: 28.720073699951172\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 323200, rl-loss: 13.388504028320312\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 323449, rl-loss: 16.451242446899414\n",
            "----------------------------------------\n",
            "  timestep     |  6600\n",
            "  reward       |  27.767684140177572\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 324200, rl-loss: 38.3706169128418\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 325200, rl-loss: 18.11707305908203\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 326200, rl-loss: 19.58729362487793\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 327200, rl-loss: 48.345558166503906\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 328200, rl-loss: 24.889850616455078\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 329200, rl-loss: 13.563044548034668\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 330200, rl-loss: 24.684587478637695\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 331200, rl-loss: 7.33804988861084\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 332200, rl-loss: 32.7153434753418\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 333200, rl-loss: 10.917373657226562\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 333249, rl-loss: 13.2410249710083\n",
            "----------------------------------------\n",
            "  timestep     |  6800\n",
            "  reward       |  27.761197705815338\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 334200, rl-loss: 22.401744842529297\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 335200, rl-loss: 5.279380798339844\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 336200, rl-loss: 73.34711456298828\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 337200, rl-loss: 28.56013298034668\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 338200, rl-loss: 28.919925689697266\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 339200, rl-loss: 13.821492195129395\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 340200, rl-loss: 2.148063898086548\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 341200, rl-loss: 37.47958755493164\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 342200, rl-loss: 59.2190055847168\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 343049, rl-loss: 27.41398811340332\n",
            "----------------------------------------\n",
            "  timestep     |  7000\n",
            "  reward       |  27.785048642446807\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 343200, rl-loss: 13.166627883911133\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 344200, rl-loss: 30.53215217590332\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 345200, rl-loss: 26.794776916503906\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 346200, rl-loss: 11.132287979125977\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 347200, rl-loss: 77.4723892211914\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 348200, rl-loss: 4.376523971557617\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 349200, rl-loss: 60.655120849609375\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 350200, rl-loss: 14.700695991516113\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 351200, rl-loss: 43.364013671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 352200, rl-loss: 27.990758895874023\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 352849, rl-loss: 2.977633476257324\n",
            "----------------------------------------\n",
            "  timestep     |  7200\n",
            "  reward       |  27.78231756695769\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 353200, rl-loss: 18.58307647705078\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 354200, rl-loss: 7.063758850097656\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 355200, rl-loss: 20.15511703491211\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 356200, rl-loss: 19.771150588989258\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 357200, rl-loss: 23.416311264038086\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 358200, rl-loss: 9.207244873046875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 359200, rl-loss: 30.864831924438477\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 360200, rl-loss: 10.878776550292969\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 361200, rl-loss: 44.220558166503906\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 362200, rl-loss: 19.60888671875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 362649, rl-loss: 12.60066032409668\n",
            "----------------------------------------\n",
            "  timestep     |  7400\n",
            "  reward       |  27.768726953723753\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 363200, rl-loss: 12.08355712890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 364200, rl-loss: 36.56871795654297\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 365200, rl-loss: 28.96438980102539\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 366200, rl-loss: 20.697338104248047\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 367200, rl-loss: 14.390628814697266\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 368200, rl-loss: 20.668434143066406\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 369200, rl-loss: 35.084232330322266\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 370200, rl-loss: 44.19007873535156\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 371200, rl-loss: 8.326176643371582\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 372200, rl-loss: 15.753711700439453\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 372449, rl-loss: 14.75854778289795\n",
            "----------------------------------------\n",
            "  timestep     |  7600\n",
            "  reward       |  27.749391004174516\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 373200, rl-loss: 19.708126068115234\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 374200, rl-loss: 22.703433990478516\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 375200, rl-loss: 7.229083061218262\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 376200, rl-loss: 17.38210678100586\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 377200, rl-loss: 8.591821670532227\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 378200, rl-loss: 32.04313278198242\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 379200, rl-loss: 12.331893920898438\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 380200, rl-loss: 12.073273658752441\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 381200, rl-loss: 26.165729522705078\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 382200, rl-loss: 26.322059631347656\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 382249, rl-loss: 7.368269443511963\n",
            "----------------------------------------\n",
            "  timestep     |  7800\n",
            "  reward       |  27.796126396010887\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 383200, rl-loss: 43.141841888427734\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 384200, rl-loss: 37.60649871826172\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 385200, rl-loss: 32.16961669921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 386200, rl-loss: 7.20658016204834\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 387200, rl-loss: 36.95814895629883\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 388200, rl-loss: 15.446666717529297\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 389200, rl-loss: 22.614748001098633\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 390200, rl-loss: 5.801665306091309\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 391200, rl-loss: 16.519521713256836\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 392049, rl-loss: 41.291961669921875\n",
            "----------------------------------------\n",
            "  timestep     |  8000\n",
            "  reward       |  27.763960645992196\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 392200, rl-loss: 34.97180938720703\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 393200, rl-loss: 35.62446594238281\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 394200, rl-loss: 55.72087097167969\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 395200, rl-loss: 22.76125717163086\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 396200, rl-loss: 21.331939697265625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 397200, rl-loss: 28.955543518066406\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 398200, rl-loss: 20.146249771118164\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 399200, rl-loss: 9.200236320495605\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 400200, rl-loss: 34.7872314453125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 401200, rl-loss: 24.728622436523438\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 401849, rl-loss: 25.145950317382812\n",
            "----------------------------------------\n",
            "  timestep     |  8200\n",
            "  reward       |  27.7548335675856\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 402200, rl-loss: 15.668274879455566\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 403200, rl-loss: 39.05492401123047\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 404200, rl-loss: 14.298035621643066\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 405200, rl-loss: 4.8741254806518555\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 406200, rl-loss: 26.77307891845703\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 407200, rl-loss: 44.51579666137695\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 408200, rl-loss: 42.91950988769531\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 409200, rl-loss: 35.506805419921875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 410200, rl-loss: 25.060335159301758\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 411200, rl-loss: 23.08576774597168\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 411649, rl-loss: 22.760520935058594\n",
            "----------------------------------------\n",
            "  timestep     |  8400\n",
            "  reward       |  27.781519655358267\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 412200, rl-loss: 21.00157928466797\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 413200, rl-loss: 19.11919593811035\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 414200, rl-loss: 47.70082092285156\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 415200, rl-loss: 2.9037296772003174\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 416200, rl-loss: 8.978832244873047\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 417200, rl-loss: 3.240844249725342\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 418200, rl-loss: 27.49843406677246\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 419200, rl-loss: 38.57280731201172\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 420200, rl-loss: 23.23150634765625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 421200, rl-loss: 2.866098403930664\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 421449, rl-loss: 16.725563049316406\n",
            "----------------------------------------\n",
            "  timestep     |  8600\n",
            "  reward       |  27.75871114915756\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 422200, rl-loss: 33.486793518066406\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 423200, rl-loss: 28.396209716796875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 424200, rl-loss: 13.50073528289795\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 425200, rl-loss: 20.149959564208984\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 426200, rl-loss: 38.11164474487305\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 427200, rl-loss: 9.192021369934082\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 428200, rl-loss: 25.233261108398438\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 429200, rl-loss: 20.573795318603516\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 430200, rl-loss: 20.76947593688965\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 431200, rl-loss: 23.58683204650879\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 431249, rl-loss: 17.19911766052246\n",
            "----------------------------------------\n",
            "  timestep     |  8800\n",
            "  reward       |  27.741020180370757\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 432200, rl-loss: 8.080608367919922\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 433200, rl-loss: 17.435457229614258\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 434200, rl-loss: 31.48514175415039\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 435200, rl-loss: 22.442638397216797\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 436200, rl-loss: 43.205230712890625\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 437200, rl-loss: 18.80277442932129\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 438200, rl-loss: 29.48659324645996\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 439200, rl-loss: 31.601991653442383\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 440200, rl-loss: 10.072160720825195\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 441049, rl-loss: 41.51551818847656\n",
            "----------------------------------------\n",
            "  timestep     |  9000\n",
            "  reward       |  27.814672444242937\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 441200, rl-loss: 19.877622604370117\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 442200, rl-loss: 29.237642288208008\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 443200, rl-loss: 30.522323608398438\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 444200, rl-loss: 44.4229736328125\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 445200, rl-loss: 24.4882755279541\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 446200, rl-loss: 63.35359573364258\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 447200, rl-loss: 20.521072387695312\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 448200, rl-loss: 59.255393981933594\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 449200, rl-loss: 15.946765899658203\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 450200, rl-loss: 15.955269813537598\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 450849, rl-loss: 63.608367919921875\n",
            "----------------------------------------\n",
            "  timestep     |  9200\n",
            "  reward       |  27.77123572225964\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 451200, rl-loss: 28.558406829833984\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 452200, rl-loss: 37.349220275878906\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 453200, rl-loss: 27.01873779296875\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 454200, rl-loss: 39.793739318847656\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 455200, rl-loss: 6.842658042907715\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 456200, rl-loss: 11.087189674377441\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 457200, rl-loss: 34.93488311767578\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 458200, rl-loss: 36.19434356689453\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 459200, rl-loss: 20.02889633178711\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 460200, rl-loss: 36.41844177246094\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 460649, rl-loss: 12.895021438598633\n",
            "----------------------------------------\n",
            "  timestep     |  9400\n",
            "  reward       |  27.775340930684884\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 461200, rl-loss: 20.51058578491211\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 462200, rl-loss: 30.438404083251953\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 463200, rl-loss: 2.626089096069336\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 464200, rl-loss: 12.48357105255127\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 465200, rl-loss: 4.762925148010254\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 466200, rl-loss: 20.87423324584961\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 467200, rl-loss: 23.061443328857422\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 468200, rl-loss: 12.035879135131836\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 469200, rl-loss: 16.70810317993164\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 470200, rl-loss: 33.55043029785156\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 470449, rl-loss: 26.13956069946289\n",
            "----------------------------------------\n",
            "  timestep     |  9600\n",
            "  reward       |  27.777100194650295\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 471200, rl-loss: 30.916933059692383\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 472200, rl-loss: 11.493307113647461\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 473200, rl-loss: 15.139543533325195\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 474200, rl-loss: 26.631851196289062\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 475200, rl-loss: 24.675621032714844\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 476200, rl-loss: 11.339523315429688\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 477200, rl-loss: 33.088600158691406\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 478200, rl-loss: 17.626766204833984\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 479200, rl-loss: 16.584699630737305\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 480200, rl-loss: 26.82120132446289\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 480249, rl-loss: 40.3700065612793\n",
            "----------------------------------------\n",
            "  timestep     |  9800\n",
            "  reward       |  27.772197772329235\n",
            "----------------------------------------\n",
            "INFO - Agent dqn, step 481200, rl-loss: 14.728497505187988\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 482200, rl-loss: 6.869356632232666\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 483200, rl-loss: 6.018985748291016\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 484200, rl-loss: 34.90007019042969\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 485200, rl-loss: 18.57071876525879\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 486200, rl-loss: 32.9914665222168\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 487200, rl-loss: 5.519649505615234\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 488200, rl-loss: 7.11904764175415\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 489200, rl-loss: 26.310741424560547\n",
            "INFO - Copied model parameters to target network.\n",
            "INFO - Agent dqn, step 490000, rl-loss: 15.732124328613281"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRzmdZr-VL5A",
        "outputId": "70dc7c56-0893-413f-9476-8b682d267fc1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        }
      },
      "source": [
        "logger.close_files()\n",
        "logger.plot('DQN')"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "./experiments/blackjack_dqn_result/performance.csv\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEGCAYAAACtqQjWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3zdZfn4/9eV1cxmdKTpSrroHrSlLZRVWqpUhiBbkaEiKgKKH4EfKvDxi/JBWcpGBJRVhCJQASnQQMsobdp0752u7DR7nev3x3knnKbnJCfJORkn1/PxOI+ec79H7jsnPde5t6gqxhhjTCCEdXYGjDHGhA4LKsYYYwLGgooxxpiAsaBijDEmYCyoGGOMCZiIzs5AR+jbt69mZGS06dry8nLi4uICm6FuwMrd8/TUslu5fcvKyspX1X6tuW+PCCoZGRmsWrWqTddmZmZy5plnBjZD3YCVu+fpqWW3cvsmIntbe19r/jLGGBMwFlSMMcYEjAUVY4wxAdMj+lS8qa2tJScnh6qqqmbPS0xMZPPmzR2Uq8CLjo5m8ODBREZGdnZWjDE9QI8NKjk5OSQkJJCRkYGI+DyvtLSUhISEDsxZ4KgqBQUF5OTkMGzYsM7OjjGmB+ixzV9VVVX06dOn2YDS3YkIffr0abE2ZowxgdJjgwoQ0gGlQU8oozGm6+jRQcUYYzrLW9kHKCyv6exsBJwFlU4UHh7OlClTGD9+PJMnT+aBBx7A5XI1Hl++fDkzZsxgzJgxjB49mscff7zx2N13301sbCy5ubmNafHx8R2af2NM2xwuqeLmV7P527JdnZ2VgLOg0oliYmLIzs5m48aNLFmyhPfee4977rkHgMOHD3PllVfy5JNPsmXLFj777DOeffZZ3nzzzcbr+/btywMPPNBZ2TfGtNGuvDIAVuwu7OScBJ4FlS6if//+PP300zz66KOoKo899hjXXHMNU6dOBdwB5P777+dPf/pT4zXXXXcdCxcupLAw9P4wjQlluwvKAViXU0xlTX0n5yaweuyQYk/3vLORTQePej1WX19PeHh4q+85bmBv7jpvfKuuGT58OPX19eTm5rJx40auvvrqY45Pnz6dTZs2Nb6Oj4/nuuuu45FHHmms4Rhjur49+e6gUluvrN5XxOyRfTs5R4FjNZVu7qabbuKFF16gtLS0s7NijPHT7vwKBiXFECawYldBZ2cnoKymAs3WKDpy8uOuXbsIDw+nf//+jBs3jqysLC644ILG41lZWUyfPv2Ya5KSkrjyyit57LHHOiSPxpj2251fxoRBvUmJi+LLEOtXCVpNRUSGiMhSEdkkIhtF5GYnfaGIZDuPPSKS7eP6XzjXbRCRV0Qk2kkfJiIrRGSHc6+oYJWhI+Xl5XHDDTdw4403IiL87Gc/4/nnnyc72/3rKSgo4M477+S3v/3tcdf+8pe/5KmnnqKurq6js22MaaV6l7K/sJJhfeOZNTyF7P3FVNWGTr9KMJu/6oBbVXUcMAv4mYiMU9XLVHWKqk4B3gAWNb1QRAYBNwHTVXUCEA5c7hz+P+AhVR0JFAE/CGIZgqqysrJxSPG8efOYP38+d911FwBpaWm8+OKLXH/99YwePZqBAwdy0003ccYZZxx3n759+3LhhRdSXV3d0UUwxrTSweJKaupdDOsby8xhfaipc7FmX3FnZytggtb8paqHgEPO81IR2QwMAjYBiHuq96XAWc3kLUZEaoFY4KBzzVnAlc45LwB3A08EqRhBVV/f/LeT008/na+++gqAxx9/nD/84Q9885vfJDk5mbvvvvuYcx988EEefPDBYGXVhIB6l3LkaBUDk2I6Oys92m6nkz6jTxxj0nojAit2F3DyiD6dnLPAEFUN/g8RyQA+BSao6lEn7XTgQVWd7uOam4F7gUrgA1X9roj0Bb50aimIyBDgPac20/T664HrAVJTU6e9+uqrxxxPTExk5MiRLea9raO/upIdO3ZQUlLSqmvKysp65GTKUC23qvLUumpWHann4TNjiY86fvmeUC17Szq63B/ureXFzTU8fGYMSdFh/O6zSuIi4bYZHRvs/Sn3nDlzsnx9RvsS9I56EYnH3cx1S0NAcVwBvOLjmmTgAmAYUAz8S0S+B7zv789V1aeBpwGmT5+uTbfN3Lx5s18d8N15leIG0dHRnHjiia26xrZYDS2PLd3Bl4e2AhA7dBxnjkk97pxQLXtOUQUPfLCN3397AvG9jv/I6+hyZ769kbio/VzwjTmICHNLN/Lyin2cfOpp9IrouC+wwSp3UIcUi0gk7oDykqou8kiPAC4CFvq4dB6wW1XzVLUWd7/LKUABkORcDzAYONDW/HVELa2z9YQyhoKSyloez9xBdV3gO2yXbDrCnz/YyoKJA4gIE1btKQr4z+jK3l57kDfXHOC99Yc6OysA7CkoJ6NvXONirzOH9aG6zsW6nNa1JnRVwRz9JcCzwGZVbdrYPw/Yoqo5Pi7fB8wSkVjnPnOd+yiwFLjYOe9q4K225C86OpqCgoKQ/tBt2E8lOjq6s7NiWvDUJzu5//2tfLgpt+WTW2HL4aPc8uoaJg5K5MFLpzB+UGKPCypZTnnf7SJBZXe+O6g0mDEsBQid+SrBbP6aDVwFrPcYNvz/qeq7uEdyHdP0JSIDgb+p6gJVXSEirwOrcY8iW4PTlAXcBrwqIv/PSX+2LZkbPHgwOTk55OXlNXteVVVVt/5Qbtj50XRdZdV1/PPLvQB8tOUI35qUFpD7FpbX8MMXVhHXK4Knr5pOdGQ409OTefHLvdTUuYiKCP25zy6XkrWviDCB5TvyKamoJTG283ZBra13kVNUyXmTBjampcRFMWZAAit2F3Jjp+UscII5+ms54HUzD1W9xkvaQWCBx+u7gLu8nLcLmNHe/EVGRvq1G2JmZmar+yOMaY1XVuyjtKqOMQMSyNyaR71LCQ9reR+c0qpaABKij/+QrKlz8ZMXs8gtrea1H5/MgET3F6Pp6ck8u3w3Gw6WMHVocmAL0gXtyi+nuKKW784cyksr9vHBpsNcMn1Ip+Vnf2EF9S5lmEdNBWDmsBReW5VDbb2LyPDuHey7d+6N6eZq6lw8u3w3Jw/vw0/njKSwvIa1OS3PWXC5lPP+upzJ93zAuX9dxj3vbOS99YfIL6tGVbn7nY2s2F3I/d+ZxJQhSY3XTctwB5KsHtIElrXXPVv92tnDGJQU0+lNYHuchSQzmgaV4X2orK0PiX4VW6bFmE709tqDHD5axX3fmciJQ5IJDxM+3pzbYi1i5Z5C9hRU8K2JaRSUV/Pyin0899keAAYnx5BTVMkNZ4zg2ycOOua6/gnRpPeJZdXeQn7E8GAVy2+qyordhQxOjmFwcmzA779qTxHJsZGM6BfHtyal8dxnuzu1CWxXnjuoNK2pNPar7C5gWnr3rkFaUDGmk7hcytOf7mTMgATOOKEfIsK09GQ+2pLLr74xutlr31p7kNiocP50ySRioyKoqXOx/kAJK/cUsnJ3IaeN6sv/+LjHtPRkPt2Wh6p2+nbTmdvyuPa5lQAMSYnhlOF9OXlEH04e0YfU3u3vy8zaW8S09GREhAUT03j6012d2gS2p6Cc3tERJDcJan3jezGyfzwrdhXy0zM7JWsBY81fpsfZX1jBx1uOdHY2WLo1l21HyvjxGcMbP9znjunP5kNHOVhc6fO6mjoX764/xPxxqcRGub8XRkWEMS09mRvOGMGz15zEHy+a5LNfZnp6CvllNewpqAh8oVrpmU93kZYYzV3njWPsgN68t+EQtyzMZuYfPuKsBzL5cFPb36fC8hp25ZczLd1dC5g8OLHdTWAFZdUcLqlq8/V78isY1i/eazCfOSyFVXsKqat3ebmy+7CgYnqUXXllXPTE51z3/CrufHM9tZ34H/ipT3YxKCmGcz1GAs0d2x9wBxxfPt2WR3FFLRdMGeTznOZMd/pVVu3p3NVxNx4s4fOdBVxzSgbXzh7G09+fzprfzWfxz0/lzgVjiQwL4+evrGHLYe97HbUka6+736ihOUlE+NakNPcosMraNt3zhhez+M4Tn7f572Z3fjnD+nhv5ps1vA/lNfVs9LG3U3dhQcX0GHsLyrnymRW4XNo4GuiqZ1dQWF7T4XnJ2lvEV3sK+cGpw44Z7TOiXzxDUmL4eLPvoPLv7AOkxEVx6qi2bew0sl88vaMjGj90O8vflu0mLiqcy2cMbUwLDxMmDErkR6cP558/mEF8dAQ3/DOrTUFg1d5CIsOFSYMTG9MWTEyjtl5Z0oYa0JbDR1m5p4gDxZW8nX2w1ddX1dZzsKTyuE76BjOHf92v0p1ZUOmGCspsNeLWyimq4MpnVlBVV89LP5rJvRdO5KHLJrN6XzHnP7q8zd+G2+qpT3aSGBPJZScd27YvIswdk8pnO/O9LodeVl3Hh5uP8K2JaW0eehoW5u67WdWJQeVQSSXvrD3IZScNJTHGe6d5/97RPPHdqeQUVfLLhdm4XK2bqJy1p4gJgxKJjvx66ZOGJrD/rGt9UHh5xT6iIsIY3jeOJz/Z2er87CusQPX4TvoG/ROiGd43jhW7mq9BqiqHS6pYtj2Pvy/fzR2L1nPTK2v4bEd+l5jMbUGlm1m6JZfp937Ypm9aPdWhkkqufGYFpVW1vPiDmYwZ0BuAC08czGs/PpmaOhffefxzPth4uEPysyO3jCWbj/D9k9OJ87IW1Vlj+lNV6+KLncd/Y12y6TBVtS4umDLwuGOtMT0jhR25ZRRXdHwtDeD5z/fgUuXa2RnNnjc9I4XfnTeOj7bk8tePd/h9/+q6etYdKGFak1F0bW0Cq6yp583VB1gwYQA3zR3F9twyPtrSutUPGlYn9hVUwF1b+Wp3IfVNAtbu/HLuf38LFz7+GZPu/oBZf/yIq579iv9dvIn3Nxxi2fY8vvu3FZz36HLeyj7Qqf0yFlS6mXfXH0IV/uf1te3qMOwpco9W8d1n3E1c//jBTCYMSjzm+JQhSbx946mM6B/P9f/M4u2dNa3+Btpaf1u2i6jwMK4+JcPr8ZnDU4iNCucjL4MJ/r3mIIOSYto9cXG608/QXBNYRU0dFz7+WcDXzCqrruPlFfs4Z2IaQ1JaHkZ81ax0Lpo6iIc/2ub3AIsNB45SU+dq7D/y1JYmsHfWHaS0uo4rZ6Zz7qQ0BifH8HjmjlbVDBr2pffV/AXudcBKq+vYfOgo5dV1/GvVfi598gvm/DmTpz51/91cNHUQv//2BF69fhZZv5nHmt/N54s75vLHiyZSUVPPza9mc8afMnnus92UV3f8xn0WVLoRl0tZujWX6enJVNe6+MXC7OO+0ZivFZRV892/reDw0Sqev/akYyYBehqQGM1rPz6ZC6YMZNH2Wr77txUcaGb0VXvkHq1i0eoDXDJ9MH3je3k9p1dEOKeO7MvHm3OP+dDKL6tm+Y58LpgykDA/Ztw3Z/KQJCLDhZXNTIJ86ct9rNlXzN+W727Xz2rqtZX7Ka2q40en+TdPRkT4w4UTGTugN7e8ms1eZwJhcxomPU71MuejLaPAXl6xj5H94zkpI5mI8DB+fPpw1uwrZkUrtgLenV9On7goentZAaFBQ7/K7YvWMePeD/mf19eRX1bNbd8cwxe3n8XCH5/MPRdM4KpZ6cwa3oc+zt9QdGQ4V8wYyoe/OIOnr5pGWmI097yziVPu+5hPtzW/FFWgWVDpRtYfKCG/rIbvzUrnnvPH88WuAp78ZGdnZ6tLqqlzcd3zK9lfVMGzV5/E9IyUZs+Pjgzn4cumcO34KNblFPPNhz7ljaycgLdRP7p0B3UuFz88tfkP1Llj+3OwpIoth0sb0/6z7hD1Lm3zqC9P0ZHhjB+Y2Pjh21RlTb37m3FEGFl7ixq/ZbdXXb2Lv3+2m5Mykn0GeV/5feqqaYSFCT/+ZxYVNc1/A8/aW8TQlFj6Jxw/18U9Z2UAy7bn+dUEtungUbL3F3PFjKGNQ4EvmT6EPnFRPJHp//+/pgtJepOWGMOYAQnsyivnW5PSeP2Gk/no1jP4yZkj6O/HvJ2wMGH++AG8/pNTeOMnpzB7ZB/GDOjYrTssqHQjH23JJUzgjBP6ccn0wZw7KY0Hl2xj9b6eseRGa/zpv1tYm1PCQ5dO8XtHPRHhjCGRvHfz6YxJS+DWf63lJy+u9jowwuVSth0p5fWsHPb5Od9j7f5i/vnlXr43K73FD5c5o91Diz/2aLd/K/sAYwYkMDpAHxLT05NZm1Pidbn9l7/aR35ZNX+6eBIi8OYa/3aYKKuuY3+h79/HfzceIaeokh/6WUvxNCQllr9cfiJbj5Ry99sbfZ6nqmTtLWps4vPmW5MG+t0E9vJXe4mKCOM7U78O5tGR4Vx36jA+2ZbHxoP+La2yp6C82f6UBv+64WRW/WYe9188mekZKW2eoDotPZnHvzvNr2AUSBZUupGlW9zLdyTHRSEi3HvhRAb0jubmV9dwtKpt4+5D0dKtuTyzbDdXzUrnnImtX/F3aJ9YXr3+ZO44Zwwfb8nlGw8v4731h8jcmsuDS7Zx1bMrmPy/HzD/oU/51b/WcvVzXzUu7uhLXb2LOxatp39CrxZny4N75NOkwYmNQWVfQQWr9xUHpJbSYHpGMjV1LjYcOHbkW1VtPU9+spNZw1O4YMogThnRh0Vr/Ku13fpaNmf8aSl3vbXhuFqAqvLMsl1k9Ill3tjjNwnzx+kn9OP604fz2qoc1vtYJ2tvQQX5ZTWN65x5428TWEVNHf9ec5BzJ6aRFBt1zLHvzUonvlcET36yq8V8l1fXceRotV9BJSE6snFSa3dkQaWbOHK0ivUHSjjLmRwHkBgTyV+umMLB4ip+++8NXWI4YWc7crSKW19by5gBCdz5rbFtvk94mPDjM0bw9s9n0y+hFz95aTXXPLeSRz/eTl5pNedNHsifL5nMo1eeyN6Ccu5YtL7Z3/9zn+1h06Gj3H3e+Gbb1D3NGd2f1fuKKCyv4e217prCeZMDsyw+0DjTvGkT2MKV+8krrebmuScAcNGJg9lfWNniEOTtR0r578YjjBnQm39+uZe5D3zCotVfB6OsvUVk7y/mB6cO82sVZl9unDOSPnFR3PvuJq+/81VNJj1649kEVl7r+317Z+1ByqrruHLm0OOOJcZE8t1ZQ/nPuoMt9vM0LiTZp+Wg0t1ZUOkmljrfWM8a0/+Y9GnpKdwydxRvZR9k0epjmyiq6+rZX1jBqj2FPaImU+9SfrEwm8qaeh698sRj5ie01ZgBvXnrZ7N55PIpvPTDmay7+xu8f8vp/OHCiVw8bTDnThrIr74xmsXrDvGisydKU/sLK3hwyTbmje3PNycM8Ptnzx3bH1XI3JrLv7MPMiMjJaCLLvZL6OVeXNKjs766rp4nMncyIyOFWU6n8TcnDCAmMvy4v6+mnvp0F9GRYbz4w5m8feOpDEqO4ZevreWyp79k6+FSnlm2i6TYSC6e1r51txKiI7ll3ii+3FV4TPNgg6y9RSRER3BC/+abCRuawF7dUuNzhvzLK/Yxqn+8zwD1g9nDiAgL46lPm6+t7Ml3Nwlm9A38opldjQWVbuLjLbkMSophdOrx/1F+OmckM4el8Nu3NvCjf6zivL8uZ/r/W8Lo37zPafcv5eInv+D372zqhFx3rCc/2cnnOwu45/zxjGzhA6U1oiLCuGDKIGaP7Ot1j/MbTh/BnNH9+P3izcc1yagqv3trAyJwzwUTWtU+PmFgIv0SevH0p7vYkVvG+e2cm+LN9PQUsvYWNX7jf21VDoePVnHT3FGNeY3rFcE3Jwxg8bqDXidkAhwsruTfaw5w+UlDSYmLYsKgRN78ySn88aKJbDtSyoK/LOODTUf43sx0YqLaH+wvnzGU4f3i+MO7m4+bk5G1t5CpQ5NbHCE3eXAiN84ZybIDdVz73Mrjmus2HChhbU4JV84c6vN96987mu9MG8zrq3LIPep7iL/VVAwAO3JL2ZAf+D3DW6u6rp7lO/KZM6af1z/u8DDh4cunMKp/PPsKKkiJi+Lscan8Yt4J3P+dScwclsIXIbJVqS+r9hTy4JJtnDd5IJdM79idLsPChAcvnULf+Ch++vKxS4q8t+EwS7fm8cuzT2BQUkyr73vW6P5sOVxKRJjwrTb0D7VkekYyBeU17M4vp9alPLF0B1OHJjF75LGDGy6aOojSqjqvNQOAZ5fvRoEfnvb1xndhYcIVM4by8a1ncsm0wQxOjuH7p6QHJN+R4WHccc5YduaV8+rK/Y3pJRW1bDtS5tfy8SLCr74xmh9MiGLF7gIufuLzYwYZvPzVPnpFhHHRic3/Pf349OHUuVz83dl6wJvd+eWk9u7ldbJrqLGg0oznP9/Dk2s7f4Lhil2FVNTUM3eM787NtMQY3rrxVP77i9N54boZ/PGiSdw8bxSXnjSE+eMHkFNUyaGS4My96GwlFbXc/Go2g5JiuPfC1tUGAiU5Loq/XjmVQ8VV/M+/1qKqHK2q5e63NzJhUG+u8THRsSVznObOM07oR3JcVAtnt17DCKlVe4v47EAdB0uquHneCcf9Dk8Z0ZfU3r1YtDrnuHsUV9Twylf7OH/yQK/NcylxUdz3nUks+/VZXof4ttW8sf2ZMSyFhz/c1jhQYvX+omPK5Y/TBkfyj+tmkltazbcf+4ysvUWUV9fx1poDnDtpYIt7r2T0jeOciWm8+OVe8kq9L6G0O7+8R9RSwIJKsxJjIqmoo9M7wD/ekkt0ZJjfQ2ObOskZBdPcRLfu7I4313HkaBV/veJEvzvBg2FaejK3nzOGDzYd4dnlu/nT+1vJL6vmjxdOIqKN63SdNqovk4ckce3slre+bosR/eJJjInky10FvLOzlslDkjjdy0KV4WHCt6cMInNr3nFDrP/xxV4qaur58Rkdu+mXiHDngrHkl9XwlDMCK2tPEeFhwuRWzIEBOHlEHxb99BTioyO44pkv+fUb6yivqefKmf71//zy7BOorqvnj+9t9np8T75/w4lDgQWVZiTFROFS99j7zqKqfLTlCLNH9G1zx/O4tN7ERoV3ylLnn+3IZ8OB4G2ReqikknfXH+YnZ45o9QdJMPzg1GHMH5fKfe9t4cUVe7n6lAwmDk5s+UIf4npF8NbPZrd5ReKWNCwu+e81ByioUm6eO9JnTe+iqYOpcynvrP16McbKmnqe/3wPZ43p37imWkeaPCSJC6YM5JlluzhUUsmqvYWMTUtoUzPTiH7xvPnT2UwZnMR/1h1idGqC38vhjOgXz49OG86i1Qf4qsks+5LKWgrKayyotJeIDBGRpSKySUQ2isjNTvpCEcl2HntEJNvLtaM9zskWkaMicotz7G4ROeBxbEGwytCwempb914IhJ15ZewvrGxsBmmLiPAwpg5NPu6PPdjqXcrPXl7Ntc+vpKTC/9+hr85gb5ZvzwfgnAmB729oCxHhT5dMJi0pmrTe0dw6v+U5KZ1tWnoyLoWM3mGNky69GT0ggXFpvVnkMRHytVX7KSyv4YYzRnREVr361fzRKPB/720he38x09ObXz2hOSlxUfzzhzO4ae4o7j5/fKuaUm88aySDkmL47b83HDOazJ81v0JJMGsqdcCtqjoOmAX8TETGqeplqjpFVacAbwCLml6oqls9zpkGVABvepzyUMNxVX03WAXo3QWCysc+hhK31kkZKWw9UtqhZVmbU0xxRS15pdXc9773ZoGmHlu6g6m/X0K+n8v7L9+RT9/4qA5fiqI5iTGRLL7xNN75+aleR4t1NaeP6keYwIWjIlv8EL1o6iDW5ZSwI7eUunoXzyzbxdShSY1NrJ1hSEos187O4N/ZB6mqdbV7j/deEeH88uwTWt3cHBsVwe/OG8fWI6W88PmexvSGkV9WU2knVT2kqqud56XAZqBxOrC4/3ovBV5p4VZzgZ2q6n0SQBB1hZrKR5tzGTMggYGtHDnU1EkZyajC6g7cQyNzax5hApdNH8IrX+33upS7py92FvDAB1upqKlvdpOqBi6X8tmOfGaP7NvuBRYDLTE2snGxv65u4uBE1vx2PpP7tRwAz58ykPAwYdHqA/xn/SFyiir5yZm+m8w6yk/PHNm473t7g0p7zB+XypzR/XhoybbGVcR355cjAkP9WJE5FHTI1ygRyQBOBFZ4JJ8GHFHV7S1cfjnHB54bReT7wCrctaHjPilF5HrgeoDU1FQyMzNbne99R93NMJ+vzKZmf8d/4yyvVVbuqWDBsMg25d9TdZ0SLvDGp9nI4ZZHEZWVlbX7Zy5eVcmw3mGclVzAxzHCLS+t4PezY4gKP/4D6Gi18rvPK+kfK1TXwavLNtK/vPnF+vYdrSe/rIa+dfntzmuDQJS7u/K37ONTwnj1y13ERQoD44XwI5vIzPWvJhpMl40KY21uONuyV7CtFdcF+j0/J9XFsu313PRcJj+dEs2KjVWk9BK+/GxZwH5GIATtb11Vg/oA4oEs4KIm6U/gDgjNXRsF5AOpHmmpQDjuWta9wN9bysO0adO0LXKKKjT9tsX66ld723R9e72dfUDTb1usq/YUBOR+5z+6XC954nO/zl26dGm7flZ+aZVm3L5YH16yTVVVl23L0/TbFuv9728+7tz6epd+729f6gl3vqubDpbonW+u0zG/eU8ra+qa/RlPfbJD029brIeKK9uVV0/tLXd35m/Z33L+LtNvW6yvrdwX3Ex1gGC85w8t2arpty3W5dvz9Py/LtPvPvNlwH9Ge/lTbmCVtvIzP6ijv0QkEne/yUuqusgjPQK4CFjYwi3OAVarauNSoqp6RFXrVdUFPAPMCHzO3Tq7+evjLbkkx0YyZUhgqvMnpSeTnVPsdVXaQFu+Ix9VOHN0PwBOHdWXi6cN5qlPdrHp4LELGD7xyU6Wbc/nrvPGMzatN3PHplJZW99ic9my7fmM7B/PgMSOXYW1p5s/LpWEXhGkJUYHdIHLUHLDGSNI7xPLb9/awK788h6xPEuDYI7+EuBZYLOqPtjk8Dxgi6oeP5PqWFfQpOlLRDyH+VwIbGhvXn2JiwonTDonqNS7lMytuZw5un+7Ft/zdNKwFGrqXD5Xdw2kzK15pMRFMdFjp8U7F4wlMSaSOxata9xc7KvdhTzwwVbOmzyQK2a45wScPLwPsa9llI8AACAASURBVFHhfLjZ97LkVbX1fLW7kFNHBmeorfEtOjKchy+fwiOXn0hUhM1K8CY6Mpy7zx/PrrxySqvqeszERwju6K/ZwFXAWV6G/x7XTyIiA0XkXY/XccDZHD867H4RWS8i64A5wC+CVQARIS6ic4JK9v4iiipq2z3qy1PDLOOvgjxfxeVSPt2Wx+mjju1AT46L4q7zx7M2p4TnP99DYXkNN72yhqEpsfzBYyZ8dGQ4p43qy8dbjt350FPW3iKq61ycFqT5G6Z5c8emMmNY24fu9gRzRvfnG+Pdq2D0lJFfEMSOelVdDnj9iq2q13hJOwgs8HhdDhw3pk9VrwpcLlsWGymUVHbs5EdV5YXP9xIRJpw+ql/A7tsnvhfD+8UdsyptMGw4WEJBeQ1njD4+7+dNSuPfaw7w5/9u5b8bD1NYXsOin55CQpOZ8HPHpvLfjUfYePDocfvKg7vpKyJMmDm8basMGNMR/veCCfSJ79Wj/k6t7tqCuEjp8JrKi1/u5e21B/nZnJEtrjvUWjMyUli1pxBXEPe2/2RrHiJ4DYgiwu+/PYEwcTd9/fbcsV6Dxllj+iOCzyaw5TvymDo0uVvMAzE9V2rvaP5w4cQe9XdqQaUFsR0cVLL2FnLPO5s4a0x/bp47KuD3n56RwtGqOrbllrZ8chtlbstj4qBEn/M0BiXF8NBlU/jFvBP43izvq9b2je/FiUOS+MjLfJXC8ho2HjwatKVLjDFtZ0GlBXERcLSDgkru0Sp+8uJqBiXH8NClU4IyoW9GhrsdPFiLSxZX1LBmXxFnntB8s9388QO4ed6oZifNzR2byvoDJRxpsk/FZ87IMgsqxnQ9FlRa0FHNX7X1Ln728mpKq+p48nvTAt7s1WBISgz9E3qxMkjrgC3fkY9L8dqf0lpnj3N3cjatrSzfnk9CdASTvDSbGWM6lwWVFjQ0f/kahRQo9/5nMyv3FHHfdyYyNi14q72KCCc5/SrBkLk1j8SYSCYPbv+KwaP6xzMkJeaYfhVVZfmOfE4Z0afNy8kbY4LH/le2IC5SqHcp5TXBmzD45pocnv98D9fNHtYhk8lOykjmYEkVOUUVLZ/cCqrKJ9vyOHVU34B84IsIc8ek8tmOfCqd3/+eggoOFFdyagBHxRljAseCSgsaWqGC1QS26eBR7li0nhnDUrhjwZig/Iympjv9KoEeWrzp0FHySqtb7E9pjbPHpVJd52L5DvcS98u35wFwmk16NKZLsqDSgrgId0dya/YDaY0HPthKfK8IHrtyKpEd1JwzNq038b0ifE6C3JlXxpcHWz8355Nt7g/8MwIYVE7KSCGhVwQfOU1gy7bnMzg5hvQ+PWfZC2O6EwsqLYiLdIJKkGoqBeU1jE3rTb+EjlsmPTxMmJqefFy/iqry2qr9nPuX5Ty5rrrVm3plbs1jXFpv+vcO3FpcURFhnD66Hx9uzqW23sUXOws4bVTfTl9q3RjjnQWVFgS7+auypp7YqLZtE9weMzKS2XakjOKKGgBKq2q5+dVsfv36OqYMSaJ3FDy6dIff9ztaVUvW3qKAjPpqat7Y/uSXVfPil3spra7j1JHWn2JMV2VBpQUNNZVgzVWpqK0jNqrjZ9t69qus3V/MuX9dzn/WH+JX80/gxR/O5BsZkXy6LY+1+4v9ut/nO/Kpd2lA+1MazHEW1Xz4w+2IwCmt3JHPGNNxLKi0INjNX5U19URHdnxNZcqQJCLDhQeXbOM7T3xOXb2y8PpZ3HjWKMLDhLOGRpIYE8ljftZWMrfmkdArgqlB2HUvKTaKaenJlFTWMmFgIslxLW8yZozpHBZUWhAd7u6DCLXmr+jIcCYOSmTToaPMHdufd286rbH2AhATIVxzSgYfbDrC1sPNL+lS73IPJZ49sm/QBhvMG+terdlm0RvTtVlQaYGI0Ds6IihBRVWpqO2coAJw13njefTKE33O4L92dgZxUeEt1lYe+XAbh0qqOG/ywGBllQUT0xiaEsu5k9JaPtkY02ksqPghMSYyKEGlus6FKsR0UlCZPCSJcycN9DmSKik2iu+dnM7idQfZnV/u9ZyPNh/hLx/v4JJpg1kwcUDQ8jo4OZZPfz2H8QNtaRZjujILKn4IVlCpcGaJx3ZCn4q/fnjqcCLDw3gi8/jayt6Ccm5ZmM34gb35/bcn2DBfY4wFFX/0DlpQcU8w7IzRX/7ql9CLK2YMZdHqAxwormxMr6yp54YXVxMmwpPfm9Ypgw2MMV2PBRU/JMZEBmVIccN6Vp3V/OWv608fjgg89clOwN0XdOeb69ly+CgPXz6FISk2u90Y42ZBxQ/Bbv6K6eLf8gcmxfCdqYN5deV+ckureHHFPhatOcAtc09gzuj+nZ09Y0wXYkHFDw1BJdDL31fWOn0qXbymAvCTM0dQV+/ittfX8b/vbGTO6H78/KyRnZ0tY0wXE7SgIiJDRGSpiGwSkY0icrOTvlBEsp3HHhHJ9nLtaI9zskXkqIjc4hxLEZElIrLd+Tfws+2aSIyJpM6ljTWLQOkuzV8A6X3iOH/yQJZuzWNAYjQPXRacnSmNMd1bMHuI64BbVXW1iCQAWSKyRFUvazhBRB4ASppeqKpbgSnOOeHAAeBN5/DtwEeqep+I3O68vi2I5SAxxj2Ho6SylrhegfuVNY7+6sId9Z5unncCuaXV3PmtsSTF2qx2Y8zxglZTUdVDqrraeV4KbAYad6AS9/jTS4FXWrjVXGCnqu51Xl8AvOA8fwH4diDz7Y1nUAmkr0d/df2aCsCwvnG8/KNZNlfEGONTh3xFFpEM4ERghUfyacARVd3ewuWXc2zgSVXVQ87zw0Cqj595PXA9QGpqKpmZma3ON0BZWRn7CjYB8MkXKzmSErgAsHavO0itWbmCnb26VlNSWVlZm39n3VlPLTf03LJbuQNMVYP6AOKBLOCiJulP4G4ea+7aKCAfdyBpSCtuck5RS3mYNm2attXSpUt1fU6xpt+2WN/fcKjN9/Hmicwdmn7bYi2vrg3ofQNh6dKlnZ2FTtFTy63ac8tu5fYNWKWt/MwPak1FRCKBN4CXVHWRR3oEcBEwrYVbnAOsVtUjHmlHRCRNVQ+JSBqQG+h8NxW85i93n0p0RPdo/jLGmJYEc/SXAM8Cm1X1wSaH5wFbVDWnhdtcwfF9Lm8DVzvPrwbeam9eW9Kw2GKgJ0BW1tQRHRlmo6iMMSEjmPNUZgNXAWd5DA1e4Bxr2k+CiAwUkXc9XscBZwOLONZ9wNkish13cLovWAVoEB8VQZgEvqZSWVvfbUZ+GWOMP4L2iaaqywGvX8FV9RovaQeBBR6vy4HjtvhT1QLcI8I6TFiYBGX9r4qa+i4/m94YY1rDZtT7KRhLtXTWBl3GGBMsFlT8FIygUmFBxRgTYiyo+ClYNZXusESLMcb4y4KKn4LSp1JbZx31xpiQYkHFT8HYU8U66o0xocaCip+Csfx9lTV/GWNCjAUVPyXGRFJbr417oARCRa111BtjQosFFT8FY6mWCqupGGNCjAUVPwU6qNS7lJo6F7GR1lFvjAkdFlT81BhUKgITVLrbXirGGOMPCyp+CnRNpTttJWyMMf6yoOKnQAeVhmXvbUixMSaUNNugLyLvAD7H0Krq+QHPURfVO0hBxZq/jDGhpKVe4j87/14EDABedF5fARzxekWISugVgUjg9lRpGJpszV/GmFDSbFBR1U8AROQBVZ3ucegdEVkV1Jx1MWFhQu/owC3VUtlYU7HRX8aY0OFvn0qciAxveCEiw4C44GSp6wrkopI2+ssYE4r8/Zp8C5ApIrtwb7yVDlwftFx1UYEMKtb8ZYwJRS0GFREJAxKBUcAYJ3mLqlYHM2NdUWBrKtZRb4wJPS02f6mqC/i1qlar6lrn0eMCCgQpqNiMemNMCPG3T+VDEfmViAwRkZSGR1Bz1gW591SpC8i9Kp0+legomypkjAkd/n5Nvsz592ceaQoM93IuACIyBPgHkOqc+7SqPiIiC4HRzmlJQLGqTvFyfRLwN2CCc/11qvqFiNwN/AjIc079/1T1XT/L0S4Ne6qoKiLSrntV1tYTHiZEhVtQMcaEDr+CiqoOa8O964BbVXW1iCQAWSKyRFUbAhQi8gBQ4uP6R4D3VfViEYkCYj2OPaSqf/ZxXdAkxkRSU++iqtbV7g72ipp6YiPD2x2cjDGmK/G7QV9EJgDjgOiGNFX9h6/zVfUQcMh5Xioim4FBwCbnfgJcCpzl5WclAqcD1zjX1wA1/uY1WDyXamlvULH96Y0xocivoCIidwFn4g4q7wLnAMtxN2/5c30GcCKwwiP5NOCIqm73cskw3M1bz4nIZCALuFlVy53jN4rI94FVuGtDRV5+5vU4w55TU1PJzMz0J6vHKSsra7w257C7H+TDTz9ncEL7mq325FQh9a425yvYPMvdk/TUckPPLbuVO8BUtcUHsB53p/5a53UqsMTPa+NxB4WLmqQ/gTsgeLtmOu7ms5nO60eA33v87HAnP/cCf28pD9OmTdO2Wrp0aePzZdvyNP22xbpiV0Gb79fgB8+v1G8+/Gm77xMsnuXuSXpquVV7btmt3L4Bq9SPz3nPh79ftyvVPbS4TkR6A7nAkJYuEpFI4A3gJVVd5JEegXs9sYU+Ls0BclS1oWbzOjAVQFWPqGq9k59ngBl+lqHdArlScWVtnc1RMcaEHH+DyipnNNYzuGsdq4EvmrvA6TN5Ftisqg82OTwP9wTKHG/XquphYL+INIwSm8vXfTFpHqdeCGzwswztFsigUlFTb8veG2NCjr+jv37qPH1SRN4HeqvquhYumw1cBawXkWwnrWH47+XAK54ni8hA4G+qusBJ+jnwkjPyaxdwrZN+v4hMwT3MeA/wY3/KEAgBranU1NM3vle772OMMV2Jvx31/wQ+BZap6hZ/rlHV5bjXCfN27BovaQeBBR6vs3H3rTQ97yp/fn4wJES7l78PTPNXvTV/GWNCjr/NX38H0oC/isguEXlDRG4OYr66pLAwIaFXRED2VKmosaBijAk9/jZ/LRWRT4GTgDnADcB43KOyepTE2MCs/1VZU0+MrftljAkx/jZ/fYR7/5QvgGXASaqaG8yMdVWBWFRSVamosdFfxpjQ42/z1zrcM9onAJOACSISE7RcdWGBCCrVdS5canupGGNCj7/NX78AcNbwugZ4Dvee9T1u+FJiTCRHjpa16x4NWwnbkGJjTKjxt/nrRtzLqkzDPYz377ibwXqcQNRUKmptgy5jTGjyt6c4GngQyFLVwGwo0k31DkBQaaypWFAxxoQYv/pU1L3MfCTuyYyISD8Racty+N1eYkwkNXUuqpzaRltUNm4lbKO/jDGhxa+g4qxSfBtwh5MUCbwYrEx1ZYGYVV/h7PpozV/GmFDj7+ivC4HzgXJonP2eEKxMdWUBCSq11vxljAlN/gaVGmcZZAUQkbjgZalrC0RQ+br5y4KKMSa0tBhUnNWGF4vIU0CSiPwI+BD3isU9TmNQqWhP85cTVGxGvTEmxLT4qaaqKiKXAL8EjgKjgd+p6pJgZ64rCkxNxd2nEh3Vvt0jjTGmq/H3U201UKyq/6Oqv+qpAQWaDyollbXcsWgdB4srm71HZa2N/jLGhCZ/g8pM4AsR2Ski6xoewcxYV5UQ7Tuo3P/+Fl75aj/Ld+Q3e48Km1FvjAlR/n5V/kZQc9GNhIcJCdERxwWVrL1FvPzVPgCKK2qavUdlTT29IsIID/O63YwxxnRb/q79tTfYGelOEmMij9lTpbbexZ1vrmdA72jyy6opLG++v8X2UjHGhCrrKW6Dput/Pbt8N1sOl3LP+eNJio1qsabiDirWn2KMCT0WVNrAM6jsL6zg4Q+3MX9cKvPHDyAlNorC8haav2rrbOKjMSYkWVBpg4agoqr87q0NhItw9/njAUiKjaS4hTksFTX11klvjAlJQQsqIjJERJaKyCYR2diwp72ILBSRbOexR0SyfVyfJCKvi8gWEdksIic76SkiskREtjv/JgerDL40BJV31x9m6dY8fjl/NAOT3HuWJcdGUeRH85fVVIwxoSiYNZU64FZVHQfMAn4mIuNU9TJVnaKqU4A3gEU+rn8EeF9VxwCTgc1O+u3AR6o6CvjIed2hEmPctZG739nIhEG9ufrk9MZjyXEtB5WqWuuoN8aEpqAFFVU9pKqrneeluIPCoIbjzvIvlwKvNL1WRBKB04FnnetrVLXYOXwB8ILz/AXg28Eqgy+9YyKpqXdRUFbNHy+cRET417/G5NhIiircTWO+2OgvY0yo6pAhSCKSAZwIrPBIPg04oqrbvVwyDMgDnhORyUAWcLOqlgOpqnrIOe8wkOrjZ14PXA+QmppKZmZmm/JeVlZ23LVH9rv7TOYOjaBgxxoyd3x9rPBQLfUu5d0PM4mL9D4PpehoBSURVW3OU0fwVu6eoKeWG3pu2a3cAaaqQX0A8biDwkVN0p/A3Tzm7ZrpuJvPZjqvHwF+7zwvbnJuUUt5mDZtmrbV0qVLj0vbk1+mv/7XWi2tqj3u2L9W7df02xbrnvwyn/eccs9/9Tdvrm9znjqCt3L3BD213Ko9t+xWbt+AVdrKz/ygjv4SkUjc/SYvqeoij/QI4CJgoY9Lc4AcVW2o2bwOTHWeHxGRNOc+aUBuMPLenPQ+cfzfxZOI73V8RS8lzr2MS1EzI8Cs+csYE6qCOfpLcPeJbFbVB5scngdsUdUcb9eq6mFgv4iMdpLmApuc528DVzvPrwbeCmjG2ykpNgqAIh9zVepdSnWdy0Z/GWNCUjBrKrNx72l/lscQ4gXOsctp0kEvIgNF5F2PpJ8DLzkLV04B/uCk3wecLSLbcQen+4JYhlZLbggqPkaANaxQbPNUjDGhKGgd9aq6HPDaU62q13hJOwgs8Hidjbtvpel5BbhrLl1SihNUfM2qt10fjTGhzGbUB1hCdARhgs9Z9Q1BJcbW/jLGhCALKgEWFiYkx0ZR6KP5q6LWveuj1VSMMaHIgkoQuNf/8hFUGmsqFlSMMaHHgkoQJMdGUeRjT5XGPhXrqDfGhCALKkHQ3PpfFTW2P70xJnRZUAkC9/pfvoKKu08lJsp+9caY0GOfbEGQHOdu/lIvi0ra6C9jTCizoBIEybFR1NS7Gpu6PDVMfrQ+FWNMKLKgEgQpzcyqt9FfxphQZkElCJJinUUlvYwAq6ypJ0ygV4T96o0xocc+2YIgOa75mkpsVATu9TaNMSa0WFAJguYWlaysrbOmL2NMyLKgEgTJjc1f3msqtkKxMSZUWVAJgsSYSESg0MuikrZBlzEmlFlQCYKI8DB6R3tf/6uqtt6av4wxIcuCSpCkxEV53VLYairGmFBmQSVIkmIjm+lTsdn0xpjQZEElSJJjvS8qWVlTZzUVY0zIsqASJO7l733NU7GgYowJTRZUgsS9UrH3GfXWUW+MCVVBCyoiMkRElorIJhHZKCI3O+kLRSTbeewRkWwf1+8RkfXOeas80u8WkQMe91gQrDK0R3JcFJW19VTVfr2opKpSUWvzVIwxoSuYPcZ1wK2qulpEEoAsEVmiqpc1nCAiDwAlzdxjjqrme0l/SFX/HOD8BpTnrPq0xBgAaupd1LvUmr+MMSEraDUVVT2kqqud56XAZmBQw3FxL351KfBKsPLQmZK9LCpZVeMCbC8VY0zo6pBPNxHJAE4EVngknwYcUdXtPi5T4AMRUeApVX3a49iNIvJ9YBXu2lCRl595PXA9QGpqKpmZmW3Ke1lZWZuu3VvobvbK/GIluX3cNZPCKndQ2b97B5l1e9uUn47S1nJ3dz213NBzy27lDjBVDeoDiAeygIuapD+BOyD4um6Q829/YC1wuvM6FQjHXcu6F/h7S3mYNm2attXSpUvbdN2WQ0c1/bbF+s7aA41pO3JLNf22xfrvNTltzk9HaWu5u7ueWm7Vnlt2K7dvwCpt5Wd+UEd/iUgk8Abwkqou8kiPAC4CFvq6VlUPOP/mAm8CM5zXR1S1XlVdwDMN6V1NcpzT/OUxAqxxK2HrqDfGhKhgjv4S4Flgs6o+2OTwPGCLqub4uDbO6dxHROKA+cAG53Wax6kXNqR3NUkxTke9x1yVhl0fY61PxRgTooJZU5kNXAWc5WX47+U06aAXkYEi8q7zMhVYLiJrga+A/6jq+86x+52hxuuAOcAvgliGNouKCCOhV8Qxs+orauoAiImy6UHGmNAUtK/Mqroc8Lq9oape4yXtILDAeb4LmOzj2qsCl8vgSoo7dv2vr5u/rKZijAlN9pU5iNzrf3n0qdQ2NH9Zn4oxJjRZUAmipotKft2nYkHFGBOaLKgEkXv9Ly/NXxZUjDEhyoJKECXHRVHsMaPeRn8ZY0KdBZUgSo6NorS6jpo690z6ito6oiLCCA/zOn7BGGO6PQsqQZQc556rUlzpbgKrtL1UjDEhzoJKEDVdVNK9lbAFFWNM6LKgEkSey9+De0ixddIbY0KZBZUgaggqxRXW/GWM6RksqARRw6KShY3NX3XE2mx6Y0wIs6ASRMc1f9n+9MaYEGdBJYiiI8OJiQxvXP+rwpq/jDEhzoJKkLln1XuM/rKgYowJYRZUgiw5LurrjvpaG1JsjAltFlSCLDk2isKKhuavOmv+MsaENAsqQeauqdTicilVtS5ibN0vY0wIs6ASZMmxkRSW11BVZ8veG2NCnwWVIEuOjeJoVS1lVe6thC2oGGNCmQWVIEuOjUQVDpVUAVhHvTEmpFlQCbKGlYoPFlcCtpeKMSa0Be0TTkSGAP8AUgEFnlbVR0RkITDaOS0JKFbVKV6u3wOUAvVAnapOd9JTgIVABrAHuFRVi4JVjvZqmFV/wAkqMVEWx40xoSuYX5vrgFtVdbWIJABZIrJEVS9rOEFEHgBKmrnHHFXNb5J2O/CRqt4nIrc7r28LdOYD5bigYmt/GWNCWNC+NqvqIVVd7TwvBTYDgxqOi4gAlwKvtPLWFwAvOM9fAL7d/twGT8Oikl83f1mfijEmdImqBv+HiGQAnwITVPWok3Y68GBDs5aXa3YDRbibzp5S1aed9GJVTXKeC1DU8LrJ9dcD1wOkpqZOe/XVV9uU97KyMuLj49t0LUBVnXLDhxVk9A5jz1EX954aw6D4rt8E1t5yd1c9tdzQc8tu5fZtzpw5Wb4+o30JeluMiMQDbwC3NAQUxxU0X0s5VVUPiEh/YImIbFHVTz1PUFUVEa9R0QlCTwNMnz5dzzzzzDblPzMzk7Ze6+SDqMz3Ka2PAGo4Y/YsBifHtvl+HaW95e6uemq5oeeW3codWEH9yiwikbgDykuqusgjPQK4CHeHu1eqesD5Nxd4E5jhHDoiImnOfdKA3ODkPjBEhOTYSAqclYpt9JcxJpQFLag4TVPPAptV9cEmh+cBW1Q1x8e1cU7nPiISB8wHNjiH3waudp5fDbwV6LwHWkNnPVifijEmtAWzpjIbuAo4S0SynccC59jlNGn6EpGBIvKu8zIVWC4ia4GvgP+o6vvOsfuAs0VkO+7gdF8QyxAQDUFFBHpFdP3+FGOMaaugtcWo6nJAfBy7xkvaQWCB83wXMNnHtQXA3IBltAM0jACLiQzHXYEzxpjQZF+bO0BDTcWavowxoc6CSgdoCCq266MxJtRZUOkADet/xdpsemNMiLOg0gGSY50+FaupGGNCnAWVDmB9KsaYnsKCSgdobP6yoGKMCXEWVDpAQ/NXtG3QZYwJcRZUOoDVVIwxPYUFlQ6Q0CuCiDCxdb+MMSHPPuU6gIjwm2+NZVp6SmdnxRhjgsqCSge5Zvawzs6CMcYEnTV/GWOMCRgLKsYYYwLGgooxxpiAsaBijDEmYCyoGGOMCRgLKsYYYwLGgooxxpiAsaBijDEmYERVOzsPQSciecDeNl7eF8gPYHa6Cyt3z9NTy27l9i1dVfu15qY9Iqi0h4isUtXpnZ2Pjmbl7nl6atmt3IFlzV/GGGMCxoKKMcaYgLGg0rKnOzsDncTK3fP01LJbuQPI+lSMMcYEjNVUjDHGBIwFFWOMMQFjQaUZIvJNEdkqIjtE5PbOzk97iMgQEVkqIptEZKOI3Oykp4jIEhHZ7vyb7KSLiPzFKfs6EZnqca+rnfO3i8jVnVWm1hCRcBFZIyKLndfDRGSFU76FIhLlpPdyXu9wjmd43OMOJ32riHyjc0rSOiKSJCKvi8gWEdksIif3hPdcRH7h/J1vEJFXRCQ6FN9zEfm7iOSKyAaPtIC9vyIyTUTWO9f8RUSkxUypqj28PIBwYCcwHIgC1gLjOjtf7ShPGjDVeZ4AbAPGAfcDtzvptwP/5zxfALwHCDALWOGkpwC7nH+TnefJnV0+P8r/S+BlYLHz+jXgcuf5k8BPnOc/BZ50nl8OLHSej3P+BnoBw5y/jfDOLpcf5X4B+KHzPApICvX3HBgE7AZiPN7ra0LxPQdOB6YCGzzSAvb+Al8554pz7Tkt5qmzfyld9QGcDPzX4/UdwB2dna8Alu8t4GxgK5DmpKUBW53nTwFXeJy/1Tl+BfCUR/ox53XFBzAY+Ag4C1js/AfJByKavtfAf4GTnecRznnS9P33PK+rPoBE58NVmqSH9HvuBJX9zodkhPOefyNU33Mgo0lQCcj76xzb4pF+zHm+Htb85VvDH2aDHCet23Oq9ycCK4BUVT3kHDoMpDrPfZW/O/5eHgZ+Dbic132AYlWtc157lqGxfM7xEuf87ljuYUAe8JzT9Pc3EYkjxN9zVT0A/BnYBxzC/R5m0TPecwjc+zvIed40vVkWVHoYEYkH3gBuUdWjnsfU/XUkpMaYi8i5QK6qZnV2XjpBBO6mkSdU9USgHHdzSKMQfc+TgQtwB9WBQBzwzU7NVCfpjPfXgopvB4AhHq8HO2ndlohE4g4oL6nqIif5iIikOcfTgFwn3Vf5u9vvZTZwvojsAV7F3QT2CJAkIhHOOZ5laCyfczwRKKD7lRvcZrGtbwAAA+BJREFU3yxzVHWF8/p13EEm1N/zecBuVc1T1VpgEe6/g57wnkPg3t8DzvOm6c2yoOLbSmCUM2IkCncH3tudnKc2c0ZtPAtsVtUHPQ69DTSM9rgad19LQ/r3nREjs4ASp0r9X2C+iCQ73wjnO2ldkqreoaqDVTUD93v4sap+F1gKXOyc1rTcDb+Pi53z1Um/3BkpNAwYhbsTs8tS1cPAfhEZ7STNBTYR4u857mavWSIS6/zdN5Q75N9zR0DeX+fYURGZ5fwev+9xL986u5OpKz9wj5bYhnvUx52dnZ92luVU3NXgdUC281iAu+34I2A78CGQ4pwvwGNO2dcD0z3udR2ww3lc29lla8Xv4Ey+Hv01HPcHxA7gX0AvJz3aeb3DOT7c4/o7nd/HVvwYBdMVHsAUYJXzvv8b9+iekH/PgXuALcAG4J+4R3CF3HsOvIK736gWd830B4F8f4Hpzu9wJ/AoTQZ9eHvYMi3GGGMCxpq/jDHGBIwFFWOMMQFjQcUYY0zAWFAxxhgTMBZUjDHGBIwFFWP85Kz4+1Pn+UAReT2IP2uKiCwI1v2NCRYLKsb4Lwn3irao6kFVvbiF89tjCu55RMZ0KzZPxRg/iciruNeU2op7YtlYVZ0gItcA38a9xtQo3IsZRgFXAdXAAlUtFJERuCef9QMqgB+p6hYRuQS4C6jHvZjhPNyT0GJwL4vxR9wr7f4VmABEAner6lvOz74Q99Iig4AXVfWeIP8qjPEpouVTjDGO24EJqjrFWel5scexCbhXfo7GHRBuU9UTReQh3MtbPAw8DdygqttFZCbwOO61yH4HfENVD4hIkqrWiMjvcM94vhFARP6Ae/mQ60QkCfhKRD50fvYM5+dXACtF5D+quiqYvwhjfLGgYkxgLFXVUqBUREqAd5z09cAkZ3XoU4B/eWye18v59zPgeRF5Dffih97Mx70w5q+c19HAUOf5ElUtABCRRbiX5LGgYjqFBRVjAqPa47nL47UL9/+zMNz7eUxpeqGq3uDUXL4FZInINC/3F+A7qrr1mET3dU3bsK1N23Qa66g3xn+luLdibjV1712z2+k/adgvfLLzfISqrlDV3+HeVGuIl5/1X+DnDXuEi8iJHsfOdvYlj8Hdt/NZW/JoTCBYUDHGT04T02cisgH4Uxtu8V3gByKyFtiIu9Mf4E8ist657+e490VfCowTkWwRuQz4Pe4O+nUistF53eAr3PvkrAPesP4U05ls9Jcx3Zgz+quxQ9+YzmY1FWOMMQFjNRVjjDEBYzUVY4wxAWNBxRhjTMBYUDHGGBMwFlSMMcYEjAUVY4wxAfP/A2MU5QT8NmFhAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "awlrQQCUpEJd",
        "outputId": "342f695d-380a-4f3c-914a-97e5c84c5d19",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "source": [
        "from google.colab import files\n",
        "files.download('./experiments/blackjack_dqn_result/performance.csv') "
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_70cd65dd-6f94-42db-87ee-8005a355127b\", \"performance.csv\", 1245)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}