RL Blog FINAL MEDIUM code 02_12_19.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code will successfully solve Continuous Mountain Car\n",
    "if the random policy finds the top of the hill reward (+100)\n",
    "in the first several episodes. Otherwise, it will fall into \n",
    "the bad local optimum of stopping at the bottom of the hill.\n",
    "In this case, re-start until the reward is found. \n",
    "This exploration process could be automated but is not\n",
    "in this version."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 0, Number of Steps : 698, Cumulative reward: 71.48\n",
      "Episode: 1, Number of Steps : 632, Cumulative reward: 73.92\n",
      "Episode: 2, Number of Steps : 688, Cumulative reward: 70.95\n",
      "Episode: 3, Number of Steps : 290, Cumulative reward: 88.38\n",
      "Episode: 4, Number of Steps : 999, Cumulative reward: -42.01\n",
      "Episode: 5, Number of Steps : 979, Cumulative reward: 58.99\n",
      "Episode: 6, Number of Steps : 999, Cumulative reward: -39.12\n",
      "Episode: 7, Number of Steps : 604, Cumulative reward: 79.44\n",
      "Episode: 8, Number of Steps : 854, Cumulative reward: 65.54\n",
      "Episode: 9, Number of Steps : 999, Cumulative reward: -36.22\n",
      "Episode: 10, Number of Steps : 776, Cumulative reward: 71.42\n",
      "Episode: 11, Number of Steps : 999, Cumulative reward: -38.01\n",
      "Episode: 12, Number of Steps : 826, Cumulative reward: 69.56\n",
      "Episode: 13, Number of Steps : 999, Cumulative reward: -35.81\n",
      "Episode: 14, Number of Steps : 999, Cumulative reward: -38.05\n",
      "Episode: 15, Number of Steps : 843, Cumulative reward: 70.65\n",
      "Episode: 16, Number of Steps : 717, Cumulative reward: 74.73\n",
      "Episode: 17, Number of Steps : 538, Cumulative reward: 80.74\n",
      "Episode: 18, Number of Steps : 416, Cumulative reward: 84.58\n",
      "Episode: 19, Number of Steps : 465, Cumulative reward: 83.26\n",
      "Episode: 20, Number of Steps : 574, Cumulative reward: 79.83\n",
      "Episode: 21, Number of Steps : 564, Cumulative reward: 79.01\n",
      "Episode: 22, Number of Steps : 654, Cumulative reward: 75.77\n",
      "Episode: 23, Number of Steps : 663, Cumulative reward: 76.65\n",
      "Episode: 24, Number of Steps : 515, Cumulative reward: 80.52\n",
      "Episode: 25, Number of Steps : 550, Cumulative reward: 79.14\n",
      "Episode: 26, Number of Steps : 363, Cumulative reward: 86.61\n",
      "Episode: 27, Number of Steps : 466, Cumulative reward: 80.67\n",
      "Episode: 28, Number of Steps : 574, Cumulative reward: 77.16\n",
      "Episode: 29, Number of Steps : 357, Cumulative reward: 85.39\n",
      "Episode: 30, Number of Steps : 460, Cumulative reward: 81.69\n",
      "Episode: 31, Number of Steps : 371, Cumulative reward: 86.09\n",
      "Episode: 32, Number of Steps : 528, Cumulative reward: 78.84\n",
      "Episode: 33, Number of Steps : 235, Cumulative reward: 90.22\n",
      "Episode: 34, Number of Steps : 315, Cumulative reward: 87.55\n",
      "Episode: 35, Number of Steps : 420, Cumulative reward: 84.81\n",
      "Episode: 36, Number of Steps : 274, Cumulative reward: 89.35\n",
      "Episode: 37, Number of Steps : 379, Cumulative reward: 84.94\n",
      "Episode: 38, Number of Steps : 285, Cumulative reward: 88.50\n",
      "Episode: 39, Number of Steps : 246, Cumulative reward: 90.09\n",
      "Episode: 40, Number of Steps : 220, Cumulative reward: 89.74\n",
      "Episode: 41, Number of Steps : 502, Cumulative reward: 77.92\n",
      "Episode: 42, Number of Steps : 192, Cumulative reward: 89.89\n",
      "Episode: 43, Number of Steps : 156, Cumulative reward: 92.48\n",
      "Episode: 44, Number of Steps : 304, Cumulative reward: 86.16\n",
      "Episode: 45, Number of Steps : 345, Cumulative reward: 86.02\n",
      "Episode: 46, Number of Steps : 240, Cumulative reward: 88.00\n",
      "Episode: 47, Number of Steps : 188, Cumulative reward: 90.51\n",
      "Episode: 48, Number of Steps : 228, Cumulative reward: 89.35\n",
      "Episode: 49, Number of Steps : 226, Cumulative reward: 89.55\n",
      "Episode: 50, Number of Steps : 195, Cumulative reward: 90.71\n",
      "Episode: 51, Number of Steps : 205, Cumulative reward: 90.12\n",
      "Episode: 52, Number of Steps : 118, Cumulative reward: 93.28\n",
      "Episode: 53, Number of Steps : 194, Cumulative reward: 90.49\n",
      "Episode: 54, Number of Steps : 119, Cumulative reward: 94.55\n",
      "Episode: 55, Number of Steps : 137, Cumulative reward: 93.40\n",
      "Episode: 56, Number of Steps : 181, Cumulative reward: 91.32\n",
      "Episode: 57, Number of Steps : 342, Cumulative reward: 85.17\n",
      "Episode: 58, Number of Steps : 221, Cumulative reward: 89.29\n",
      "Episode: 59, Number of Steps : 218, Cumulative reward: 90.23\n",
      "Episode: 60, Number of Steps : 190, Cumulative reward: 91.46\n",
      "Episode: 61, Number of Steps : 154, Cumulative reward: 91.05\n",
      "Episode: 62, Number of Steps : 191, Cumulative reward: 89.66\n",
      "Episode: 63, Number of Steps : 177, Cumulative reward: 91.77\n",
      "Episode: 64, Number of Steps : 183, Cumulative reward: 91.03\n",
      "Episode: 65, Number of Steps : 111, Cumulative reward: 93.91\n",
      "Episode: 66, Number of Steps : 264, Cumulative reward: 88.06\n",
      "Episode: 67, Number of Steps : 120, Cumulative reward: 94.47\n",
      "Episode: 68, Number of Steps : 125, Cumulative reward: 93.39\n",
      "Episode: 69, Number of Steps : 265, Cumulative reward: 88.03\n",
      "Episode: 70, Number of Steps : 213, Cumulative reward: 90.65\n",
      "Episode: 71, Number of Steps : 219, Cumulative reward: 90.65\n",
      "Episode: 72, Number of Steps : 211, Cumulative reward: 88.74\n",
      "Episode: 73, Number of Steps : 118, Cumulative reward: 93.34\n",
      "Episode: 74, Number of Steps : 160, Cumulative reward: 92.85\n",
      "Episode: 75, Number of Steps : 177, Cumulative reward: 91.58\n",
      "Episode: 76, Number of Steps : 117, Cumulative reward: 93.44\n",
      "Episode: 77, Number of Steps : 123, Cumulative reward: 92.86\n",
      "Episode: 78, Number of Steps : 202, Cumulative reward: 90.92\n",
      "Episode: 79, Number of Steps : 174, Cumulative reward: 91.66\n",
      "Episode: 80, Number of Steps : 186, Cumulative reward: 90.53\n",
      "Episode: 81, Number of Steps : 115, Cumulative reward: 92.94\n",
      "Episode: 82, Number of Steps : 159, Cumulative reward: 92.30\n",
      "Episode: 83, Number of Steps : 116, Cumulative reward: 92.78\n",
      "Episode: 84, Number of Steps : 160, Cumulative reward: 92.17\n",
      "Episode: 85, Number of Steps : 156, Cumulative reward: 91.89\n",
      "Episode: 86, Number of Steps : 196, Cumulative reward: 90.69\n",
      "Episode: 87, Number of Steps : 209, Cumulative reward: 90.27\n",
      "Episode: 88, Number of Steps : 121, Cumulative reward: 93.14\n",
      "Episode: 89, Number of Steps : 179, Cumulative reward: 89.48\n",
      "Episode: 90, Number of Steps : 115, Cumulative reward: 93.51\n",
      "Episode: 91, Number of Steps : 148, Cumulative reward: 91.49\n",
      "Episode: 92, Number of Steps : 194, Cumulative reward: 89.65\n",
      "Episode: 93, Number of Steps : 222, Cumulative reward: 91.31\n",
      "Episode: 94, Number of Steps : 108, Cumulative reward: 93.41\n",
      "Episode: 95, Number of Steps : 150, Cumulative reward: 92.91\n",
      "Episode: 96, Number of Steps : 203, Cumulative reward: 89.71\n",
      "Episode: 97, Number of Steps : 146, Cumulative reward: 92.09\n",
      "Episode: 98, Number of Steps : 186, Cumulative reward: 91.17\n",
      "Episode: 99, Number of Steps : 111, Cumulative reward: 92.91\n",
      "Episode: 100, Number of Steps : 173, Cumulative reward: 90.82\n",
      "Episode: 101, Number of Steps : 188, Cumulative reward: 91.14\n",
      "Episode: 102, Number of Steps : 197, Cumulative reward: 90.01\n",
      "Episode: 103, Number of Steps : 170, Cumulative reward: 91.73\n",
      "Episode: 104, Number of Steps : 176, Cumulative reward: 90.60\n",
      "Episode: 105, Number of Steps : 157, Cumulative reward: 91.61\n",
      "Episode: 106, Number of Steps : 160, Cumulative reward: 92.60\n",
      "Episode: 107, Number of Steps : 178, Cumulative reward: 90.91\n",
      "Episode: 108, Number of Steps : 221, Cumulative reward: 90.80\n",
      "Episode: 109, Number of Steps : 177, Cumulative reward: 93.43\n",
      "Episode: 110, Number of Steps : 184, Cumulative reward: 90.55\n",
      "Episode: 111, Number of Steps : 163, Cumulative reward: 90.75\n",
      "Episode: 112, Number of Steps : 233, Cumulative reward: 87.93\n",
      "Episode: 113, Number of Steps : 147, Cumulative reward: 93.69\n",
      "Episode: 114, Number of Steps : 192, Cumulative reward: 91.13\n",
      "Episode: 115, Number of Steps : 161, Cumulative reward: 92.31\n",
      "Episode: 116, Number of Steps : 189, Cumulative reward: 92.09\n",
      "Episode: 117, Number of Steps : 165, Cumulative reward: 92.86\n",
      "Episode: 118, Number of Steps : 223, Cumulative reward: 89.34\n",
      "Episode: 119, Number of Steps : 153, Cumulative reward: 91.49\n",
      "Episode: 120, Number of Steps : 114, Cumulative reward: 93.24\n",
      "Episode: 121, Number of Steps : 128, Cumulative reward: 93.56\n",
      "Episode: 122, Number of Steps : 116, Cumulative reward: 94.20\n",
      "Episode: 123, Number of Steps : 155, Cumulative reward: 90.86\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.01\n",
      "Episode: 124, Number of Steps : 113, Cumulative reward: 92.84\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.14\n",
      "Episode: 125, Number of Steps : 135, Cumulative reward: 92.27\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.27\n",
      "Episode: 126, Number of Steps : 188, Cumulative reward: 89.85\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.30\n",
      "Episode: 127, Number of Steps : 144, Cumulative reward: 92.11\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.41\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 128, Number of Steps : 186, Cumulative reward: 91.21\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.55\n",
      "Episode: 129, Number of Steps : 121, Cumulative reward: 93.82\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.64\n",
      "Episode: 130, Number of Steps : 119, Cumulative reward: 94.29\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.77\n",
      "Episode: 131, Number of Steps : 189, Cumulative reward: 91.90\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.82\n",
      "Episode: 132, Number of Steps : 163, Cumulative reward: 90.12\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.94\n",
      "Episode: 133, Number of Steps : 182, Cumulative reward: 89.53\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.93\n",
      "Episode: 134, Number of Steps : 287, Cumulative reward: 88.27\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:90.94\n",
      "Episode: 135, Number of Steps : 142, Cumulative reward: 92.76\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.02\n",
      "Episode: 136, Number of Steps : 211, Cumulative reward: 88.65\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.01\n",
      "Episode: 137, Number of Steps : 181, Cumulative reward: 89.42\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.05\n",
      "Episode: 138, Number of Steps : 198, Cumulative reward: 89.88\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.07\n",
      "Episode: 139, Number of Steps : 141, Cumulative reward: 93.67\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.10\n",
      "Episode: 140, Number of Steps : 212, Cumulative reward: 89.79\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.10\n",
      "Episode: 141, Number of Steps : 154, Cumulative reward: 91.61\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.24\n",
      "Episode: 142, Number of Steps : 153, Cumulative reward: 90.90\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.25\n",
      "Episode: 143, Number of Steps : 185, Cumulative reward: 89.79\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.22\n",
      "Episode: 144, Number of Steps : 110, Cumulative reward: 93.14\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.29\n",
      "Episode: 145, Number of Steps : 218, Cumulative reward: 88.95\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.32\n",
      "Episode: 146, Number of Steps : 212, Cumulative reward: 90.04\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.34\n",
      "Episode: 147, Number of Steps : 110, Cumulative reward: 93.51\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.37\n",
      "Episode: 148, Number of Steps : 122, Cumulative reward: 94.71\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.43\n",
      "Episode: 149, Number of Steps : 120, Cumulative reward: 94.39\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.48\n",
      "Episode: 150, Number of Steps : 172, Cumulative reward: 90.87\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.48\n",
      "Episode: 151, Number of Steps : 174, Cumulative reward: 92.16\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.50\n",
      "Episode: 152, Number of Steps : 110, Cumulative reward: 94.53\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.51\n",
      "Episode: 153, Number of Steps : 119, Cumulative reward: 94.93\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.55\n",
      "Episode: 154, Number of Steps : 223, Cumulative reward: 88.31\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.49\n",
      "Episode: 155, Number of Steps : 117, Cumulative reward: 94.57\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.50\n",
      "Episode: 156, Number of Steps : 186, Cumulative reward: 92.59\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.52\n",
      "Episode: 157, Number of Steps : 112, Cumulative reward: 92.59\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.59\n",
      "Episode: 158, Number of Steps : 112, Cumulative reward: 93.68\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.63\n",
      "Episode: 159, Number of Steps : 187, Cumulative reward: 90.26\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.63\n",
      "Episode: 160, Number of Steps : 154, Cumulative reward: 92.11\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.64\n",
      "Episode: 161, Number of Steps : 150, Cumulative reward: 91.98\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.65\n",
      "Episode: 162, Number of Steps : 268, Cumulative reward: 89.22\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.65\n",
      "Episode: 163, Number of Steps : 126, Cumulative reward: 93.99\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.67\n",
      "Episode: 164, Number of Steps : 151, Cumulative reward: 92.85\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.69\n",
      "Episode: 165, Number of Steps : 200, Cumulative reward: 89.77\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.64\n",
      "Episode: 166, Number of Steps : 115, Cumulative reward: 92.82\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.69\n",
      "Episode: 167, Number of Steps : 198, Cumulative reward: 92.19\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.67\n",
      "Episode: 168, Number of Steps : 141, Cumulative reward: 92.86\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.66\n",
      "Episode: 169, Number of Steps : 188, Cumulative reward: 89.18\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.68\n",
      "Episode: 170, Number of Steps : 174, Cumulative reward: 92.16\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.69\n",
      "Episode: 171, Number of Steps : 208, Cumulative reward: 91.16\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.70\n",
      "Episode: 172, Number of Steps : 112, Cumulative reward: 94.03\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.75\n",
      "Episode: 173, Number of Steps : 260, Cumulative reward: 90.30\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.72\n",
      "Episode: 174, Number of Steps : 117, Cumulative reward: 93.53\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.73\n",
      "Episode: 175, Number of Steps : 156, Cumulative reward: 92.38\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.73\n",
      "Episode: 176, Number of Steps : 116, Cumulative reward: 94.02\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.74\n",
      "Episode: 177, Number of Steps : 146, Cumulative reward: 91.96\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.73\n",
      "Episode: 178, Number of Steps : 130, Cumulative reward: 93.85\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.76\n",
      "Episode: 179, Number of Steps : 212, Cumulative reward: 92.11\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.76\n",
      "Episode: 180, Number of Steps : 151, Cumulative reward: 92.66\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.79\n",
      "Episode: 181, Number of Steps : 215, Cumulative reward: 90.20\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.76\n",
      "Episode: 182, Number of Steps : 280, Cumulative reward: 88.38\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.72\n",
      "Episode: 183, Number of Steps : 107, Cumulative reward: 93.70\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.73\n",
      "Episode: 184, Number of Steps : 150, Cumulative reward: 93.11\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.74\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 185, Number of Steps : 237, Cumulative reward: 90.82\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.73\n",
      "Episode: 186, Number of Steps : 148, Cumulative reward: 93.54\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.76\n",
      "Episode: 187, Number of Steps : 149, Cumulative reward: 92.21\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.77\n",
      "Episode: 188, Number of Steps : 195, Cumulative reward: 91.74\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.76\n",
      "Episode: 189, Number of Steps : 98, Cumulative reward: 94.95\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.82\n",
      "Episode: 190, Number of Steps : 155, Cumulative reward: 91.74\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.80\n",
      "Episode: 191, Number of Steps : 214, Cumulative reward: 89.73\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.78\n",
      "Episode: 192, Number of Steps : 110, Cumulative reward: 94.70\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.83\n",
      "Episode: 193, Number of Steps : 150, Cumulative reward: 91.41\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.83\n",
      "Episode: 194, Number of Steps : 125, Cumulative reward: 94.00\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.84\n",
      "Episode: 195, Number of Steps : 255, Cumulative reward: 89.20\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.80\n",
      "Episode: 196, Number of Steps : 190, Cumulative reward: 91.14\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.81\n",
      "Episode: 197, Number of Steps : 150, Cumulative reward: 92.48\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.82\n",
      "Episode: 198, Number of Steps : 117, Cumulative reward: 93.27\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.84\n",
      "Episode: 199, Number of Steps : 112, Cumulative reward: 94.81\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.86\n",
      "Episode: 200, Number of Steps : 161, Cumulative reward: 92.98\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.88\n",
      "Episode: 201, Number of Steps : 111, Cumulative reward: 94.58\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.91\n",
      "Episode: 202, Number of Steps : 119, Cumulative reward: 94.73\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.96\n",
      "Episode: 203, Number of Steps : 201, Cumulative reward: 90.45\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.95\n",
      "Episode: 204, Number of Steps : 151, Cumulative reward: 93.15\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.97\n",
      "Episode: 205, Number of Steps : 156, Cumulative reward: 91.62\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.97\n",
      "Episode: 206, Number of Steps : 159, Cumulative reward: 90.90\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.96\n",
      "Episode: 207, Number of Steps : 153, Cumulative reward: 93.72\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.99\n",
      "Episode: 208, Number of Steps : 176, Cumulative reward: 93.59\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.01\n",
      "Episode: 209, Number of Steps : 203, Cumulative reward: 91.07\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:91.99\n",
      "Episode: 210, Number of Steps : 148, Cumulative reward: 91.91\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.00\n",
      "Episode: 211, Number of Steps : 162, Cumulative reward: 93.21\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.03\n",
      "Episode: 212, Number of Steps : 199, Cumulative reward: 92.77\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.08\n",
      "Episode: 213, Number of Steps : 154, Cumulative reward: 92.29\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.06\n",
      "Episode: 214, Number of Steps : 189, Cumulative reward: 91.00\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.06\n",
      "Episode: 215, Number of Steps : 151, Cumulative reward: 92.77\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.07\n",
      "Episode: 216, Number of Steps : 220, Cumulative reward: 92.15\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.07\n",
      "Episode: 217, Number of Steps : 164, Cumulative reward: 93.63\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.07\n",
      "Episode: 218, Number of Steps : 106, Cumulative reward: 94.36\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.12\n",
      "Episode: 219, Number of Steps : 146, Cumulative reward: 92.14\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.13\n",
      "Episode: 220, Number of Steps : 133, Cumulative reward: 94.99\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.15\n",
      "Episode: 221, Number of Steps : 255, Cumulative reward: 91.06\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.12\n",
      "Episode: 222, Number of Steps : 151, Cumulative reward: 92.25\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.10\n",
      "Episode: 223, Number of Steps : 120, Cumulative reward: 93.75\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.13\n",
      "Episode: 224, Number of Steps : 148, Cumulative reward: 92.13\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.13\n",
      "Episode: 225, Number of Steps : 167, Cumulative reward: 92.56\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.13\n",
      "Episode: 226, Number of Steps : 189, Cumulative reward: 92.03\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.15\n",
      "Episode: 227, Number of Steps : 176, Cumulative reward: 90.14\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.13\n",
      "Episode: 228, Number of Steps : 144, Cumulative reward: 92.93\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.15\n",
      "Episode: 229, Number of Steps : 147, Cumulative reward: 92.06\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.13\n",
      "Episode: 230, Number of Steps : 201, Cumulative reward: 90.27\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.09\n",
      "Episode: 231, Number of Steps : 144, Cumulative reward: 92.57\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.10\n",
      "Episode: 232, Number of Steps : 118, Cumulative reward: 93.91\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.13\n",
      "Episode: 233, Number of Steps : 143, Cumulative reward: 94.92\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.19\n",
      "Episode: 234, Number of Steps : 113, Cumulative reward: 94.27\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.25\n",
      "Episode: 235, Number of Steps : 150, Cumulative reward: 91.74\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.24\n",
      "Episode: 236, Number of Steps : 111, Cumulative reward: 94.95\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.30\n",
      "Episode: 237, Number of Steps : 214, Cumulative reward: 91.24\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.32\n",
      "Episode: 238, Number of Steps : 147, Cumulative reward: 93.84\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.36\n",
      "Episode: 239, Number of Steps : 153, Cumulative reward: 90.46\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.33\n",
      "Episode: 240, Number of Steps : 149, Cumulative reward: 91.97\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.35\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 241, Number of Steps : 134, Cumulative reward: 92.89\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.36\n",
      "Episode: 242, Number of Steps : 168, Cumulative reward: 93.51\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.39\n",
      "Episode: 243, Number of Steps : 117, Cumulative reward: 94.78\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.44\n",
      "Episode: 244, Number of Steps : 154, Cumulative reward: 93.75\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.44\n",
      "Episode: 245, Number of Steps : 184, Cumulative reward: 91.18\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.47\n",
      "Episode: 246, Number of Steps : 147, Cumulative reward: 91.68\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.48\n",
      "Episode: 247, Number of Steps : 140, Cumulative reward: 93.00\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.48\n",
      "Episode: 248, Number of Steps : 146, Cumulative reward: 91.99\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.45\n",
      "Episode: 249, Number of Steps : 126, Cumulative reward: 95.47\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.46\n",
      "Episode: 250, Number of Steps : 148, Cumulative reward: 92.70\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.48\n",
      "Episode: 251, Number of Steps : 159, Cumulative reward: 93.00\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.49\n",
      "Episode: 252, Number of Steps : 155, Cumulative reward: 93.45\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.48\n",
      "Episode: 253, Number of Steps : 143, Cumulative reward: 92.60\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.45\n",
      "Episode: 254, Number of Steps : 148, Cumulative reward: 92.14\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.49\n",
      "Episode: 255, Number of Steps : 195, Cumulative reward: 91.53\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.46\n",
      "Episode: 256, Number of Steps : 156, Cumulative reward: 93.84\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.47\n",
      "Episode: 257, Number of Steps : 144, Cumulative reward: 93.74\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.49\n",
      "Episode: 258, Number of Steps : 149, Cumulative reward: 95.14\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.50\n",
      "Episode: 259, Number of Steps : 154, Cumulative reward: 95.12\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.55\n",
      "Episode: 260, Number of Steps : 150, Cumulative reward: 92.08\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.55\n",
      "Episode: 261, Number of Steps : 236, Cumulative reward: 91.15\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.54\n",
      "Episode: 262, Number of Steps : 146, Cumulative reward: 92.25\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.57\n",
      "Episode: 263, Number of Steps : 145, Cumulative reward: 94.57\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.58\n",
      "Episode: 264, Number of Steps : 189, Cumulative reward: 92.48\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.57\n",
      "Episode: 265, Number of Steps : 152, Cumulative reward: 93.27\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.61\n",
      "Episode: 266, Number of Steps : 145, Cumulative reward: 93.74\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.62\n",
      "Episode: 267, Number of Steps : 146, Cumulative reward: 92.05\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.62\n",
      "Episode: 268, Number of Steps : 110, Cumulative reward: 94.92\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.64\n",
      "Episode: 269, Number of Steps : 140, Cumulative reward: 91.74\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.66\n",
      "Episode: 270, Number of Steps : 157, Cumulative reward: 91.31\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.65\n",
      "Episode: 271, Number of Steps : 149, Cumulative reward: 93.18\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.67\n",
      "Episode: 272, Number of Steps : 154, Cumulative reward: 91.57\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.65\n",
      "Episode: 273, Number of Steps : 175, Cumulative reward: 92.19\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.67\n",
      "Episode: 274, Number of Steps : 162, Cumulative reward: 92.71\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.66\n",
      "Episode: 275, Number of Steps : 143, Cumulative reward: 94.48\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.68\n",
      "Episode: 276, Number of Steps : 147, Cumulative reward: 92.96\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.67\n",
      "Episode: 277, Number of Steps : 145, Cumulative reward: 91.99\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.67\n",
      "Episode: 278, Number of Steps : 265, Cumulative reward: 90.55\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.64\n",
      "Episode: 279, Number of Steps : 211, Cumulative reward: 91.69\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.63\n",
      "Episode: 280, Number of Steps : 140, Cumulative reward: 92.84\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.63\n",
      "Episode: 281, Number of Steps : 129, Cumulative reward: 95.22\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.68\n",
      "Episode: 282, Number of Steps : 146, Cumulative reward: 91.92\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.72\n",
      "Episode: 283, Number of Steps : 131, Cumulative reward: 95.38\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.74\n",
      "Episode: 284, Number of Steps : 212, Cumulative reward: 91.09\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.72\n",
      "Episode: 285, Number of Steps : 112, Cumulative reward: 94.72\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.76\n",
      "Episode: 286, Number of Steps : 162, Cumulative reward: 90.72\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.73\n",
      "Episode: 287, Number of Steps : 144, Cumulative reward: 94.38\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.75\n",
      "Episode: 288, Number of Steps : 171, Cumulative reward: 91.54\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.75\n",
      "Episode: 289, Number of Steps : 79, Cumulative reward: 95.21\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.75\n",
      "Episode: 290, Number of Steps : 142, Cumulative reward: 93.26\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.77\n",
      "Episode: 291, Number of Steps : 142, Cumulative reward: 91.50\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.78\n",
      "Episode: 292, Number of Steps : 149, Cumulative reward: 94.09\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.78\n",
      "Episode: 293, Number of Steps : 113, Cumulative reward: 93.71\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.80\n",
      "Episode: 294, Number of Steps : 148, Cumulative reward: 92.08\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.78\n",
      "Episode: 295, Number of Steps : 144, Cumulative reward: 91.73\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.81\n",
      "Episode: 296, Number of Steps : 211, Cumulative reward: 89.22\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.79\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 297, Number of Steps : 112, Cumulative reward: 94.08\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.80\n",
      "Episode: 298, Number of Steps : 112, Cumulative reward: 94.46\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.81\n",
      "Episode: 299, Number of Steps : 110, Cumulative reward: 94.04\n",
      "****************Solved***************\n",
      "Mean cumulative reward over 100 episodes:92.81\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import gym  #requires OpenAI gym installed\n",
    "env = gym.envs.make(\"MountainCarContinuous-v0\") \n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "input_dims = 2\n",
    "state_placeholder = tf.placeholder(tf.float32, [None, input_dims]) \n",
    "\n",
    "def value_function(state):\n",
    "    n_hidden1 = 400  \n",
    "    n_hidden2 = 400\n",
    "    n_outputs = 1\n",
    "    \n",
    "    with tf.variable_scope(\"value_network\"):\n",
    "        init_xavier = tf.contrib.layers.xavier_initializer()\n",
    "        \n",
    "        hidden1 = tf.layers.dense(state, n_hidden1, tf.nn.elu, init_xavier)\n",
    "        hidden2 = tf.layers.dense(hidden1, n_hidden2, tf.nn.elu, init_xavier) \n",
    "        V = tf.layers.dense(hidden2, n_outputs, None, init_xavier)\n",
    "    return V\n",
    "\n",
    "\n",
    "def policy_network(state):\n",
    "    n_hidden1 = 40\n",
    "    n_hidden2 = 40\n",
    "    n_outputs = 1\n",
    "    \n",
    "    with tf.variable_scope(\"policy_network\"):\n",
    "        init_xavier = tf.contrib.layers.xavier_initializer()\n",
    "        \n",
    "        hidden1 = tf.layers.dense(state, n_hidden1, tf.nn.elu, init_xavier)\n",
    "        hidden2 = tf.layers.dense(hidden1, n_hidden2, tf.nn.elu, init_xavier)\n",
    "        mu = tf.layers.dense(hidden2, n_outputs, None, init_xavier)\n",
    "        sigma = tf.layers.dense(hidden2, n_outputs, None, init_xavier)\n",
    "        sigma = tf.nn.softplus(sigma) + 1e-5\n",
    "        norm_dist = tf.contrib.distributions.Normal(mu, sigma)\n",
    "        action_tf_var = tf.squeeze(norm_dist.sample(1), axis=0)\n",
    "        action_tf_var = tf.clip_by_value(\n",
    "            action_tf_var, env.action_space.low[0], \n",
    "            env.action_space.high[0])\n",
    "    return action_tf_var, norm_dist\n",
    "\n",
    "################################################################\n",
    "#sample from state space for state normalization\n",
    "import sklearn\n",
    "import sklearn.preprocessing\n",
    "                                    \n",
    "state_space_samples = np.array(\n",
    "    [env.observation_space.sample() for x in range(10000)])\n",
    "scaler = sklearn.preprocessing.StandardScaler()\n",
    "scaler.fit(state_space_samples)\n",
    "\n",
    "#function to normalize states\n",
    "def scale_state(state):                 #requires input shape=(2,)\n",
    "    scaled = scaler.transform([state])\n",
    "    return scaled                       #returns shape =(1,2)   \n",
    "###################################################################\n",
    "\n",
    "lr_actor = 0.00002  #set learning rates\n",
    "lr_critic = 0.001\n",
    "\n",
    "# define required placeholders\n",
    "action_placeholder = tf.placeholder(tf.float32)\n",
    "delta_placeholder = tf.placeholder(tf.float32)\n",
    "target_placeholder = tf.placeholder(tf.float32)\n",
    "\n",
    "action_tf_var, norm_dist = policy_network(state_placeholder)\n",
    "V = value_function(state_placeholder)\n",
    "\n",
    "# define actor (policy) loss function\n",
    "loss_actor = -tf.log(norm_dist.prob(action_placeholder) + 1e-5) * delta_placeholder\n",
    "training_op_actor = tf.train.AdamOptimizer(\n",
    "    lr_actor, name='actor_optimizer').minimize(loss_actor)\n",
    "\n",
    "# define critic (state-value) loss function\n",
    "loss_critic = tf.reduce_mean(tf.squared_difference(\n",
    "                             tf.squeeze(V), target_placeholder))\n",
    "training_op_critic = tf.train.AdamOptimizer(\n",
    "        lr_critic, name='critic_optimizer').minimize(loss_critic)\n",
    "################################################################\n",
    "#Training loop\n",
    "gamma = 0.99        #discount factor\n",
    "num_episodes = 300\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    episode_history = []\n",
    "    for episode in range(num_episodes):\n",
    "        #receive initial state from E\n",
    "        state = env.reset()   # state.shape -> (2,)\n",
    "        reward_total = 0 \n",
    "        steps = 0\n",
    "        done = False\n",
    "        while (not done):\n",
    "                \n",
    "            #Sample action according to current policy\n",
    "            #action.shape = (1,1)\n",
    "            action  = sess.run(action_tf_var, feed_dict={\n",
    "                          state_placeholder: scale_state(state)})\n",
    "            #Execute action and observe reward & next state from E\n",
    "            # next_state shape=(2,)    \n",
    "            #env.step() requires input shape = (1,)\n",
    "            next_state, reward, done, _ = env.step(\n",
    "                                    np.squeeze(action, axis=0)) \n",
    "            steps +=1\n",
    "            reward_total += reward\n",
    "            #V_of_next_state.shape=(1,1)\n",
    "            V_of_next_state = sess.run(V, feed_dict = \n",
    "                    {state_placeholder: scale_state(next_state)})  \n",
    "            #Set TD Target\n",
    "            #target = r + gamma * V(next_state)     \n",
    "            target = reward + gamma * np.squeeze(V_of_next_state) \n",
    "            \n",
    "            # td_error = target - V(s)\n",
    "            #needed to feed delta_placeholder in actor training\n",
    "            td_error = target - np.squeeze(sess.run(V, feed_dict = \n",
    "                        {state_placeholder: scale_state(state)})) \n",
    "            \n",
    "            #Update actor by minimizing loss (Actor training)\n",
    "            _, loss_actor_val  = sess.run(\n",
    "                [training_op_actor, loss_actor], \n",
    "                feed_dict={action_placeholder: np.squeeze(action), \n",
    "                state_placeholder: scale_state(state), \n",
    "                delta_placeholder: td_error})\n",
    "            #Update critic by minimizinf loss  (Critic training)\n",
    "            _, loss_critic_val  = sess.run(\n",
    "                [training_op_critic, loss_critic], \n",
    "                feed_dict={state_placeholder: scale_state(state), \n",
    "                target_placeholder: target})\n",
    "            \n",
    "            state = next_state\n",
    "            #end while\n",
    "        episode_history.append(reward_total)\n",
    "        print(\"Episode: {}, Number of Steps : {}, Cumulative reward: {:0.2f}\".format(\n",
    "            episode, steps, reward_total))\n",
    "        \n",
    "        if np.mean(episode_history[-100:]) > 90 and len(episode_history) >= 101:\n",
    "            print(\"****************Solved***************\")\n",
    "            print(\"Mean cumulative reward over 100 episodes:{:0.2f}\" .format(\n",
    "                np.mean(episode_history[-100:])))\n",
    "            \n",
    "            "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}