testSeqGAN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alexchen/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import sys\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import mxnet as mx\n",
    "from mxnet import gluon\n",
    "from mxnet import ndarray as nd\n",
    "from mxnet import symbol as sym\n",
    "from mxnet.gluon import nn, utils\n",
    "from mxnet import autograd\n",
    "from scipy.stats import norm\n",
    "import matplotlib.mlab as mlab\n",
    "from math import e\n",
    "import math\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "----\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "----\n",
      "9\n",
      "10\n",
      "9\n",
      "9\n",
      "----\n"
     ]
    }
   ],
   "source": [
    "x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\n",
    "a = iter(x)\n",
    "for i in a:\n",
    "    first = i\n",
    "    second = next(a, first)\n",
    "    third = next(a,first)\n",
    "    four = next(a, first)\n",
    "    print(first)\n",
    "    print(second)\n",
    "    print(third)\n",
    "    print(four)\n",
    "    print(\"----\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import tee\n",
    "def pairwise(iterable):\n",
    "    \"s -> (s0,s1,s2,s3), (s1,s2,s3,s4), (s2, s3,s4,s5), ...\"\n",
    "    a, b, c, d = tee(iterable, n=4)\n",
    "    next(b, None)\n",
    "    return zip(a, b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "n_mixture = 8\n",
    "std = 0.025\n",
    "radius = 1.0\n",
    "latent_z_size =100\n",
    "epochs = 5000\n",
    "centers = [5]\n",
    "number_samples = 65536\n",
    "sample_size = 64\n",
    "\n",
    "use_gpu = True\n",
    "ctx = mx.gpu() if use_gpu else mx.cpu()\n",
    "\n",
    "lr = 0.00002\n",
    "beta1 = 0.5\n",
    "dropout = 0.5\n",
    "data_type = \"arrivals\"\n",
    "#data_type = \"frequencies\"\n",
    "#data_type = \"freqOfFreq\"\n",
    "\n",
    "#unroll_steps = 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.43122860794147, 2.15614303970735, 5.390357599268375, 8.983929332113957, 11.229911665142447, 11.229911665142447, 9.35825972095204, 6.684471229251457, 4.177794518282161, 2.3209969546012004, 1.1604984773006, 0.5274993078639092, 0.2197913782766288, 0.08453514549101108, 0.030191123389646815, 0.01006370779654894, 0.0031449086864215434, 0.0009249731430651598, 0.00025693698418476665, 6.761499583809648e-05, 1.6903748959524117e-05, 4.0247021332200284e-06, 9.147050302772792e-07, 1.9884891962549546e-07, 4.142685825531156e-08, 8.285371651062311e-09, 1.5933407021273675e-09, 2.950630929865496e-10, 5.2689838033312416e-11, 9.084454833329728e-12, 1.5140758055549546e-12, 2.4420577508950884e-13]\n",
      "[0.215614303970735, 1.078071519853675, 2.6951787996341876, 4.491964666056979, 5.614955832571224, 5.614955832571224, 4.67912986047602, 3.3422356146257286, 2.0888972591410804, 1.1604984773006002, 0.5802492386503, 0.2637496539319546, 0.1098956891383144, 0.04226757274550554, 0.015095561694823408, 0.00503185389827447, 0.0015724543432107717, 0.0004624865715325799, 0.00012846849209238333, 3.380749791904824e-05, 8.451874479762058e-06, 2.0123510666100142e-06, 4.573525151386396e-07, 9.942445981274773e-08, 2.071342912765578e-08, 4.142685825531156e-09, 7.966703510636838e-10, 1.475315464932748e-10, 2.6344919016656208e-11, 4.542227416664864e-12, 7.570379027774773e-13, 1.2210288754475442e-13]\n",
      "[0.1078071519853675, 0.5390357599268375, 1.3475893998170938, 2.2459823330284894, 2.807477916285612, 2.807477916285612, 2.33956493023801, 1.6711178073128643, 1.0444486295705402, 0.5802492386503001, 0.29012461932515, 0.1318748269659773, 0.0549478445691572, 0.02113378637275277, 0.007547780847411704, 0.002515926949137235, 0.0007862271716053858, 0.00023124328576628995, 6.423424604619166e-05, 1.690374895952412e-05, 4.225937239881029e-06, 1.0061755333050071e-06, 2.286762575693198e-07, 4.9712229906373864e-08, 1.035671456382789e-08, 2.071342912765578e-09, 3.983351755318419e-10, 7.37657732466374e-11, 1.3172459508328104e-11, 2.271113708332432e-12, 3.7851895138873864e-13, 6.105144377237721e-14]\n"
     ]
    }
   ],
   "source": [
    "#Model Data\n",
    "lamb = 5\n",
    "model_128 = []\n",
    "for k in range(32):\n",
    "    model_128.append(sample_size*((e ** (-1 * lamb)) * (lamb ** k))/math.factorial(k))\n",
    "    \n",
    "model_64 = []\n",
    "for k in range(32):\n",
    "    model_64.append((sample_size/2)*((e ** (-1 * lamb)) * (lamb ** k))/math.factorial(k))\n",
    "    \n",
    "model_32 = []\n",
    "for k in range(32):\n",
    "    model_32.append((sample_size/4)*((e ** (-1 * lamb)) * (lamb ** k))/math.factorial(k))\n",
    "    \n",
    "    \n",
    "print(model_128)\n",
    "print(model_64)\n",
    "print(model_32)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[11  5  5 ...  7  6  2]\n",
      " [ 3  8  7 ...  3  4  6]\n",
      " [ 4  4  4 ...  7  4  4]\n",
      " ...\n",
      " [ 6  3  3 ...  3  7  5]\n",
      " [ 5  6  0 ...  6  8  4]\n",
      " [ 5  7  4 ...  8  5  8]]\n"
     ]
    }
   ],
   "source": [
    "#arrivals data\n",
    "if data_type == \"arrivals\":\n",
    "    samples = []\n",
    "    for i in range(number_samples):\n",
    "        for c in centers:\n",
    "            samples.append(np.random.poisson(lam = c, size = sample_size).tolist())\n",
    "    #for s in range(len(samples)):\n",
    "    #    samples[s] = tf.convert_to_tensor(samples[s])\n",
    "    #for z in range(len(samples)):\n",
    "    #   samples[z] = [samples[z]]\n",
    "\n",
    "    #test output shape    \n",
    "    #o = np.array(samples)\n",
    "    #print(o.shape)\n",
    "\n",
    "    random.shuffle(samples)\n",
    "    samples = np.asarray(samples)\n",
    "    print(samples)\n",
    "    #print(samples.T[0:3])\n",
    "    train_data = mx.io.NDArrayIter(data = samples, batch_size = batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#frequencies\n",
    "if data_type == \"frequencies\":\n",
    "    samples = []\n",
    "    for i in range(number_samples):\n",
    "        for c in centers:\n",
    "            arrivals = np.random.poisson(lam = c, size = sample_size).tolist()\n",
    "            freq = np.zeros(32)\n",
    "            for count in arrivals:\n",
    "                freq[count] += 1\n",
    "            samples.append(freq)\n",
    "    \n",
    "    random.shuffle(samples)\n",
    "    samples = np.asarray(samples)\n",
    "    print(samples)\n",
    "    print(samples[1])\n",
    "    train_data = mx.io.NDArrayIter(data = samples, batch_size = batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#freqOfFreq\n",
    "if data_type == \"freqOfFreq\":\n",
    "    samples = []\n",
    "    for i in range(number_samples):\n",
    "        for c in centers:\n",
    "            arrivals = np.random.poisson(lam = c, size = sample_size).tolist()\n",
    "            freq = np.zeros(32)\n",
    "            for count in arrivals:\n",
    "                freq[count] += 1\n",
    "            \n",
    "            freqOfFreq = np.zeros(25)\n",
    "            for frequences in freq:\n",
    "                freqOfFreq[int(frequences)] += 1\n",
    "            samples.append(freqOfFreq)\n",
    "    \n",
    "    random.shuffle(samples)\n",
    "    samples = np.asarray(samples)\n",
    "    print(samples)\n",
    "    train_data = mx.io.NDArrayIter(data = samples, batch_size = batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "65536\n",
      "64\n",
      "[ 7  6  9  5  3  5  6  5  9  6  5  1  4  7  3  1  5  4  3  5  3  4  5 12\n",
      "  5  6  2  2  6  4  2  3  5  3  8  7  7  4  7  5  1  6  6  4  3  8  3  6\n",
      "  3  4 10  7  7  4  8  6  3  2  5  7  5  3  3  2]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADhlJREFUeJzt3X+MZXV9xvH3UxYri1hQBmNZtgMJoRoiPzohKA1tWW1Xl4A2NYGooZV2/7EKxkSXkNT4H43GaNJGswGEFIKJiNWAIhuU0iZCu8sPXVwQqiusrOxYolhJBMqnf9xDsh1ZZueec+4u375fyeT+mDP3+ZzZuc+ee+4996aqkCS9/P3WgR5AkjQMC12SGmGhS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEZY6JLUiFWzDDv66KNrfn5+lpGS9LK3bdu2n1XV3HLLzbTQ5+fn2bp16ywjJellL8mP92c5d7lIUiMsdElqhIUuSY2w0CWpERa6JDVi2UJPcnWSPUm273XdJ5M8mOS7Sb6S5Mhxx5QkLWd/ttCvAdYvuW4LcHJVvQn4AXDZwHNJklZo2UKvqjuBJ5dcd1tVPdddvAtYM8JskqQVGGIf+vuBbwxwO5KkHnodKZrkcuA54PqXWGYjsBFg7dq1feKaMb/plv1abucVG0aepL+W1kV6uZt6Cz3JRcC5wHuqqva1XFVtrqqFqlqYm1v2rQgkSVOaags9yXrgY8AfVdXTw44kSZrG/rxs8QbgO8BJSXYluRj4B+AIYEuS+5J8fuQ5JUnLWHYLvaoufJGrrxphFklSDx4pKkmNsNAlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqRHLFnqSq5PsSbJ9r+tek2RLkoe706PGHVOStJz92UK/Bli/5LpNwO1VdSJwe3dZknQALVvoVXUn8OSSq88Hru3OXwu8c+C5JEkrNO0+9NdV1W6A7vSYfS2YZGOSrUm2Li4uThknSVrO6E+KVtXmqlqoqoW5ubmx4yTp/61pC/2JJK8H6E73DDeSJGka0xb614CLuvMXAV8dZhxJ0rT252WLNwDfAU5KsivJxcAVwNuSPAy8rbssSTqAVi23QFVduI9vrRt4FklSDx4pKkmNsNAlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNWLZ93KRDkbzm27Zr+V2XrFh5Emkg4db6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY3oVehJPpzkgSTbk9yQ5JVDDSZJWpmpCz3JscCHgIWqOhk4BLhgqMEkSSvTd5fLKuCwJKuA1cDj/UeSJE1j6kKvqp8AnwIeBXYDv6iq24YaTJK0MlN/YlGSo4DzgeOBnwNfSvLeqrpuyXIbgY0Aa9eu7TGq+vJTfvrb398h+HvU7PXZ5fJW4EdVtVhVzwI3AW9ZulBVba6qhapamJub6xEnSXopfQr9UeDMJKuTBFgH7BhmLEnSSvXZh343cCNwD/C97rY2DzSXJGmFpt6HDlBVHwc+PtAskqQePFJUkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSI3oVepIjk9yY5MEkO5K8eajBJEkrs6rnz38WuLWq/iLJK4DVA8wkSZrC1IWe5NXA2cBfAlTVM8Azw4wlSVqpPrtcTgAWgS8kuTfJlUkOX7pQko1JtibZuri42CNOkvRS+hT6KuB04HNVdRrwK2DT0oWqanNVLVTVwtzcXI84SdJL6VPou4BdVXV3d/lGJgUvSToApi70qvop8FiSk7qr1gHfH2QqSdKK9X2VyweB67tXuPwQ+Kv+I0mSptGr0KvqPmBhoFkkST14pKgkNcJCl6RGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGtH33RZftuY33bLfy+68YsOIk0jSMNxCl6RGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1Ijehd6kkOS3Jvk5iEGkiRNZ4gt9EuAHQPcjiSph16FnmQNsAG4cphxJEnT6ruF/hngo8DzA8wiSeph6kJPci6wp6q2LbPcxiRbk2xdXFycNk6StIw+W+hnAecl2Ql8ETgnyXVLF6qqzVW1UFULc3NzPeIkSS9l6kKvqsuqak1VzQMXAN+qqvcONpkkaUV8HbokNWKQD4muqjuAO4a4LUnSdNxCl6RGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXCQpekRljoktQIC12SGmGhS1IjBnlzrlmY33TLfi2384oNI08iHRjeB7Qct9AlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaMXWhJzkuybeT7EjyQJJLhhxMkrQyfd4+9zngI1V1T5IjgG1JtlTV9weaTZK0AlNvoVfV7qq6pzv/S2AHcOxQg0mSVmaQD7hIMg+cBtz9It/bCGwEWLt27RBxkgbgB2a0p/eTokleBXwZuLSqnlr6/araXFULVbUwNzfXN06StA+9Cj3JoUzK/PqqummYkSRJ0+jzKpcAVwE7qurTw40kSZpGny30s4D3Aeckua/7esdAc0mSVmjqJ0Wr6t+ADDiLJKkHjxSVpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRFrokNcJCl6RGWOiS1AgLXZIaYaFLUiMG+cQiSVqJVj4taX/XA2azLm6hS1IjLHRJaoSFLkmNsNAlqREWuiQ1wkKXpEZY6JLUCAtdkhphoUtSIyx0SWqEhS5JjbDQJakRvQo9yfokDyV5JMmmoYaSJK3c1IWe5BDgH4G3A28ELkzyxqEGkyStTJ8t9DOAR6rqh1X1DPBF4PxhxpIkrVSfQj8WeGyvy7u66yRJB0CqarofTN4N/FlV/XV3+X3AGVX1wSXLbQQ2dhdPAh6actajgZ9N+bPmHbis1vNaXrdZ57W8bn3zfq+q5pZbqM8nFu0Cjtvr8hrg8aULVdVmYHOPHACSbK2qhb63Y17b6zbrvJbXbdZ5La/brPL67HL5D+DEJMcneQVwAfC1YcaSJK3U1FvoVfVckr8FvgkcAlxdVQ8MNpkkaUV6fUh0VX0d+PpAsyyn924b8w5IVut5La/brPNaXreZ5E39pKgk6eDiof+S1IiDvtCTXJ1kT5LtM8g6Lsm3k+xI8kCSS0bOe2WSf09yf5f3iTHzusxDktyb5OYZZO1M8r0k9yXZOoO8I5PcmOTB7t/wzSNmndSt1wtfTyW5dMS8D3d/I9uT3JDklWNldXmXdFkPjLFeL3a/TvKaJFuSPNydHjVy3ru79Xs+yWCvPtlH1ie7v8vvJvlKkiOHytvbQV/owDXA+hllPQd8pKreAJwJfGDktzP4NXBOVZ0CnAqsT3LmiHkAlwA7Rs7Y259U1akzennYZ4Fbq+r3gVMYcT2r6qFuvU4F/gB4GvjKGFlJjgU+BCxU1clMXoRwwRhZXd7JwN8wORr8FODcJCcOHHMNv3m/3gTcXlUnArd3l8fM2w78OXDngDn7ytoCnFxVbwJ+AFw2cCbwMij0qroTeHJGWbur6p7u/C+ZFMJoR7/WxH93Fw/tvkZ7UiPJGmADcOVYGQdKklcDZwNXAVTVM1X18xnFrwP+s6p+PGLGKuCwJKuA1bzIMR8DegNwV1U9XVXPAf8CvGvIgH3cr88Hru3OXwu8c8y8qtpRVdMe6LjSrNu63yXAXUyO2xncQV/oB0qSeeA04O6Rcw5Jch+wB9hSVWPmfQb4KPD8iBl7K+C2JNu6I4bHdAKwCHyh26V0ZZLDR858wQXADWPdeFX9BPgU8CiwG/hFVd02Vh6TLdezk7w2yWrgHfzfgwjH8rqq2g2TjSvgmBlkHgjvB74xxg1b6C8iyauALwOXVtVTY2ZV1f90D9vXAGd0D3cHl+RcYE9VbRvj9vfhrKo6nck7cn4gydkjZq0CTgc+V1WnAb9i2IfsL6o7qO484EsjZhzFZOv1eOB3gcOTvHesvKraAfw9k90EtwL3M9kdqZ6SXM7kd3n9GLdvoS+R5FAmZX59Vd00q9xu98AdjPd8wVnAeUl2MnlnzHOSXDdSFgBV9Xh3uofJ/uUzRozbBeza6xHOjUwKfmxvB+6pqidGzHgr8KOqWqyqZ4GbgLeMmEdVXVVVp1fV2Ux2Hzw8Zl7niSSvB+hO98wgc2aSXAScC7ynRnq9uIW+lyRhsg92R1V9egZ5cy88253kMCZ33AfHyKqqy6pqTVXNM9lF8K2qGm0rL8nhSY544Tzwp0weyo+iqn4KPJbkpO6qdcD3x8rby4WMuLul8yhwZpLV3d/oOkZ+YjvJMd3pWiZPHI69jjB565CLuvMXAV+dQeZMJFkPfAw4r6qeHi2oqg7qLyZ/SLuBZ5lshV08YtYfMtnv+13gvu7rHSPmvQm4t8vbDvzdjH6nfwzcPHLGCUweqt8PPABcPoP1OhXY2v0+/xk4auS81cB/Ab8zg3X7BJP/7LcD/wT89sh5/8rkP8T7gXUj3P5v3K+B1zJ5dcvD3elrRs57V3f+18ATwDdHzHqEyduNv9Arnx/j380jRSWpEe5ykaRGWOiS1AgLXZIaYaFLUiMsdElqhIUuSY2w0CWpERa6JDXifwEiVh8DQYGZyQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(len(samples))\n",
    "print(len(samples[0]))\n",
    "#print(len(samples[0][0]))\n",
    "\n",
    "sample_num = 5\n",
    "\n",
    "data = samples[sample_num]\n",
    "\n",
    "count = 0\n",
    "\n",
    "bin_count = 30\n",
    "\n",
    "print(data)\n",
    "\n",
    "plt.hist(data, bins=bin_count)\n",
    "plt.xticks(np.arange(min(data), max(data)+1, 1.0))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if data_type == \"arrivals\":\n",
    "    from scipy import stats\n",
    "\n",
    "\n",
    "    def change_to_freq(data, max_freq):\n",
    "        frequencies = np.zeros(max_freq)\n",
    "        for i in data:\n",
    "            frequencies[i] += 1\n",
    "        return frequencies\n",
    "\n",
    "\n",
    "    def distribution_tester(data, model_128, model_64, model_32):\n",
    "\n",
    "        pScoreVals = []\n",
    "\n",
    "        data_128 = change_to_freq(data, 32)\n",
    "        _, p = stats.chisquare(f_obs = data_128, f_exp = model_128)\n",
    "        pScoreVals.append(p)\n",
    "\n",
    "        data_64_1 = change_to_freq(data[:int(sample_size)], 32)\n",
    "        _, p = stats.chisquare(f_obs = data_64_1, f_exp = model_64)\n",
    "        pScoreVals.append(p)\n",
    "\n",
    "        data_64_2 = change_to_freq(data[int(sample_size/2):], 32)\n",
    "        _, p = stats.chisquare(f_obs = data_64_2, f_exp = model_64)\n",
    "        pScoreVals.append(p)\n",
    "\n",
    "        data_32_1 = change_to_freq(data[:int(sample_size/4)], 32)\n",
    "        _, p = stats.chisquare(f_obs = data_32_1, f_exp = model_32)\n",
    "        pScoreVals.append(p)\n",
    "\n",
    "        data_32_2 = change_to_freq(data[int(sample_size/4):int(sample_size/2)], 32)\n",
    "        _, p = stats.chisquare(f_obs = data_32_2, f_exp = model_32)\n",
    "        pScoreVals.append(p)\n",
    "\n",
    "        data_32_3 = change_to_freq(data[int(sample_size/2):int(sample_size*3/4)], 32)\n",
    "        _, p = stats.chisquare(f_obs = data_32_3, f_exp = model_32)\n",
    "        pScoreVals.append(p)\n",
    "\n",
    "        data_32_4 = change_to_freq(data[int(sample_size*3/4):], 32)\n",
    "        _, p = stats.chisquare(f_obs = data_32_4, f_exp = model_32)\n",
    "        pScoreVals.append(p)\n",
    "        \n",
    "        count = 0\n",
    "        for x in pScoreVals:\n",
    "            if x < 0.05:\n",
    "                count+= 1\n",
    "        \n",
    "        return count/7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#total = 0\n",
    "#for x in samples:\n",
    "#    z = distribution_tester(x, model_128, model_64, model_32)\n",
    "#    total += z\n",
    "#print(total/len(samples))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "netG = mx.gluon.rnn.SequentialRNNCell()\n",
    "with netG.name_scope():\n",
    "\n",
    "    netG.add(mx.gluon.rnn.LSTMCell(20))\n",
    "    netG.add(mx.gluon.rnn.LSTMCell(20))\n",
    "    netG.add(mx.gluon.rnn.LSTMCell(1))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "netD1 = nn.Sequential()\n",
    "with netD1.name_scope():\n",
    "    \n",
    "    #Convolutional\n",
    "    #input is 256 x 1 x 2\n",
    "#     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "#     netD.add(nn.LeakyReLU(0.2))\n",
    "#     # should still be 256 x 1 x 2\n",
    "#     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "#     netD.add(nn.BatchNorm())\n",
    "#     netD.add(nn.LeakyReLU(0.2))\n",
    "#     # should still be 256 x 1 x 2\n",
    "#     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "#     netD.add(nn.BatchNorm())\n",
    "#     netD.add(nn.LeakyReLU(0.2))\n",
    "#     # should still be 256 x 1 x 2\n",
    "#     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "#     netD.add(nn.BatchNorm())\n",
    "#     netD.add(nn.LeakyReLU(0.2))\n",
    "#     # should still be 256 x 1 x 2\n",
    "#     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, strides = 2,padding = 2, in_channels = 1))\n",
    "#     # should still be 256 x 1 x 1\n",
    "\n",
    "\n",
    "    #Dense\n",
    "    netD1.add(nn.Dense(200))\n",
    "    netD1.add(nn.Dropout(0.5))\n",
    "    netD1.add(nn.LeakyReLU(0.2))\n",
    "    #netD.add(nn.Dense(100))\n",
    "    #netD.add(nn.LeakyReLU(0.2))\n",
    "    netD1.add(nn.Dense(200))\n",
    "    netD1.add(nn.LeakyReLU(0.2))\n",
    "    #netD.add(nn.Dropout(0.5))\n",
    "    netD1.add(nn.Dense(1))\n",
    "    \n",
    "    \n",
    "    \n",
    "    #Try three smh\n",
    "    \n",
    "#     netD1.add(nn.Dense(128, activation = \"tanh\"))\n",
    "#     netD1.add(nn.Dense(128, activation = \"tanh\"))\n",
    "#     netD1.add(nn.Dense(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "netD2 = nn.Sequential()\n",
    "with netD2.name_scope():\n",
    "    \n",
    "    #Convolutional\n",
    "    #input is 256 x 1 x 2\n",
    "#     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "#     netD.add(nn.LeakyReLU(0.2))\n",
    "#     # should still be 256 x 1 x 2\n",
    "#     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "#     netD.add(nn.BatchNorm())\n",
    "#     netD.add(nn.LeakyReLU(0.2))\n",
    "#     # should still be 256 x 1 x 2\n",
    "#     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "#     netD.add(nn.BatchNorm())\n",
    "#     netD.add(nn.LeakyReLU(0.2))\n",
    "#     # should still be 256 x 1 x 2\n",
    "#     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "#     netD.add(nn.BatchNorm())\n",
    "#     netD.add(nn.LeakyReLU(0.2))\n",
    "#     # should still be 256 x 1 x 2\n",
    "#     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, strides = 2,padding = 2, in_channels = 1))\n",
    "#     # should still be 256 x 1 x 1\n",
    "\n",
    "\n",
    "    #Dense\n",
    "    netD2.add(nn.Dense(256))\n",
    "    netD2.add(nn.Dropout(0.5))\n",
    "    netD2.add(nn.LeakyReLU(0.2))\n",
    "    #netD.add(nn.Dense(100))\n",
    "    #netD.add(nn.LeakyReLU(0.2))\n",
    "    netD2.add(nn.Dense(200))\n",
    "    netD2.add(nn.LeakyReLU(0.2))\n",
    "    #netD.add(nn.Dropout(0.5))\n",
    "    netD2.add(nn.Dense(1))\n",
    "\n",
    "\n",
    "\n",
    "    #Try three smh\n",
    "    #netD2.add(nn.Dense(128, activation = \"tanh\"))\n",
    "    #netD2.add(nn.Dense(128, activation = \"tanh\"))\n",
    "    #netD2.add(nn.Dense(1))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# netD3 = nn.Sequential()\n",
    "# with netD3.name_scope():\n",
    "    \n",
    "#     #Convolutional\n",
    "#     #input is 256 x 1 x 2\n",
    "# #     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "# #     netD.add(nn.LeakyReLU(0.2))\n",
    "# #     # should still be 256 x 1 x 2\n",
    "# #     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "# #     netD.add(nn.BatchNorm())\n",
    "# #     netD.add(nn.LeakyReLU(0.2))\n",
    "# #     # should still be 256 x 1 x 2\n",
    "# #     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "# #     netD.add(nn.BatchNorm())\n",
    "# #     netD.add(nn.LeakyReLU(0.2))\n",
    "# #     # should still be 256 x 1 x 2\n",
    "# #     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, padding = 2, in_channels = 1))\n",
    "# #     netD.add(nn.BatchNorm())\n",
    "# #     netD.add(nn.LeakyReLU(0.2))\n",
    "# #     # should still be 256 x 1 x 2\n",
    "# #     netD.add(nn.Conv1D(channels = 1, kernel_size = 5, strides = 2,padding = 2, in_channels = 1))\n",
    "# #     # should still be 256 x 1 x 1\n",
    "\n",
    "\n",
    "# #     #Dense\n",
    "# #     netD2.add(nn.Dense(256))\n",
    "# #     netD2.add(nn.Dropout(0.5))\n",
    "# #     netD2.add(nn.LeakyReLU(0.2))\n",
    "# #     #netD.add(nn.Dense(100))\n",
    "# #     #netD.add(nn.LeakyReLU(0.2))\n",
    "# #     netD2.add(nn.Dense(200))\n",
    "# #     netD2.add(nn.LeakyReLU(0.2))\n",
    "# #     #netD.add(nn.Dropout(0.5))\n",
    "# #     netD2.add(nn.Dense(1))\n",
    "\n",
    "\n",
    "\n",
    "#     #Try three smh\n",
    "#     netD3.add(nn.Dense(128, activation = \"tanh\"))\n",
    "#     netD3.add(nn.Dense(128, activation = \"tanh\"))\n",
    "#     netD3.add(nn.Dense(1))\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = gluon.loss.SigmoidBinaryCrossEntropyLoss()\n",
    "\n",
    "netG.initialize(mx.init.Normal(0.5), ctx = ctx)\n",
    "netD1.initialize(mx.init.Normal(0.095), ctx = ctx)\n",
    "netD2.initialize(mx.init.Normal(0.095), ctx = ctx)\n",
    "\n",
    "trainerG = gluon.Trainer(netG.collect_params(), 'adam', {'learning_rate': lr, 'beta1':beta1})\n",
    "trainerD1 = gluon.Trainer(netD1.collect_params(), 'adam', {'learning_rate': lr, 'beta1':beta1})\n",
    "trainerD2 = gluon.Trainer(netD2.collect_params(), 'adam', {'learning_rate': lr, 'beta1':beta1})\n",
    "#unrolledtrainerD = gluon.Trainer(netD.collect_params(), 'adam', {'learning_rate': lr, 'beta1':beta1})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def makeOutput(netG, inputs):\n",
    "    states = netG.begin_state(batch_size=batch_size, ctx = ctx, func = mx.ndarray.ones)\n",
    "    outputs = []\n",
    "    for i in range(sample_size):\n",
    "        output, states = netG(inputs, states)\n",
    "        outputs.append(output.asnumpy())\n",
    "    outputs = mx.nd.array(outputs, ctx = ctx)\n",
    "    return mx.ndarray.floor(mx.ndarray.abs(10*outputs.T[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sigh\n",
      "<class 'mxnet.ndarray.ndarray.NDArray'>\n",
      "<class 'mxnet.ndarray.ndarray.NDArray'>\n",
      "<class 'mxnet.ndarray.ndarray.NDArray'>\n",
      "\n",
      "[5. 5. 5. 5. 4. 4. 4. 4. 4. 4. 4. 4. 4. 4. 4. 3. 3. 3. 3. 3. 3. 3. 3. 3.\n",
      " 3. 3. 3. 3. 3. 3. 3. 3. 3. 3. 3. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2.\n",
      " 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2. 2.]\n",
      "<NDArray 64 @gpu(0)>\n",
      "128\n",
      "64\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.ndarray'>\n",
      "<class 'numpy.int64'>\n",
      "[11  5  5  2  2  4  5  6  7  3  5  4  3  5  6  5  7  1  8  2  5  9  4  6\n",
      "  2  5  8  3  4  5  5  6  6  3  6  6  9  5  7  5  6  4  5  7  6  5  7  4\n",
      "  7  4  4  7  3  1  3  4  4  5  5  8  6  7  6  2]\n",
      "65536\n",
      "64\n"
     ]
    }
   ],
   "source": [
    "latent1 = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "fake = makeOutput(netG, latent1)\n",
    "print(\"sigh\")\n",
    "print(type(fake))\n",
    "print(type(fake[0]))\n",
    "print(type(fake[0][0]))\n",
    "print(fake[0])\n",
    "print(len(fake))\n",
    "print(len(fake[0]))\n",
    "print(type(samples))\n",
    "print(type(samples[0]))\n",
    "print(type(samples[0][0]))\n",
    "print(samples[0])\n",
    "print(len(samples))\n",
    "print(len(samples[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Begin\n",
      "BEFORE THE FIRE:\n",
      "Plot 0\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACzVJREFUeJzt3WGo3fddx/H3x2RlWle62tsQmtVbIYz1ydpxKZXAYK0b1Y41DzrZUAkSyJMpFYWZ+UzwQfvEzQcihHZ6wc21VEtKN+pK1iKCdLux1bXLRmqIW0jtvdMWNx84sn19cP+V0N70/O895+TefPd+QTjn/7+/k//3wck7f/45/5NUFZKkK9/PbPcAkqTZMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkprYfTkPdv3119fi4uLlPKQkXfFOnjz5/apamLTusgZ9cXGRlZWVy3lISbriJfn3Meu85CJJTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNXNY7RSWpk8WjXx617uwD98x5knWeoUtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2Smhh163+Ss8APgB8DF6pqKcl1wCPAInAW+PWqem0+Y0qSJtnMGfqHqurWqloato8CJ6pqP3Bi2JYkbZNpLrncCywPz5eBg9OPI0naqrFBL+CrSU4mOTLs21NVrwAMjzfMY0BJ0jhjvz73QFWdT3ID8HSSb489wPAXwBGAm266aQsjSpLGGHWGXlXnh8dV4HHgduDVJHsBhsfVS7z2WFUtVdXSwsLCbKaWJL3FxKAnuTrJu954DnwEeBF4Ajg0LDsEHJ/XkJKkycZcctkDPJ7kjfVfrKqnknwDeDTJYeC7wMfnN6YkaZKJQa+qM8D7N9j/n8Bd8xhKkrR53ikqSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmRgc9ya4kzyd5cti+OclzSU4neSTJVfMbU5I0yWbO0O8HTl20/SDw2araD7wGHJ7lYJKkzRkV9CT7gHuAh4btAHcCjw1LloGD8xhQkjTO2DP0zwGfBn4ybP8C8HpVXRi2zwE3bvTCJEeSrCRZWVtbm2pYSdKlTQx6ko8Cq1V18uLdGyytjV5fVceqaqmqlhYWFrY4piRpkt0j1hwAPpbk14B3AtewfsZ+bZLdw1n6PuD8/MaUJE0y8Qy9qj5TVfuqahH4BPC1qvoN4BngvmHZIeD43KaUJE00zefQ/xD4/SQvs35N/eHZjCRJ2ooxl1z+X1U9Czw7PD8D3D77kSRJW+GdopLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNTAx6kncm+XqSf0nyUpI/HvbfnOS5JKeTPJLkqvmPK0m6lDFn6P8L3FlV7wduBe5OcgfwIPDZqtoPvAYcnt+YkqRJJga91v1w2HzH8KuAO4HHhv3LwMG5TChJGmXUNfQku5K8AKwCTwP/BrxeVReGJeeAG+czoiRpjFFBr6ofV9WtwD7gduB9Gy3b6LVJjiRZSbKytra29UklSW9rU59yqarXgWeBO4Brk+wefrQPOH+J1xyrqqWqWlpYWJhmVknS2xjzKZeFJNcOz38W+BXgFPAMcN+w7BBwfF5DSpIm2z15CXuB5SS7WP8L4NGqejLJt4AvJfkT4Hng4TnOKUmaYGLQq+pfgds22H+G9evpkqQdwDtFJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpqYGPQk70nyTJJTSV5Kcv+w/7okTyc5PTy+e/7jSpIuZcwZ+gXgD6rqfcAdwKeS3AIcBU5U1X7gxLAtSdomE4NeVa9U1T8Pz38AnAJuBO4Flodly8DBeQ0pSZpsU9fQkywCtwHPAXuq6hVYjz5ww6yHkySNNzroSX4e+Fvg96rqvzfxuiNJVpKsrK2tbWVGSdIIo4Ke5B2sx/wLVfV3w+5Xk+wdfr4XWN3otVV1rKqWqmppYWFhFjNLkjYw5lMuAR4GTlXVn170oyeAQ8PzQ8Dx2Y8nSRpr94g1B4DfAr6Z5IVh3x8BDwCPJjkMfBf4+HxGlCSNMTHoVfWPQC7x47tmO44kaau8U1SSmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU1MDHqSzydZTfLiRfuuS/J0ktPD47vnO6YkaZIxZ+h/Bdz9pn1HgRNVtR84MWxLkrbRxKBX1T8A//Wm3fcCy8PzZeDgjOeSJG3SVq+h76mqVwCGxxsutTDJkSQrSVbW1ta2eDhJ0iRz/0fRqjpWVUtVtbSwsDDvw0nST62tBv3VJHsBhsfV2Y0kSdqKrQb9CeDQ8PwQcHw240iStmrMxxb/Bvgn4L1JziU5DDwAfDjJaeDDw7YkaRvtnrSgqj55iR/dNeNZJElT8E5RSWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITE2/93ykWj3551LqzD9wz50kkaWfyDF2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU1cMR9blObJj8WqA8/QJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJamKqoCe5O8l3kryc5OishpIkbd6Wg55kF/DnwK8CtwCfTHLLrAaTJG3ONGfotwMvV9WZqvoR8CXg3tmMJUnarGmCfiPwvYu2zw37JEnbYJrvQ88G++oti5IjwJFh84dJvrPF410PfH/iUA9u8XfXTzvfX5qbPDju/fU2fnHMommCfg54z0Xb+4Dzb15UVceAY1McB4AkK1W1NO3vI23E95fm6XK9v6a55PINYH+Sm5NcBXwCeGI2Y0mSNmvLZ+hVdSHJ7wB/D+wCPl9VL81sMknSpkz1f4pW1VeAr8xolkmmvmwjvQ3fX5qny/L+StVb/h1TknQF8tZ/SWriigh6kl1Jnk/y5HbPon6SnE3yzSQvJFnZ7nnUS5JrkzyW5NtJTiX55Xkda6pr6JfR/cAp4JrtHkRtfaiqpvmcsHQpfwY8VVX3DZ8I/Ll5HWjHn6En2QfcAzy03bNI0mYkuQb4IPAwQFX9qKpen9fxdnzQgc8BnwZ+st2DqK0Cvprk5HBnszQrvwSsAX85XDZ+KMnV8zrYjg56ko8Cq1V1crtnUWsHquoDrH9z6KeSfHC7B1Ibu4EPAH9RVbcB/wPM7avGd3TQgQPAx5KcZf3bHO9M8tfbO5K6qarzw+Mq8Djr3yQqzcI54FxVPTdsP8Z64OdiRwe9qj5TVfuqapH1rxb4WlX95jaPpUaSXJ3kXW88Bz4CvLi9U6mLqvoP4HtJ3jvsugv41ryOd6V8ykWalz3A40lg/c/DF6vqqe0dSc38LvCF4RMuZ4DfnteBvFNUkprY0ZdcJEnjGXRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpif8DTAhADABkF8QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plot 1\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADYJJREFUeJzt3X+o3fV9x/Hna0mcog4tnkpQs1uGSKXQWC5BCJTOame1TAsrTJgIc6QDHcrKtrT/rO4HZLDq/hll6XRmzOmkKkp1XYNVROi0NzbG2LSzc9mWmpmIE/Ufh/reH/crhJDrOfec870n93OfDzicc77ne+55fxGf+fK93++5qSokSavfL8x6AEnSdBh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRqxfyQ8755xzam5ubiU/UpJWvT179rxWVYNh661o0Ofm5lhYWFjJj5SkVS/Jf46ynodcJKkRBl2SGmHQJakRBl2SGmHQJakRQ4Oe5NQkzyZ5PsmLSW7rlt+d5D+S7O1um/sfV5K0lFFOW3wHuKyq3k6yAXg6yT93r/1BVX27v/EkSaMaGvRa/Bt1b3dPN3Q3/26dJJ1kRjqGnmRdkr3AEWB3VT3TvfTnSfYluSPJL/Y2pSRpqJGuFK2q94DNSc4CHkryCeCrwP8ApwA7gT8C/uT49ybZBmwD2LRp05TGVqvmtj860noHd1zd8yTS6rOss1yq6g3gSeDKqjpci94B/g7YssR7dlbVfFXNDwZDv4pAkjSmUc5yGXR75iQ5Dbgc+EmSjd2yANcC+/scVJL04UY55LIR2JVkHYv/ANxfVd9J8v0kAyDAXuB3e5xTkjTEKGe57AMuOcHyy3qZSJI0Fq8UlaRGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGDA16klOTPJvk+SQvJrmtW/6xJM8keSnJPyU5pf9xJUlLGWUP/R3gsqr6JLAZuDLJpcBfAHdU1YXA/wI39jemJGmYoUGvRW93Tzd0twIuA77dLd8FXNvLhJKkkYx0DD3JuiR7gSPAbuDfgTeq6t1ulUPAeUu8d1uShSQLR48encbMkqQTGCnoVfVeVW0Gzge2AB8/0WpLvHdnVc1X1fxgMBh/UknSh1rWWS5V9QbwJHApcFaS9d1L5wOvTHc0SdJyjHKWyyDJWd3j04DLgQPAE8BvdKvdADzc15CSpOHWD1+FjcCuJOtY/Afg/qr6TpIfA/cl+TPgR8CdPc4pSRpiaNCrah9wyQmWv8zi8XRJ0knAK0UlqREGXZIaYdAlqREGXZIaYdAlqREGXZIaYdAlqREGXZIaYdAlqREGXZIaYdAlqREGXZIaYdAlqREGXZIaYdAlqREGXZIaYdAlqREGXZIaYdAlqREGXZIaMTToSS5I8kSSA0leTHJLt/zrSX6eZG93u6r/cSVJS1k/wjrvAl+pqueSnAnsSbK7e+2OqvrL/saTJI1qaNCr6jBwuHv8VpIDwHl9DyZJWp5lHUNPMgdcAjzTLbo5yb4kdyU5e8qzSZKWYeSgJzkDeAC4tareBL4J/AqwmcU9+G8s8b5tSRaSLBw9enQKI0uSTmSkoCfZwGLM76mqBwGq6tWqeq+q3ge+BWw50XuramdVzVfV/GAwmNbckqTjjHKWS4A7gQNVdfsxyzces9oXgf3TH0+SNKpRznLZClwPvJBkb7fsa8B1STYDBRwEvtzLhJKkkYxylsvTQE7w0mPTH0eSNC6vFJWkRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRgwNepILkjyR5ECSF5Pc0i3/SJLdSV7q7s/uf1xJ0lJG2UN/F/hKVX0cuBS4KcnFwHbg8aq6EHi8ey5JmpGhQa+qw1X1XPf4LeAAcB5wDbCrW20XcG1fQ0qShlvWMfQkc8AlwDPAuVV1GBajD3x0ifdsS7KQZOHo0aOTTStJWtLIQU9yBvAAcGtVvTnq+6pqZ1XNV9X8YDAYZ0ZJ0ghGCnqSDSzG/J6qerBb/GqSjd3rG4Ej/YwoSRrFKGe5BLgTOFBVtx/z0iPADd3jG4CHpz+eJGlU60dYZytwPfBCkr3dsq8BO4D7k9wI/BfwpX5GlCSNYmjQq+ppIEu8/NnpjiNJGpdXikpSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDViaNCT3JXkSJL9xyz7epKfJ9nb3a7qd0xJ0jCj7KHfDVx5guV3VNXm7vbYdMeSJC3X0KBX1VPA6yswiyRpApMcQ785yb7ukMzZU5tIkjSW9WO+75vAnwLV3X8D+O0TrZhkG7ANYNOmTWN+nNSuue2PjrTewR1X9zyJVrux9tCr6tWqeq+q3ge+BWz5kHV3VtV8Vc0PBoNx55QkDTFW0JNsPObpF4H9S60rSVoZQw+5JLkX+AxwTpJDwB8Dn0mymcVDLgeBL/c4oyRpBEODXlXXnWDxnT3MIkmagFeKSlIjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjDLokNcKgS1IjhgY9yV1JjiTZf8yyjyTZneSl7v7sfseUJA0zyh763cCVxy3bDjxeVRcCj3fPJUkzNDToVfUU8Ppxi68BdnWPdwHXTnkuSdIyjXsM/dyqOgzQ3X90eiNJksbR+y9Fk2xLspBk4ejRo31/nCStWeMG/dUkGwG6+yNLrVhVO6tqvqrmB4PBmB8nSRpm3KA/AtzQPb4BeHg640iSxjXKaYv3Aj8ALkpyKMmNwA7giiQvAVd0zyVJM7R+2ApVdd0SL312yrNIkibglaKS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNGPrlXJqdue2PjrTewR1X9zyJpNXAPXRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGTHRhUZKDwFvAe8C7VTU/jaEkScs3jStFf7WqXpvCz5EkTcBDLpLUiEn30Av4XpIC/qaqdh6/QpJtwDaATZs2jf1Bfq+JJH24SffQt1bVp4DPAzcl+fTxK1TVzqqar6r5wWAw4cdJkpYyUdCr6pXu/gjwELBlGkNJkpZv7KAnOT3JmR88Bj4H7J/WYJKk5ZnkGPq5wENJPvg5/1hV353KVJKkZRs76FX1MvDJKc4iSZqApy1KUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1wqBLUiMMuiQ1YpK/KSpJY5nb/uhI6x3ccXXPk0xm1O2AldkW99AlqRETBT3JlUl+muRnSbZPayhJ0vKNHfQk64C/Bj4PXAxcl+TiaQ0mSVqeSfbQtwA/q6qXq+r/gPuAa6YzliRpuSYJ+nnAfx/z/FC3TJI0A6mq8d6YfAn4tar6ne759cCWqvq949bbBmzrnl4E/HTMWc8BXhvzvauV27w2uM1rwyTb/MtVNRi20iSnLR4CLjjm+fnAK8evVFU7gZ0TfA4ASRaqan7Sn7OauM1rg9u8NqzENk9yyOWHwIVJPpbkFOA3gUemM5YkabnG3kOvqneT3Az8C7AOuKuqXpzaZJKkZZnoStGqegx4bEqzDDPxYZtVyG1eG9zmtaH3bR77l6KSpJOLl/5LUiNWRdDX2lcMJLkryZEk+2c9y0pIckGSJ5IcSPJikltmPVPfkpya5Nkkz3fbfNusZ1opSdYl+VGS78x6lpWQ5GCSF5LsTbLQ62ed7Idcuq8Y+DfgChZPlfwhcF1V/Ximg/UoyaeBt4G/r6pPzHqeviXZCGysqueSnAnsAa5t/L9xgNOr6u0kG4CngVuq6l9nPFrvkvw+MA/8UlV9Ydbz9C3JQWC+qno/73417KGvua8YqKqngNdnPcdKqarDVfVc9/gt4ACNX3Vci97unm7obif33tUUJDkfuBr421nP0qLVEHS/YmANSTIHXAI8M9tJ+tcdetgLHAF2V1Xz2wz8FfCHwPuzHmQFFfC9JHu6K+d7sxqCnhMsa35PZi1KcgbwAHBrVb0563n6VlXvVdVmFq+y3pKk6cNrSb4AHKmqPbOeZYVtrapPsfjNtDd1h1R7sRqCPtJXDGh1644jPwDcU1UPznqelVRVbwBPAlfOeJS+bQV+vTumfB9wWZJ/mO1I/auqV7r7I8BDLB5G7sVqCLpfMdC47heEdwIHqur2Wc+zEpIMkpzVPT4NuBz4yWyn6ldVfbWqzq+qORb/P/5+Vf3WjMfqVZLTu1/0k+R04HNAb2evnfRBr6p3gQ++YuAAcH/rXzGQ5F7gB8BFSQ4luXHWM/VsK3A9i3tse7vbVbMeqmcbgSeS7GNxp2V3Va2J0/jWmHOBp5M8DzwLPFpV3+3rw0760xYlSaM56ffQJUmjMeiS1AiDLkmNMOiS1AiDLkmNMOiS1AiDLkmNMOiS1Ij/B/fhUWIKpqylAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plot 2\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACp1JREFUeJzt3E+spfVdx/HPt0xJTbWhyIUQpjgsSFM2peaGNCExLViDQgqL1rQxZhYks6kJRk1FdxoXsLHduHBSGifRCqRKIG1TSxBiTAz20lYtjgQkWBHCDAqRbjTQr4s5NRM6wzn3z+HOfPt6JeSc5zm/557v6s0zz32eW90dAM5/79jvAQDYG4IOMISgAwwh6ABDCDrAEIIOMISgAwxxYJVFVfVckteSvJHk9e7erKqLk9yX5FCS55L8cne/sp4xAVhmO2foH+3ua7t7c7F9Z5JHuvvqJI8stgHYJ7XKk6KLM/TN7n75tH1PJflId79YVZcneay73/9WP+eSSy7pQ4cO7W5igB8zTzzxxMvdvbFs3UqXXJJ0km9UVSf54+4+muSy7n4xSRZRv3TZDzl06FC2trZW/EoAkqSq/m2VdasG/frufmER7Yer6l+2MciRJEeS5Morr1z1MAC2aaVr6N39wuL1RJIHklyX5KXFpZYsXk+c5dij3b3Z3ZsbG0v/xQDADi0NelW9u6p+6ofvk/xCku8meSjJ4cWyw0keXNeQACy3yiWXy5I8UFU/XP+l7v56VX0zyf1VdXuS7yX55PrGBGCZpUHv7meTfPAM+/8zyY3rGAqA7fOkKMAQgg4whKADDCHoAEOs+mARAG9y6M6vrrTuubtuXvMkpzhDBxhC0AGGEHSAIQQdYAhBBxhC0AGGEHSAIQQdYAhBBxhC0AGGEHSAIQQdYAhBBxhC0AGGEHSAIQQdYAhBBxhC0AGGEHSAIQQdYAhBBxhC0AGGEHSAIQQdYAhBBxhC0AGGEHSAIQQdYAhBBxhC0AGGWDnoVXVBVX27qr6y2L6qqh6vqqer6r6qunB9YwKwzHbO0O9Icvy07buTfK67r07ySpLb93IwALZnpaBX1cEkNyf5wmK7ktyQ5MuLJceS3LaOAQFYzapn6J9P8tkkP1hs/3SSV7v79cX280muONOBVXWkqraqauvkyZO7GhaAs1sa9Kq6JcmJ7n7i9N1nWNpnOr67j3b3Zndvbmxs7HBMAJY5sMKa65N8vKp+Kcm7krwnp87YL6qqA4uz9INJXljfmAAss/QMvbt/p7sPdvehJJ9K8tfd/StJHk3yicWyw0keXNuUACy1m/vQfzvJb1TVMzl1Tf2evRkJgJ1Y5ZLL/+vux5I8tnj/bJLr9n4kAHbCk6IAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBBLg15V76qqv6+qf6iqJ6vq9xb7r6qqx6vq6aq6r6ouXP+4AJzNKmfo/5Pkhu7+YJJrk9xUVR9OcneSz3X31UleSXL7+sYEYJmlQe9Tvr/YfOfiv05yQ5IvL/YfS3LbWiYEYCUrXUOvqguq6jtJTiR5OMm/Jnm1u19fLHk+yRVnOfZIVW1V1dbJkyf3YmYAzmCloHf3G919bZKDSa5L8oEzLTvLsUe7e7O7Nzc2NnY+KQBvaVt3uXT3q0keS/LhJBdV1YHFRweTvLC3owGwHavc5bJRVRct3v9Ekp9PcjzJo0k+sVh2OMmD6xoSgOUOLF+Sy5Mcq6oLcup/APd391eq6p+T3FtVf5Dk20nuWeOcACyxNOjd/Y9JPnSG/c/m1PV0AM4BnhQFGELQAYYQdIAhBB1gCEEHGELQAYYQdIAhBB1gCEEHGELQAYYQdIAhBB1gCEEHGELQAYYQdIAhBB1gCEEHGELQAYYQdIAhBB1gCEEHGELQAYYQdIAhBB1gCEEHGELQAYYQdIAhBB1gCEEHGELQAYYQdIAhBB1gCEEHGELQAYZYGvSqel9VPVpVx6vqyaq6Y7H/4qp6uKqeXry+d/3jAnA2q5yhv57kN7v7A0k+nOQzVXVNkjuTPNLdVyd5ZLENwD5ZGvTufrG7v7V4/1qS40muSHJrkmOLZceS3LauIQFYblvX0KvqUJIPJXk8yWXd/WJyKvpJLt3r4QBY3cpBr6qfTPIXSX69u/97G8cdqaqtqto6efLkTmYEYAUrBb2q3plTMf+z7v7Lxe6XquryxeeXJzlxpmO7+2h3b3b35sbGxl7MDMAZrHKXSyW5J8nx7v7D0z56KMnhxfvDSR7c+/EAWNWBFdZcn+RXk/xTVX1nse93k9yV5P6quj3J95J8cj0jArCKpUHv7r9NUmf5+Ma9HQeAnfKkKMAQgg4whKADDCHoAEMIOsAQgg4whKADDCHoAEMIOsAQgg4whKADDCHoAEOs8tcWzwmH7vzqSuueu+vmNU8CcG5yhg4whKADDCHoAEMIOsAQgg4whKADDCHoAEMIOsAQgg4whKADDCHoAEMIOsAQgg4whKADDCHoAEMIOsAQgg4whKADDCHoAEMIOsAQgg4wxNKgV9UXq+pEVX33tH0XV9XDVfX04vW96x0TgGVWOUP/kyQ3vWnfnUke6e6rkzyy2AZgHy0Nenf/TZL/etPuW5McW7w/luS2PZ4LgG3a6TX0y7r7xSRZvF66dyMBsBNr/6VoVR2pqq2q2jp58uS6vw7gx9ZOg/5SVV2eJIvXE2db2N1Hu3uzuzc3NjZ2+HUALLPToD+U5PDi/eEkD+7NOADs1Cq3Lf55kr9L8v6qer6qbk9yV5KPVdXTST622AZgHx1YtqC7P32Wj27c41kA2AVPigIMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQwg6wBCCDjCEoAMMIegAQ+wq6FV1U1U9VVXPVNWdezUUANu346BX1QVJ/ijJLya5Jsmnq+qavRoMgO3ZzRn6dUme6e5nu/t/k9yb5Na9GQuA7dpN0K9I8u+nbT+/2AfAPjiwi2PrDPv6RxZVHUlyZLH5/ap6aoffd0mSl5cOdfcOfzrAmtTdq/XrLfzMKot2E/Tnk7zvtO2DSV5486LuPprk6C6+J0lSVVvdvbnbnwPwdnu7+rWbSy7fTHJ1VV1VVRcm+VSSh/ZmLAC2a8dn6N39elX9WpK/SnJBki9295N7NhkA27KbSy7p7q8l+doezbLMri/bAOyTt6Vf1f0jv8cE4Dzk0X+AIXZ1yeXtsngqdSvJf3T3Lfs9D8Cqquq5JK8leSPJ6+u82+W8CHqSO5IcT/Ke/R4EYAc+2t27uQ99Jef8JZeqOpjk5iRf2O9ZAM5l53zQk3w+yWeT/GC/BwHYgU7yjap6YvHk/Nqc00GvqluSnOjuJ/Z7FoAdur67fzan/jLtZ6rq59b1Red00JNcn+Tji18q3Jvkhqr60/0dCWB13f3C4vVEkgdy6i/VrsV5cx96VX0kyW+5ywU4X1TVu5O8o7tfW7x/OMnvd/fX1/F958tdLgDno8uSPFBVyanefmldMU/OozN0AN7auX4NHYAVCTrAEIIOMISgAwwh6ABDCDrAEIIOMISgAwzxfzadKhp1nRvXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plot 3\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADZRJREFUeJzt3V+InfWdx/H3pxmlra1E6xiCkY2FIJVCVYbgEpBdU7suSs1FLZVdCd2U3HTF0oU29WYp7EV607oXSyEkdmdZWxWtRGzpVlKlLexaJ2rXP9HVDUFDUme6KtVerGi/ezFP2SATzzPnzPHM/Hy/IJzzPPOcnO+D5J0nv3nOmKpCkrT2fWDSA0iSVoZBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJaoRBl6RGGHRJasRUn4OSrAf2A58ECvgb4DngLmAzcAz4fFW9+m6/z3nnnVebN28eflpJeh86fPjwb6tqetBx6fPR/ySzwC+qan+SM4EPA7cCr1TV3iR7gHOq6uvv9vvMzMzU3NxcvzOQJAGQ5HBVzQw6buCSS5KzgSuBAwBV9WZVvQZcD8x2h80CO4YfV5I0qj5r6B8HFoDvJXk8yf4kZwEbquokQPd4/hjnlCQN0CfoU8DlwHer6jLg98Cevm+QZHeSuSRzCwsLQ44pSRqkT9CPA8er6pFu+x4WA/9yko0A3eP8Ui+uqn1VNVNVM9PTA9f0JUlDGhj0qvoN8FKSi7td24FngPuBnd2+ncDBsUwoSeql122LwM3AHd0dLkeBL7L4l8HdSXYBLwI3jGdESVIfvYJeVU8AS90ys31lx5EkDctPikpSIwy6JDWi7xr6xG3e86Nexx3be+2YJ5Gk1ckrdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEZM9TkoyTHgdeBt4K2qmklyLnAXsBk4Bny+ql4dz5iSpEGWc4X+51V1aVXNdNt7gENVtQU41G1LkiZklCWX64HZ7vkssGP0cSRJw+ob9AJ+muRwkt3dvg1VdRKgezx/qRcm2Z1kLsncwsLC6BNLkpbUaw0d2FZVJ5KcDzyY5Nm+b1BV+4B9ADMzMzXEjJKkHnpdoVfVie5xHrgP2Aq8nGQjQPc4P64hJUmDDQx6krOSfPSPz4HPAE8B9wM7u8N2AgfHNaQkabA+Sy4bgPuS/PH471fVT5I8CtydZBfwInDD+MaUJA0yMOhVdRT41BL7/wfYPo6hJEnL5ydFJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGmHQJakRBl2SGtE76EnWJXk8yQPd9kVJHknyfJK7kpw5vjElSYMs5wr9FuDIKdvfAr5TVVuAV4FdKzmYJGl5egU9ySbgWmB/tx3gKuCe7pBZYMc4BpQk9dP3Cv024GvAH7rtjwGvVdVb3fZx4IIVnk2StAwDg57kOmC+qg6funuJQ+s0r9+dZC7J3MLCwpBjSpIG6XOFvg34bJJjwJ0sLrXcBqxPMtUdswk4sdSLq2pfVc1U1cz09PQKjCxJWsrAoFfVN6pqU1VtBr4A/Kyq/gp4CPhcd9hO4ODYppQkDTTKfehfB76a5AUW19QPrMxIkqRhTA0+5P9V1cPAw93zo8DWlR9JkjQMPykqSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY0YGPQkH0zyqyS/TvJ0km92+y9K8kiS55PcleTM8Y8rSTqdPlfo/wtcVVWfAi4FrklyBfAt4DtVtQV4Fdg1vjElSYMMDHoteqPbPKP7VcBVwD3d/llgx1gmlCT10msNPcm6JE8A88CDwH8Dr1XVW90hx4ELxjOiJKmPXkGvqrer6lJgE7AV+MRShy312iS7k8wlmVtYWBh+UknSu1rWXS5V9RrwMHAFsD7JVPelTcCJ07xmX1XNVNXM9PT0KLNKkt5Fn7tcppOs755/CPg0cAR4CPhcd9hO4OC4hpQkDTY1+BA2ArNJ1rH4F8DdVfVAkmeAO5P8A/A4cGCMc0qSBhgY9Kr6T+CyJfYfZXE9XZK0CvhJUUlqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqhEGXpEYYdElqxMCgJ7kwyUNJjiR5Oskt3f5zkzyY5Pnu8ZzxjytJOp0+V+hvAX9XVZ8ArgC+nOQSYA9wqKq2AIe6bUnShAwMelWdrKrHuuevA0eAC4DrgdnusFlgx7iGlCQNtqw19CSbgcuAR4ANVXUSFqMPnH+a1+xOMpdkbmFhYbRpJUmn1TvoST4C3At8pap+1/d1VbWvqmaqamZ6enqYGSVJPfQKepIzWIz5HVX1w273y0k2dl/fCMyPZ0RJUh997nIJcAA4UlXfPuVL9wM7u+c7gYMrP54kqa+pHsdsA24CnkzyRLfvVmAvcHeSXcCLwA3jGVGS1MfAoFfVL4Gc5svbV3YcSdKw/KSoJDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwy6JDXCoEtSIwYGPcntSeaTPHXKvnOTPJjk+e7xnPGOKUkapM8V+j8D17xj3x7gUFVtAQ5125KkCRoY9Kr6OfDKO3ZfD8x2z2eBHSs8lyRpmYZdQ99QVScBusfzV24kSdIwxv5N0SS7k8wlmVtYWBj320nS+9awQX85yUaA7nH+dAdW1b6qmqmqmenp6SHfTpI0yLBBvx/Y2T3fCRxcmXEkScPqc9viD4B/By5OcjzJLmAvcHWS54Gru21J0gRNDTqgqm48zZe2r/AskqQR+ElRSWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRhh0SWqEQZekRkxNeoD3o817ftTruGN7rx3zJJJa4hW6JDXCoEtSI0ZacklyDfCPwDpgf1XtXZGppPdYS8tgnst7Z7XNN/QVepJ1wD8BfwlcAtyY5JKVGkyStDyjLLlsBV6oqqNV9SZwJ3D9yowlSVquUYJ+AfDSKdvHu32SpAlIVQ33wuQG4C+q6kvd9k3A1qq6+R3H7QZ2d5sXA88NOet5wG+HfO1q08q5tHIe4LmsVq2cy6jn8SdVNT3ooFG+KXocuPCU7U3AiXceVFX7gH0jvA8ASeaqambU32c1aOVcWjkP8FxWq1bO5b06j1GWXB4FtiS5KMmZwBeA+1dmLEnScg19hV5VbyX5W+DfWLxt8faqenrFJpMkLctI96FX1Y+BH6/QLIOMvGyzirRyLq2cB3guq1Ur5/KenMfQ3xSVJK0ufvRfkhqxJoKe5JokzyV5IcmeSc8zrCS3J5lP8tSkZxlFkguTPJTkSJKnk9wy6ZmGleSDSX6V5NfduXxz0jONIsm6JI8neWDSs4wiybEkTyZ5IsncpOcZRZL1Se5J8mz3Z+ZPx/Zeq33JpfsRA/8FXM3irZKPAjdW1TMTHWwISa4E3gD+pao+Oel5hpVkI7Cxqh5L8lHgMLBjjf43CXBWVb2R5Azgl8AtVfUfEx5tKEm+CswAZ1fVdZOeZ1hJjgEzVbXm70FPMgv8oqr2d3cEfriqXhvHe62FK/RmfsRAVf0ceGXSc4yqqk5W1WPd89eBI6zRTwnXoje6zTO6X6v7Kuc0kmwCrgX2T3oWLUpyNnAlcACgqt4cV8xhbQTdHzGwiiXZDFwGPDLZSYbXLVM8AcwDD1bVWj2X24CvAX+Y9CAroICfJjncfdp8rfo4sAB8r1sK25/krHG92VoIepbYtyavoFqT5CPAvcBXqup3k55nWFX1dlVdyuKnnbcmWXPLYUmuA+ar6vCkZ1kh26rqchZ/muuXu+XKtWgKuBz4blVdBvweGNv3AddC0Hv9iAG9t7r15nuBO6rqh5OeZyV0/xR+GLhmwqMMYxvw2W7t+U7gqiT/OtmRhldVJ7rHeeA+Fpde16LjwPFT/tV3D4uBH4u1EHR/xMAq030j8QBwpKq+Pel5RpFkOsn67vmHgE8Dz052quWrqm9U1aaq2szin5GfVdVfT3isoSQ5q/tmO93yxGeANXlnWFX9BngpycXdru3A2G4eWPX/k+iWfsRAkh8Afwacl+Q48PdVdWCyUw1lG3AT8GS39gxwa/fJ4bVmIzDb3U31AeDuqlrTt/w1YANw3+J1A1PA96vqJ5MdaSQ3A3d0F6RHgS+O641W/W2LkqR+1sKSiySpB4MuSY0w6JLUCIMuSY0w6JLUCIMuSY0w6JLUCIMuSY34PxFqaeIfBZDRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plot 4\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADCxJREFUeJzt3V+IZoV5x/Hvr7tKgkkxxlEWVzpeLBIpRMuwCAuFakxtlbgXsURa2Yste5MUQwrppneBXpibxF70ZlHplqZRiRFFQ5plowQhVWf9k2g2qVa26bLiTholetOy5unFHOkis3nfnZkz786z3w8M73vOnNf3OSz79XDe855NVSFJ2vx+Z9YDSJLWh0GXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktTE1o18s0svvbTm5+c38i0ladM7cuTIL6tqbtJ2Gxr0+fl5FhcXN/ItJWnTS/Kf02znKRdJamKqI/Qkx4B3gPeAU1W1kOQS4EFgHjgG/FlVvTXOmJKkSc7mCP2PquraqloYlvcDh6tqB3B4WJYkzchaTrncBhwcnh8Edq99HEnSak0b9AK+n+RIkn3Dusur6g2A4fGylV6YZF+SxSSLS0tLa59YkrSiaa9y2VVVJ5JcBhxK8rNp36CqDgAHABYWFvzXNCRpJFMdoVfVieHxJPAIsBN4M8k2gOHx5FhDSpImmxj0JBcl+ej7z4FPAy8DjwF7hs32AI+ONaQkabJpTrlcDjyS5P3t/6WqvpfkOeChJHuBXwC3jzemJGmSiUGvqteBT66w/r+BG8cYaiXz+5+Yartjd98y8iSSdG7ym6KS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUxNRBT7IlyQtJHh+Wr0ryTJJXkzyY5MLxxpQkTXI2R+h3AUdPW/4a8I2q2gG8Bexdz8EkSWdnqqAn2Q7cAtw7LAe4Afj2sMlBYPcYA0qSpjPtEfo9wJeB3wzLHwferqpTw/Jx4Ip1nk2SdBYmBj3JrcDJqjpy+uoVNq0zvH5fksUki0tLS6scU5I0yTRH6LuAzyQ5BjzA8qmWe4CLk2wdttkOnFjpxVV1oKoWqmphbm5uHUaWJK1kYtCr6itVtb2q5oHPAT+oqj8HngQ+O2y2B3h0tCklSROt5Tr0vwG+lOQ1ls+p37c+I0mSVmPr5E3+X1U9BTw1PH8d2Ln+I0mSVsNvikpSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITE4Oe5ENJnk3yUpJXknx1WH9VkmeSvJrkwSQXjj+uJOlMpjlC/x/ghqr6JHAtcHOS64GvAd+oqh3AW8De8caUJE0yMei17N1h8YLhp4AbgG8P6w8Cu0eZUJI0lanOoSfZkuRF4CRwCPgP4O2qOjVschy4YpwRJUnTmCroVfVeVV0LbAd2Ap9YabOVXptkX5LFJItLS0urn1SS9Fud1VUuVfU28BRwPXBxkq3Dr7YDJ87wmgNVtVBVC3Nzc2uZVZL0W0xzlctckouH5x8GPgUcBZ4EPjtstgd4dKwhJUmTbZ28CduAg0m2sPw/gIeq6vEkPwUeSPJ3wAvAfSPOKUmaYGLQq+rHwHUrrH+d5fPpkqRzgN8UlaQmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1MTHoSa5M8mSSo0leSXLXsP6SJIeSvDo8fmz8cSVJZzLNEfop4K+r6hPA9cDnk1wD7AcOV9UO4PCwLEmakYlBr6o3qur54fk7wFHgCuA24OCw2UFg91hDSpImO6tz6EnmgeuAZ4DLq+oNWI4+cNkZXrMvyWKSxaWlpbVNK0k6o6mDnuQjwMPAF6vq19O+rqoOVNVCVS3Mzc2tZkZJ0hSmCnqSC1iO+Ter6jvD6jeTbBt+vw04Oc6IkqRpTHOVS4D7gKNV9fXTfvUYsGd4vgd4dP3HkyRNa+sU2+wC7gR+kuTFYd3fAncDDyXZC/wCuH2cESVJ05gY9Kp6GsgZfn3j+o4jSVotvykqSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQmDLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDVh0CWpCYMuSU0YdElqwqBLUhMGXZKaMOiS1IRBl6QmDLokNWHQJakJgy5JTUwMepL7k5xM8vJp6y5JcijJq8Pjx8YdU5I0yTRH6P8I3PyBdfuBw1W1Azg8LEuSZmhi0Kvqh8CvPrD6NuDg8PwgsHud55IknaXVnkO/vKreABgeL1u/kSRJqzH6h6JJ9iVZTLK4tLQ09ttJ0nlrtUF/M8k2gOHx5Jk2rKoDVbVQVQtzc3OrfDtJ0iSrDfpjwJ7h+R7g0fUZR5K0WtNctvgt4EfA1UmOJ9kL3A3clORV4KZhWZI0Q1snbVBVd5zhVzeu8yySpDXwm6KS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1MfFeLpqd+f1PTLXdsbtvGXkSSZuBR+iS1IRBl6QmDLokNWHQJakJgy5JTRh0SWrCoEtSEwZdkpow6JLUhEGXpCYMuiQ1YdAlqQlvzqVzijckOz/45zwOj9AlqQmDLklNGHRJasJz6NKMeT5Z68UjdElqwqBLUhMGXZKaMOiS1MSaPhRNcjPw98AW4N6quntdppKkTeBc+0B71UfoSbYA/wD8CXANcEeSa9ZrMEnS2VnLKZedwGtV9XpV/S/wAHDb+owlSTpbawn6FcB/nbZ8fFgnSZqBVNXqXpjcDvxxVf3lsHwnsLOq/uoD2+0D9g2LVwM/X+WslwK/XOVrNyv3+fzgPve31v39vaqam7TRWj4UPQ5cedryduDEBzeqqgPAgTW8DwBJFqtqYa3/nc3EfT4/uM/9bdT+ruWUy3PAjiRXJbkQ+Bzw2PqMJUk6W6s+Qq+qU0m+APwry5ct3l9Vr6zbZJKks7Km69Cr6rvAd9dplknWfNpmE3Kfzw/uc38bsr+r/lBUknRu8av/ktTEpgh6kpuT/DzJa0n2z3qesSW5P8nJJC/PepaNkOTKJE8mOZrklSR3zXqmsSX5UJJnk7w07PNXZz3TRkmyJckLSR6f9SwbIcmxJD9J8mKSxVHf61w/5TLcYuDfgZtYvlTyOeCOqvrpTAcbUZI/BN4F/qmqfn/W84wtyTZgW1U9n+SjwBFgd/M/4wAXVdW7SS4Angbuqqp/m/Foo0vyJWAB+N2qunXW84wtyTFgoapGv+5+Mxyhn3e3GKiqHwK/mvUcG6Wq3qiq54fn7wBHaf6t41r27rB4wfBzbh9drYMk24FbgHtnPUtHmyHo3mLgPJJkHrgOeGa2k4xvOPXwInASOFRV7fcZuAf4MvCbWQ+ygQr4fpIjwzfnR7MZgp4V1rU/kjkfJfkI8DDwxar69aznGVtVvVdV17L8LeudSVqfXktyK3Cyqo7MepYNtquq/oDlO9N+fjilOorNEPSpbjGgzW04j/ww8M2q+s6s59lIVfU28BRw84xHGdsu4DPDOeUHgBuS/PNsRxpfVZ0YHk8Cj7B8GnkUmyHo3mKgueEDwvuAo1X19VnPsxGSzCW5eHj+YeBTwM9mO9W4quorVbW9quZZ/nv8g6r6ixmPNaokFw0f9JPkIuDTwGhXr53zQa+qU8D7txg4CjzU/RYDSb4F/Ai4OsnxJHtnPdPIdgF3snzE9uLw86ezHmpk24Ank/yY5YOWQ1V1XlzGd565HHg6yUvAs8ATVfW9sd7snL9sUZI0nXP+CF2SNB2DLklNGHRJasKgS1ITBl2SmjDoktSEQZekJgy6JDXxf5ch0+HTxXSJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Good luck bud\n",
      "1-100\n",
      "0\n",
      "1-100\n",
      "1\n",
      "1-100\n",
      "2\n",
      "1-100\n",
      "3\n",
      "1-100\n",
      "4\n",
      "1-100\n",
      "5\n",
      "1-100\n",
      "6\n",
      "1-100\n",
      "7\n",
      "1-100\n",
      "8\n",
      "1-100\n",
      "9\n",
      "1-100\n",
      "10\n",
      "1-100\n",
      "11\n",
      "1-100\n",
      "12\n",
      "1-100\n",
      "13\n",
      "1-100\n",
      "14\n",
      "1-100\n",
      "15\n",
      "1-100\n",
      "16\n",
      "1-100\n",
      "17\n",
      "1-100\n",
      "18\n",
      "1-100\n",
      "19\n",
      "1-100\n",
      "20\n",
      "1-100\n",
      "21\n",
      "1-100\n",
      "22\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "import time\n",
    "import logging\n",
    "\n",
    "real_label = nd.ones((batch_size * 4,), ctx = ctx)\n",
    "fake_label = nd.zeros((batch_size * 4,), ctx = ctx)\n",
    "\n",
    "def facc(label, pred):\n",
    "    pred = pred.ravel()\n",
    "    label = label.ravel()\n",
    "    return ((pred>0.5) == label).mean()\n",
    "metric = mx.metric.CustomMetric(facc)\n",
    "\n",
    "\n",
    "stamp =  datetime.now().strftime('%Y_%m_%d-%H_%M')\n",
    "logging.basicConfig(level=logging.DEBUG)\n",
    "print(\"Begin\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "print(\"BEFORE THE FIRE:\")\n",
    "latent1 = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "fake = makeOutput(netG, latent1)\n",
    "for i in range(5):\n",
    "    latent = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "    fakeadd = makeOutput(netG, latent)\n",
    "    print(\"Plot %d\" % i)\n",
    "    fake = mx.ndarray.concat(fake, fakeadd, dim = 0)\n",
    "    data = fakeadd[i].asnumpy().tolist()\n",
    "    plt.hist(data, bins=bin_count)\n",
    "    plt.xticks(np.arange(min(data), max(data)+1, 1.0))\n",
    "\n",
    "    plt.show()\n",
    "    \n",
    "print(\"Good luck bud\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#set up Discriminator first\n",
    "for i in range(100):\n",
    "    \n",
    "    tic = time.time()\n",
    "    btic = time.time()\n",
    "    train_data.reset()\n",
    "    print(\"1-100\")\n",
    "    print(i)\n",
    "    iter = 0\n",
    "    #print(\"RUNNING\")\n",
    "    for batch1 in train_data:\n",
    "        #print(\"batch\")\n",
    "        #print(iter)\n",
    "        batch2 = next(train_data, batch1)\n",
    "        batch3 = next(train_data, batch1)\n",
    "        batch4 = next(train_data, batch1)\n",
    "        ############################\n",
    "        # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))\n",
    "        ###########################\n",
    "\n",
    "        data = (mx.ndarray.concat(batch1.data[0], batch2.data[0], batch3.data[0], batch4.data[0], dim = 0)).as_in_context(ctx)\n",
    "        \n",
    "        #test new shape for data\n",
    "        \n",
    "        \n",
    "        \n",
    "        #print(data)\n",
    "        #if iter == 0:\n",
    "            #x = data.T[0].asnumpy().tolist()\n",
    "            #y = data.T[1].asnumpy().tolist()\n",
    "            #print(x)\n",
    "            #print(y)\n",
    "            #plt.scatter(x,y)\n",
    "            #plt.show()\n",
    "            \n",
    "        \n",
    "        \n",
    "        noise = mx.ndarray.random_normal(0, 0.1, batch_size * 4, ctx = ctx)\n",
    "        real_label_noise = mx.ndarray.add(real_label, noise)\n",
    "        noise = mx.ndarray.random_normal(0, 0.1, batch_size * 4, ctx = ctx)\n",
    "        fake_label_noise = mx.ndarray.add(fake_label, noise)\n",
    "        #print(\"TESTING 123\")\n",
    "        #print(real_label_noise)\n",
    "        #print(fake_label_noise)\n",
    "        #print(\"THIS IS A CHECK\")\n",
    "        #print(data)\n",
    "        #print(len(data))\n",
    "        #print(len(data[0]))\n",
    "        #print(len(data[0][0]))\n",
    "        #print(len(batch1.data[0]))\n",
    "        #print(len(batch1.data[0][0]))\n",
    "        #print(len(batch1.data[0][0][0]))\n",
    "        #print(len(batch2.data[0]))\n",
    "        #print(len(batch2.data[0][0]))\n",
    "        #print(len(batch2.data[0][0][0]))\n",
    "        #print(data)\n",
    "        \n",
    "        \n",
    "        latent_z1 = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "        latent_z2 = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "        latent_z3 = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "        latent_z4 = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "\n",
    "        with autograd.record():\n",
    "            # train with real image\n",
    "            #print(\"Real Data\")\n",
    "            #print(data)\n",
    "            output = netD1(data).reshape((-1, 1))\n",
    "            #print(\"Output of Discriminator\")\n",
    "            #print(output)\n",
    "            errD1_real = loss(output, real_label_noise)\n",
    "            #print(\"This is the guess for real\")\n",
    "            #print(output)\n",
    "            metric.update([real_label,], [output,])\n",
    "\n",
    "            # train with fake image\n",
    "            firstFake = makeOutput(netG, latent_z1)\n",
    "            secondFake = makeOutput(netG, latent_z2)\n",
    "            thirdFake = makeOutput(netG, latent_z3)\n",
    "            fourthFake = makeOutput(netG, latent_z4)\n",
    "            #print(\"testing 1\")\n",
    "            #print(firstFake)\n",
    "            \n",
    "            #only add if using dense\n",
    "            #firstFake = firstFake.reshape((128, 1, 2))\n",
    "            #secondFake = secondFake.reshape((128, 1, 2))  \n",
    "            #print(\"testing 2\")\n",
    "            #print(firstFake)\n",
    "            \n",
    "\n",
    "            fake = mx.ndarray.concat(firstFake, secondFake, thirdFake, fourthFake, dim = 0)\n",
    "            #print(fake)\n",
    "            #print(fake)\n",
    "            #print(\"TESTING\")\n",
    "            #print(len(fake))\n",
    "            output = netD1(fake.detach()).reshape((-1, 1))\n",
    "            errD1_fake = loss(output, fake_label_noise)\n",
    "            errD1 = errD1_real + errD1_fake\n",
    "            errD1.backward()\n",
    "            metric.update([fake_label,], [output,])\n",
    "\n",
    "        trainerD1.step(data.shape[0])\n",
    "        \n",
    "        with autograd.record():\n",
    "            # train with real image\n",
    "            #print(\"Real Data\")\n",
    "            #print(data)\n",
    "            output = netD2(data).reshape((-1, 1))\n",
    "            #print(\"Output of Discriminator\")\n",
    "            #print(output)\n",
    "            errD2_real = loss(output, real_label_noise)\n",
    "            #print(\"This is the guess for real\")\n",
    "            #print(output)\n",
    "            metric.update([real_label,], [output,])\n",
    "\n",
    "            # train with fake image\n",
    "            firstFake = makeOutput(netG, latent_z1)\n",
    "            secondFake = makeOutput(netG, latent_z2)\n",
    "            thirdFake = makeOutput(netG, latent_z3)\n",
    "            fourthFake = makeOutput(netG, latent_z4)\n",
    "            #print(\"testing 1\")\n",
    "            #print(firstFake)\n",
    "            \n",
    "            #only add if using dense\n",
    "            #firstFake = firstFake.reshape((128, 1, 2))\n",
    "            #secondFake = secondFake.reshape((128, 1, 2))  \n",
    "            #print(\"testing 2\")\n",
    "            #print(firstFake)\n",
    "            \n",
    "\n",
    "            fake = mx.ndarray.concat(firstFake, secondFake, thirdFake, fourthFake, dim = 0)\n",
    "            #print(\"TESTING\")\n",
    "            #print(len(fake))\n",
    "            output = netD2(fake.detach()).reshape((-1, 1))\n",
    "            errD2_fake = loss(output, fake_label_noise)\n",
    "            errD2 = errD2_real + errD2_fake\n",
    "            errD2.backward()\n",
    "            metric.update([fake_label,], [output,])\n",
    "\n",
    "        trainerD2.step(data.shape[0])\n",
    "        iter+=1\n",
    "        \n",
    "        \n",
    "    name, acc = metric.get()\n",
    "    metric.reset()\n",
    "    \n",
    "print(\"Done setting up Discriminator\")\n",
    "for epoch in range(epochs+1):\n",
    "    print(epoch)\n",
    "    train_data.reset()\n",
    "    tic = time.time()\n",
    "    btic = time.time()\n",
    "    count = 0\n",
    "    iter = 0\n",
    "    #print(\"RUNNING\")\n",
    "    for batch1 in train_data:\n",
    "        #print(iter)\n",
    "        batch2 = next(train_data, batch1)\n",
    "        batch3 = next(train_data, batch1)\n",
    "        batch4 = next(train_data, batch1)\n",
    "        ############################\n",
    "        # (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))\n",
    "        ###########################\n",
    "\n",
    "        data = (mx.ndarray.concat(batch1.data[0], batch2.data[0], batch3.data[0], batch4.data[0], dim = 0)).as_in_context(ctx)\n",
    "        #if count == 0:\n",
    "            #print(\"Real DATA\")\n",
    "            #x = \n",
    "            #plt.scatter(x,y)\n",
    "            #plt.show()\n",
    "            \n",
    "            \n",
    "            #print(\"END of Real Data\")\n",
    "        \n",
    "        \n",
    "        noise = mx.ndarray.random_normal(0, 0.1, batch_size * 4, ctx = ctx)\n",
    "        real_label_noise = mx.ndarray.add(real_label, noise)\n",
    "        noise = mx.ndarray.random_normal(0, 0.1, batch_size * 4, ctx = ctx)\n",
    "        fake_label_noise = mx.ndarray.add(fake_label, noise)\n",
    "        #print(\"THIS IS A CHECK\")\n",
    "        #print(data)\n",
    "        #print(len(data))\n",
    "        #print(len(data[0]))\n",
    "        #print(len(data[0][0]))\n",
    "        #print(len(batch1.data[0]))\n",
    "        #print(len(batch1.data[0][0]))\n",
    "        #print(len(batch1.data[0][0][0]))\n",
    "        #print(len(batch2.data[0]))\n",
    "        #print(len(batch2.data[0][0]))\n",
    "        #print(len(batch2.data[0][0][0]))\n",
    "        #print(data)\n",
    "        \n",
    "        \n",
    "        latent_z1 = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "        latent_z2 = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "        latent_z3 = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "        latent_z4 = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "        with autograd.record():\n",
    "            # train with real image\n",
    "            #print(\"Real Data\")\n",
    "            #print(data)\n",
    "            output = netD1(data).reshape((-1, 1))\n",
    "            #print(\"Output of Discriminator\")\n",
    "            #print(output)\n",
    "            errD1_real = loss(output, real_label_noise)\n",
    "            #print(\"This is the guess for real\")\n",
    "            #print(output)\n",
    "            metric.update([real_label], [output,])\n",
    "\n",
    "            # train with fake image\n",
    "            firstFake = makeOutput(netG, latent_z1)\n",
    "            secondFake = makeOutput(netG, latent_z2)\n",
    "            thirdFake = makeOutput(netG, latent_z3)\n",
    "            fourthFake = makeOutput(netG, latent_z4)\n",
    "            #print(\"testing 1\")\n",
    "            #print(firstFake)\n",
    "\n",
    "            #only add if using dense\n",
    "            #firstFake = firstFake.reshape((128, 1, 2))\n",
    "            #secondFake = secondFake.reshape((128, 1, 2))  \n",
    "            #print(\"testing 2\")\n",
    "            #print(firstFake)\n",
    "\n",
    "\n",
    "            fake = mx.ndarray.concat(firstFake, secondFake, thirdFake, fourthFake, dim = 0)\n",
    "            #print(\"TESTING\")\n",
    "            #print(len(fake))\n",
    "            output = netD1(fake.detach()).reshape((-1, 1))\n",
    "            errD1_fake = loss(output, fake_label_noise)\n",
    "            errD1 = errD1_real + errD1_fake\n",
    "            errD1.backward()\n",
    "            metric.update([fake_label,], [output,])\n",
    "\n",
    "        trainerD1.step(data.shape[0])\n",
    "        \n",
    "        with autograd.record():\n",
    "            # train with real image\n",
    "            #print(\"Real Data\")\n",
    "            #print(data)\n",
    "            output = netD2(data).reshape((-1, 1))\n",
    "            #print(\"Output of Discriminator\")\n",
    "            #print(output)\n",
    "            errD2_real = loss(output, real_label_noise)\n",
    "            #print(\"This is the guess for real\")\n",
    "            #print(output)\n",
    "            metric.update([real_label], [output,])\n",
    "\n",
    "            # train with fake image\n",
    "            firstFake = makeOutput(netG, latent_z1)\n",
    "            secondFake = makeOutput(netG, latent_z2)\n",
    "            thirdFake = makeOutput(netG, latent_z3)\n",
    "            fourthFake = makeOutput(netG, latent_z4)\n",
    "            #print(\"testing 1\")\n",
    "            #print(firstFake)\n",
    "\n",
    "            #only add if using dense\n",
    "            #firstFake = firstFake.reshape((128, 1, 2))\n",
    "            #secondFake = secondFake.reshape((128, 1, 2))  \n",
    "            #print(\"testing 2\")\n",
    "            #print(firstFake)\n",
    "\n",
    "\n",
    "            fake = mx.ndarray.concat(firstFake, secondFake, thirdFake, fourthFake, dim = 0)\n",
    "            #print(\"TESTING\")\n",
    "            #print(len(fake))\n",
    "            output = netD2(fake.detach()).reshape((-1, 1))\n",
    "            errD2_fake = loss(output, fake_label_noise)\n",
    "            errD2 = errD2_real + errD2_fake\n",
    "            errD2.backward()\n",
    "            metric.update([fake_label,], [output,])\n",
    "\n",
    "        trainerD2.step(data.shape[0])\n",
    "            \n",
    "        \n",
    "        #print(\"Generator\")\n",
    "        ############################\n",
    "        # (2) Update G network: maximize log(D(G(z)))\n",
    "        ###########################\n",
    "        with autograd.record():\n",
    "            firstFake = makeOutput(netG, latent_z1)\n",
    "            secondFake = makeOutput(netG, latent_z2)\n",
    "            thirdFake = makeOutput(netG, latent_z3)\n",
    "            fourthFake = makeOutput(netG, latent_z4)\n",
    "            #print(fake1)\n",
    "            #rint(fake1.T)\n",
    "            #print(fake1.T[0])\n",
    "           # print(fake1.T[0][1])\n",
    "            \n",
    "            \n",
    "            #only add if using dense\n",
    "            #fake1 = fake1.reshape((128, 1, 2))\n",
    "            #fake2 = fake2.reshape((128, 1, 2))\n",
    "            \n",
    "            output = netD1(mx.ndarray.concat(firstFake, secondFake, thirdFake, fourthFake, dim = 0)).reshape((-1, 1))\n",
    "            errG = loss(output, real_label_noise)\n",
    "            errG.backward()\n",
    "        trainerG.step(mx.ndarray.concat(batch1.data[0], batch2.data[0], batch3.data[0], batch4.data[0]).shape[0], ignore_stale_grad = True)\n",
    "        with autograd.record():\n",
    "            firstFake = makeOutput(netG, latent_z1)\n",
    "            secondFake = makeOutput(netG, latent_z2)\n",
    "            thirdFake = makeOutput(netG, latent_z3)\n",
    "            fourthFake = makeOutput(netG, latent_z4)\n",
    "            #print(fake1)\n",
    "            #rint(fake1.T)\n",
    "            #print(fake1.T[0])\n",
    "           # print(fake1.T[0][1])\n",
    "            \n",
    "            \n",
    "            #only add if using dense\n",
    "            #fake1 = fake1.reshape((128, 1, 2))\n",
    "            #fake2 = fake2.reshape((128, 1, 2))\n",
    "            \n",
    "            output = netD2(mx.ndarray.concat(firstFake, secondFake, thirdFake, fourthFake, dim = 0)).reshape((-1, 1))\n",
    "            errG = loss(output, real_label_noise)\n",
    "            errG.backward()\n",
    "\n",
    "        trainerG.step(mx.ndarray.concat(batch1.data[0], batch2.data[0], batch3.data[0], batch4.data[0]).shape[0], ignore_stale_grad = True)\n",
    "        \n",
    "        \n",
    "        \n",
    "\n",
    "        # Print log infomation every ten batches\n",
    "        if iter % 10 == 0:\n",
    "            name, acc = metric.get()\n",
    "            #logging.firstFake info('speed: {} samples/s'.format(batch_size / (time.time() - btic)))\n",
    "            #logging.info('discriminator loss = %f, generator loss = %f, binary training acc = %f at iter %d epoch %d'\n",
    "            #         %(nd.mean(errD).asscalar(),\n",
    "            #           nd.mean(errG).asscalar(), acc, iter, epoch))\n",
    "        iter = iter + 1\n",
    "        btic = time.time()\n",
    "\n",
    "    name, acc = metric.get()\n",
    "    if acc == 1.0 and epoch >201:\n",
    "        print(\"FAIL\")\n",
    "        sys.exit(\"D too good\")\n",
    "    metric.reset()\n",
    "    #logging.info('\\nbinary training acc at epoch %d: %s=%f' % (epoch, name, acc))\n",
    "    #logging.info('time: %f' % (time.time() - tic))\n",
    "\n",
    "    #Visualize one generated image for each epoch\n",
    "    fake_img = firstFake[0]\n",
    "    #print(\"testing\")\n",
    "    #print(\"Fake data\")\n",
    "    #print(fake1)\n",
    "    ##print(\"fake data transposed\")\n",
    "    #print(fake1.T)\n",
    "    #print(len(fake))0\n",
    "    #print(len(fake[0]))\n",
    "    #print(len(fake[0][0]))\n",
    "    #print(fake)\n",
    "    \n",
    "    \n",
    "    #test small print\n",
    "    #print(\"epoch %d\" % (epoch))\n",
    "    #print(\"X: %s   Y: %s  \" % (fake_img[0][0],fake_img[0][1]))\n",
    "    #x= fake.T[0][0].asnumpy().tolist()\n",
    "    #y = fake.T[1][0].asnumpy().tolist()\n",
    "    #print(\"Plot\")\n",
    "    #plt.scatter(x,y)\n",
    "    #plt.show()\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    #real print\n",
    "    if(epoch%100 ==0):\n",
    "        print(\"Epoch: %d\" % epoch)\n",
    "    if(epoch%200 == 0):# or epoch % 200 == 1 or epoch % 200 == 2 or epoch % 200 == 3):\n",
    "        logging.info('\\nbinary training acc at epoch %d: %s=%f' % (epoch, name, acc))\n",
    "        logging.info('time: %f' % (time.time() - tic))\n",
    "        logging.info('time: %f' % (time.time() - tic))\n",
    "        print(\"epoch %d\" % (epoch))\n",
    "        \n",
    "        #For convolution?\n",
    "        #print(\"X: %s   Y: %s  \" % (fake_img[0][0],fake_img[0][1]))\n",
    "        #x= fake1.T[0][0].asnumpy().tolist()\n",
    "        #y = fake1.T[0][1].asnumpy().tolist()\n",
    "        latent1 = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "        firstFake = makeOutput(netG, latent1)\n",
    "        for i in range(5):\n",
    "            latent = mx.nd.random_normal(loc = 0, scale = 3, shape=(batch_size, latent_z_size, 2), ctx=ctx)\n",
    "            fakeadd = makeOutput(netG, latent)\n",
    "            \n",
    "            data = fakeadd[i].asnumpy().tolist()\n",
    "            plt.hist(data, bins=bin_count)\n",
    "            plt.xticks(np.arange(min(data), max(data)+1, 1.0))\n",
    "            plt.show()\n",
    "            fake = mx.ndarray.concat(fake, fakeadd, dim = 0)\n",
    "        #fake = mx.ndarray.concat(fake1, fake2, fake3, fake4, dim = 0)\n",
    "        \n",
    "\n",
    "        #print(\"X: \")\n",
    "        #print(fake.T[0][0])\n",
    "        #print(\"Y: \")\n",
    "        #print(fake.T[0][1])\n",
    "        #print(\"\")\n",
    "        #print(\"\")\n",
    "        print(fake)\n",
    "\n",
    "\n",
    "        #plt.show()   \n",
    "    \n",
    "    # visualize(fake_img)\n",
    "    # plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}