09_up_and_running_with_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPython 3.6.6\n",
      "IPython 6.5.0\n",
      "\n",
      "numpy 1.15.2\n",
      "scipy 1.1.0\n",
      "matplotlib 2.2.3\n",
      "sklearn 0.20.0\n",
      "tensorflow 1.12.0\n"
     ]
    }
   ],
   "source": [
    "%load_ext watermark\n",
    "%watermark -v -p numpy,scipy,matplotlib,sklearn,tensorflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**9장 – 텐서플로 시작하기**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_이 노트북은 9장에 있는 모든 샘플 코드와 연습문제 해답을 가지고 있습니다._"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 설정"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "파이썬 2와 3을 모두 지원합니다. 공통 모듈을 임포트하고 맷플롯립 그림이 노트북 안에 포함되도록 설정하고 생성한 그림을 저장하기 위한 함수를 준비합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 파이썬 2와 파이썬 3 지원\n",
    "from __future__ import division, print_function, unicode_literals\n",
    "\n",
    "# 공통\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# 일관된 출력을 위해 유사난수 초기화\n",
    "def reset_graph(seed=42):\n",
    "    tf.reset_default_graph()\n",
    "    tf.set_random_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "# 맷플롯립 설정\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['axes.labelsize'] = 14\n",
    "plt.rcParams['xtick.labelsize'] = 12\n",
    "plt.rcParams['ytick.labelsize'] = 12\n",
    "\n",
    "# 한글출력\n",
    "plt.rcParams['font.family'] = 'NanumBarunGothic'\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 그림을 저장할 폴더\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"tensorflow\"\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True):\n",
    "    path = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID, fig_id + \".png\")\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format='png', dpi=300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 계산 그래프 만들고 실행하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "reset_graph()\n",
    "\n",
    "x = tf.Variable(3, name=\"x\")\n",
    "y = tf.Variable(4, name=\"y\")\n",
    "f = x*x*y + y + 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'add_1:0' shape=() dtype=int32>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "sess.run(x.initializer)\n",
    "sess.run(y.initializer)\n",
    "result = sess.run(f)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    x.initializer.run()\n",
    "    y.initializer.run()\n",
    "    result = f.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    result = f.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n"
     ]
    }
   ],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "init.run()\n",
    "result = f.eval()\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 그래프 다루기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reset_graph()\n",
    "\n",
    "x1 = tf.Variable(1)\n",
    "x1.graph is tf.get_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    x2 = tf.Variable(2)\n",
    "\n",
    "x2.graph is graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x2.graph is tf.get_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "w = tf.constant(3)\n",
    "x = w + 2\n",
    "y = x + 5\n",
    "z = x * 3\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    print(y.eval())  # 10\n",
    "    print(z.eval())  # 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    y_val, z_val = sess.run([y, z])\n",
    "    print(y_val)  # 10\n",
    "    print(z_val)  # 15"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 선형 회귀"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 정규방정식을 사용해서"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "reset_graph()\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "m, n = housing.data.shape\n",
    "housing_data_plus_bias = np.c_[np.ones((m, 1)), housing.data]\n",
    "\n",
    "X = tf.constant(housing_data_plus_bias, dtype=tf.float32, name=\"X\")\n",
    "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float32, name=\"y\")\n",
    "XT = tf.transpose(X)\n",
    "theta = tf.matmul(tf.matmul(tf.matrix_inverse(tf.matmul(XT, X)), XT), y)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    theta_value = theta.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-3.7185181e+01],\n",
       "       [ 4.3633747e-01],\n",
       "       [ 9.3952334e-03],\n",
       "       [-1.0711310e-01],\n",
       "       [ 6.4479220e-01],\n",
       "       [-4.0338000e-06],\n",
       "       [-3.7813708e-03],\n",
       "       [-4.2348403e-01],\n",
       "       [-4.3721911e-01]], dtype=float32)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta_value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "넘파이와 비교"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-3.69419202e+01]\n",
      " [ 4.36693293e-01]\n",
      " [ 9.43577803e-03]\n",
      " [-1.07322041e-01]\n",
      " [ 6.45065694e-01]\n",
      " [-3.97638942e-06]\n",
      " [-3.78654265e-03]\n",
      " [-4.21314378e-01]\n",
      " [-4.34513755e-01]]\n"
     ]
    }
   ],
   "source": [
    "X = housing_data_plus_bias\n",
    "y = housing.target.reshape(-1, 1)\n",
    "theta_numpy = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(y)\n",
    "\n",
    "print(theta_numpy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "사이킷런과 비교"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-3.69419202e+01]\n",
      " [ 4.36693293e-01]\n",
      " [ 9.43577803e-03]\n",
      " [-1.07322041e-01]\n",
      " [ 6.45065694e-01]\n",
      " [-3.97638942e-06]\n",
      " [-3.78654265e-03]\n",
      " [-4.21314378e-01]\n",
      " [-4.34513755e-01]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "lin_reg = LinearRegression()\n",
    "lin_reg.fit(housing.data, housing.target.reshape(-1, 1))\n",
    "\n",
    "print(np.r_[lin_reg.intercept_.reshape(-1, 1), lin_reg.coef_.T])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 배치 경사 하강법을 사용해서"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "경사 하강법은 먼저 특성 벡터의 스케일을 조정해야 합니다. 텐서플로를 사용해 할 수 있지만 그냥 여기서는 사이킷런을 사용하겠습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "scaled_housing_data = scaler.fit_transform(housing.data)\n",
    "scaled_housing_data_plus_bias = np.c_[np.ones((m, 1)), scaled_housing_data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 1.00000000e+00  6.60969987e-17  5.50808322e-18  6.60969987e-17\n",
      " -1.06030602e-16 -1.10161664e-17  3.44255201e-18 -1.07958431e-15\n",
      " -8.52651283e-15]\n",
      "[ 0.38915536  0.36424355  0.5116157  ... -0.06612179 -0.06360587\n",
      "  0.01359031]\n",
      "0.11111111111111005\n",
      "(20640, 9)\n"
     ]
    }
   ],
   "source": [
    "print(scaled_housing_data_plus_bias.mean(axis=0))\n",
    "print(scaled_housing_data_plus_bias.mean(axis=1))\n",
    "print(scaled_housing_data_plus_bias.mean())\n",
    "print(scaled_housing_data_plus_bias.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 수동으로 그래디언트 계산하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "에포크 0 MSE = 9.161542\n",
      "에포크 100 MSE = 0.7145004\n",
      "에포크 200 MSE = 0.56670487\n",
      "에포크 300 MSE = 0.5555718\n",
      "에포크 400 MSE = 0.5488112\n",
      "에포크 500 MSE = 0.5436363\n",
      "에포크 600 MSE = 0.5396291\n",
      "에포크 700 MSE = 0.5365092\n",
      "에포크 800 MSE = 0.53406775\n",
      "에포크 900 MSE = 0.5321473\n"
     ]
    }
   ],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.constant(scaled_housing_data_plus_bias, dtype=tf.float32, name=\"X\")\n",
    "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float32, name=\"y\")\n",
    "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n",
    "y_pred = tf.matmul(X, theta, name=\"predictions\")\n",
    "error = y_pred - y\n",
    "mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n",
    "gradients = 2/m * tf.matmul(tf.transpose(X), error)\n",
    "training_op = tf.assign(theta, theta - learning_rate * gradients)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        if epoch % 100 == 0:\n",
    "            print(\"에포크\", epoch, \"MSE =\", mse.eval())\n",
    "        sess.run(training_op)\n",
    "    \n",
    "    best_theta = theta.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2.0685523 ],\n",
       "       [ 0.8874027 ],\n",
       "       [ 0.14401656],\n",
       "       [-0.34770882],\n",
       "       [ 0.36178368],\n",
       "       [ 0.00393811],\n",
       "       [-0.04269556],\n",
       "       [-0.6614529 ],\n",
       "       [-0.6375279 ]], dtype=float32)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 자동미분 사용하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`gradients = ...` 라인만 빼고 위와 동일합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.constant(scaled_housing_data_plus_bias, dtype=tf.float32, name=\"X\")\n",
    "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float32, name=\"y\")\n",
    "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n",
    "y_pred = tf.matmul(X, theta, name=\"predictions\")\n",
    "error = y_pred - y\n",
    "mse = tf.reduce_mean(tf.square(error), name=\"mse\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "gradients = tf.gradients(mse, [theta])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "에포크 0 MSE = 9.161542\n",
      "에포크 100 MSE = 0.71450037\n",
      "에포크 200 MSE = 0.56670487\n",
      "에포크 300 MSE = 0.5555718\n",
      "에포크 400 MSE = 0.54881126\n",
      "에포크 500 MSE = 0.5436363\n",
      "에포크 600 MSE = 0.53962916\n",
      "에포크 700 MSE = 0.5365092\n",
      "에포크 800 MSE = 0.53406775\n",
      "에포크 900 MSE = 0.5321473\n",
      "best_theta:\n",
      "[[ 2.0685523 ]\n",
      " [ 0.8874027 ]\n",
      " [ 0.14401656]\n",
      " [-0.3477088 ]\n",
      " [ 0.36178365]\n",
      " [ 0.00393811]\n",
      " [-0.04269556]\n",
      " [-0.66145283]\n",
      " [-0.6375278 ]]\n"
     ]
    }
   ],
   "source": [
    "training_op = tf.assign(theta, theta - learning_rate * gradients)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        if epoch % 100 == 0:\n",
    "            print(\"에포크\", epoch, \"MSE =\", mse.eval())\n",
    "        sess.run(training_op)\n",
    "    \n",
    "    best_theta = theta.eval()\n",
    "\n",
    "print(\"best_theta:\")\n",
    "print(best_theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`a`와 `b`에 대한 다음 함수의 편도함수를 어떻게 구할 수 있나요?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_func(a, b):\n",
    "    z = 0\n",
    "    for i in range(100):\n",
    "        z = a * np.cos(z + i) + z * np.sin(b - i)\n",
    "    return z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.21253923284754914"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_func(0.2, 0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "a = tf.Variable(0.2, name=\"a\")\n",
    "b = tf.Variable(0.3, name=\"b\")\n",
    "z = tf.constant(0.0, name=\"z0\")\n",
    "for i in range(100):\n",
    "    z = a * tf.cos(z + i) + z * tf.sin(b - i)\n",
    "\n",
    "grads = tf.gradients(z, [a, b])\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$a=0.2$와 $b=0.3$일 때 함수 값을 계산하고 그 다음 $a$와 $b$에 대한 편미분을 구합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.21253741\n",
      "[-1.1388495, 0.19671397]\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    print(z.eval())\n",
    "    print(sess.run(grads))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `GradientDescentOptimizer` 사용하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.constant(scaled_housing_data_plus_bias, dtype=tf.float32, name=\"X\")\n",
    "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float32, name=\"y\")\n",
    "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n",
    "y_pred = tf.matmul(X, theta, name=\"predictions\")\n",
    "error = y_pred - y\n",
    "mse = tf.reduce_mean(tf.square(error), name=\"mse\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(mse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "에포크 0 MSE = 9.161542\n",
      "에포크 100 MSE = 0.7145004\n",
      "에포크 200 MSE = 0.56670487\n",
      "에포크 300 MSE = 0.5555718\n",
      "에포크 400 MSE = 0.54881126\n",
      "에포크 500 MSE = 0.5436363\n",
      "에포크 600 MSE = 0.53962916\n",
      "에포크 700 MSE = 0.5365092\n",
      "에포크 800 MSE = 0.53406775\n",
      "에포크 900 MSE = 0.5321473\n",
      "best_theta:\n",
      "[[ 2.0685523 ]\n",
      " [ 0.8874027 ]\n",
      " [ 0.14401656]\n",
      " [-0.3477088 ]\n",
      " [ 0.36178365]\n",
      " [ 0.00393811]\n",
      " [-0.04269556]\n",
      " [-0.66145283]\n",
      " [-0.6375278 ]]\n"
     ]
    }
   ],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        if epoch % 100 == 0:\n",
    "            print(\"에포크\", epoch, \"MSE =\", mse.eval())\n",
    "        sess.run(training_op)\n",
    "    \n",
    "    best_theta = theta.eval()\n",
    "\n",
    "print(\"best_theta:\")\n",
    "print(best_theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 모멘텀 옵티마이저 사용하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.constant(scaled_housing_data_plus_bias, dtype=tf.float32, name=\"X\")\n",
    "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float32, name=\"y\")\n",
    "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n",
    "y_pred = tf.matmul(X, theta, name=\"predictions\")\n",
    "error = y_pred - y\n",
    "mse = tf.reduce_mean(tf.square(error), name=\"mse\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate,\n",
    "                                       momentum=0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_theta:\n",
      "[[ 2.068558  ]\n",
      " [ 0.82962847]\n",
      " [ 0.11875335]\n",
      " [-0.26554456]\n",
      " [ 0.3057109 ]\n",
      " [-0.00450249]\n",
      " [-0.03932662]\n",
      " [-0.8998645 ]\n",
      " [-0.8705207 ]]\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        sess.run(training_op)\n",
    "    \n",
    "    best_theta = theta.eval()\n",
    "\n",
    "print(\"best_theta:\")\n",
    "print(best_theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 훈련 알고리즘에 데이터 주입하기"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 플레이스홀더 노드"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[6. 7. 8.]]\n"
     ]
    }
   ],
   "source": [
    "reset_graph()\n",
    "\n",
    "A = tf.placeholder(tf.float32, shape=(None, 3))\n",
    "B = A + 5\n",
    "with tf.Session() as sess:\n",
    "    B_val_1 = B.eval(feed_dict={A: [[1, 2, 3]]})\n",
    "    B_val_2 = B.eval(feed_dict={A: [[4, 5, 6], [7, 8, 9]]})\n",
    "\n",
    "print(B_val_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 9. 10. 11.]\n",
      " [12. 13. 14.]]\n"
     ]
    }
   ],
   "source": [
    "print(B_val_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 미니배치 경사 하강법"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 1000\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n + 1), name=\"X\")\n",
    "y = tf.placeholder(tf.float32, shape=(None, 1), name=\"y\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n",
    "y_pred = tf.matmul(X, theta, name=\"predictions\")\n",
    "error = y_pred - y\n",
    "mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 100\n",
    "n_batches = int(np.ceil(m / batch_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_batch(epoch, batch_index, batch_size):\n",
    "    np.random.seed(epoch * n_batches + batch_index)  # not shown in the book\n",
    "    indices = np.random.randint(m, size=batch_size)  # not shown\n",
    "    X_batch = scaled_housing_data_plus_bias[indices] # not shown\n",
    "    y_batch = housing.target.reshape(-1, 1)[indices] # not shown\n",
    "    return X_batch, y_batch\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for batch_index in range(n_batches):\n",
    "            X_batch, y_batch = fetch_batch(epoch, batch_index, batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "\n",
    "    best_theta = theta.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2.0703337 ],\n",
       "       [ 0.8637145 ],\n",
       "       [ 0.12255152],\n",
       "       [-0.31211877],\n",
       "       [ 0.38510376],\n",
       "       [ 0.00434168],\n",
       "       [-0.0123295 ],\n",
       "       [-0.83376896],\n",
       "       [-0.8030471 ]], dtype=float32)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 모델의 저장과 복원"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "에포크 0 MSE = 9.161542\n",
      "에포크 100 MSE = 0.7145004\n",
      "에포크 200 MSE = 0.56670487\n",
      "에포크 300 MSE = 0.5555718\n",
      "에포크 400 MSE = 0.54881126\n",
      "에포크 500 MSE = 0.5436363\n",
      "에포크 600 MSE = 0.53962916\n",
      "에포크 700 MSE = 0.5365092\n",
      "에포크 800 MSE = 0.53406775\n",
      "에포크 900 MSE = 0.5321473\n"
     ]
    }
   ],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_epochs = 1000                                                                       # 책에는 없습니다.\n",
    "learning_rate = 0.01                                                                  # 책에는 없습니다.\n",
    "\n",
    "X = tf.constant(scaled_housing_data_plus_bias, dtype=tf.float32, name=\"X\")            # 책에는 없습니다.\n",
    "y = tf.constant(housing.target.reshape(-1, 1), dtype=tf.float32, name=\"y\")            # 책에는 없습니다.\n",
    "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n",
    "y_pred = tf.matmul(X, theta, name=\"predictions\")                                      # 책에는 없습니다.\n",
    "error = y_pred - y                                                                    # 책에는 없습니다.\n",
    "mse = tf.reduce_mean(tf.square(error), name=\"mse\")                                    # 책에는 없습니다.\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)            # 책에는 없습니다.\n",
    "training_op = optimizer.minimize(mse)                                                 # 책에는 없습니다.\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        if epoch % 100 == 0:\n",
    "            print(\"에포크\", epoch, \"MSE =\", mse.eval())                                # 책에는 없습니다.\n",
    "            save_path = saver.save(sess, \"/tmp/my_model.ckpt\")\n",
    "        sess.run(training_op)\n",
    "    \n",
    "    best_theta = theta.eval()\n",
    "    save_path = saver.save(sess, \"/tmp/my_model_final.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2.0685523 ],\n",
       "       [ 0.8874027 ],\n",
       "       [ 0.14401656],\n",
       "       [-0.3477088 ],\n",
       "       [ 0.36178365],\n",
       "       [ 0.00393811],\n",
       "       [-0.04269556],\n",
       "       [-0.66145283],\n",
       "       [-0.6375278 ]], dtype=float32)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_theta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /tmp/my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"/tmp/my_model_final.ckpt\")\n",
    "    best_theta_restored = theta.eval() # 책에는 없습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.allclose(best_theta, best_theta_restored)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`theta`를 `\"weights\"`와 같은 다른 이름으로 저장하고 복원하는 Saver 객체를 원할 경우엔:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "saver = tf.train.Saver({\"weights\": theta})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "기본적으로 Saver 객체는 `.meta` 확장자를 가진 두 번째 파일에 그래프 구조도 저장합니다. `tf.train.import_meta_graph()` 함수를 사용하여 그래프 구조를 복원할 수 있습니다. 이 함수는 저장된 그래프를 기본 그래프로 로드하고 상태(즉, 변수 값)를 복원할 수 있는 `Saver` 객체를 반환합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from /tmp/my_model_final.ckpt\n"
     ]
    }
   ],
   "source": [
    "reset_graph()\n",
    "# 빈 그래프로 시작합니다\n",
    "\n",
    "saver = tf.train.import_meta_graph(\"/tmp/my_model_final.ckpt.meta\")  # 그래프 구조를 로드합니다.\n",
    "theta = tf.get_default_graph().get_tensor_by_name(\"theta:0\") # 책에는 없습니다.\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, \"/tmp/my_model_final.ckpt\")  # 그래프 상태를 로드합니다.\n",
    "    best_theta_restored = theta.eval() # 책에는 없습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.allclose(best_theta, best_theta_restored)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이를 사용하면 그래프를 만든 파이썬 코드가 없이도 미리 훈련된 모델을 임포트할 수 있습니다. 모델을 저장하고 변경할 때도 매우 편리합니다. 이전에 저장된 모델을 구축한 코드의 버전을 찾지 않아도 로드할 수 있습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 그래프 시각화\n",
    "## 쥬피터 노트북안에서"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "주피터 노트북에서 그래프를 나타내기 위해 https://tensorboard.appspot.com/ 에 서비스 중인 텐서보드 서버를 사용하겠습니다(즉, 인터넷 연결이 안되면 작동되지 않습니다). 제가 아는 한 이 코드는 Alex Mordvintsev가 [딥드림 튜토리얼](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/deepdream/deepdream.ipynb)에서 처음 사용했습니다. 또는 [tfgraphviz](https://github.com/akimach/tfgraphviz)를 사용할 수도 있습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow_graph_in_jupyter import show_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe seamless style=\"width:1200px;height:620px;border:0\" srcdoc=\"\n",
       "        <script src=&quot;//cdnjs.cloudflare.com/ajax/libs/polymer/0.3.3/platform.js&quot;></script>\n",
       "        <script>\n",
       "          function load() {\n",
       "            document.getElementById(&quot;graph0.3745401188473625&quot;).pbtxt = 'node {\\n  name: &quot;X&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 20640\\n          }\\n          dim {\\n            size: 9\\n          }\\n        }\\n        tensor_content: &quot;<stripped 743040 bytes>&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;y&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n          dim {\\n            size: 20640\\n          }\\n          dim {\\n            size: 1\\n          }\\n        }\\n        tensor_content: &quot;<stripped 82560 bytes>&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\t\\\\000\\\\000\\\\000\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/min&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: -1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/max&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/RandomUniform&quot;\\n  op: &quot;RandomUniform&quot;\\n  input: &quot;random_uniform/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;seed&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n  attr {\\n    key: &quot;seed2&quot;\\n    value {\\n      i: 42\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;random_uniform/max&quot;\\n  input: &quot;random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform/mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;random_uniform/RandomUniform&quot;\\n  input: &quot;random_uniform/sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;random_uniform&quot;\\n  op: &quot;Add&quot;\\n  input: &quot;random_uniform/mul&quot;\\n  input: &quot;random_uniform/min&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;theta&quot;\\n  op: &quot;VariableV2&quot;\\n  attr {\\n    key: &quot;container&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;shape&quot;\\n    value {\\n      shape {\\n        dim {\\n          size: 9\\n        }\\n        dim {\\n          size: 1\\n        }\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;shared_name&quot;\\n    value {\\n      s: &quot;&quot;\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;theta/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;theta&quot;\\n  input: &quot;random_uniform&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@theta&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;theta/read&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;theta&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@theta&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;predictions&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;theta/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;sub&quot;\\n  op: &quot;Sub&quot;\\n  input: &quot;predictions&quot;\\n  input: &quot;y&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Square&quot;\\n  op: &quot;Square&quot;\\n  input: &quot;sub&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\000\\\\000\\\\000\\\\000\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;mse&quot;\\n  op: &quot;Mean&quot;\\n  input: &quot;Square&quot;\\n  input: &quot;Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tidx&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;keep_dims&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n          }\\n        }\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/grad_ys_0&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 1.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Fill&quot;\\n  op: &quot;Fill&quot;\\n  input: &quot;gradients/Shape&quot;\\n  input: &quot;gradients/grad_ys_0&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;index_type&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Reshape/shape&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\001\\\\000\\\\000\\\\000\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Reshape&quot;\\n  op: &quot;Reshape&quot;\\n  input: &quot;gradients/Fill&quot;\\n  input: &quot;gradients/mse_grad/Reshape/shape&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tshape&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_INT32\\n        tensor_shape {\\n          dim {\\n            size: 2\\n          }\\n        }\\n        tensor_content: &quot;\\\\240P\\\\000\\\\000\\\\001\\\\000\\\\000\\\\000&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Tile&quot;\\n  op: &quot;Tile&quot;\\n  input: &quot;gradients/mse_grad/Reshape&quot;\\n  input: &quot;gradients/mse_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;Tmultiples&quot;\\n    value {\\n      type: DT_INT32\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/Const_1&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 20640.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/mse_grad/truediv&quot;\\n  op: &quot;RealDiv&quot;\\n  input: &quot;gradients/mse_grad/Tile&quot;\\n  input: &quot;gradients/mse_grad/Const_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Square_grad/Const&quot;\\n  op: &quot;Const&quot;\\n  input: &quot;^gradients/mse_grad/truediv&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 2.0\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Square_grad/Mul&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;sub&quot;\\n  input: &quot;gradients/Square_grad/Const&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/Square_grad/Mul_1&quot;\\n  op: &quot;Mul&quot;\\n  input: &quot;gradients/mse_grad/truediv&quot;\\n  input: &quot;gradients/Square_grad/Mul&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/Neg&quot;\\n  op: &quot;Neg&quot;\\n  input: &quot;gradients/Square_grad/Mul_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/Square_grad/Mul_1&quot;\\n  input: &quot;^gradients/sub_grad/Neg&quot;\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/Square_grad/Mul_1&quot;\\n  input: &quot;^gradients/sub_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/Square_grad/Mul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/sub_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/sub_grad/Neg&quot;\\n  input: &quot;^gradients/sub_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/sub_grad/Neg&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/predictions_grad/MatMul&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;gradients/sub_grad/tuple/control_dependency&quot;\\n  input: &quot;theta/read&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/predictions_grad/MatMul_1&quot;\\n  op: &quot;MatMul&quot;\\n  input: &quot;X&quot;\\n  input: &quot;gradients/sub_grad/tuple/control_dependency&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_a&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;transpose_b&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/predictions_grad/tuple/group_deps&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^gradients/predictions_grad/MatMul&quot;\\n  input: &quot;^gradients/predictions_grad/MatMul_1&quot;\\n}\\nnode {\\n  name: &quot;gradients/predictions_grad/tuple/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/predictions_grad/MatMul&quot;\\n  input: &quot;^gradients/predictions_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/predictions_grad/MatMul&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;gradients/predictions_grad/tuple/control_dependency_1&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;gradients/predictions_grad/MatMul_1&quot;\\n  input: &quot;^gradients/predictions_grad/tuple/group_deps&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@gradients/predictions_grad/MatMul_1&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/learning_rate&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_FLOAT\\n        tensor_shape {\\n        }\\n        float_val: 0.009999999776482582\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent/update_theta/ApplyGradientDescent&quot;\\n  op: &quot;ApplyGradientDescent&quot;\\n  input: &quot;theta&quot;\\n  input: &quot;GradientDescent/learning_rate&quot;\\n  input: &quot;gradients/predictions_grad/tuple/control_dependency_1&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@theta&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: false\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;GradientDescent&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^GradientDescent/update_theta/ApplyGradientDescent&quot;\\n}\\nnode {\\n  name: &quot;init&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^theta/Assign&quot;\\n}\\nnode {\\n  name: &quot;save/Const&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n        }\\n        string_val: &quot;model&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;theta&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/SaveV2&quot;\\n  op: &quot;SaveV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/SaveV2/tensor_names&quot;\\n  input: &quot;save/SaveV2/shape_and_slices&quot;\\n  input: &quot;theta&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/control_dependency&quot;\\n  op: &quot;Identity&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;^save/SaveV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@save/Const&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/tensor_names&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;theta&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2/shape_and_slices&quot;\\n  op: &quot;Const&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtype&quot;\\n    value {\\n      type: DT_STRING\\n    }\\n  }\\n  attr {\\n    key: &quot;value&quot;\\n    value {\\n      tensor {\\n        dtype: DT_STRING\\n        tensor_shape {\\n          dim {\\n            size: 1\\n          }\\n        }\\n        string_val: &quot;&quot;\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/RestoreV2&quot;\\n  op: &quot;RestoreV2&quot;\\n  input: &quot;save/Const&quot;\\n  input: &quot;save/RestoreV2/tensor_names&quot;\\n  input: &quot;save/RestoreV2/shape_and_slices&quot;\\n  device: &quot;/device:CPU:0&quot;\\n  attr {\\n    key: &quot;dtypes&quot;\\n    value {\\n      list {\\n        type: DT_FLOAT\\n      }\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/Assign&quot;\\n  op: &quot;Assign&quot;\\n  input: &quot;theta&quot;\\n  input: &quot;save/RestoreV2&quot;\\n  attr {\\n    key: &quot;T&quot;\\n    value {\\n      type: DT_FLOAT\\n    }\\n  }\\n  attr {\\n    key: &quot;_class&quot;\\n    value {\\n      list {\\n        s: &quot;loc:@theta&quot;\\n      }\\n    }\\n  }\\n  attr {\\n    key: &quot;use_locking&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n  attr {\\n    key: &quot;validate_shape&quot;\\n    value {\\n      b: true\\n    }\\n  }\\n}\\nnode {\\n  name: &quot;save/restore_all&quot;\\n  op: &quot;NoOp&quot;\\n  input: &quot;^save/Assign&quot;\\n}\\n';\n",
       "          }\n",
       "        </script>\n",
       "        <link rel=&quot;import&quot; href=&quot;https://tensorboard.appspot.com/tf-graph-basic.build.html&quot; onload=load()>\n",
       "        <div style=&quot;height:600px&quot;>\n",
       "          <tf-graph-basic id=&quot;graph0.3745401188473625&quot;></tf-graph-basic>\n",
       "        </div>\n",
       "    \"></iframe>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "show_graph(tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 텐서보드 사용하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "from datetime import datetime\n",
    "\n",
    "now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "root_logdir = \"tf_logs\"\n",
    "logdir = \"{}/run-{}/\".format(root_logdir, now)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n + 1), name=\"X\")\n",
    "y = tf.placeholder(tf.float32, shape=(None, 1), name=\"y\")\n",
    "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n",
    "y_pred = tf.matmul(X, theta, name=\"predictions\")\n",
    "error = y_pred - y\n",
    "mse = tf.reduce_mean(tf.square(error), name=\"mse\")\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "mse_summary = tf.summary.scalar('MSE', mse)\n",
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 100\n",
    "n_batches = int(np.ceil(m / batch_size))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:                                                        # 책에는 없습니다.\n",
    "    sess.run(init)                                                                # 책에는 없습니다.\n",
    "\n",
    "    for epoch in range(n_epochs):                                                 # 책에는 없습니다.\n",
    "        for batch_index in range(n_batches):\n",
    "            X_batch, y_batch = fetch_batch(epoch, batch_index, batch_size)\n",
    "            if batch_index % 10 == 0:\n",
    "                summary_str = mse_summary.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "                step = epoch * n_batches + batch_index\n",
    "                file_writer.add_summary(summary_str, step)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "\n",
    "    best_theta = theta.eval()                                                     # 책에는 없습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2.0703337 ],\n",
       "       [ 0.8637145 ],\n",
       "       [ 0.12255152],\n",
       "       [-0.31211877],\n",
       "       [ 0.38510376],\n",
       "       [ 0.00434168],\n",
       "       [-0.0123295 ],\n",
       "       [-0.83376896],\n",
       "       [-0.8030471 ]], dtype=float32)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 이름 범위"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "root_logdir = \"tf_logs\"\n",
    "logdir = \"{}/run-{}/\".format(root_logdir, now)\n",
    "\n",
    "n_epochs = 1000\n",
    "learning_rate = 0.01\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n + 1), name=\"X\")\n",
    "y = tf.placeholder(tf.float32, shape=(None, 1), name=\"y\")\n",
    "theta = tf.Variable(tf.random_uniform([n + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n",
    "y_pred = tf.matmul(X, theta, name=\"predictions\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.name_scope(\"loss\") as scope:\n",
    "    error = y_pred - y\n",
    "    mse = tf.reduce_mean(tf.square(error), name=\"mse\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(mse)\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "mse_summary = tf.summary.scalar('MSE', mse)\n",
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best_theta:\n",
      "[[ 2.0703337 ]\n",
      " [ 0.8637145 ]\n",
      " [ 0.12255152]\n",
      " [-0.31211877]\n",
      " [ 0.38510376]\n",
      " [ 0.00434168]\n",
      " [-0.0123295 ]\n",
      " [-0.83376896]\n",
      " [-0.8030471 ]]\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10\n",
    "batch_size = 100\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for batch_index in range(n_batches):\n",
    "            X_batch, y_batch = fetch_batch(epoch, batch_index, batch_size)\n",
    "            if batch_index % 10 == 0:\n",
    "                summary_str = mse_summary.eval(feed_dict={X: X_batch, y: y_batch})\n",
    "                step = epoch * n_batches + batch_index\n",
    "                file_writer.add_summary(summary_str, step)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "\n",
    "    best_theta = theta.eval()\n",
    "\n",
    "file_writer.flush()\n",
    "file_writer.close()\n",
    "print(\"best_theta:\")\n",
    "print(best_theta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss/sub\n"
     ]
    }
   ],
   "source": [
    "print(error.op.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss/mse\n"
     ]
    }
   ],
   "source": [
    "print(mse.op.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a\n",
      "a_1\n",
      "param/a\n",
      "param_1/a\n"
     ]
    }
   ],
   "source": [
    "reset_graph()\n",
    "\n",
    "a1 = tf.Variable(0, name=\"a\")      # name == \"a\"\n",
    "a2 = tf.Variable(0, name=\"a\")      # name == \"a_1\"\n",
    "\n",
    "with tf.name_scope(\"param\"):       # name == \"param\"\n",
    "    a3 = tf.Variable(0, name=\"a\")  # name == \"param/a\"\n",
    "\n",
    "with tf.name_scope(\"param\"):       # name == \"param_1\"\n",
    "    a4 = tf.Variable(0, name=\"a\")  # name == \"param_1/a\"\n",
    "\n",
    "for node in (a1, a2, a3, a4):\n",
    "    print(node.op.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 모듈화"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "중복이 많습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "n_features = 3\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n",
    "\n",
    "w1 = tf.Variable(tf.random_normal((n_features, 1)), name=\"weights1\")\n",
    "w2 = tf.Variable(tf.random_normal((n_features, 1)), name=\"weights2\")\n",
    "b1 = tf.Variable(0.0, name=\"bias1\")\n",
    "b2 = tf.Variable(0.0, name=\"bias2\")\n",
    "\n",
    "z1 = tf.add(tf.matmul(X, w1), b1, name=\"z1\")\n",
    "z2 = tf.add(tf.matmul(X, w2), b2, name=\"z2\")\n",
    "\n",
    "relu1 = tf.maximum(z1, 0., name=\"relu1\")\n",
    "relu2 = tf.maximum(z1, 0., name=\"relu2\")  # Oops, cut&paste error! Did you spot it?\n",
    "\n",
    "output = tf.add(relu1, relu2, name=\"output\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "relu() 함수를 사용해 더 나아졌습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "def relu(X):\n",
    "    w_shape = (int(X.get_shape()[1]), 1)\n",
    "    w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")\n",
    "    b = tf.Variable(0.0, name=\"bias\")\n",
    "    z = tf.add(tf.matmul(X, w), b, name=\"z\")\n",
    "    return tf.maximum(z, 0., name=\"relu\")\n",
    "\n",
    "n_features = 3\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n",
    "relus = [relu(X) for i in range(5)]\n",
    "output = tf.add_n(relus, name=\"output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_writer = tf.summary.FileWriter(\"logs/relu1\", tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이름 범주를 사용하면 훨씬 더 낫습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "def relu(X):\n",
    "    with tf.name_scope(\"relu\"):\n",
    "        w_shape = (int(X.get_shape()[1]), 1)                          # 책에는 없습니다.\n",
    "        w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")    # 책에는 없습니다.\n",
    "        b = tf.Variable(0.0, name=\"bias\")                             # 책에는 없습니다.\n",
    "        z = tf.add(tf.matmul(X, w), b, name=\"z\")                      # 책에는 없습니다.\n",
    "        return tf.maximum(z, 0., name=\"max\")                          # 책에는 없습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_features = 3\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n",
    "relus = [relu(X) for i in range(5)]\n",
    "output = tf.add_n(relus, name=\"output\")\n",
    "\n",
    "file_writer = tf.summary.FileWriter(\"logs/relu2\", tf.get_default_graph())\n",
    "file_writer.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 변수 공유"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`threshold` 변수를 공유하는 기본적인 방법은 `relu()` 함수 밖에서 정의한 후 매개변수를 통해 전달하는 것입니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "def relu(X, threshold):\n",
    "    with tf.name_scope(\"relu\"):\n",
    "        w_shape = (int(X.get_shape()[1]), 1)                        # 책에는 없습니다.\n",
    "        w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")  # 책에는 없습니다.\n",
    "        b = tf.Variable(0.0, name=\"bias\")                           # 책에는 없습니다.\n",
    "        z = tf.add(tf.matmul(X, w), b, name=\"z\")                    # 책에는 없습니다.\n",
    "        return tf.maximum(z, threshold, name=\"max\")\n",
    "\n",
    "threshold = tf.Variable(0.0, name=\"threshold\")\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n",
    "relus = [relu(X, threshold) for i in range(5)]\n",
    "output = tf.add_n(relus, name=\"output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "def relu(X):\n",
    "    with tf.name_scope(\"relu\"):\n",
    "        if not hasattr(relu, \"threshold\"):\n",
    "            relu.threshold = tf.Variable(0.0, name=\"threshold\")\n",
    "        w_shape = int(X.get_shape()[1]), 1                          # 책에는 없습니다.\n",
    "        w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")  # 책에는 없습니다.\n",
    "        b = tf.Variable(0.0, name=\"bias\")                           # 책에는 없습니다.\n",
    "        z = tf.add(tf.matmul(X, w), b, name=\"z\")                    # 책에는 없습니다.\n",
    "        return tf.maximum(z, relu.threshold, name=\"max\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n",
    "relus = [relu(X) for i in range(5)]\n",
    "output = tf.add_n(relus, name=\"output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "with tf.variable_scope(\"relu\"):\n",
    "    threshold = tf.get_variable(\"threshold\", shape=(),\n",
    "                                initializer=tf.constant_initializer(0.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"relu\", reuse=True):\n",
    "    threshold = tf.get_variable(\"threshold\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.variable_scope(\"relu\") as scope:\n",
    "    scope.reuse_variables()\n",
    "    threshold = tf.get_variable(\"threshold\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "def relu(X):\n",
    "    with tf.variable_scope(\"relu\", reuse=True):\n",
    "        threshold = tf.get_variable(\"threshold\")\n",
    "        w_shape = int(X.get_shape()[1]), 1                          # 책에는 없습니다.\n",
    "        w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")  # 책에는 없습니다.\n",
    "        b = tf.Variable(0.0, name=\"bias\")                           # 책에는 없습니다.\n",
    "        z = tf.add(tf.matmul(X, w), b, name=\"z\")                    # 책에는 없습니다.\n",
    "        return tf.maximum(z, threshold, name=\"max\")\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n",
    "with tf.variable_scope(\"relu\"):\n",
    "    threshold = tf.get_variable(\"threshold\", shape=(),\n",
    "                                initializer=tf.constant_initializer(0.0))\n",
    "relus = [relu(X) for relu_index in range(5)]\n",
    "output = tf.add_n(relus, name=\"output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_writer = tf.summary.FileWriter(\"logs/relu6\", tf.get_default_graph())\n",
    "file_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "def relu(X):\n",
    "    with tf.variable_scope(\"relu\"):\n",
    "        threshold = tf.get_variable(\"threshold\", shape=(), initializer=tf.constant_initializer(0.0))\n",
    "        w_shape = (int(X.get_shape()[1]), 1)\n",
    "        w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")\n",
    "        b = tf.Variable(0.0, name=\"bias\")\n",
    "        z = tf.add(tf.matmul(X, w), b, name=\"z\")\n",
    "        return tf.maximum(z, threshold, name=\"max\")\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n",
    "with tf.variable_scope(\"\", default_name=\"\") as scope:\n",
    "    first_relu = relu(X)     # 공유 변수를 만든 후\n",
    "    scope.reuse_variables()  # 재사용합니다.\n",
    "    relus = [first_relu] + [relu(X) for i in range(4)]\n",
    "output = tf.add_n(relus, name=\"output\")\n",
    "\n",
    "file_writer = tf.summary.FileWriter(\"logs/relu8\", tf.get_default_graph())\n",
    "file_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()\n",
    "\n",
    "def relu(X):\n",
    "    threshold = tf.get_variable(\"threshold\", shape=(),\n",
    "                                initializer=tf.constant_initializer(0.0))\n",
    "    w_shape = (int(X.get_shape()[1]), 1)                        # 책에는 없습니다.\n",
    "    w = tf.Variable(tf.random_normal(w_shape), name=\"weights\")  # 책에는 없습니다.\n",
    "    b = tf.Variable(0.0, name=\"bias\")                           # 책에는 없습니다.\n",
    "    z = tf.add(tf.matmul(X, w), b, name=\"z\")                    # 책에는 없습니다.\n",
    "    return tf.maximum(z, threshold, name=\"max\")\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_features), name=\"X\")\n",
    "relus = []\n",
    "for relu_index in range(5):\n",
    "    with tf.variable_scope(\"relu\", reuse=(relu_index >= 1)) as scope:\n",
    "        relus.append(relu(X))\n",
    "output = tf.add_n(relus, name=\"output\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_writer = tf.summary.FileWriter(\"logs/relu9\", tf.get_default_graph())\n",
    "file_writer.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 추가 내용"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x0: my_scope/x\n",
      "x1: my_scope/x_1\n",
      "x2: my_scope/x_2\n",
      "x3: my_scope/x\n",
      "x4: my_scope_1/x\n",
      "x5: my_scope/x\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "reset_graph()\n",
    "\n",
    "with tf.variable_scope(\"my_scope\"):\n",
    "    x0 = tf.get_variable(\"x\", shape=(), initializer=tf.constant_initializer(0.))\n",
    "    x1 = tf.Variable(0., name=\"x\")\n",
    "    x2 = tf.Variable(0., name=\"x\")\n",
    "\n",
    "with tf.variable_scope(\"my_scope\", reuse=True):\n",
    "    x3 = tf.get_variable(\"x\")\n",
    "    x4 = tf.Variable(0., name=\"x\")\n",
    "\n",
    "with tf.variable_scope(\"\", default_name=\"\", reuse=True):\n",
    "    x5 = tf.get_variable(\"my_scope/x\")\n",
    "\n",
    "print(\"x0:\", x0.op.name)\n",
    "print(\"x1:\", x1.op.name)\n",
    "print(\"x2:\", x2.op.name)\n",
    "print(\"x3:\", x3.op.name)\n",
    "print(\"x4:\", x4.op.name)\n",
    "print(\"x5:\", x5.op.name)\n",
    "print(x0 is x3 and x3 is x5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "첫 번째 `variable_scope()` 블럭은 이름이 `my_scope/x`인 공유 변수 `x0`를 만듭니다. 공유 변수 이외의 모든 연산에 대해서는 (공유되지 않는 변수를 포함하여) 변수 범위가 일반적인 이름 범위처럼 작동합니다. 그래서 두 변수 `x1`과 `x2`에 접두사 `my_scope/`가 붙습니다. 하지만 텐서플로는 이름을 고유하게 만들기 위해 `my_scope/x_1`, `my_scope/x_2`처럼 인덱스를 추가시킵니다.\n",
    "\n",
    "두 번째 `variable_scope()` 블럭은 `my_scope` 범위에 있는 공유 변수를 재사용합니다. 그래서 `x0 is x3`가 참입니다. 여기에서도 공유 변수를 제외한 모든 연산은 이름 범주와 같이 작동합니다. 첫 번째 블럭과 다르기 때문에 텐서플로가 고유한 범주 이름을 만듭니다(`my_scope_1`). 변수 `x4`의 이름은 `my_scope_1/x`가 됩니다.\n",
    "\n",
    "세 번째 블럭은 공유 변수 `my_scope/x`를 다루는 다른 방식을 보여 줍니다. 루트 범위(이름이 빈 문자열입니다)에서 `variable_scope()`를 만들고 공유 변수의 전체 이름(즉, `\"my_scope/x\"`)으로 `get_variable()`을 호출합니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 문자열"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[b'Do' b'you' b'want' b'some' b'caf\\xc3\\xa9?']\n"
     ]
    }
   ],
   "source": [
    "reset_graph()\n",
    "\n",
    "text = np.array(\"Do you want some café?\".split())\n",
    "text_tensor = tf.constant(text)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    print(text_tensor.eval())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Autodiff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "노트: 자동 미분 내용은 [extra_autodiff.ipynb](extra_autodiff.ipynb) 노트북으로 옮겨졌습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 연습문제 해답"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. to 11."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "부록 A 참조."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 12. 텐서플로를 사용한 미니배치 경사 하강법으로 구현한 로지스틱 회귀"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "먼저 사이킷런의 `make_moons()` 함수를 사용해 moons 데이터셋을 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_moons\n",
    "\n",
    "m = 1000\n",
    "X_moons, y_moons = make_moons(m, noise=0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "데이터를 잠깐 들여다 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD+CAYAAADRRMnDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJztvXuYXFWZ7/99u9OVVCehpStcMkJ34BDAyQQiaSHIGcJM/OmPIKjxcNEmdNCcSLcjUXF4wuQ4GNoW9ZGR6BEDSjCk2wtnjEYkmRkNR0BIxMYkRMIgw6VjnjSYdKAhSSfV6XrPH7tW165da+299qWqdlWtz/Psp7v2ddWuvde71nslZobBYDAYDHbqyt0Ag8FgMMQPIxwMBoPBUIARDgaDwWAowAgHg8FgMBRghIPBYDAYCjDCwWAwGAwFGOFgMBgMhgKMcDAYDAZDAUY4GAwGg6GACeVuQFCmTZvGM2bMKHczDAaDoaJ45plnDjDzSV77VaxwmDFjBvr7+8vdDIPBYKgoiGhAZz+jVjIYDAZDAaGFAxE1ENEXiGiUiK5T7HMKEd1LRM8T0dNE9AQRzc5uSxHRYSLaZlu+EbZdBoPBYAhOFGql/wmAAWxz2ecCAP/OzJ8CACK6BcBdAN4PYBqAp5j5/4ugLQaDwWCIgNDCgZnvAQAiutJln82OVYO2a08DcB4Rbc2u2wbgy8z8eti2GQwGgw6jo6PYu3cvjh49Wu6mRMakSZNw2mmnoaGhIdDxJTdIE9EpAO4A8Mnsqn4Af8XMY0Q0ObttExG1saPYBBEtA7AMAFpaWkrYaoPBUM3s3bsXU6dOxYwZM0BE5W5OaJgZQ0ND2Lt3L84444xA5yipQZqIUgA2AfgSMz8GAMx8jJnHsv8fBnArgHMBnOU8npnvY+Y2Zm476SRPTyyDwT+Dg8D8+cBrr5W7JYYScvToUaRSqaoQDABAREilUqFmQiUTDkQ0HcAWAHcxc6/brrDa9VZJGmYw2OnuBn77W+uvoaaoFsEgCPt9iiIcsh5ITxLRzOznVliCoZuZf+jY96qs4ABZ3+YOAL8xNgdDyRkcBB54AMhkrL9m9mAoEaOjo1i6dCnmzp2L+fPn4/nnnwcAfPWrX8X3v/995XGvvvoq3ve+9xWlTcWaOTQCaAXQlP18F4BTAPyjzV31sew2BrCBiPoB/A7AyQDai9Qug0FNd7clGABgbMzMHgxK+nb1YcbdM1C3qg4z7p6Bvl19oc73ve99D6lUCs888wy+9a1vYenSpePb7rzzzvHP3//+97Fy5cpQ19IlMoM0M19m+//PAE6zff4fLsc9DODhqNphMARCzBrSaetzOm19/uIXgVNPLW/bDLGib1cflj28DEdGjwAABoYHsOzhZQCA9tnBxrW7d+/GRz/6UQDA+eefj4MHD45v++IXv4glS5YAALZt24apU6cCAC655BK8/fbbOPnkk4N+FVdMhLQhXpTLIGyfNQjM7MEgYeWWleOCQXBk9AhWbgk+or/00kvx05/+FMyMxx9/HDNnzhzf1t3djaVLl2Lz5s1gZkyePBl33XUXHn/8cfziF78IfE0vjHAwxIuoDcJOYaMSPlu35mYNgnQaeOqpaNphqBr2DO/xtV6Ha665BmeffTY+/OEPo6+vD/fff//4tttuuw2zZ8/GQw89hG9/+9vo7u5GY2Mj/uVf/iXw9XSo2MR7hirEaRCOQqVjFzbf+U7h58FB4LrrgM2bjfrIoEVLUwsGhgtz17U0hYu9uvnmm3HZZZfhW9/6Fq699lq8853vxPvf/368733vQ3NzMyZOnDi+b2dnJwBgeHg4zz4RJWbmUCtUgv9+1AZhp7DZubPQG8m4rhp80rOgB40NjXnrGhsa0bOgJ9R5BwYGcN1112Hp0qV45JFHcOutt+Luu+/Gq6++Oi4YfvCDH2DevHm4+OKLcdFFF+GTn/wk3vve94a6rhJmrshl7ty5bPBBZydzXR1zV1e5W5Jj3z7mSy9lHhy0/p80iRnILcmktS0onZ3MiYR1rkSCedas/M8dHblrhr2WoaLZvXu3r/17n+3l1m+2Mn2JuPWbrdz7bG/oNvz4xz/mz3/+83nrVq9ezT09PczM/Pzzz/MFF1zAw8PD49s3bNjAH/zgB5XnlH0vAP2s0ceamUMtELX/flSzEPuoXWYQPn4cuOCCYNeReR8991z+595ea4YCBJupVMJszFAU2me349XPvorM7Rm8+tlXA3sp2bnooouwefNm9Pf349ixY3j++efx4IMP4m//9m8BAFOnTsWRI0fwpz/9Cel0Gm+88Qb++Mc/omjZInQkSBwXM3PwgXMEHXb2EMUsxD5TSCatUb191mBfOjr8n/+GG5iJ1OeULX5nD3GcjRkC4XfmUCz6+/t5yZIlfNlll/HHPvYx/o//+I+87U8//TTfeOONfNlll/EVV1zBX/nKV/jw4cPK84WZOZS9kw+6GOGgSdTqGmenHvQ8MoFlX9fQkOvc6+u9r2NXUTEzp1L+BINfwRnVfTDEgrgIh6gxaiWDmqj9972MxipVi329TOWzdq21iHWjo1aXLa4zZ467+sauohocBA4fttYnk9bnffuASZNy62bNKjyHH9dVE01tqHKMcKh2ovTfV0UR2zttlfePl30hnbYEgorXXwduu829XcKmcttthR23szOfP18+f9i+PZr7YDBUOjrTizguRq1kw6lSKRZ2tY9MFbNvH/PEidb6SZNy7fFjX3BbVOolp4qqvj7/uEmTolWted0HQ8Vh1EpGrVRZ6HrDlMpX32sW0t2dG/2n07n22EftR48CbW1Wl7pvH3Dppdb3tHe1qZT8+mNjwIoVwMUXW8vOncC8eYWjeOGBZG+js91hVEEmmtpQC+hIkDguNTFz0PGGKbVhVOX5ZJ812JfZswvXixmA7Pvt22d9D7fZg/hfzEDq6oLNRObM0fvOpZqZGcpGHGYO6XSa//Ef/5Hf85738MUXX8zvec97eMWKFXz8+HHlMa+88govWLBAud3MHKoR3diEUhpG3XTt9lmDnV27CtePjQHLl8u/X3c3MDKiboN9VvDcc9Zfp/0CsAzYQgyIGUpHB0AETJ9ufRcd+4Jok4miNjiJOM7lvvvuw4EDB7Bt2zY89dRTeOqpp/Dyyy9j3bp14/uUMmW3EQ5xRafTL7VhVOX5dMEFwKOPyjtpQL5+48bC7ye+TxC6uvKFwQkn5AucJ56wgt6YreuojNtOggQQmuC42iDiQcOJJ56Iv/zlL3jttdfAzBgcHMSBAwfQ3Nw8vs+2bdtw5IiVEfaSSy7BVVddFcm1pehML+K4VLVaSTc2obPTigcolWF0zhy1imbWLLV6p66u0EgsC0Dr6Cg09IqFyD2obeLE3P2xq6tk91IsO3d6f2c/AYRC/dTRYYLjKgzfaqUiqXPXrl3LV155Jb/3ve/lq666int7c2k5Nm3axJ/4xCd45cqV/I1vfIOPHz9eVLUSWftWHm1tbdzf31/uZhSHri7g/vvzjZ6JBLB0qZVJVPDudwM7dhQeP2eOvsokCCKT6be+ZRmEjx611DVhnqWGBmt07jQm+2H6dCu7qmhTMglccw3wwx/KVV7/7b8B//Vf8nMNDgIf+Yhl9LYXaa+rs+7teecVHtPVBaxZY+0zNmZd/+WXTbbXCuD555/Hu971Lv0D7O+o7N30yde+9jW88cYbyu3JZBKvvvoqvvOd7yCZTGLNmjU4dOgQrr76aixduhS//vWvpcfJvhcRPcPMbZ6N0pEgbguABgBfADAK4DrFPgSgG8ALAHYD6AUw2Wub21LVMwfVCN1pQC3G6EXH+HrDDdY1zz47N6quqwtuGI5ycSbX85qxqGYPnZ257+U85pxz5PfNOUMRyf2MMTv2+Jo5FCFJ5HPPPcfbt29XLv/5n/8pPe7NN9/kH/3oR8rzljV9BoAuALcAeMJFOCwB8AyAZPbzAwD+t9c2t6WqhYMufnMm6XT8Tg8i5zH79nl3uLreQfZzu6msirnodvTORUfFB1j3isiomGKOL+FQpDiXb3/72zxr1iy+6KKLxpe/+Zu/4a9+9avj+zzwwAN80UUX8bx58/jCCy/kj370ozwwMKA8ZyxyKwH4jYtw2Axgme3zHABDXtvclpoXDkFGLyrXUVnabHEu5zFi1uC2JJPMP/6xd7tk7ZGdP5HIF05RCwhZR+8Uus522ZMB6ggTk38p1vgSDroze5/ceeed/L3vfS9v3Y9+9CNeuXIlM5c+ZXephMMLABbYPk8FwACa3LZJzrMMQD+A/paWFo9bXeX4Hb3YO7BJk5jnzSvs/O2ZTGX1Dnbs0Js1JBL5bZO1SyaI3GYl4njZ9/ZaZPEXzo5eCMkdOwo7+kmTCttlj9bWaZOJoI41cYhzuPPOO7m1tZXnzp07vpx55pnjwmHv3r187rnn8u9//3s+duwYHzx4kO+44w6+8cYbleesBOHwJwCX2T4nswLgRLdtbter+ZmD39GLvQMTevSOjlzHKetA6+ryR9Bnny2/5jXX6Hkj2UfOqqysquNnzLA6b53UG061mFfq7ubmnJC02yzs51MJFbffwsweKoY4CAcdKjJlt4dweBTADbbP5wB4O2uMVm5zu17NCwc/qNQeQh/uZxSuGhXr7GPvsGWjczfVTCIhdw9VCYtZs6ztOqP6E0/0VgvJlmnTCu+1m9rNzB5iS6UIB7/ELkKaiFJE9CQRzcyuWg9gKRElsp8/A2BDtqFu22oXnUAqsc/Onfl/ncfIgtcAy90yitus437qzMHkbM/Ro8CxY+7HZzJWWm/797v0UsuV0E4iYd0HQJ4Hyclbb+V/h1mz8rt1Va6nU07J/zw4CPT1uX+HMPmXTHCdoYQUK0K6EUArLJsCADwIy5vpaSLqB3ACLCHgta120Ym+FPu0t+f/XbEivxPR6SCDMmGC3n6zZuViL1Tt0RFU9oR+qnPZO+Ht24ETT3Q/59hYfhzEc88Bzz5r/S+rDdHZacUyCAEkWLFCLihPOSWXXDBM/IlJ41FUqm08Gvr76Ewv4rhUtVpJJ37BzUNGqIucPvZBjLliufLKcCqoiRPV31XYK4SR10vFY08HrnMvg7jenn124T1LJJivvjpng3D+Nm7V54KUOnV+D1N5rmi8/PLLvH//fs5kMuVuSiRkMhnev38/v/zyywXboKlW0hz2GUqKLK+SM/pSpSoSxwBWLqFMJnd8mBnEww9bUcyjo/nRzKrI6MmTrejjO+4A7r0X+OQn5ee1j7ZFSu6XXwbOPDM/MtmOmD3oRKR2dweLuv7Tnyw1nTN31f/5P7l9nL/N6acDQ0Py8/X2Ap/7HHDzzcBPfuI/alrnmTAE5rTTTsPevXuxf//+cjclMiZNmoTTTjst+Al0JEgcl6qdOejEL+j41XuNtKMKOpNFAQvPH6crrLMNslF9fb11nCygzL7IgtecbN+uF7VdVyefFdkjwN2OFVHWzpiRk08u/B39BMS5udea2YMhICi1t1Kpl6oVDjrxC37VQ6KjlkVH28/V0BBMBWOPArYLrvp69whulWdPU5P3NYU3kkAW/R204pz9e+ns19gojxnREShez4LKvdZ4PhkCYoRDpaITvxBk1C9L4+B3BuK1eGVW1dXRT5xouZe6XYuoMG7CGd8QxfexR2a72Vyuvjp3L72C7mTCzYn9t1FdN2RErqE2McKhFggyMnZGRwc1UMsWr5mHc7Tr1n7ZeewqIvu5tm/PdaDOtB/iuK4u/0LVfg3dKGg/53fOHuyzH795swwGTYxwqAWCjPpFhzl9urpznjzZW/C4RUy7LfbRbljhJASBva3CBuIcvas8nLxyRYn2hlVRyRbn7KGz0xJyJ59sbAyGoqErHEwluEpmZCTXfXR2WuvqPH5S4fEyOAi85z353dW+fVZNhCNHLB/+zs7CADNBOm159OhAJPfzDxt/IcqNinKhol29vYX1G5zxEYBVC+PBB9Xnt9fFuPRS697OmpX7TmHZvTsXiyIqzjEDf/lL4X0pdglYg8GJjgSJ42JmDjb27dPTczsXe/I45vxRdDKpHi3PmeOeB2nWLGtmIjyOvNQiYWYQfr63c6TuNhuw72tXW0W5+FVbGRuDIQJgZg41RHe3vNKZF2LkDRSmfjh+3Jo9yLqpBx6wKp7ZEdHDzMDcudb/ok1eta2DzCCIgF//Wv84e0oNwGrf7t3q/e37Xn+99b2iRkRyO2uBC5Ysyb/vxazuZzA40ZEgcVyqfuagU5hH7Kdre1D5/A8OynXvKj23bMQtRsGqiGQdo6rbbES2nHhi4Wi7ocFKiOc18naO1Ovr8+MJxHffvl19fT8R0s7FbgNRzRqcMzvZM6H7nBgMWWAM0jHH66WWFcJR7RfW40iVcruhQV6Hwc210q2Dd1OLROlW66V+UV3rnHMKPYTcVE9OgefHG8r+27odN326JbQuusgyVDvdkXWfE4MhixEOccftpfaTRyeKSGc3vb2zo3VzsQxTW1fXVdRZI9oZeOd1vX378u0hXqN7r31mzQqev8reVrff8Zxz5MeZfEuGABjhEGdUVdBUPu46Rep1R96yUb9ukJVX5x+mtq6ukHO21Rl4p2P89jO6d6riZNHqurMA2XdRJeRzSxhoL45kYiEMPjHCIc6oqqA58xGJxa1I/b59VlDbySerR6yi87IHhgXpwDs7C0fc9mOLVFtXet/EIgu808lkG8b7SHwft5G7EPZXX+1+rhNPzBf8usdNmlQ44zOzB4MGRjjEFVUVNPGi2/MRyUaaziRvHR16HdrEie4zC3sHrjJ8Tp5c3M7fyw6jOypXCTudkbaXDl8nilmornSEkFPwi0A4r++oM6sxGCQY4RBXZKNf2YuuWuzlL4nyR82TJlnCQnUu3c5EeC4tWZLfbuf17Nujujd+jau6sxW/2W5Vo3C3GV4yaRmPndlYdRZxbFijvImFMHhQMuEAYD6APwB4FkA/gHmSff4NwDbbsh3AK9ltcwEcdGy/xeu6FSscdEa/opN3dsZimT1bbkSuq1O7cep2JrLiO241qP0U3XGbFRTbuOo3261McDozzspUbE7jse7iNLZ7LamU8VIyBKIkwgHAOwAMAbg4+/kyAK8DaPQ47nMA7sn+/wEA3/N77YoVDk5knZZKKDj3ka2fODGXWC+ID7wz3mHJErmtwe/sQTUrsKvHimlc9Zph6Mwsok5UGHRWYLyUDCEolXC4FsBTjnU7AFzpckwSwACA07Of27Off5ddegBM9bp21QiHqIruOJfp03MqJh1vJ2a5h0xdnbuqQ2f24NaZydRV5ej0vGYWKluRKv1IFIvdU23HDpOx1RAJpRIOtwFY71j3UwDLXY75PIDv2j4nAVD2/2YAPwTwM8Wxy7Kqq/6Wlpai3byyUIysn8LG4ObtZCdo5+YnUM+rw5V1zKXAa2ahshXZv0uQQklei1AfzZrlbuswsweDJqUSDv8E4AeOdT8B8DnF/kkAewC0uJxzOoDjAJJu147lzCFoKgO3FA26y5lnendO9loOMvykf5B1oKp7ourMvNQ0cTKuqoSH3UEgqvsmBgpnny13a3ZzJzYYPNAVDmET7+0F0OJY15JdL6MLwGZm3uNyznoARwEcC9m20tPdDfz2t/5TK19/vfc+qZQ6fTZgFbYfG3M/RzoNbNumbt/pp6uPtSfWcy5uCeG6u3NpwgVjY8CKFYXJ5pzXiFOiue3brTbZ05jbk/lt3So/bs4ctXjYvt36vvPn55IS/upXuRTkf/pT4W86NiZPR/7UU9F8T4NBoCNBVAuAJgAHAMzOfr4QwBsAUgCeBDDTtm8jgD8DaHWc4zoA78j+PwHAetjUTqoldjOHoEZCt1xF06bl9ovSNqEqfOMkCt22qt2pVOXVRQ6THkSF3VCvq5oyaiRDCFCKmQMzDwO4GsBaInoawN0AFmYFQWtWeAi6APwbMw84TpMEsIWIfg/gKQCvwbJLxBvniM8+QvZTmKW7G2hokG87fDh3/u3bgbPPDtdmgazwjZPBQWDt2tzI3ivttgox4nYup59emKI67iNg1SwoaBEekao7k7H+dnR4z/7ENS+4wP9vYTD4QUeCxHEp+8zBOeILOqJ0mxE4jbdRzRx02idLtRH3kX2xiTo9iH1mppMI0LnU8m9hCAxMsZ8i4hzx3XZb8BGlGFmL8pN27CPpFSvyt51zjtwG0dFR2I3Iyn16te/xxwu/U9xH9sVGNQsKYhtxFvjxKtZUX1/4f5CZnMGgiREOQXCqkH75S7mK5LHH9M8pahR3dVkdzr591rrNmwurtAHACy/Iq6A98kjhOlmlNa+O/tJL8w2vol1xMhLHncFB4OKLrcXZictUVG7Y1U3ifz8qLaca1GDwQmd6EcelbGollQrJHuEr1DF2lZBwcVUltRPnFO6mIoCtq0sdf3DNNcXxeS+G4bUWsbu3OlVAUTkY+KmXYdJtGFhfrVT2Tj7oUjbhoIqklXmZCK8g+4spe0nt57QHromX/8QT5R3DxInF8fgJU5fBYLFvX37+KzcPsTCCwu13EQMRWflTQ82iKxyMWskvKhWNzMsknc7582cylveP3Vbx2muFume7usr+V0UxPH6CqKEM+XR359sRVB5ig4PACScAO3YAEyf6v47b7yLibtrbg3nSGWobHQkSx6Ws3kp+qn/ZI1rtqbkbGqz8R1dfrZfzXzWSN0nY4oN9pC7LmiubPYhnSaTH8DNrkHlJyWYLMlWUPVeToaaAUSsVAVF1Tbz4UWTtdKvfrNM5mCRs8cGro3eqE4NWpiNSd+qiDWefrT6nSA9ubBCh6X22l1u/2cr0JeLWb7Zy77O9WtvKia5wEAnvKo62tjbu7+8v7UW7uoDvftfyKspkLC+ej30MeOUV4Cc/AS6/3FIPBKWhQe3SOGsW8Mc/5q8bHATOPBM4ejS3LpkEXn4ZOPXU4O0w+Mf+WxBZ3bCMOXNyHl9dXcD991uqoUQCmDkTeP55oLkZOHDA/XpXXQVs3Ji/bscOYO5cPS8o0UbzvASmb1cflj28DEdGj4yva2xoxH1X3gcAym3ts9tL3lY7RPQMM7d57WdsDrqIiGEg9/Kl00BvL/DEE5Ye1+kHP2eOv2uoBIM9h4/dJTHqiF1DcOy/RUNDzvXXuQjB4LQ1pdNWTqVMxsqTNXeutU9npzyC/he/yNmsxPNw/fVqwWB3R7af0zwvgVm5ZWVe5w8AR0aPYOWWla7bKgUjHHRxGhgFY2PWCycLSBLC4oYbcuuI8gOanOzcaR1HlFtnT11hT+5nDMfxQNbRewWoucU5MAPPPJP7jVWDhk99ynJ4ePxxYNmyXMI+GeK5CNJWg5Q9w/L8oXuG97huqxSMcNDBHhGtQjUCGxy0ZhcCZncPpI9/3BoBOtUSY2PAzTcDa9bkvJ02b3YfnVYRfbv6MOPuGahbVYcZd89A364+74NKRZAZnEywO7n/fus3PvFE+fZf/jIXHPnww+rziGy327eb2WaEtDQ5E1JbNCebUUfyrlV1jIxyP/NGOOgge6ESifwZgGoEtmKFv0jY554Ddu8uXJ9OW6oEITRq6IUWut2B4QEwGAPDA1j28DJ0PdKFaV+fBlpFoFWEaV+fVh6h4WcGJ9RAdsEuS28CAMeOWQn2VGQy+on6xLNiZpuR0bOgB40NjXnrEvUJvHXsLYxx4e/S2NCIngU9WudWPfOlfL6NQVqHd79bz9CcSABLlwLf+U5uXXMz8MYb+teqy8prYfAW59uxw2qHnRoxJs64ewYGhp3JfNWkkimsvnx12Q1/Urq6gHvvBW66KfeceD1fwgEiDHZDuCEy+nb1YeWWldgzvActTS04lD6EoZGhgv3qqR7rPrJO+5lUPfOtTa149bOvhmqzMUhHyfbtuVxHohiNzNjsHIENDgJvveXvWplMvsF77dqcsdFJGWYP5Zjq+tXTDo0MlXyUpYUzYaM9HXtnZ25g4CSoYLAXGpIVFjIU4Pf5bp/djlc/+yrWL1oPAFLBAAAZzvgarMTBZmGEgy7OKm/C2OwUGvbRWXe33rRfeLfI1AsiylqlaiqhOqBcU10/elrBkdEj6PhZR7wEhFvNj61bw88OBNOn52wMzusHqVRYIwR9vu3HqdB9hoVwYsg1OkHehaAY4aCDasQHuL9wqtKRTkZHrU5epg/OZCxbg917CchlcC2hqqBc7nky3S6BFHvnGOOx+MwgvLyExOx00qRoruVM8e72DBsABH++ZcfZ0bU1eAkZPzaLKAgtHIhoPhH9gYieJaJ+Ipon2WcuER0kom225ZbsNiKibiJ6gYh2E1EvEU0O265IUY34vF448cKr1AV21q3L7T99en66bKBwVJnJ+EsJHgHFnuqqpvTts9tx35X3obWpFQRCa1Mr/v6Mv9c6Z2x8y2VODceP51d085vGO5GwgiNlxuze3sJBjMmv5ErQ59tte2tTa17gm5vayk3IOM9TCkIJByJ6B4ANAD7NzOcB+AKAjUTU6Nh1GoCfMvM823JXdlsHrNKic5j5rwGMAvhamHZFituIT+eF0/VWWrTIutacOdZf+/WGhwv3twfGlQjVlDaKqa7XlF7odjO3Z9CzoAdb92rOyhAT33LZrHB01Pqt3TyJ7MyaVeght3u3/BjZIEb2DBs7xDhBn2/VdmE8tgsGt2dc9ZwSKO88pSLszOEDAF5g5q0AwMy/ATAIYIFjv2kA3k9Ev8suPUQ0NbvtWgD3MvNI9vNqAB8L2a7oUPmFi2yrzhdu5878l83N/9zOSy9ZQU1/+UvhNplwKYP7oUy9E8VUt29XHzp+1qE9pfeaxjsppZ5WiTN63q5CEh21qtKcWObOLbRhCXuVTCW1Zg3w7LPusQ3GDjFO0Odb9zgvtVUxB19BCCsczgTwkmPdS9n1djYAmMHMFwG4HMAZAB5UnOMlAM1E1OS8GBEty6qu+vfv3x+y6Zqo/MJ/+Uv5C9fennvZ/Hor6QoSe1BTCZGpd8JOdcVoSuYXDshHU24zgWIIr6IQRM0jq/InBgkyAZDJWEGVqmf4gQeMHcJG0Odb9zjVczswPIC6VXU4lD6Ehrr8VCnlfH5DxTkQ0T8BOJuZl9jW/QTANmb+pstx0wHKh9P5AAAgAElEQVT8GcBUADsBLMvOOkBESQBHADQzszJAoGRxDoODwHXXWYn1Tj0193loSJ6uwJ7Q7JprLFtCUMQ5fvSj/JdbFk9RoXjFMMj8w918wHsW9OT5nfcs6IlfvEOQhIluxzBbSfsOHy48jsiaVTDnjrc/m/YkklXyTMUVnXidRH0CUxNTcXDkYNGe31LFOewF4JzztGTXu1EP4CiAY5JztAA4BODNkG2LBue0W3yeP79w2u9MaCYb6flBnKOKI1q97AEyjyOV99LCmQvzbBPl0NNq4TeFxeCgXKV09Chw223WcUeO5FxYnS7RK1bkX/P4cWC95ZefF1NjZg9FRfbcOkmPpTElMSUWz29Y4bARwHlENBsAiOhCAOcCeJSIniSimdn112WN1yCiCQDuBLCemTMA1gNYSkTiaf4MgA0ch9BtpzfSzp2Fn4V9QWb0O3w4F/+gclNsaFDnzkmngdNOq+r8Sc3JZs99nLaH9tnt6Di/I8+dlcFYt3NdPNxWvfCbwkKoKJ0J+JittN0PPGD9PzgILF+e/xwyW55La9fm1o2Oyu1YxospMDrBc071k4pYOFAgpHBg5mEAVwNYS0RPA7gbludRI4BWAMJukASwhYh+D+ApAK8B+Hx224MAngDwNBH1AzgBloAoP069sLPcot2+4DUaVAXEjY6q02s4Ux5UmWdJ364+DB+TeGJJcL4wm17cVBAoFBu3VS9UhmeZwBeDDsAaXFxwgbWIkqJvvZX/XD30kJWTyc7YmHeSP8Da57HHquoZC4JulLTYj1YRFm9YnOeFdP2G66W5vuwz29amVul5Y+FAAZNbSY1Mx6ti0iRLd3tE4kEjOnjd/ExATv/7v/5Xvr1DlpengvGTM8mZU6ZuVZ00ipRAyNweUaRxHLAXBLLnWIoi35LAbm+osmfML24FfOwqHtl+MggEBo/bw7zOUYqCQCa3Ulj8BCSl05ZgEMVUREqNHTus4vGvvQZs2qQf/Wr3QLF7PlWZZ4nu9FnYE+yjOZU6Ki6jrkhwqirtz6NfwZBMAh0d8m3OWg9V9Iz5RTdKWtedWgxgBoYHsHjDYnQ90jW+rRjef1FihIMKnXz7AvGiiiR5olNXqZ3sVblktohkEvjBD/Jf1Ntuq7oIV92OXNgT7NN2WYKz2LqtBsVvxLQbY2P5dUWAnEu0UGmZKGpXd1O7iiiIXYDBWNO/Ju88cXagMMJBhV0vbO/AxQhMlrJAJMkTnboo+7h2rV6UtcBp3zh+3Hqxq6x618KZC7X2q6d65ShNGPbiNuqKBD8DFC/S6UKbl24UdQ3hNmCxe83pOFLIYLBvu1i5iv4Y4aCDc0T1y1/KX9pMxurEnS9hOl24v3gxVZ4r9rQIo6PuL3aF8tBzD3nu09jQqAyQAzCuz43bqCsSxADFLZ23DpMmWak3nNg9pEyFOADuA5Yjo0ewfPNyTPv6NGVqbh38zDrKWfTHCAcvZCOqI0es9bIXbmys0OXQXqNBIF5MmeeKqqi87PiY4Hd007erz/MFE7OBVDLlup+IMI1d+dCoCJvOO52Wx+XYPaRMhTj07erDup3uQatDI0PK53ZKYgo62zpRTy414pGbnei8M+XKhAwY4eCNW24lWY0FFXY7g1cBFpU6wV68JUaxDkFGN14Pt91+8NYx7xQk5SqlWBJUAwhdMhkrGtpNReTHvbZK8Zuzy8mh9CHcv/1+15mueK5135lyFv0xwsGObkctcit5je6dx8hGYbLEZxX2ogYZ3Xg93GIKv3LLSoxmRl339XPdisL5PNo/69YKIQIaG63Z7gUXFAZu1nhMg50oOtz0mNpGlEqmxu1iuu9MOZPx1bZwcL4cso7a7oIqvDv27bPiH1TGwiVL9Dr3KnEdVMUqhK2MNTQy5Kt2tCAuEaahUaVu6e7OlRb1gtkSDCKCetEi4PHHc2k3TEbWcYrd4Y4cHxn/X3dGUKxMyDrUtnDQiSNwGqNXrLDy3MiSnAnWr9fr6KvEdVClY3XTverkmQlKVcQ66KRuEZHTgvp6b8P1yy9bfx98sCoGJlGieiZTyZSn3UuHIOm5yxkLUbvCwfnyyeIIZMbo9eut9W7Yj1dN26vIdVClY3XTvdof+jA4c9RUTayDTuoWmQebruE6k8k9exU8MIkSWUfcu6gXB249gNWXry5Ipx0EMTPwMyMoVyxE7QoH58sniyOwCwyB7svnjHC2o8qyWaEvqaqD9+r4xUPPt3PgkZlwZSUQUskUkhOSWLxhcWV7LskGDs89l/959+5Crzgg57Swb19+1TgZzoyszkJVNYhbR0zOOu4BEPERcY+OBmpVOMhePllHrYpnUGG3NWzapJ62q7JsVqjrYBR60dWXrw6kZhIxDusXrcfI8REMjQxVvueSTmT0hAmWsRmwhIA90hmw1J+yRI8qnIkkDXms3LLS1disy1vH3pKWvo1jnE5tCgedly+dBk4/Pec2KIuIdrJxo/waskhUID99Qcw9ktyIYhQkS8OtgxBA5fQHjxydyOjRUet5AXK2MDt+a4mI2YiuDaLGPJ2icnIYzYxWzDNZm8LBTxyBc5bhhpjGy2Ym9rxLMttGhb9oXqMgWcBP364+TPv6NNAqAq0irOlfI820qmJyw+Tx65TTHzxy3OpNJ5NWQkenyqi3N9/lVThMiP1VKpFzzpEXqvKaPVSwp1OQdBQqA3I91aOzrdOXWrRSnsnaFA5+4gj8JD877TT1MUePAjffLDdC33Zbxb5oOsgCfm78+Y1Y8vMledGmfgQDgLyZQtyKs0eKc0Bx7bVyNaiYPciM2SrhMGGCf+eICnbBDpqOQpVWY9ncZbjnintw4NYD2rPeSnkma1M4+MFril9XZ0U+79uXS8+tOmbjRnm0dW9vRb5oushUPqOZURzPHNc6XuUSa3/Jehb0IFGfr/pL1Ccq33NJ1nG/8IJ830ceURuzVQOc3bvljhdus4cKdsEOqn7c9OImz/W6nb5uwkkZpUzCF1o4ENF8IvoDET1LRP1ENE+yzylEdC8RPU9ETxPRE7bSoikiOkxE22zLN8K2KzLss4w5cwq3ZzJW9Sz7NFtVu0GWgM9uDK+wF02XMNPoxoZGLJu7TKvjdxauqtRCVnn4nbn6TfPd0OCvTnmFu2AHVT/qHLdw5kKt2YNK0HhR6iR8oYRDti70BgCfZubzAHwBwEYicrqdXADg35n5Xcx8IYCfA7gru20agKeYeZ5t+UKYdhWFwUFrZtDRkR9oVFcHtLWpYyacdHTIdclAxb1ouvidRtdT/bhhu+P8Djz03EMFniLpsTSWb14+/mLI0mxUkvFPia59TKhF/ab5ttc6txepuvRSYPPmwv0rPHtrUPWj13EiaZ+OajToYKnUThdhZw4fAPACM28FAGb+DYBBAAvsOzHzZmbeYFs1CGBC9v9pAM4joq1E9Hsi+jYRnRKyXdHT3Q088UROBSQQabp1UnoD+YbDCn/RdPETDd1Q14B1H1mHzO0Z9Czowf3b71dmwRwaGRofOVWVQdqO3zxbYv99+4Dp0y1bwymn5GpOyxDPnKxIlZMKz97q1+1aqHFkaVzsx/lJ2icTNDrqolI/42GFw5kAXnKseym7Xkq2478DwKrsqn4Af8XMFwO4DEAawCaSRJwQ0bKs6qp///79IZvuAzGVZpb7jtsLuIuU3rJ03mJf8dJV+Iumi3B19fLoSCVTeODDD4x7IC3fvNzTt1yMnKraIB2EFStybtKvvw4cO6beN522VKPOIlWyWWyFJYV04sft2q7GcUIgdJzf4ekt50QmiHTVRaV+xsMKBwLg7C2Pq85LRCkAmwB8iZkfAwBmPsZs5Vlg5sMAbgVwLoCznMcz833M3MbMbSeddFLIpvvArx53bMxyTZXZKIBc5799e24qb493qJAXzQ/ts9sxJTFFuq21qRV8O+PArQfGXzadeg+CgeEBHEofKrBLVE0qDb8MDhaWBFUhUslfemlNzGIB/eAzt9kAg30bowk0Ppixd/y66qJSJ+ELKxz2AnDelZbs+jyIaDqALQDuYma3J5ey7fJO4l8K/MQ5CNwK+Tg7/wr2F/eL7rS4b1cfPrHxE77OPTQyBGYrDUdc0xGUjBUr9Acz9lmDbBZbhTYwXfwYqXU8kIQ9wjkz0H0vSp1yI6xw2AjLXiA8jy6ENep/lIieJKKZ2fWtsARDNzP/0H4CIroqKziQVSXdAeA3zPx6yLZFg86soaEhX6+bTMqNeU4q2F88CLrT4qCpCkYzo5iSmBLbdAQlYccOK+OqF3aDtmzWIKjS2YMOfozUfj2QRL0St+vI1pcy5UYo4cDMwwCuBrCWiJ4GcDeAhQAaAbQCaMrueheAUwD8o81d9TFxGgAbiKgfwO8AnAwgPm+1bioDu15X94WqYH/xIOhOi71qOLjZLoLUf6gqrr/efbvIw2Sfvbo94+k08OtfV3wEfxDcHCmcz20Qo/DQyBD6dvWVtWaDG6HjHJj5/zLze5j5QmZ+LzNvZeY/M/NpzNyf3ed/MHPK4a46P7vtYWa+OGtLuJCZlzLzwbDtigy794fTNiDWOz1BdKbjFe4vHgSdaXHfrj5PX/EpiSmY3DBZuo1AlZlsLwoGB71L18oGIZs2yZ/tzk7LVbuhoWZUn3acaeVFMKbsuQ1qFF65ZWVsM7SaCGldVLaB7m556uSjR614B7fz1YgB0O6mt3LLSvQs6FFOi1duWenpKz4wPIDDo/JiSwyu/NiGoHR3q9Nk2HF6w8mebbvK0817qQpxPq8LZy5Ea1MrMpxBa1Mrehb0FDy3QUf5YsYRxwytVKlRpG1tbdzf31+aiw0OAmeeaXX4yaRVTevUU/PXy0ilgAMH5Nve/W5LP+xkzpyq8lYSbnp2b4zGhkbpjGH55uXaHkpeEAgtTS3SF7kq8XoWBcmkpUa6+WbgJz+xZgiyZ7urC7j//nx1UyIBLF0KfOc7xf0uZUT2vDqRPb8AkPxyEkfHPO6/A5FyvpQQ0TPM3Oa1n5k56KCyDXgZq48cURdyr3B/cV2Wb17u6abXt6sPN2y4ITLBAGDcX3zxhsWgVVRZxX+CZOnt7tar3+Cs26DKElyj3ks6wWwyN9O+XX2ulQ9lxMGu4IaZOXghG5GJEdbll8tH/wIx0mIG7r0XuOmmqh51Oenb1YfrN8gNpATC+kXrsXLLypIZkVUjvtjR1aV+XgYHgeuus0b9p56aW6+aibqRSBQKgEmTrDQxb74pN1JX+eyhblWddnbg1qZW7Bneg+ZkM944+gYyrB8LlUqmsPry1WV5Fs3MISrcbAPbt6sD3QDr5br/fuC7360pna3ATfffnGxWRp8Wi4oo/uPl3qyyfW3a5F4WVBSWEkZmQN75p9PAX/7i7r1UZRH8dnQNywQaj2geGhnyJRgAy6ki7oMUIxy8UKW4eCzribt9u/XCTZhQeCyQn7agSg3OMvp29Xl2/Lq5aKLEy+WwlCmRpbi5N7sJDi+1kqj3sHatuypUbBNFgmReTFWm+rSjmwfMb+0RJ5WQ88sIBy+ctgEx8po/39ouXtjjGrUJakBnC+SMeipSyRQOjpTHW9kt6RmtIizesLhkKZEL8HJv1ik9qyKdthJCyjzrAKv++Q035LydarSmtMyttLOtc/yzn4pvblRCzi8jHFTGP9l62citu1stGKZOLVwnRnBVHFTkZtRrbGjE6stXl+Xl8Ep6BhSOCEuqinJTYboJDi/HiCVLrIHNqaeq91u/3srHJGyQ9iJBNTCgseN0K73ninvGP6vyg8lw5voSxN0QLTDCwS1+wbleNnLbulUtHN5+u3CdGMFV8YjMbcosDMKy6btumUVdEvUJz1xLOt4pJVMBuGXpdRMcXlH8Gzdafy+91DIoyxgbq/kUGqo65/Z1fmxkaz+0ViuALq7UtreSTvyCWG/3BxeIbe98p16is0QC+NjHLE8T5zWrCNVL5PTp7tvVh5VbVmLP8J7xmISovZdSyRQO3KqINYGed0o5fNEL0ImLEV5O73oX8Kc/5auQdu60Ck359WgSVNizKnu2nB2y2GdgeAB1VFdgVE7UJ8DMBUWkdIjFM6PAeCvpoBO/YC+EIhu5+c2A2ddX9fmUdHPFyKJC/RQG0kHkr1Hhpd6KjQrAKy7GGdHstC1cc03+OZxVCL2ooGdVVh9h8YbF6HqkS7oPAKm3UXosHUgwxOaZCUntCgeVDnfnTvn6xx+XT/l/8Qv1NeyZL/ftA04+2VJBVXk+pbC5YpITkuP/T0lMQUNdQ972xobGAiNhHakfZTebQc+CHqVuuJJUAJ52hxdeAJ59Vn9/JxXkwipTFTIYa/rX5JWVjcJbrp7q855F1TNTdi+4ACj8L2sA1UygvV2+fv584I9/zF8/OAicfrr6GuvW5f6/+WbLf9yJGJFVWVBR++x2ZaeqmvLLUhdkOIOlFyzFphc3SVUE4hg3P3Mvm4FTtdpQ15BXkS726NYc+fjHc8+wbq3pCgx6U/3eIu9W++z2yOxIGc7gnivukW6zq60IVFDPAUCsn7HamzkILyTVTOCll/RLd3r5ln/849bfHTuAf/1X+T4VNCKLAreSiKqKWJte3KRMSqYzAmxpalGO3FZuWVmgOhjNjMY3WE7mRac7C9i9O3ecXcXkFchZYbNbN1WhEApRecupzhMrL7iA1J5wEF5I8+fLdbgjI/o5j7Zudb+WeBk/+tHCbSJitcqDipy4lUR0q4il6ty9RoAEwlnNZxUIpOs3XI8pX5miNH7HNkhJ5kWnOwsgktsNhKBQPYcVZG8ALFWhyvNNdOZR2LbcbAuynGJOYvuMZakt4RB15bXt263AIRUNDZY66eWXC7dV2AsXFW4CQDUKs6facM42vEaADMajrzwqfVFVab+BmAYpyZ7fwUErF5IYaLgZmjOZXGS/DFWhoAqb3bbPbsdNbTcVCAh7Z+6s1eAXN3uUbv3zWD5jNkILByKaT0R/IKJniaifiOZJ9iEi6iaiF4hoNxH1EtFkr22RE3XltcFBy/tIRTqd8zGXbaugFy4q3EoiqrycgMJUG0dGj6DjZx3j+lw3vFxV3TqRWCF7flessFSkK1YU7pNIALNm5WIbEolcZL8Tt0JBFZhG/p4r7sH6ReuVhmK73SuVTBU4PbiRSqZcay6I8p9uxPYZsxFKOBDROwBsAPBpZj4PwBcAbCQi53ytA1b50DnM/NcARgF8TWNbdOhUXvObKtnN5jBnjjWKU033Tzyx4l64KHBzc1V5OalSbYgUyQwOFUDH4NhV4SpA9vyuXZsbnPT2yj3tnntO7pEns1s0SDrI+nq9eugxRFVAx2n3GhoZ8uWyOjQyhGlfn6b0OPKaNcT2GXMQ1lvpAwBeYOatAMDMvyGiQQALADxs2+9aAPcy80j282oAWwD8g8e26HCLMBWeGHZ9rpd3hiyfzaRJwCuv5AKFuroKjxO4ZdCsYsQLoQpQknk56QTGhUmEFueApXFkz++xY7l0F2NjwLXXehumhUfe88/nP+cqu0UVetNF4cY6NDIUyOOIb6+coOOwaqUzAbzkWPdSdr3bfi8BaCaiJo9t0eGWmgDwb4+QvazpdKGhUMXhwxXlARIlfksihjUees0qDqUPxd//XPb8OrMbvPCCt2E6nbbUR87n3F4r3WmzqDBvJRl2h4aoIvBVHkeq/EtRJe0rFWGFAwFw6lWOS87r3E8kI6rz2JZ/EqJlWbtG//79+/211CvC1K89QvayOg1+27dbL5tsllCjBukgONVNIk+NLkJtpGJoZKg8WVj94Hx+VY4Qp5xSmGLbmVVYqI9kz2AV1jZ3qpGixOlg8b4H34dD6UMF+02om4DVl6+O9NrFJqxw2AvAaWFsya53268FwCEAb3psy4OZ72PmNmZuO+mkk0I23YaOPcLJpk1WIrOODneDn8ouUaMG6Sh4x6R3KKOaZRAIC2cu1Er0Vwn+5wCARx6Rr3/9dXVHrvOce82wK5CooqFl2B0suh7pwpZXtkj3a5rYFHsbg5OwwmEjgPOIaDYAENGFAM4F8CgRPUlEM7P7rQewlIjEG/0ZABvYCk1121YagoyWuruBJ56wDIGql01ml6jR+IYw9O3qw40/vzHfgDimb0BkMDa9uKnA2K0aRQ4MD8RfzeQWma8a2Og851VY27xY8QROj6P7nrlPuW+56peEIZRwYOZhAFcDWEtETwO4G5bnUSOAVgDCbvAggCcAPE1E/QBOgCUEvLaVBr+jJdHpMxfOCuwv24oV+ZXgnNsNWizfvLzAm8SvemDP8J4CW4ebqqni1EydnbkZrOoZq8JZgQ5RxhPY7Qn2HGBAznuu2G0oFbWdsjsoXV1WbWiV8U/4hU+bBgxJ3Noq0G+8nNCqaOo8tDa1SvMyeakcYu/NZE8xL/CbYntwELjuOiudfIWk5dalb1cfFm9YHIm9oaGuIW+gQiDc1HYT7rniHky4Y4JSQPQu6o2NWsmk7A6KV6yDLMmZXVUkpuCDg5ZHkmq7oeQMDA/gExs/gWlfn4a6VXVYuWUlOs7vGFc1uR0XazVTFEZkVdGrCsSZagWANGI6CLIZrMj2umyuvDTugjMWxEYw+MEIBydeL4nsRTx+HLjgAnUyNKNKCkzUnXF6LJ3nnbRu5zr0LOhB5vZM5aqZwqqLok4rU0ZUiR0vablkPGI6ahiM5ZuX454r7kFnW+e4N51I5/3rG34d+TVLgVEr2VFVhhPbPvIRYNcu4IhCDdHVZQULRTHNNwBQV5UDrJfPTc+ri1AbVY2ayS92NWkFpui2o1OFUKf6XxDipDpyw6iVguA22u/uBn73O0swdHXJK2q5FXyXzS4Mnrh5mkQhGOzXcMZTBGlTxRHEjTvGuCV2BKyZRTEEA+BeVKoSMcJB4PaSDA5aeWwEa9fmXh6ZQJFN80dHrfMY9ZIvSuHl0ZxsHtdRr9yy0lPNVImeJ0qqLOjNLbEj4L8D9xNwGWXt8zhghIPA7SXp7s6vySvSZKgEyubN+W6GstmFQQtVor6oUhEk6hN469hb0nTgurWwY4HfpJGCKnNv9frN/M76ls1dht5FvVrpWwgUP3tUCIxwEKheksces2YKdsGRyVjrbrtNb9RljNOBUWVqXX356tDFWoBC10Qglw588YbFSE5IIpVMxTtjKxDc26gKg97s8QepZCrvN/M76xOBbfZnUDUwEWVIqwVjkPaiqwu4995CIVBXZ6Xd9opjqFHjtKpOdDGuUezpfGNDY3yFAuDuSFFDyBwKnL9d364+XL9BUdRIgez3V8XeEAiZ2zVKtpYRY5COiq1b5WmQMxkrhYHXqKvKdLo6uNWJjhIR8cy3M3oX9RYt62Xs8y3Zn7EadnxwK0EraJ/d7vs5kf3+uvYoVXnbSsAIBy9U027dqXeV6XR10HlJo0TMIA6OHERrUyt6F/VGEvBkJ7YeSk67l3B8EJXhqgy3ztbLU0kQRCXpnJ3q2KNKNUgqFkY4FJsq1Ol6ofuSRoHqBWxONkd6HZWuuuwjQ9nMFLASQsZ89uD33rl1tn27+lBH8u6MwQXnd+ZF8ovTFpZKppCckMTiDYvHr1XqQVLUGJuDIXJ0ApF08bJdqK5VR3XIcDS6X5XNQUfHXXTe/W5gxw75NhGUGUOC3DvVb51KpjByfMQzeLGxoREd53dg3c51gVJ4q6q4qb6L6hrltksYm4OhbETlAqozLVfNRqISDG4eSm4jw5LNKCq0gpufUbW4lyrHg6GRIa3O/sjoEdz3zH2BBINbvIPqu6iOqZQ4GSMcDJGjcj/1O5rW6UCK/aItnLlQ2W6VYBJCrKS65gpzfNBVPdoHCFEQNKpelVQPUH+XMR6rnDgZCUY4GIqC3zrRMnQ6kLD1pb34bv93QatIOvpXCaZ6qi+9rrnCHB9U966O6vLus1cVt0R9oui1mRecsQD3XHGPcrvqu4hBUdhBUrkwwsEQW7xSIQDh60vrIhv9q9RnqtFpUT2eKszxYeHMhVKPsjEey7vPXvdMZNl10lDX4KuUrBv/dfC/XLe7qVGjGCSVCyMcDLFF13bRPrsdPQt60NLUoqU2UHm1eCHzmZeNDFU+8PYcTpXm8x4lfbv6sG7nOmUCPPt9DqI2rKd6jGZG0VDXEKqdAi8BFZUaNW6E8lYiovkAvglgAoA0gH9g5m2S/U4BcAeASwG8DeAYgC5m3kVEKQB7AOyyHfJbZv6C27WNt1JtoBNprZtqOypUXitu7UnUJ8DMeak6ovJsKkU0epRtcjMuC4RHT6l/WxnVlqK96N5KRPQOABsAfJqZzwPwBQAbiUimAL4AwL8z87uY+UIAPwdwV3bbNABPMfM82+IqGAy1g3NaDqBg9O2ll46Seqp39USyt0eouFqbWjE1MVWawymsHSKOgVZebdJRr4kZgxiVF9uuoKKSDMhRE3jmQETXAljOzO+1rdsB4IvM/LDHsR8HsJSZ/56ILoElZF6GNQPZBuDLzPy62znMzKH28OtPXiyc1xQzgCf3PIk1/Wvy1CVim6qGcVif9yhjSqLCq01eMwfVjGrKV6bg8OjhyNvrRmdbp6sxuhKJbOZARAki2uZcAJwL4CXH7i8BONPjfELFtCq7qh/AXzHzxQAug6We2kREBdYqIlpGRP1E1L9//36vphuqDL/+5MVA5Ym0fPPyAsFg36ZjXA9CKaPRdfFqk8yWJIzTTn29mKXRKiq5YACA7//h+zVrG/IUDsycdqh85jHzPACjAJzWv+Nu58zaFzYB+BIzP5Y9/zFmy4rIzIcB3ApL8Jwlact9zNzGzG0nnXSS5lc0VAt+/MmLgZsnkqhLrdq2cOZCbZ93PwF0xRI6YfBqk8yAu37RevDtnOfRE3WMAwCph5Qbo5nRikl3ETVhvJX2AnA+BS3Z9QUQ0XQAWwDcxcy9LuelbLveCtE2QxWi408O5KJZU8lUZO6MANBxfkfgAvWbXtyk5dHi14agcgk9lDaqbv4AABJtSURBVD5UkhGvTJDJZgaJ+gQOpQ+N7wegLLakm9pu8v0bxjbpYpEJIxw2AjiPiGYDABFdCGvE/ysiShHRk0Q0M7utFZZg6GbmH9pPQkRXZQUHsqqkOwD8xsvmYKg9dPzJ+XbG8X8+Dr6dceDWA1j7obXjnUHYTK0PPfcQehb0BDqPWwdj72A7ftYhVVt1/KyjYCbh5hI6NDJUdMO0SpABhcVxmHl8duUUeKrzBJ0xiN9ncsPkgm3rdq6TzuLcqJR0F1ET1pX17wB8HQDDUindwsxbieh0AFsBfJiZ+4noXwH8HYAXbYcfY+b5RHQlgH8C0AAgA+BZALcy80G3axuDdG0Sxm0ziuJAvYt6fReLAazYCuF/LwiaCE4YbP18l9am1shcXL3uYz3VI8OZ8d9Hta+Xgbqe6gOnuxADArfzir+tTa2u97F3UW/ZXYOjRNcgbbKyGmoOHT97FW6dThCCdoCtTa3YM7xHaeeQoRNX4SV8/cYdeHmT8e2srKqmc7wKMXvwuj9egjaVTOHArQd8Xz/OmKysBoOCMDrkPcN7Is3nFHRkLDpvP3jFVejYO/zaAdz21fEy6zi/I5Aar6WpRev+iHuiUlmuvny172tXC0Y4GGqOMDrklqaWPG+bsKhSedRRnWuuKDGq9yuk3ASjTgryKD2HdATjup3rcFPbTZhQN0H7vA11DehZ0IOeBT1aKTQGhgeweMNiJCckkUqmqioFRhiMcDBUFFHUSQgz8l84cyFm3D0DizcsDnS8E1VFMqGzXzZ3masR3u/I2k0w6qQgjxKnd5mMI6NHsOnFTfjBh3+gLYztIVKScCkpDMtgPnJ8BOsXra+4JHnFwNgcDBVDlJXX+nb1oeNnHb7VOgTK02M7P/shlUxJM4o6mdwwGZMmTMLBkYOB8hQJglZa87KLBLkH4pjWplac1XwWtryyxXVfEUWu+33D2IaqLZeSE2NzMFQdUdbkbZ/d7lotTjUad3aCQQUDgbQEAwAcHj2Mt469JR3RetlP7PmdvISo3xTk4rxB7oE4ZmB4AE/seQILzlig3Nc+29Gd9e0Z3hPYtlSrcQ1OjHAwVAxRp4pwC6pbv2i9L5uC38RwfjvU0cwort9wfYEqze072GM+dNQkflOQA8BZzWeFTl+SHkvj0VceBVAolJ1R5LqJ+LwM0gTClMQU6bbmZDOmfX0aaBWBVhGmfX1aTabQMMLBUDFEnSpCJ6hOV58/JTEFvYt6I43IluH0IApbr7tvV19eR7h883L0LOjJK06zcOZC5fFbXtkinVn4vQ9CWNqFZiqZks522me3Kzt2IPf93WYZDMbE+onSSO43j76ZN6sbGhnCjT+/seYEhBEOhopB9bIHTRWhU6RFV/DsGd6D9tntWPuhtb7b4ReRzA8IV2imb1cfbvz5jQUd4Sc2fiLvfm56cZOv9tVTPdZ+aC062zp9Hedk5PiIcpvbbFF8f2GwV3Fw5GDBvWuoa5AKu1rMsWQM0oaKom9XH5ZvXl6grw9imNYtJKQTEZ1KpjAlMWW80wpqi/BD2HTS074+TWn3sBtl61bV+fo+9kI9QaLJVe2wo5Oq3Ctgz3lur/aGTa8eF4xB2lCVqFQKfg3TugnuvFQYgOVX/3b67fFzlUIwAMCa/jWBVR19u/pcDeL2kblftV1LU8v4/Q3LwPCA9DvqqNPcAvZkqjev56fWciwZ4WCoOKIwTPvxfJpYP1F5ntamVpww8QSkx9La144KBgdWdeh2hH27+nAofUj7vI0NjVg4c6E0gWBQhNC2x7is3LJyPEuuSp2mo3qy47a/CKyrJfTDDg2GmNDS1CJVKfgZ2fkRMAdH5DkgCYRXP/sq6laVb4xVLHfNhTMXauVRIhCak804OHIQzclmHBs7hu/2fzdQm1QIG8vI8ZHxtgwMD2DdznWuqkTVc9La1Co9RrU/gfDAhx+ouaA4M3MwVBxhPXQAy11Rd72Xl1QcC+t4ofr+gnU712H55uWeo38G442jb4xHGOvOMlRpQ1QMjQz5jnHx+5yo9l+/aH3NCQbACAdDBRLGQycIXp1MlIn4BDoutDqqDrsqZuqdU1G3qg60yjsA78joEe0gPbdgQhWnn3B6JLmp3GZAfp+TUj9Xccd4KxlqEpUHjsojRSeVtdjenGzW7ljD4lanwW967TijSjUivn/QGh+1iKnnYDC4oOMKGQa3GgVRk6hPjMdX2DvJQ+lDJRNSxSSVTGH15aulebVkxZKC5tuqFUriykpE84noD0T0LBH1E9E8xX5zieggEW2zLbdktxERdRPRC0S0m4h6iaiwvp/BECFR2C1UeLmXppIp3+k23EiPpbF4w2Lc+PMb81xzq0EwiJoKKpXPphc3RZZvy5BPYOFARO8AsAHAp5n5PABfALCRiGTK12kAfsrM82zLXdltHQAWApjDzH8NYBTA14K2y2DQQUe/HDQ9uFfHJFJDRykgGJxXgrSSUf0mIqWJSO0BqLOumuR54QnjyvoBAC8w81YAYObfENEggAUAHnbsOw3A+4nod9nPvwbwVWZ+G8C1AO5lZhErvxrAFgD/EKJtBoMnIsWCDKe+XgTJiePc0OmYjowewZHRI6FSfseFOqoLZJSWoavW8wqyq7WAtWLgOXMgooRDHbSNiLYBOBfAS47dXwJwpuQ0GwDMYOaLAFwO4AwAD2a3nek4z0sAmomoSdKWZVn1Vf/+/fs9v5zBEJQw6cH9dExegqGhriHSGUYxYGarFnSAcp52/Kj1/EY/G/zjKRyYOe1QB81j5nmw1D/ODFXHZedk5hHOWr6Z+SCAWwBcSURJAOQ4z3FV25j5PmZuY+a2k046Sef7GQyBCBOFHcS1VdWxjmZGlUF4cSFovEcd6gKX5fQb/WzwTxiD9F4AzqehJbvei3oARwEck5ynBcAhAG+GaJvBEIow6cGd9oxUMuWZwtptBhG12insCN9O0HiP1qZWPLjoQRy49UBeenBd3OpYGMEQDWGEw0YA5xHRbAAgogthqZp+RUQpInqSiGZmt12XNWCDiCYAuBPAembOAFgPYCkRibfnMwA2cKX62BqqgrDeTHbj6erLV2NqYmoxmlmAlwqqnupxU9tN0joGDXUNvq5VT/UFBmOv4kCpZEqr+JCXM0Axvc0MFoEN0sw8TERXA1hLRAxLHbSQmd8kotMBtAIQdoMkgC1ElAHAAB4D8M/ZbQ8COAvA00R0HMBuGGO0ocyIjitscFWpA9G83FcznME9V9yDS1ouKfhugPV9deouq2IJhJFfFWSooyJTOQM8uedJbHpx03ibO87vyPtsgt+ixQTBGQxFRBVsV0/1rrWZi4WON5CqzfZzeHXEqloRYa7v9OwywW7BMPUcDIYy4FSHqDrZDGcC5xaakpjiWwUEWJ3rwPCAZ8yGm2qmjuo8BUPfrj68nX67YL1u2muVsdk5EzHBbsXFCAeDISJkBYRUxl+hBvHr1ZSoT2DNB9fggQ8/gHqq1z7OPupWFTYStM9uV5b4zHDG9VjAUk3J6lucMPEErVF+FKnXDeExwsFgiAiZ7z2j0P9fGE51DLhAfsTw2g+tHdfrr/vIOk/Po9amVrQ2tSpH3SrD7yUtlyiN224j9r5dfcrZkq5LrkxouglZQ3EwNgeDISLcai23NrW6Gk6DJgLseqQLa/rXKDPM3tR2k3K72Me5bXLDZKTH0q7pOGTZa/3WbHbDmQV34cyFJsFeROjaHEwlOIMhItwqj3l1ij0LeqRZR7109HbPI+e1GexZlU0mNA6PHnY9BpCP2KOMWpalNpF5WBnBUDzMzMFgiAjZyNnP6NarZoQXXl5GUaH6Tm4zp95FvaYjjwlm5mAwlJiwsRFuiQB1KIVx1s2N1W/NZkO8MQZpgyEiwo78g1zPbkz2qgsdhsaGRvQu6nWNbI4iajlomnRD9JiZg8EQAWFSfEd1vYa6BiTqE1I3Ujt+04SLSmxe3yPszKnU99DgjrE5GAwRUOyyo7rXSyVTmJKYorQ9JOoT+OS7P1mQdgIAOn7WIY3aLtZ3cFLqe1irGJuDwVBCwqT4jvJ6B0cO4sCtBwBYI/Hlm5ePp7HwmgEs3rDY17WiptT30OCOEQ4GQwSojLHFCtLSuZ5fA3epv0Pcrm/IxxikDYYIKHUK6WJcr9xpsMt9fUM+RjgYDBHgLPDjt7JZHK5X6u8Qt+sb8jEGaYPBYKghTMpug8HgiYkrMKgIJRyIaD4R/YGIniWifiKap9jv34hom23ZTkSvZLfNJaKDju23hGmXwWDwRpZi3Csdt/N4I1iql8DeStma0BsAfJCZtxLRZQA2EtEZzJyXfYuZ/3/HsZ8DMDP7cRqAnzLz/wzaFoPB4B9ZojyRjttLz28C1qqfMDOHDwB4gZm3AgAz/wbAIIAFbgcRURLAZwHcmV01DcD7ieh32aWHiEpTjd1gqGHCxBW4CRZDdeApHIgo4VD5bCOibQDOBfCSY/eXAJzpccpOAJuY+c/ZzxsAzGDmiwBcDuAMAA8q2rIsq77q379/v1fTDQaDC6r4AZ24AhOwVv14CgdmTjPzPOcCYBSAM9b+uNs5JbMGMPMIZ12mmPkggFsAXJnd19mW+5i5jZnbTjrpJJ3vZzAYFISJKwgjWAyVQRi10l4AziehJbteRReAzczsNryoB3AUwLEQbTMYDB6EiSswAWvVT+A4ByJqgqVG+jtm3kVEFwL4d1hqoXoAvwCwhJlfzO7fCOAFAP+dmQds57kOwL8x85tENAHAAwAOMbO8wnkWE+dgMJSXUqcoN0RD0RPvMfMwEV0NYC0RMSyV0sJsJ386gFYATbZDumAJAWfylCSALUSUAcAAHgPwz0HbZTAYSkPY4kSGeGMipA0Gg6GGMBHSBoPBYAiMEQ4Gg8FgKMAIB4PBYDAUYISDwWAwGAqoWIM0Ee0HIC+UW1qmAThQ7kb4oJLaW0ltBUx7i0kltRWId3tbmdkzirhihUNcIKJ+Hct/XKik9lZSWwHT3mJSSW0FKq+9MoxayWAwGAwFGOFgMBgMhgKMcAjPfeVugE8qqb2V1FbAtLeYVFJbgcprbwHG5mAwGAyGAszMwWAwGAwFGOFgMBgMhgKMcPAJETUQ0ReIaDSbbly1HxFRNxG9QES7iaiXiCaXsq3Zdswnoj8Q0bPZKnrzFPvNJaKDjop/t8ShfXG5lz7aW5Z7KWmH57Mas3ur095Y3NtsWz5FRDuzz8GzRNSl2O/TRPSfRPRHIvolEZ1S6rYGgpnN4mOBlXr8FgBPALjOZb8lAJ4BkMx+fgDA/y5xW98BYAjAxdnPlwF4HUCjZN8PAPheHNsXh3vps70lv5eK9no+q3G5tz7aG5d7Ww/gGwCmZD+/E8AIgHc69rsMwB4AJ2U/fwnAL8vdfq3vWO4GVOoC4DcewmEzgGW2z3MADJW4jdcCeMqxbgeAKyX7tsOKOP9ddukBMDUO7YvDvfTZ3pLfS492K5/VuNxbH+2N1b21tWsSgGEALY713wXwFdvnE2HVvmkqd5u9FqNWkkBECce0VSwJH6c5E1alPMFLAJqzFfQiRdVeAOc62iDacabkNBsAzGDmiwBcDqui34NRt9WB8x4B8vaV7F56oNvectzLoMTl3uoS13t7N4CfcGEJ5Lz7y8xvwBIiM0rXtGAErgRXzTBzGoBUN+8DAjBm+3w8+zdygaxqLxH9k6MNoh0FbWDmEdv/B7N63D8TUdK+LWKc90jVvpLdSw+02lumexmUuNxbLeJ4b4noy7DUSh+VbYbmOxg3Yt/ACmYvgBbb5xYAhwC8WcY2iHbs1Ti2HsBRAMeibpQN3fbF4V7K2iHa4nU/S3EvgxKXexuUst5bIvoGgFkAPpodpDnJu79E1AggBb13sKwY4RARRJQioieJaGZ21XoAS22qqM8A2MBZxWOJ2AjgPCKanW3jhbBUTb9ytpeIriOid2T/nwDgTgDrmTlThvY9GsN7qd3eMt1LLWL6nCqJyXMqa1cdEa0BcDqAq4VgIKJ6ItpCRH+b3XU9gHabmu7TAJ5k5v2lbG8QjFopOhoBtAIQD8GDAM4C8DQRHQewG8A/lLJBzDxMRFcDWEtEDGs6u5CZ3ySi0x3tTQLYQkQZAAzgMQD/XI72IYb30md7S34vfRDLe+tCXO/tQgCfAtAP4LdEJNZ/GZYdpBkAmPn/EtF3ADxGRKMA9gFQusDHCZM+w2AwGAwFGLWSwWAwGAowwsFgMBgMBRjhYDAYDIYCjHAwGAwGQwFGOBgMBoOhACMcDAaDwVCAEQ4Gg8FgKMAIB4PBYDAUYISDwWAwGAr4f4HGO6dcrqjEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(X_moons[y_moons == 1, 0], X_moons[y_moons == 1, 1], 'go', label=\"양성\")\n",
    "plt.plot(X_moons[y_moons == 0, 0], X_moons[y_moons == 0, 1], 'r^', label=\"음성\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "모든 샘플에 추가적인 편향 특성($x_0 = 1$)을 추가해야 합니다. 이렇게 하려면 입력 행렬 $\\mathbf{X}$의 왼쪽에 1로 채워진 열을 추가해야 합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_moons_with_bias = np.c_[np.ones((m, 1)), X_moons]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "확인해 보죠:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.        , -0.05146968,  0.44419863],\n",
       "       [ 1.        ,  1.03201691, -0.41974116],\n",
       "       [ 1.        ,  0.86789186, -0.25482711],\n",
       "       [ 1.        ,  0.288851  , -0.44866862],\n",
       "       [ 1.        , -0.83343911,  0.53505665]])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_moons_with_bias[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "좋네요. 이제 `y_train`의 크기를 바꾸어 열 벡터로 만들겠습니다(즉, 하나의 열이 있는 2D 배열입니다):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_moons_column_vector = y_moons.reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 데이터셋을 훈련 세트와 테스트 세트로 나눕니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ratio = 0.2\n",
    "test_size = int(m * test_ratio)\n",
    "X_train = X_moons_with_bias[:-test_size]\n",
    "X_test = X_moons_with_bias[-test_size:]\n",
    "y_train = y_moons_column_vector[:-test_size]\n",
    "y_test = y_moons_column_vector[-test_size:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "좋습니다. 이제 훈련 배치를 생성하기 위한 간단한 함수를 만들겠습니다. 이 함수는 각 배치를 위해 훈련 세트에서 랜덤하게 샘플을 선택합니다. 하나의 배치에 동일한 샘플이 여러번 들어갈 수 있고 한 번의 에포크에 모든 훈련 샘플이 포함되지 않을 수 있습니다(사실 샘플의 3분의 2 정도가 포함됩니다). 하지만 실전에서 별 문제가 되지 않고 코드가 간단해 집니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_batch(X_train, y_train, batch_size):\n",
    "    rnd_indices = np.random.randint(0, len(X_train), batch_size)\n",
    "    X_batch = X_train[rnd_indices]\n",
    "    y_batch = y_train[rnd_indices]\n",
    "    return X_batch, y_batch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "작은 배치 하나를 만들어 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.        ,  1.93189866,  0.13158788],\n",
       "       [ 1.        ,  1.07172763,  0.13482039],\n",
       "       [ 1.        , -1.01148674, -0.04686381],\n",
       "       [ 1.        ,  0.02201868,  0.19079139],\n",
       "       [ 1.        , -0.98941204,  0.02473116]])"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_batch, y_batch = random_batch(X_train, y_train, 5)\n",
    "X_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1],\n",
       "       [0],\n",
       "       [0],\n",
       "       [1],\n",
       "       [0]])"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_batch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "좋습니다! 모델에 주입할 데이터가 준비되었으므로 모델을 만들 차례입니다. 간단하게 시작해서 기능을 점차 추가해 보겠습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "먼저  기본 그래프를 리셋합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_moons_ 데이터셋은 두 개의 입력 특성을 가지므로 각 샘플은 평면 위의 한 점입니다(즉, 2차원입니다):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "로지스틱 회귀 모델을 만들어 보겠습니다. 4장에서 보았던 것처럼 이 모델은 먼저 (선형 회귀 모델과 동일하게) 입력의 가중치 합을 계산하고 그 결과를 시그모이드 함수에 적용하여 양성 클래스에 대한 추정 확률을 만듭니다:\n",
    "\n",
    "$\\hat{p} = h_\\mathbf{\\theta}(\\mathbf{x}) = \\sigma(\\mathbf{\\theta}^T \\cdot \\mathbf{x})$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\\mathbf{\\theta}$는 편향 $\\theta_0$와 가중치 $\\theta_1, \\theta_2, \\dots, \\theta_n$를 포함한 파라미터 벡터입니다. 입력 벡터 $\\mathbf{x}$는 상수 항 $x_0 = 1$과 입력 특성 $x_1, x_2, \\dots, x_n$을 포함합니다.\n",
    "\n",
    "한 번에 여러 샘플에 대한 예측을 만들 수 있어야 하므로 하나의 입력 벡터보다는 입력 행렬 $\\mathbf{X}$를 사용합니다. $i^{th}$ 번째 행이 $i^{th}$ 번째 입력 벡터의 전치$(\\mathbf{x}^{(i)})^T$입니다. 다음 식을 사용하여 각 샘플이 양성 클래스에 속할 확률을 추정할 수 있습니다:\n",
    "\n",
    "$ \\hat{\\mathbf{p}} = \\sigma(\\mathbf{X} \\cdot \\mathbf{\\theta})$\n",
    "\n",
    "모델을 만들기 위해 준비를 마쳤습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs + 1), name=\"X\")\n",
    "y = tf.placeholder(tf.float32, shape=(None, 1), name=\"y\")\n",
    "theta = tf.Variable(tf.random_uniform([n_inputs + 1, 1], -1.0, 1.0, seed=42), name=\"theta\")\n",
    "logits = tf.matmul(X, theta, name=\"logits\")\n",
    "y_proba = 1 / (1 + tf.exp(-logits))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "사실 텐서플로는 `tf.sigmoid()` 함수를 가지고 있어 마지막 라인을 다음과 같이 바꿀 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_proba = tf.sigmoid(logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "4장에서 보았듯이 로그 손실은 로지스틱 회귀에 사용하기 좋은 비용 함수입니다:\n",
    "\n",
    "$J(\\mathbf{\\theta}) = -\\dfrac{1}{m} \\sum\\limits_{i=1}^{m}{\\left[ y^{(i)} log\\left(\\hat{p}^{(i)}\\right) + (1 - y^{(i)}) log\\left(1 - \\hat{p}^{(i)}\\right)\\right]}$\n",
    "\n",
    "직접 구현하는 것도 한가지 방법입니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "epsilon = 1e-7  # 로그를 계산할 때 오버플로우를 피하기 위해\n",
    "loss = -tf.reduce_mean(y * tf.log(y_proba + epsilon) + (1 - y) * tf.log(1 - y_proba + epsilon))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "하지만 텐서플로의 `tf.losses.log_loss()` 함수를 사용할 수 있습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss = tf.losses.log_loss(y, y_proba)  # 기본적으로 epsilon = 1e-7 가 사용됩니다"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "나머지는 아주 기본적입니다. 옵티마이저를 만들고 비용 함수를 최소화시키도록 합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "training_op = optimizer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(이 간단한 예에서) 남은 것은 변수 초기화입니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "모델을 훈련하고 예측을 만들 준비가 되었습니다!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "다음 코드에는 특별한 것은 없습니다. 앞서 선형 회귀에서 사용했던 것과 사실상 동일합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "에포크: 0 \tLoss: 0.79260236\n",
      "에포크: 100 \tLoss: 0.3434635\n",
      "에포크: 200 \tLoss: 0.30754042\n",
      "에포크: 300 \tLoss: 0.29288897\n",
      "에포크: 400 \tLoss: 0.28533572\n",
      "에포크: 500 \tLoss: 0.2804781\n",
      "에포크: 600 \tLoss: 0.27808297\n",
      "에포크: 700 \tLoss: 0.27615443\n",
      "에포크: 800 \tLoss: 0.27551997\n",
      "에포크: 900 \tLoss: 0.27491236\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 1000\n",
    "batch_size = 50\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for batch_index in range(n_batches):\n",
    "            X_batch, y_batch = random_batch(X_train, y_train, batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        loss_val = loss.eval({X: X_test, y: y_test})\n",
    "        if epoch % 100 == 0:\n",
    "            print(\"에포크:\", epoch, \"\\tLoss:\", loss_val)\n",
    "\n",
    "    y_proba_val = y_proba.eval(feed_dict={X: X_test, y: y_test})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "노트: 배치를 만들 때 에포크 수를 사용하지 않았으므로 두 개의 `for` 반복을 중첩하지 않고 하나의 `for` 반복을 사용할 수 있습니다. 하지만 훈련 시간을 에포크의 개수로 생각하는게 편리합니다(즉, 알고리즘이 훈련 세트를 모두 훑고 지나가는 횟수)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "테스트 세트에 있는 각 샘플에 대해서 `y_proba_val`은 해당 샘플이 양성 클래스에 속할 모델의 추정 확률을 담고 있습니다. 예를 들어 다음은 첫 번째 다섯 개 샘플의 추정 확률입니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.54895616],\n",
       "       [0.7072437 ],\n",
       "       [0.51900256],\n",
       "       [0.9911136 ],\n",
       "       [0.50859046]], dtype=float32)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_proba_val[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "각 샘플을 분류하기 위해서 최대 가능도 방법(maximum likelihood)을 사용합니다. 추정 확률이 0.5보다 크거나 같으면 양성으로 분류합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ True],\n",
       "       [ True],\n",
       "       [ True],\n",
       "       [ True],\n",
       "       [ True]])"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = (y_proba_val >= 0.5)\n",
    "y_pred[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "경우에 따라 0.5말고 다른 임계값을 사용해야 할 수 있습니다. 가령 높은 정밀도(대신 낮은 재현율)를 원한다면 임계값을 높이고 재현율을 높이려면(대신 낮은 정밀도) 임계값을 낮춥니다. 자세한 내용은 3장을 참고하세요."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "모델의 정밀도와 재현율을 계산해 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8627450980392157"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "precision_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8888888888888888"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이 예측이 어떻게 보이는지 그래프로 나타내 보겠습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD+CAYAAADRRMnDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJztnXuQHWWZ8H9PLmMmCMFM8rkuMjOkjKWLA5GkQrwl7BdLJIrFSiFZBytgZbPMgCu7+FFiyusYUb9lARGFIEGSmbXYXfPJiomrRrlIEtjBQKIB1GCCqYxuMoFRyCSZyTzfH+ecSU+f7j59OZfuc55fVVdyut/T5+n3nHmf931ur6gqhmEYhuFkUq0FMAzDMNKHKQfDMAyjCFMOhmEYRhGmHAzDMIwiTDkYhmEYRZhyMAzDMIow5WAYhmEUYcrBMAzDKMKUg2EYhlHElFoLEJdZs2Zpe3t7rcUwDMPIFE8++eQhVZ1dql1mlUN7ezv9/f21FsMwDCNTiMi+MO3MrGQYhmEUYcrBMAzDKMKUg2EYhlFEZn0OhmEY5WJkZIT9+/dz9OjRWotSNqZNm8brX/96pk6dGuv9phwMw2h49u/fz6mnnkp7ezsiUmtxEqOqDA4Osn//fs4666xY9zCzkpFNBgZgyRL4wx9qLYlRBxw9epSWlpa6UAwAIkJLS0uilZApByOb9PTAz3+e+9cwykC9KIYCSZ/HlIORPQYG4N57YWws92+p1YOtMoyUMzIywsqVK5k/fz5LlizhmWeeAeDLX/4y3/rWt3zft3fvXt797ndXRCZTDkb26OnJKQaAEydKrx5slWGUmb5dfbTf2s6kz0+i/dZ2+nb1Jbrf3XffTUtLC08++SRf+9rXWLly5fi1m266afz1t771LVavXp3os8JiysHIFoVVw/HjudfHj8O6df6rgqirDMMoQd+uPlZ9fxX7hvahKPuG9rHq+6sSKYjdu3fz3ve+F4Bzzz2Xw4cPj1/79Kc/Pb562L59O0eOHAHgHe94Bx/4wAcSPEkwphyMbOFcNRQ4ftx/VRB1lWEYJVi9ZTVHRo5MOHdk5Airt8Sf0S9evJjvfve7qCqPPPIIc+fOHb/W09PDypUr2bx5M6rKKaecws0338wjjzzCf/7nf8b+zFKYcjCyxbZtJ1cNBcbG4OGHc/93+he8Vhm2ejAS8sLQC5HOh+FDH/oQb3zjG7nkkkvo6+vjnnvuGb9244030tHRwb/9279x++2309PTw/Tp0/mXf/mX2J8XBstzMLLFjh25f7u74Z57cgN+U1NOIQwMwPz5ucG/pwdUi1cZhdXDHXdUX3ajLmid0cq+oeLada0zWhPd9x/+4R+44IIL+NrXvsbll1/OGWecwXve8x7e/e53M3PmTF71qleNt+3q6gJgaGhogn+inJhyMLKH34rg4MHcNci9njOneJVx/Dhs3ep/3+XL4f774S/+onLyG5lmzdI1rPr+qgmmpelTp7Nm6ZpE9923bx/Lly9n3bp1nHvuufz2t7/lyiuv5A1veAOve93rAPj2t7/NnXfeiYgwNjbGmWeeWbEVhJmVjOzh5Xc4cQL+/d9Pvh4dza0mVIuPwurD674W1WSUoLOjk7UXr6VtRhuC0DajjbUXr6WzozPRfbdv385FF13EokWLaG5upqOjgxUrVvCzn/0MgGeffZbbb7+dH/3oR2zbto3HH3+czs5OrrnmmnI8VhGmHIzs4eV3cL8eGYnmX7CoJiMCnR2d7L1uL2OfHWPvdXsTKwaA888/n82bN9Pf38+xY8d45plnWL9+Pe9617sAOPXUUzly5Ai//vWvOX78OC+++CK//OUvmT275L49sTDlUC80UqLXpk2weHHumVXhwAGYPLm43eho+FVAT09u9RH1fYZRJtrb29mwYQN33HEH733ve+np6eGmm24aVw5nnHEG69ev5xvf+AYXXnghH/nIR5gyZQpf//rXKyOQqmbymD9/vhoOurpUJ01S7e6utSSVx/2sXV1exqPcMW9e6fsdOKA6bdrE9zU3qw4MVPY5jNSwe/fuWotQEbyeC+jXEGNs4pWDiEwVkU+IyIiILPdpIyLSIyLPichuEekVkVNKXTNC0kgmEa9n3bbNu+28ef7+BSfOVUMBWz0YDU45zEp/ByiwPaDNCmAZME9V/woYAb4S4poRhqiJXlk0QRVkvvHG4mfdsSOa49nNtm05H4WTkRH/qCbDaAASKwdV/Yaq3gycCGh2OXCXqg7nX98G/G2Ia0Yp4iR61SoqJ4lS6umBRx+F3t7yJ7Vt2gTTpk08N21a7siSAjWMMlIth/QcYI/j9R5gpojMKHFtAiKySkT6RaT/4MGDFRU4M/iFdfoN/G6zzNNPV28VEVcpFWRWLTb/lKMkhl9Jju3bzbRkNCzVUg7CxJXFqOPzg65NQFXXquoCVV1QqfCtzOEX1ulnEnGboDo7q7OKSOIX8Rq8CwQ9a1j8SnJA/ftwjNQwMjLCDTfcwMKFC3n729/OwoULufHGGznhnhA5qIeS3fsBZ255K/Ay8FKJa0YpotjbvUxQv/pVdRzZcQvguWUGaG4+GcYaxbfgh7sPu7pyJTmiymo0FmX23a1du5ZDhw6xfft2tm7dytatW3n++ee57777xttkvmS3iLSIyGMiUigtuAFYKSL5vzg+BmzMh1UFXTPKSdAMvJKDYNwCeIVaSX6mpEo41q1YnxGWMvvuXvOa1/A///M//OEPf0BVGRgY4NChQ8ycOXO8TT2U7J4OtAEFv8F64FHgCRHpB04jpwRKXTPKiZf5pEDQIJh0EI7qF3G+b2CgOJKoYEqqhGM9rqxeZDEqzAhHBcLHP/zhD3PppZdy9dVX8853vpNrr72Wj370o1xyySUAVS/ZLVmdoC9YsED7+/trLUZ2cVY1LdDUBCtXFlcs7e6Gu+6Cq6+OV830rW+Fp54qPh+UhzAwkCucd/Rozoz0/PMTi+GVuh6XOLL6kbTfjKrxzDPP8OY3vzn8G9xVgb3+biLwla98hRdffNH3enNzM3v37uWOO+6gubmZO++8k5dffpnLLruMlStX8pOf/MTzfV7PJSJPquqCkkKFyZRL42EZ0gmZNy9cRrEze7hcWcMHDqguXhx8r64u1aam3Oc2NRVnfpe6noRyZJvv2JG7h2VbZ4JIGdIVyKj/1a9+pTt27PA9nn32Wc/3vfTSS/qd73zH975JMqRrPsjHPUw5VIlKDMKlBt9Sf3yVLHdRLmV49tknZXP3m1M5hlGURsWJpBycfxN+33EMbr/9dj377LP1/PPPHz/e8pa36Je//OXxNvfee6+ef/75umjRIl24cKFeeumlum/fPt97mnIwKoPXIDxtmuqiRfEHszCDb6k/vgr9cRbdO2hQD2LHjomyuZ/VqRwbqSZWiomkHMKuuiNy00036d133z3h3He+8x1dvXq1qqo+88wzet555+nQ0ND49Y0bN+r73/9+33vWtLaSUcdUIjksTEhrqdyNqLkdYSkVqRTWAX7FFcXnnBFWBUfmunW5oxFqYtUTScu1BPDFL36RBQsWjB/OsNVql+yu+Qog7mErhxIUZrlPPRXfbOE3Q4prckl79dOgFUlYc9OBA6oi/jNL52dMmnTSL1Fuv4kRiaxUZX3iiSf0qquu0gsuuEDf97736Ze+9CV95ZVXfNubWckopmCuOPvs8pktkvofKmkOKgdB5oKwzx5WwZRL4RplISvKISpmVjIm4jRdlCsDuhzJYZUyB5ULP3PBpk3hnz3oGYOSECFnevrkJy03wkgFphzqEa9BaHQUzjuv+olsTipoq60oUZ496BmDkhAhd+3BB0/6NSyJrqrkJtX1Q9LnMeVQb3jVIoJclvHAQG5mGoe0z/orSbme3U9xFI4DB+CVV06u9G68sTal1RuQadOmMTg4WDcKQlUZHBxkmrsUfQQsQ7re8Mp8djJ5MuzfX55s4kakklnP7qzbEydyRzkzwA1PRkZG2L9/P0ePHq21KGVj2rRpvP71r2fq1KkTzofNkDblUG/4lX9w0t3dWOUcBgZg+XK4//5kA2ylSna47+2mDOUZDKNAWOVgZqVaUSl7stt0ceBA8S5n1d7kp9aUq0Bf3LLjUe/txirDGjXAlEOtqNZWnZ/8JBw7NvFcNTf5qTVRq2f6Ke1Kl/Iu5ay2fSWMKmPKoRZUoNyvLz/4QW4F4eT4cdi9uzEyc6PO9v2UdjlLeXtRWPF1dXlfbxTnv5EaTDnUgkqYJ7xmvAMDuegXmLh7WlcXFJxU9TwjjTrbD1La1YjWKnw+FO92V8i38PqOG8U8aFSXMJlyaTwymyGdpIREUOE3rwJuXlm9aS9hUU6iZmRXsgx4VHn9ypR7fcdWuM+IAFY+I6UkKSHhNxB41f3xUwIrVqS7hEU5iVI9s9ZKM0qZcq/vuF4VvFF2wiqHxGYlEVkiIr8QkZ0i0i8iizza/FBEtjuOHSLyu/y1+SJy2HX9+qRypZa45okgk4eXmcrPRv7gg42TzBYlI7vSPoVSlPr8Ut9xPZsHjdoQRoP4HcDpwCDwtvzrC4A/AtNLvO8fgW/k/38hcHfUz87syiEufiYHvxmnc7OZUrNmo2I1+svy+X77ajSKedAoK1Rp5XAh8JyqbssrmoeAAWCp3xtEpBm4Drgpf2oW8B4ReTx/rBGRUxPKVV8EOVb9ZpxLlmSzjlGtqHXdp02bivNRmpth82b/fTXcK0Ar3GeUkaTKYQ6wx3VuT/68H13AJlX9ff71RqBdVc8HLgLOAtZ7vVFEVuVNV/0HDx5MJnmWCDI5NHLNo3oi6nc8NuatMJyF+wwjAUmVgwAnXOdG/e7rsWpAVYfzSx1U9TBwPXBxvu0EVHWtqi5Q1QUV2/0ojQQpgFrPeI3yEOc7dh/uwn22ejASkFQ57AdaXeda8+e96AY2q+oLAfecDBwFjgW0aSxMAdQ/5fiOzUFtlJGkyuEB4BwR6QAQkYXAm4CfishjIjK30FBEppNbNXzJeQMRWS4ip+f/P4XcqmKDqgbsitLgWOKT4abS5T2MhiORclDVIeAyYJ2IPAHcCiwDpgNtwAxH827gh6q6z3WbZmCLiPw3sBX4A/BPSeSqe6pVl8lIN85JQq1DcY26w0p2Z41Klo02soVzb4mtW71Ltc+bZ+ZHYwJWsrteMbuyAcVJkZs3m1/KKCumHLLEwACsW2d2ZaM6kwTzbTU0phyyRE9Pbi9oJ7Z6aDyq5Xw231ZDY8ohjfjN2B55xDvxyRLeGotqOJ+rueeIkUpMOaQRvxnb4sW5/YQh9293t9mVG5FqZMWbb6vhsWiltOEXjeS1Ab1FKzUWAwOwfDncf3+47zxqe+f77LdWt1i0Ulbxm7FZHLsR1QcQ12cQ9FszJ3XDYMohTQQ5Gq3AXmMT1QeQxGcQ9FszJ3XDYMohDRRmYzfe6D9js/pKjU1UH4Cz/dGjuVLeYfH7rW3aZE7qBsKUQxoozMZ+8ANbHRjFBK0ovcw87vaq0NubfDA3J3VDYcqh1jiX/6+8kntdKL+8eHHuta0OGpsgH4CXmcevfZTVgxsr7NdwmHKoNUEOaC/brjkEGw8/H8DDD3ubebzaQ25lCvF+QxYQ0XCYcqglfrOxp5/2t+2aQzAa9aBM/XwAixdPnFicd17uOXfsyK083duOvvLKyQquUX9DFhDReITZaDqNx/z58+Pur50eurpUm5om/sk3NameffbJ801Nqt3dufbOjeZtM/lwdHWpTpqkumKF6uLF9dNnzt+C87jyytx1v9/WihX2G2pwgH4NMcbayqGW+M3Gdu/2tu2aQzAaTn9Oby88+mj99JmXmQdgw4bg0OcHH7TfkBEKUw5hqJRpwstc0NUFU6dObFdwJppDMBpuZapaP33m51fwC30+cAAWLYKXX/Y2Y2bd9GaUHVMOYfCy0cZVGKXeF2bGV8Bmfv64/TkF6qXPCoO/l2/BSwH29MD27d5VfTs7zY9lFGHKoRR+maZRnHru7RwL7/NSFH7OxzPPNIdgFPzMLvW24goTRVT4DYN3Vd/duy2xzSgmjGMi6ACWAL8AdgL9wCKPNvOBw8B2x3F9/poAPcBzwG6gFzil1OdWzSHtdOwVnMNRHcNOp6jzfStW5M4XHM5hZInSvpGZN89LxRY7+bOO33POm3eyjddvOMw1oy4hpEM6qWI4HRgE3pZ/fQHwR2C6q92FwN0+97gSeBJozr++F/h6qc+uinLwiggpDOph/6Cc95g8WXXq1Nz/p07NvQ6rYCxSKR5hBs96xu83PDAQfC3Mfesp+quBCKsckpqVLgSeU9Vt+VXIQ8AAsNTVbhbwHhF5PH+sEZFT89cuB+5S1eH869uAv00oV3nwWrKPjuYiX8I6ht1O0YLNd2Qk97pwPkqtnHqxm1eDRq9JVSq7Oq4fy/Jt6p6kymEOsMd1bk/+vJONQLuqng9cBJwFrPe5xx5gpojMcH+YiKwSkX4R6T948GBC0UPg5Rx2DuoF/P6g/JyibkopGCtdYMQlKHktbmKb7RLXECRVDgK4RkpG3fdV1eH8cgZVPQxcD1wsIs0e9xj1k01V16rqAlVdMHv27ISih8Br1jlvXnE7vz8oP6eoF0EzNitdYMQlqMLqaaedrOUVZVVlq9iGIKly2A+0us615s8HMRk4ChzzuEcr8DLwUkLZKkMUM4VfLLo79BCCZ2xWuiAe9VA6o1LENQvZKrZhSKocHgDOEZEOABFZCLwJ+KmIPCYic/Pnl4vI6fn/TwFuAjao6hiwAVgpIvnNkfkYsLGw0sg0fopkeDiaHbwR7eblGNjNLu5NErOQrWIbhkTKQVWHgMuAdSLyBHArsAyYDrQBBb9BM7BFRP4b2Ar8Afin/LX1wKPAEyLSD5xGTkHUHzaTDU/Sgd3s4v4kMQvZKjYyfbv6aL+1nUmfn0T7re307eqrtUihkKxO0BcsWKD9/f21FiMa3d1w111w9dVwxx21lia9ODe4j7uxfXc33HNPbuBqaoKVK63PYWLfFojbx0ZJ+nb1ser7qzgycmT83PSp01l78Vo6OzprIpOIPKmqC0q1swzpamEz2fAkdXiaXdwfMwtVldVbVk9QDABHRo6wesvqGkkUHlMO1cIiPMJRjoHdBkB/zCxUVV4YesHz/L6hfak3L5lyqAY2kw1POQZ2GwD9cQY3dHXBpEk5E1w9BzfUkNYZ7mDOk6z6/qpUKwhTDtXAZrLhKcfA3ojRXVExM2dVWDZ3me+1tJuXTDlUA5vJhscG9upgZs6K07erj/uevi+wzb6hfamNZLJoJcNoNMJELA0MwPLlcP/9FsUUk/Zb29k3tC+wjSAoWvS6bUYba5auqUhEk0UrGYbhTRgzp0+eSVZj9muBnzPaiVMxOF/vG9pXc59E4ykHS0QzGp1SZk4ff0QhZn/f0D4UTcUAlmaCnNGClHx/rX0SjaccrKSC0eiU8uv4+COyHLNfC9YsXcP0qdOLzrc0t7Dhgxtom9FW8h5hVh+VorGUg0Vo1C+2IixJKJNQQNi130BVywEszXR2dLL24rW0zWhDENpmtNH7wV4O3XCIzo5Ols1dVnIFEbT6qDRTavbJtcBrRmQlFeoD54rQvtMi3GUcCiYhYKLTM8Af0Tq31dPBWssBLO10dnR6OpULkUxun4OT6VOns2bpmkqKF0jjrBwsEa1+sRVhSUKbhAL8EV5mkloPYFnF6/sAmCyTx1cZtay/BI2kHCwRrX6xmP2SlDIJjZucLnma9lva6NvZW+SP8DKT1HoAyyp+38eYjjH22TH2Xre35v3aOGYlS0SrT/xWhJ/+tMXnO2id4W8SCm1ywt9MYkQj6PtIC42zcrDM2/rEVoShCDIJWRRSMuLkfmTBRNc4ysGoT2xFGIogk5BFIcUnbu5HFkx0Vj6jHFipASPD+JV5aJvRxt7r9lZfoAyRxb6rWvkMEVkiIr8QkZ0i0i8iizzavFZE7hKRZ0TkCRF51LHvdIuIvCIi2x3HPyeVq6pYYp2RYbJg4kgr9bzqSqQcROR0YCNwjaqeA3wCeEBE3GmB5wH/papvVtWFwPeAm/PXZgFbVXWR4/hEErmqioVRGhmnYOJoaW4ZP9c8pTnyfRqx7pKfAzlNjuW4JF05XAg8p6rbAFT1IWAAWOpspKqbVXWj49QAJyOlZgHniMg2EflvEbldRF6bUK7qYWGURp0wPDo8/v/B4cFIdZMate5SPa+6kiqHOcAe17k9+fOe5Af+LwCfz5/qB/5SVd8GXAAcBzaJSFFeuYisypuu+g8ePJhQ9DJgiXVGnZA0YqlRI56y4FiOS1LlIMAJ17lRv/uKSAuwCficqj4MoKrHVPVE/v+vADcAbwLe4H6/qq5V1QWqumD27NkJRS8D5QqjtLpARo1JajuvZ9t7KTo7Otl73d7UJK+Vi6TKYT/gNq615s9PQEReB2wBblbV3oB7Sl6uPyWUrfKUK4zSHNpGjUlqO69n23ujklQ5PEDOX1CIPFpIbtb/UxF5TETm5s+3kVMMPar6r84biMgH8oqDvCnpC8BDqvrHhLJVnh074MABWLwYVqyIt1m7ObSNFJDUdl7PtvdGJZFyUNUh4DJgnYg8AdwKLAOmA23AjHzTm4HXAv/HEa76cOE2wEYR6QceB/4XkJ11WU8PPPoo9PbGG+DNoR2PsKY4M9mFIqntvJ5t7w2LqmbymD9/vtacAwdUp02bWJCjqUm1uzv++5ubVQcGKit3PdDVpTppkn9fHzigunix6ooVwe2MstK7s1fbbmlT+Zxo2y1t2ruzt9YiGS6Afg0xxlr5jCR4OaSPH4dvfhN27oz3fls9lCaMKS7piq4BSZqn0KjhrPWKKYe4uMNYnajChz9c+h5WFyg8TvNQKVNc4btRzV33a2eMU2pgD6M4GjWctV4x5RAXr1m/k927S89UrVJseAoRXZ/8ZOncEr8Vna0efAka2MOuCBo5nLUeMeUQF69Zv5OpU22mWi6eegruvDM34Pf2nlwNFHCuCoJWdLZ68CVoYA+7IrBw1vrClENc3LP+N75x4vXCTPXppy1aJilXXJHrY8gN8CMjE687TXFBKzoz2fkSNLCHXRFYOGt9YcqhHDz1FPz618XnT5yAzk5LcEvCU0/Br3414dSRyTA8Of//KTDnk830rc/XavRb0c2bZya7AIIG9pnNMz3f4z5v4az1he3nUA7e8paiAWwckdyg1NwMzz9v+z1ExaNvRwEEpigcnQz3vBX+b2eufr57y0vIDXI2SJWmb1cfq7es5oWhF2id0cqapWvo7Ohk1ldnMTg8WNS+pbmFQzccqoGkRhKqtp9Dw+MxswVgyxbo6sr5HsDs3XEYGMg59l1MIacYAKadgKuegmP7cxuuWMRM+Tk8fNjz/ODwYMOU5m5ETDkk5YorvM9/8IP+UTWWtRuOnp6TyjXPKDDqqtc7SeEr218NBDtWG3G/gbAERSQFOZQtl6F+MeWQlN/+1vv80JB/gpsV2guHh//AuWooMO0EvP9QbqMav4FsZvNMS9AKIGjF5eWP8Gpn1BemHJLy0Y/mCu554ZXg9vDDVmgvLK6IsL6dvZyyZjryOcaPSZ8Tuh/sYuazewF/xypg5qYAglZcTkdz1Pcb2cWUQ1K2bfMPnSxEyDiPxYut0F5MvKJhNnxwA9943zcC26y9eK2v3dwGtRylchQKexb4KQjLZag/TDkkZceOnOO5qSn3uqkpV7bbK2zSdo5LTJiNVZxt1ixdw+otq1G8o/JsUMsRNkfBchnikUV/lymHpEQZ8K3QXlVxOlm9cA5qWfzjLSdhcxQslyE6WS1IaHkOSenuhnvumehfaGqClSvhjjsmtn3rW3Ohr27mzbPkrArQfmu7r2Jom9E2HsdvuRFGJfH7HbbNyOXmVJuweQ6mHJJiA35qmfT5Sb7mpLYZbePJXi8ff9mSvIyK4fc7FISxzwYU76wQlgRXLayyamrx8ycIMmGJ76UYIJfklfalv5F+ylmQsJrmT1MORt3i5TwVxHc14YWFuhpJKZcTv9q+i8TKQUSWiMgvRGSniPSLyCKPNiIiPSLynIjsFpFeETml1DXDSIKX8zSKYgB8fRaNRqM77JMQ5MSP0q/VLg2TyOcgIqcDe4D3q+o2EbkAuB84S1WPONpdCXwMeKeqDovIvcArqnpt0LWgz06Nz8HIDH27+vjIxo9EUhCTZTKjnxmtoFTpxxz2lSFqv5bLd1Etn8OFwHOqug1AVR8CBoClrnaXA3ep6nD+9W3A34a4Zhhlwy/fQRCP1jlO6Anfa42CFTOsDFH7tdqbKSVVDnPIrRyc7MmfD2q3B5gpIjNKXJuAiKzKm676Dx48mFB0o9Hwy4ZW1DfzN6hkRKNg239Whqj9Wu0ExKTKQQD31GrU477udoV1+qQS1yagqmtVdYGqLpg9e3ZsoY3GxG+GVch5sMxfb2z7z8oQtV+rnYCYVDnsB9xP0po/H9SuFXgZeKnENcMoG0EKIMofXqM5Z01xVoY4/RqmfEzZUNXYBzADOAR05F8vBF4EWoDHgLn581cBjwBN+ddfB+4rdS3omD9/vhpG785ebbulTeVzom23tGnvzt6ytvd6//Q105XPMX5MXzM98n2yRtJ+M7ypRb8C/RpifE+cIS0ifw18FVByJqHrya0GtgGXqGq/iEwGvgC8L99mN3Ctqv4p6FrQ51q0klGLKJq0lUIwjKhY+YyoDAzA8uVw//22z3NGqMVAnbZSCIYRFSufERXbnS1z1CKKxpyzRqNgygFOlt223dkyRS0GanPOGo2CKQeYuM+C1/4KAwOwaBG87W2mOFJELQZq28/AaBTM5zAwAHPmwNGjJ881N8Pzz5/0PXR3wze/efL/7n0ajJrRt6uP1VtWj5ffLoSlGobhjTmkw1Jqs56BATjrLDh2LHdt2jT43e/MaW0YRiBpnbiYQzos27ZNVAyQe711a+7/PT0wMjLxmjmtG56sJ8JlXf60k9WtQZ3YyiEI96qhgK0eGpqsVynNuvzVIsnMP835MLZyKAfuVUMBWz00NFmvUpp1+atB0pl/PRQrNOUQxLZtJ6OYnIyNnTQ7GQ1H1v/w/eTcN7QvU2aPSpJUgfqFU0+SSZnpY1PWwdXIAAARKElEQVQOQfjtD217RDc0WU+EC5Iza3bxuJTyuSSdAHiFWUNuf5Cs9LEpB8OISJz8ijQ5gP0GLsieeSlOv4YxGSWd+RfyYSbL5KJrWeljUw6GEZGoiXBpi1wpyO9HVsxjcfs1jMmoHDP/zo5OxtS73lYW+tiilQyjwqQ1ciWtcoUlqvyF6COv90Bx8cS+XX2s+H8rPLeKDdtHaexji1YyjJSQVgd21utERelX5yrDD7cpqRwz/yz3sSkHw0hAGJt3Wh3YWa8TFaVfvUxJTvwG7KTfXZb72MxKhhGTsMlklnRWGaL0q98+HHByD3G/LWHr7bszs5JhVJiwsfBZnj2mmSj96jfTL9j+/b6LRv7uEq0cRGQJcAswBThObnvP7R7tXktuK9DFwJ+BY0C3qu4SkRbgBWCX4y0/V9VPBH22rRyMWmO7wmWHpCuAtBbRi0PFVw4icjqwEbhGVc8BPgE8ICJeAdTnAf+lqm9W1YXA94Cb89dmAVtVdZHjCFQMhpEG0upLMIpJsgIIEzKbpjyWchF75SAilwMfV9W3O849BXxaVb9f4r0fBlaq6v8WkXeQUzLPk1uBbAe+qKp/DLqHrRyMWlOP9mijmFLhqFn7HZRt5SAiTSKy3X0AbwL2uJrvAeaUuF/BxPT5/Kl+4C9V9W3ABeTMU5tERDzeu0pE+kWk/+DBg6VEN4yKEnc2Wo+zzCwRtf9LhczWayHDJCuHTwFvVNUrHefuB7ar6i0+72kBfgTcoqq9Pm0mA38C5qnqb/w+31YORhbJ2iyzXnAmwAkywVc0ddJUXjXlVbx8/GUAWppbuO2i28a/j1Irh6z5nqoRrbQfcBtXW/PnvQR6HbAFuNlPMRSa5uX6UwLZDKPixFkB1OssM824E+DcA/nI2Mi4YgAYHB7kqu9dNf59lkpkC6rDlOXVYRLl8ABwjoh0AIjIQnKmph+LSIuIPCYic/PX2sgphh5V/VfnTUTkA3nFQd6U9AXgoVI+B8OoJKUG/rh1fdKaLV3PlEqA82JkbGRcYZcyHwbVYUpDLa24JA1l/Wvgq4ACo8D1qrpNRM4EtgGXqGq/iPwH8NeA00x0TFWXiMjFwKeAqcAYsBO4QVUPB322mZWMShHG9BO3Zk41au3UU9hlOQhKgAsiilnI2eeTZFKiekyVJqxZyTKkDcNFmAE8rp3ZS/FMnTSV0151GoeHDycezNPu06iF4vL7PksRdzBPuw/CMqQNIyZhTD9+duaZzTMD7+02UbQ0tyAiDA4PlsUEUSufRhj/S61Kl3uZfYSiYEjP98WhXvJfTDkYhoswf9xrlq6haXJTUZs/HftTycGus6OTvdftZeyzY7y66dUcP3F8wvUkg3ktfBphB/1aKS4vn8HVC6723fDI+b44ZLkSqxNTDobhIswfd2dHJ6c2nVr0XqcjMwzlHszDzlrLmWsRdtCvpTPeqZD3XreXTb/ZFOikbpvRVnQu7Oqo0B+FXeCyWo/JlINhuAib3HZ42Dtm4oWhF0IPvklNEO7PWTZ3WUnFVm7zTthBP86zViphMEghec3yw5bQcIbMntAT4/fKmmIAc0gbRmz8HJ0tzS0Mjw6HcgoncSD7vXfFuSvY9JtNvk7fckdMhb1f1GetpHPdT+bJMpn7/ua+ovuHecY07vrmhTmkDaPC+JmfgNC29SQF4fzMOZt+s2mCCcV9r3Kbd8La2KM+ayV9FH4yeykGCNdn9ZbDMqXWAhhGVikMIu7QzI9s/Ihne79BorOjM9ZMOO5g1Dqj1XOGGzeaxq8fvJ4pyrP6PUecsFQvOSCczBCuz8rdr7XGVg6GkQC3o7Ozo7NqoYxBnxNkq69ENI1XPyTF7/kEKYvvIYrMYfqsXqKUCphyMIwyE3eQiOp89fucZXOXcdX3rprgPHXWCsrK7mZrlq7xzEdQtOq1qML0WVb6NSzmkDaMChA1Eziu89Xrcz6++eMMDg8WtW1pbuHQDYeSPViVkc97J6ulJds4i1j5DMPIEOWMdPEbUAH0s9n6e89KBFCWsGglw8gQQc7XrJZ8LpAkVyGJHd82VUqGRSsZRgrwi3QBxhOuIFxJh5bmFl+zUjXp29VXZOKK+ixRo4qcn+000zk/N879GhEzKxlGCvDyObgJa0rp29XHld+7ktGx0fFzUyZN4duXfLtqg2Cp56l0ifLCrm9uoiQo1itmVjKMjNE8pTnwepRkKneUT5gqpOWk1AY75UoM8ytr4bcKGxwetJ34QmLKwTBqTGGA8zIFOQmbJ7F6y2pGxkYmnBsZG+Hjmz8eW8aohEnEKwd+WdSFondhyWoWcyUx5WAYNSbMNpZOJ2wpR6vfQDc4PIh8XqrinA0a/OMmhnk9t9+zForeuT/Xz++S1SzmSmLKwTBqTNCs1Z1MFaY6aKmBrhqb7PjtdwEnzThRPt/vuf02Vyr0mTsh7baLbqurLOZKknQP6SXALeSino4D16rqdo9284EfA792nP53Vb1ZRAT4AvAh4ATwC+DvVfWVoM82h7RRL0SJ5Q/Ttm9XH1dsvKLk5/pVIC0Xs746K9BUFsURXI4KuAUafY/tijukReR0YCNwjaqeA3wCeEBEvLZXmgV8V1UXOY6b89dWAMuAear6V8AI8JW4chlG1ogSyx+m2F5nR2eosNUTemLCCqLceQF++10UiOII9nvuw8OHI5esqEQdqHokSZ7DhcBzqroNQFUfEpEBYCnwfVfbWcB7ROTx/OufAF9W1T8DlwN3qepw/tptwBbg2gSyGUZmiBLLH7by520X3VYyNBYmDtB+eQFxB8+g3I0CYR3BQc8dt6qtEUzJlYOINInIdvcBvAnY42q+B5jjcZuNQLuqng9cBJwFrM9fm+O6zx5gpojM8JBllYj0i0j/wYMHSz6cYWSFsLPZOHsnQHAo6wtDL1Rk7wQvWd2EdQTXW8XTLFBSOajqcZc5aJGqLiJn/jnhaj7qdU9VHda8c0NVDwPXAxeLSDMgrvsUMne87rNWVReo6oLZs2eHeT7DqCuiVP4sKBz9rLLhgxt8wztbZ7SWbaMap2lq9ZbVrDh3ha+CijK411vF0yyQxKy0H3i361wr8B8h3jsZOAocy9/HOX1oBV4GXkogm2HULXHMKIX2XpVfgzKKo4R4epWsuO/p+yZEWiVxBJv5qLokCWV9ADhHRDoARGQhOVPTj0WkRUQeE5G5+WvL8w5sRGQKcBOwQVXHgA3AShEpxL19DNioScKoDCODVLpQXNDsuxxmm1KmKXMEZ4vYKwdVHRKRy4B1IqLkzEHLVPUlETkTaAMKfoNmYIuIjAEKPAx8Jn9tPfAG4AkRGQV2Y85oo8EIKhRXzkHUb/Ydt8CdkyzsodzoYaxRsMJ7hpEC6mHfgnI+QyUG8bgbKtUbVnjPMFKM24TkF/KZpll3KcoVURQmCzwOlYjIqmdMORhGlfEa/PxCTbNU86dcEUWVGsSzYPZKE7bZj2FUGa/BT1EEQTlp5k1zHL+f2accEUWVGsTDJhAaOWzlYBhVxm+QUzQTcfyVMvsU8Buskw7ilkgXDVMOhlFl/Aa5guO21qGepUJq/cw+V2y8oiwhuJUaxC2RLhpmVjKMKrNm6RrfZLRaEyakNsi8U44Q3HKE1Qbd25RBOCyU1TBqQFrj7cOEowZFV3m1N9KFhbIahhGZMM7gMAX1ahkBVOlM80bBlINhVJlKO3STEMYZ7K746oXfDm2VJs19mzVMORhGlUlzMlaUkuB7r9tL7wd7mTppatF9/nz8zzUZkNPct1nDlINhVJk0J2NFjejp7OjktFedVnT++InjNRmQ09y3WcOilQyjyqQ9GStqRI/fdqC1GJDT3rdZwlYOhlFl6i0Zq1JJa3Got76tJaYcDKPK1FsyVpoG5Hrr21pieQ6GYSQmrXkbRjFh8xxMORiGYTQQlgRnGIZhxCaRchCRJSLyCxHZKSL9IrLIp90PRWS749ghIr/LX5svIodd169PIpdhGOnGspjTT+xQVhE5HdgIvF9Vt4nIBcADInKWqk7IQlHV97re+4/A3PzLWcB3VfXv4spiGEZ2qNZ+2UYykqwcLgSeU9VtAKr6EDAALA16k4g0A9cBN+VPzQLeIyKP5481InJqArkMw0gxlsWcDUquHESkCXjE49JmYI/r3B5gTolbdgGbVPX3+dcbgX9VVRWRmcDXgfXA33jIsgpYBdDaakkthpFFLIs5G5RUDqp6HCjyJYjIp4ATrtOjBKxGHKuGdzruP+z4/+G8v+H3ItLsvJa/vhZYC7lopVKyG4aRPiyLORskMSvtB9zfZmv+vB/dwGZVDZoiTAaOAscSyGYYRkpJU9Kc4U8S5fAAcI6IdACIyELgTcCPRaRFRB4TkYLTGRGZTm7V8CXnTURked65jYhMIeeL2KCqYwlkMwwjpVgWczaIHa2kqkMichmwTkSUnElpmaq+JCJnAm3ADMdbuoEfqqp7PdkMbBGRMUCBh4HPxJXLMIz0Y9t1ph/LkDYMw2ggLEPaMAzDiI0pB8MwDKMIUw6GYRhGEaYcDMMwjCIy65AWkYNAcSZNdZkFHKqxDHHJsuyQbfmzLDtkW36THdpUdXapRplVDmlARPrDeP3TSJZlh2zLn2XZIdvym+zhMbOSYRiGUYQpB8MwDKMIUw7JWFtrARKQZdkh2/JnWXbItvwme0jM52AYhmEUYSsHwzAMowhTDoZhGEYRphwiICJTReQTIjIiIssD2omI9IjIcyKyW0R6ReSUasrqIdMSEfmFiOwUkX4RKdrAKd9uvogcFpHtjuP6NMqbxn4uEFL+VPS1h1wlf+cp7/sw8qe17/9eRJ7O/2Z2iki3T7trRORZEfmliDwoIq8tuzCqakfIg1zZ8euBR4HlAe2uBJ4EmvOv7wW+XkO5TwcGgbflX18A/BGY7tH2QuDuGvdzKHnT1s8x5K95X/vIX/J3nta+jyB/6vqe3EZn/wy8Ov/6DGAYOMPV7gLgBWB2/vXngAfLLk+tOySLB/BQCeWwGVjleD0PGKyhvJcDW13nngIu9mjbSS7z/PH8sQY4NY3ypq2fY8hf874u8Ry+v/O09n0E+VPd93kZpwFDQKvr/DeBLzlev4bcfjozyvn5ZlZyISJNrqVm4WiKcJs5wB7H6z3ATBGZ4dO+LPjJTm6Hvj2u5nvycrrZCLSr6vnARcBZwPpKyu2Bu//AW96a9HMIwsqfhr6OS1r7PixZ6Ptbgfu1eFvlCX2vqi+SUyLt5fzw2DvB1SuqehzwtMdHQIATjtej+X8rqoz9ZBeRT7nkKchUJI+qDjv+fzhvh/29iDQ7r1UYd/+Bt7w16ecQhJI/JX0dl7T2fSjS3vci8kVyZqVLvS4T8u85CZn4IjPIfqDV8boVeBl4qTbiFMlD/vX+EO+dDBwFjpVbqADCypu2fi4Qt79r0ddxSWvfxyU1fS8i/wycDVyan/C5mdD3IjIdaCHc33NoTDmUARFpEZHHRGRu/tQGYKXDFPUxYKPmDYQ14AHgHBHpABCRheRMTT92yy4iy0Xk9Pz/pwA3ARtUdSwF8v405f1cIJT8KenrUGTgNx5ISn/nbhknicidwJnAZQXFICKTRWSLiLwr33QD0Okw4V0DPKaqB8spj5mVysN0oA0ofFnrgTcAT4jIKLAbuLZGsqGqQyJyGbBORJTcEnSZqr4kImcyUfZmYIuIjAEKPAx8Jg3ykvJ+LhBB/pr3dQQy0fcBZKHvlwF/D/QDPxeRwvkvkvOJzARQ1Z+JyB3AwyIyAhwAfEPr42LlMwzDMIwizKxkGIZhFGHKwTAMwyjClINhGIZRhCkHwzAMowhTDoZhGEYRphwMwzCMIkw5GIZhGEWYcjAMwzCKMOVgGIZhFPH/AYTWjSYwuKPiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_pred_idx = y_pred.reshape(-1) # 열 벡터를 1차원 배열로 바꿉니다\n",
    "plt.plot(X_test[y_pred_idx, 1], X_test[y_pred_idx, 2], 'go', label=\"양성\")\n",
    "plt.plot(X_test[~y_pred_idx, 1], X_test[~y_pred_idx, 2], 'r^', label=\"음성\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "음 결과가 좋지 않네요. 그렇죠? 하지만 로지스틱 회귀 모델은 선형적인 결정 경계를 가지므로 최선에 가까운 것 같습니다(잠시 후에 보겠지만 특성을 더 추가하지 않는다면 말이죠)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "다시 시작해 보죠. 이번에는 연습문제에 나열된 모든 부가 기능을 추가해 보겠습니다:\n",
    "* 재사용이 용이하도록 `logistic_regression()` 함수 안에서 그래프를 정의합니다.\n",
    "* 훈련하는 동안 일정한 간격으로 `Saver` 객체를 사용해 체크포인트를 저장하고 훈련이 끝날 때 최종 모델을 저장합니다.\n",
    "* 훈련이 중지되고 다시 시작할 때 마지막 체크포인트를 복원합니다.\n",
    "* 텐서보드에서 그래프가 잘 정돈되어 보이도록 이름 범위를 사용하여 그래프를 정의합니다.\n",
    "* 서머리(summary)를 추가해 텐서보드에서 학습 곡선을 나타냅니다.\n",
    "* 학습률, 미니배치 크기 같은 하이퍼파라미터를 바꾸어 보면서 학습 곡선의 모양을 관찰합니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "시작하기 전에 입력에 ${x_1}^2$, ${x_2}^2$, ${x_1}^3$ 그리고 ${x_2}^3$ 네 개의 특성을 추가합니다. 연습문제에 포함되어 있지는 않지만 특성을 추가하면 모델의 성능이 향상되는 것을 확인할 수 있습니다. 여기서는 수동으로 특성을 추가하지만 `sklearn.preprocessing.PolynomialFeatures`을 사용할 수 있습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_enhanced = np.c_[X_train,\n",
    "                         np.square(X_train[:, 1]),\n",
    "                         np.square(X_train[:, 2]),\n",
    "                         X_train[:, 1] ** 3,\n",
    "                         X_train[:, 2] ** 3]\n",
    "X_test_enhanced = np.c_[X_test,\n",
    "                        np.square(X_test[:, 1]),\n",
    "                        np.square(X_test[:, 2]),\n",
    "                        X_test[:, 1] ** 3,\n",
    "                        X_test[:, 2] ** 3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "특성이 추가된 훈련 세트는 다음과 같습니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.00000000e+00, -5.14696757e-02,  4.44198631e-01,\n",
       "         2.64912752e-03,  1.97312424e-01, -1.36349734e-04,\n",
       "         8.76459084e-02],\n",
       "       [ 1.00000000e+00,  1.03201691e+00, -4.19741157e-01,\n",
       "         1.06505890e+00,  1.76182639e-01,  1.09915879e+00,\n",
       "        -7.39511049e-02],\n",
       "       [ 1.00000000e+00,  8.67891864e-01, -2.54827114e-01,\n",
       "         7.53236288e-01,  6.49368582e-02,  6.53727646e-01,\n",
       "        -1.65476722e-02],\n",
       "       [ 1.00000000e+00,  2.88850997e-01, -4.48668621e-01,\n",
       "         8.34348982e-02,  2.01303531e-01,  2.41002535e-02,\n",
       "        -9.03185778e-02],\n",
       "       [ 1.00000000e+00, -8.33439108e-01,  5.35056649e-01,\n",
       "         6.94620746e-01,  2.86285618e-01, -5.78924095e-01,\n",
       "         1.53179024e-01]])"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_enhanced[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "좋습니다. 이제 기본 그래프를 초기화합니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "그래프를 만들기 위해 `logistic_regression()` 함수를 정의합니다. 입력 `X`와 타깃 `y`의 정의를 포함하지 않았습니다. 이 함수에서 정의할 수도 있지만 그렇게 하지 않아야 다양한 경우에 이 함수를 사용할 수 있습니다(예를 들어, 로지스틱 회귀 모델에 주입하기 전에 입력에 대해 전처리 단계를 추가할 수 있습니다)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "def logistic_regression(X, y, initializer=None, seed=42, learning_rate=0.01):\n",
    "    n_inputs_including_bias = int(X.get_shape()[1])\n",
    "    with tf.name_scope(\"logistic_regression\"):\n",
    "        with tf.name_scope(\"model\"):\n",
    "            if initializer is None:\n",
    "                initializer = tf.random_uniform([n_inputs_including_bias, 1], -1.0, 1.0, seed=seed)\n",
    "            theta = tf.Variable(initializer, name=\"theta\")\n",
    "            logits = tf.matmul(X, theta, name=\"logits\")\n",
    "            y_proba = tf.sigmoid(logits)\n",
    "        with tf.name_scope(\"train\"):\n",
    "            loss = tf.losses.log_loss(y, y_proba, scope=\"loss\")\n",
    "            optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate)\n",
    "            training_op = optimizer.minimize(loss)\n",
    "            loss_summary = tf.summary.scalar('log_loss', loss)\n",
    "        with tf.name_scope(\"init\"):\n",
    "            init = tf.global_variables_initializer()\n",
    "        with tf.name_scope(\"save\"):\n",
    "            saver = tf.train.Saver()\n",
    "    return y_proba, loss, training_op, loss_summary, init, saver"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서보드를 위해 서머리를 저장할 로그 디렉토리 이름을 생성하는 함수를 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "def log_dir(prefix=\"\"):\n",
    "    now = datetime.utcnow().strftime(\"%Y%m%d%H%M%S\")\n",
    "    root_logdir = \"tf_logs\"\n",
    "    if prefix:\n",
    "        prefix += \"-\"\n",
    "    name = prefix + \"run-\" + now\n",
    "    return \"{}/{}/\".format(root_logdir, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 `logistic_regression()` 함수를 사용해 그래프를 만듭니다. 텐서보드용 서머리를 로그 디렉토리에 저장하기 위해 `FileWriter`도 만듭니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 2 + 4\n",
    "logdir = log_dir(\"logreg\")\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs + 1), name=\"X\")\n",
    "y = tf.placeholder(tf.float32, shape=(None, 1), name=\"y\")\n",
    "\n",
    "y_proba, loss, training_op, loss_summary, init, saver = logistic_regression(X, y)\n",
    "\n",
    "file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "드디어 모델을 학습시킬 수 있습니다! 이전에 훈련 세션이 중지되었는지부터 검사하고 그렇다면 체크포인트를 로드하고 저장된 에포크 횟수부터 훈련을 이어갑니다. 이 예에서는 별도의 파일에 에포트 횟수를 저장했지만 11장에서 모델에 일부로 훈련 스텝을저장하는 방법을 배우겠습니다. 예를 들어 `global_step`이란 훈련되지 않는 변수를 옵티마이저의 `minimize()` 메서드에 전달합니다.\n",
    "\n",
    "다시 시작할 때 마지막 체크포인트가 제대로 복원되는지 확인하기 위해 훈련을 중지시켜 볼 수 있습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "에포크: 0 \t손실: 0.62998503\n",
      "에포크: 500 \t손실: 0.16122366\n",
      "에포크: 1000 \t손실: 0.11903212\n",
      "에포크: 1500 \t손실: 0.097329214\n",
      "에포크: 2000 \t손실: 0.083697945\n",
      "에포크: 2500 \t손실: 0.07437584\n",
      "에포크: 3000 \t손실: 0.067502156\n",
      "에포크: 3500 \t손실: 0.06220691\n",
      "에포크: 4000 \t손실: 0.058026787\n",
      "에포크: 4500 \t손실: 0.05456298\n",
      "에포크: 5000 \t손실: 0.051708277\n",
      "에포크: 5500 \t손실: 0.04923773\n",
      "에포크: 6000 \t손실: 0.047167283\n",
      "에포크: 6500 \t손실: 0.045376636\n",
      "에포크: 7000 \t손실: 0.043818742\n",
      "에포크: 7500 \t손실: 0.042374216\n",
      "에포크: 8000 \t손실: 0.041089155\n",
      "에포크: 8500 \t손실: 0.039970912\n",
      "에포크: 9000 \t손실: 0.038920246\n",
      "에포크: 9500 \t손실: 0.038010743\n",
      "에포크: 10000 \t손실: 0.0371557\n"
     ]
    }
   ],
   "source": [
    "n_epochs = 10001\n",
    "batch_size = 50\n",
    "n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "checkpoint_path = \"/tmp/my_logreg_model.ckpt\"\n",
    "checkpoint_epoch_path = checkpoint_path + \".epoch\"\n",
    "final_model_path = \"./my_logreg_model\"\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    if os.path.isfile(checkpoint_epoch_path):\n",
    "        # 체크포인트 파일이 있으면 모델을 복원하고 에포크 횟수를 로드합니다\n",
    "        with open(checkpoint_epoch_path, \"rb\") as f:\n",
    "            start_epoch = int(f.read())\n",
    "        print(\"중지되었던 훈련입니다. 에포크를 이어갑니다.\", start_epoch)\n",
    "        saver.restore(sess, checkpoint_path)\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        sess.run(init)\n",
    "\n",
    "    for epoch in range(start_epoch, n_epochs):\n",
    "        for batch_index in range(n_batches):\n",
    "            X_batch, y_batch = random_batch(X_train_enhanced, y_train, batch_size)\n",
    "            sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "        loss_val, summary_str = sess.run([loss, loss_summary], feed_dict={X: X_test_enhanced, y: y_test})\n",
    "        file_writer.add_summary(summary_str, epoch)\n",
    "        if epoch % 500 == 0:\n",
    "            print(\"에포크:\", epoch, \"\\t손실:\", loss_val)\n",
    "            saver.save(sess, checkpoint_path)\n",
    "            with open(checkpoint_epoch_path, \"wb\") as f:\n",
    "                f.write(b\"%d\" % (epoch + 1))\n",
    "\n",
    "    saver.save(sess, final_model_path)\n",
    "    y_proba_val = y_proba.eval(feed_dict={X: X_test_enhanced, y: y_test})\n",
    "    os.remove(checkpoint_epoch_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "여기에서도 추정 확률이 0.5보다 크거나 같은 샘플을 모두 양성으로 분류하면 예측이 됩니다:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = (y_proba_val >= 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9797979797979798"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9797979797979798"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD+CAYAAADRRMnDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJztnX2QXGWZ6H9PkhkyYSGYSa56kZkhJZYuOxBJKsSvhL2xRKJYrBSSdaAC3myWGXBhL15KTPm1Y0S3lhVEVIIE8zFrsbvmyoqJq0b5kCSwgwGiQdRggqmMbj4gCvmYmcxz/+ju5Mzpc06fr+4+p/v5VZ1K+py3T7/nne73eZ/PV1QVwzAMw3Ayod4dMAzDMLKHCQfDMAyjDBMOhmEYRhkmHAzDMIwyTDgYhmEYZZhwMAzDMMow4WAYhmGUYcLBMAzDKMOEg2EYhlHGpHp3IC7Tp0/Xrq6uenfDMAwjVzz11FP7VXVGpXa5FQ5dXV0MDg7WuxuGYRi5QkR2h2lnZiXDMAyjDBMOhmEYRhkmHAzDMIwycutzMAzDSIuRkRH27NnD0aNH692V1Jg8eTJveMMbaGlpifV+Ew6GYTQ9e/bs4bTTTqOrqwsRqXd3EqOqHDhwgD179nD22WfHuoeZlYx8MjQECxbA739f754YDcDRo0dpb29vCMEAICK0t7cn0oRMOBj5pL8ffvrTwr+GkQKNIhhKJH0eEw5G/hgagvvvh7Gxwr+VtAfTMoyMMzIywtKlS5k9ezYLFizgueeeA+ALX/gC3/jGN3zft2vXLt797ndXpU8mHIz80d9fEAwAx49X1h5MyzBSZmD7AF13dDHhsxPouqOLge0Die5377330t7ezlNPPcWXv/xlli5deuLabbfdduL1N77xDZYvX57os8JiwsHIFyWtYXi48Hp4GFat8tcKomoZhlGBge0DLPvuMnYf2o2i7D60m2XfXZZIQOzYsYP3vve9AJx//vkcPHjwxLVPfvKTJ7SHrVu3cvjwYQDe8Y538IEPfCDBkwRjwsHIF06tocTwsL9WEFXLMIwKLN+0nMMjh8edOzxymOWb4q/o58+fz7e//W1UlUcffZRzzjnnxLX+/n6WLl3Kxo0bUVVOPfVUbr/9dh599FH+4z/+I/ZnVsKEg5Evtmw5qTWUGBuDRx4p/N/pX/DSMkx7MBLy4qEXI50Pw4c+9CHe9KY3cdlllzEwMMB999134tqtt95Kd3c3//qv/8pdd91Ff38/U6ZM4Z//+Z9jf14YLM/ByBfbthX+7euD++4rTPitrQWBMDQEs2cXJv/+flAt1zJK2sPdd9e+70ZD0DG1g92HymvXdUztSHTfv/u7v+Oiiy7iy1/+MldeeSVnnnkm73nPe3j3u9/NtGnTOOWUU0607e3tBeDQoUPj/BNpYsLByB9+GsG+fYVrUHg9c2a5ljE8DJs3+9938WJ44AF43euq138j16xYuIJl3102zrQ0pWUKKxauSHTf3bt3s3jxYlatWsX555/Pb37zG6655hre+MY38vrXvx6Ab37zm3z9619HRBgbG+Oss86qmgZhZiUjf3j5HY4fh3/7t5OvR0cL2oRq+VHSPrzua1FNRgV6untYeelKOqd2IgidUztZeelKerp7Et1369atXHLJJcybN4+2tja6u7tZsmQJP/nJTwD45S9/yV133cUPfvADtmzZwhNPPEFPTw/XX399Go9VhgkHI394+R3cr0dGovkXLKrJiEBPdw+7btrF2KfH2HXTrsSCAeDCCy9k48aNDA4OcuzYMZ577jnWrFnDu971LgBOO+00Dh8+zK9+9SuGh4d56aWX+PnPf86MGRX37YmFCYdGoZkSvTZsgPnzC8+sCnv3wsSJ5e1GR8NrAf39Be0j6vsMIyW6urpYu3Ytd999N+9973vp7+/ntttuOyEczjzzTNasWcNXv/pVLr74Yq6++momTZrEV77ylep0SFVzecyePVsNB729qhMmqPb11bsn1cf9rL29XsajwjFrVuX77d2rOnny+Pe1takODVX3OYzMsGPHjnp3oSp4PRcwqCHm2MSag4i0iMjHRGRERBb7tBER6ReR50Vkh4isE5FTK10zQtJMJhGvZ92yxbvtrFn+/gUnTq2hhGkPRpOThlnpbwAFtga0WQIsAmap6p8DI8AXQ1wzwhA10SuPJqhSn2+9tfxZt22L5nh2s2VLwUfhZGTEP6rJMJqAxMJBVb+qqrcDxwOaXQnco6pHiq/vBP46xDWjEnESveoVlZNEKPX3w2OPwbp16Se1bdgAkyePPzd5cuHIkwA1jBSplUN6JrDT8XonME1Epla4Ng4RWSYigyIyuG/fvqp2ODf4hXX6Tfxus8wzz9ROi4grlEp9Vi03/6RREsOvJMfWrWZaMpqWWgkHYbxmMer4/KBr41DVlao6R1XnVCt8K3f4hXX6mUTcJqientpoEUn8Il6Td4mgZw2LX0kOaHwfjpEZRkZGuOWWW5g7dy5vf/vbmTt3LrfeeivH3QsiB41QsnsP4Mwt7wBeAV6ucM2oRBR7u5cJ6he/qI0jO24BPHefAdraToaxRvEt+OEew97eQkmOqH01mouUfXcrV65k//79bN26lc2bN7N582ZeeOEFVq9efaJN7kt2i0i7iDwuIqXSgmuBpSJS/MXxUWB9Mawq6JqRJkEr8GpOgnEL4JVqJfmZkqrhWLdifUZYUvbdveY1r+G///u/+f3vf4+qMjQ0xP79+5k2bdqJNo1QsnsK0AmU/AZrgMeAJ0VkEDidghCodM1IEy/zSYmgSTDpJBzVL+J839BQeSRRyZRUDcd63L56kceoMCMcVQgf//CHP8zll1/Oddddxzvf+U5uuOEGPvKRj3DZZZcB1Lxkt+R1gT5nzhwdHBysdzfyi7OqaYnWVli6tLxiaV8f3HMPXHddvGqmb30rPP10+fmgPIShoULhvKNHC2akF14YXwyv0vW4xOmrH0nHzagZzz33HG95y1vCv8FdFdjrdxOBL37xi7z00ku+19va2ti1axd33303bW1tfP3rX+eVV17hiiuuYOnSpfzoRz/yfJ/Xc4nIU6o6p2KnwmTKZfGwDOmEzJoVLqPYmT2cVtbw3r2q8+cH36u3V7W1tfC5ra3lmd+VrichjWzzbdsK97Bs61wQKUO6Chn1v/jFL3Tbtm2+xy9/+UvP97388sv6rW99y/e+STKk6z7Jxz1MONSIakzClSbfSj++apa7SEsYnnvuyb65x80pHMMISqPqRBIOzt+E3984BnfddZeee+65euGFF544/uIv/kK/8IUvnGhz//3364UXXqjz5s3TuXPn6uWXX667d+/2vacJB6M6eE3CkyerzpsXfzILM/lW+vFV6cdZdu+gST2IbdvG9839rE7h2Ew1sTJMJOEQVuuOyG233ab33nvvuHPf+ta3dPny5aqq+txzz+kFF1yghw4dOnF9/fr1+v73v9/3nnWtrWQ0MNVIDgsT0lopdyNqbkdYKkUqhXWAX3VV+TlnhFXJkblqVeFohppYjUTSci0BfO5zn2POnDknDmfYaq1LdtddA4h7mOZQgdIq9+mn45st/FZIcU0uWa9+GqSRhDU37d2rKuK/snR+xoQJJ/0SaftNjEjkpSrrk08+qddee61edNFF+r73vU8///nP66uvvurb3sxKRjklc8W556Zntkjqf6imOSgNgswFYZ89rIBJS+AaqZAX4RAVMysZ43GaLtLKgE4jOaxa5qC08DMXbNgQ/tmDnjEoCREKpqePf9xyI4xMYMKhEfGahEZH4YILap/I5qSKttqqEuXZg54xKAkRCtceeuikX8OS6GpKYVHdOCR9HhMOjYZXLSIoZBkPDRVWpnHI+qq/mqT17H6Co3Ts3QuvvnpS07v11vqUVm9CJk+ezIEDBxpGQKgqBw4cYLK7FH0ELEO60fDKfHYycSLs2ZNONnEzUs2sZ3fW7fHjhSPNDHDDk5GREfbs2cPRo0fr3ZXUmDx5Mm94wxtoaWkZdz5shrQJh0bDr/yDk76+5irnMDQEixfDAw8km2CrVbLDfW83KZRnMIwSYYWDmZXqRbXsyW7Txd695buc1XqTn3qTVoG+uGXHo97bjVWGNeqACYd6UautOj/+cTh2bPy5Wm7yU2+iVs/0E9rVLuVdyVlt+0oYNcaEQz2oQrlfX773vYIG4WR4GHbsaI7M3KirfT+hnWYpby9KGl9vr/f1ZnH+G5nBhEM9qIZ5wmvFOzRUiH6B8bun9fZCyUnVyCvSqKv9IKFdi2it0udD+W53pXwLr79xs5gHjdoSJlMui0duM6STlJAIKvzmVcDNK6s36yUs0iRqRnY1y4BH7a9fmXKvv7EV7jMigJXPyChJSkj4TQRedX/8hMCSJdkuYZEmUapn1ltoRilT7vU3blQBb6ROWOGQ2KwkIgtE5Gci8qyIDIrIPI823xeRrY5jm4j8tnhttogcdF2/OWm/Mktc80SQycPLTOVnI3/ooeZJZouSkV1tn0IlKn1+pb9xI5sHjfoQRoL4HcAZwAHgbcXXFwF/AKZUeN/fA18t/v9i4N6on51bzSEufiYHvxWnc7OZSqtmo2o1+lP5fL99NZrFPGikCjXSHC4GnlfVLUVB8zAwBCz0e4OItAE3AbcVT00H3iMiTxSPFSJyWsJ+NRZBjlW/FeeCBfmsY1Qv6l33acOG8nyUtjbYuNF/Xw23BmiF+4wUSSocZgI7Xed2Fs/70QtsUNXfFV+vB7pU9ULgEuBsYI3XG0VkWdF0Nbhv375kPc8TQSaHZq551EhE/RuPjXkLDGfhPsNIQFLhIMBx17lRv/t6aA2o6pGiqoOqHgRuBi4tth2Hqq5U1TmqOqdqux9lkSABUO8Vr5EOcf7G7sNduM+0ByMBSYXDHqDDda6jeN6LPmCjqr4YcM+JwFHgWECb5sIEQOOTxt/YHNRGiiQVDg8C54lIN4CIzAXeDPxYRB4XkXNKDUVkCgWt4fPOG4jIYhE5o/j/SRS0irWqGrArSpNjiU+Gm2qX9zCajkTCQVUPAVcAq0TkSeAOYBEwBegEpjqa9wHfV9Xdrtu0AZtE5L+AzcDvgf+TpF8NT63qMhnZxrlIqHcortFwWMnuvFHNstFGvnDuLbF5s3ep9lmzzPxojMNKdjcqZlc2oDwpcuNG80sZqWLCIU8MDcGqVWZXNmqzSDDfVlNjwiFP9PcX9oJ2YtpD81Er57P5tpoaEw5ZxG/F9uij3olPlvDWXNTC+VzLPUeMTGLCIYv4rdjmzy/sJwyFf/v6zK7cjNQiK958W02PRStlDb9oJK8N6C1aqbkYGoLFi+GBB8L9zaO2d77PvmsNi0Ur5RW/FZvFsRtRfQBxfQZB3zVzUjcNJhyyRJCj0QrsNTdRfQBJfAZB3zVzUjcNJhyyQGk1duut/is2q6/U3ET1ATjbHz1aKOUdFr/v2oYN5qRuIkw4ZIHSaux73zPtwCgnSKP0MvO426vCunXJJ3NzUjcVJhzqjVP9f/XVwutS+eX58wuvTTtoboJ8AF5mHr/2UbQHN1bYr+kw4VBvghzQXrZdcwg2H34+gEce8TbzeLWHgmYK8b5DFhDRdJhwqCd+q7FnnvG37ZpDMBqNIEz9fADz549fWFxwQeE5t20raJ7ubUdfffVkBdeo3yELiGg+wmw0ncVj9uzZcffXzg69vaqtreN/8q2tqueee/J8a6tqX1+hvXOjedtMPhy9vaoTJqguWaI6f37jjJnzu+A8rrmmcN3vu7VkiX2HmhxgUEPMsaY51BO/1diOHd62XXMIRsPpz1m3Dh57rHHGzMvMA7B2bXDo80MP2XfICIUJhzBUyzThZS7o7YWWlvHtSs5EcwhGwy1MVRtnzPz8Cn6hz3v3wrx58Mor3mbMvJvejNQx4RAGLxttXIFR6X1hVnwlbOXnj9ufU6JRxqw0+Xv5FrwEYH8/bN3qXdW3p8f8WEYZJhwq4ZdpGsWp597OsfQ+L0Hh53w86yxzCEbBz+zSaBpXmCii0ncYvKv67thhiW1GOWEcE0EHsAD4GfAsMAjM82gzGzgIbHUcNxevCdAPPA/sANYBp1b63Jo5pJ2OvZJzOKpj2OkUdb5vyZLC+ZLDOUxforRvZmbN8hKx5U7+vOP3nLNmnWzj9R0Oc81oSAjpkE4qGM4ADgBvK76+CPgDMMXV7mLgXp97XAM8BbQVX98PfKXSZ9dEOHhFhJQm9bA/KOc9Jk5UbWkp/L+lpfA6rICxSKV4hJk8Gxm/7/DQUPC1MPdtpOivJiKscEhqVroYeF5VtxS1kIeBIWChq9104D0i8kTxWCEipxWvXQnco6pHiq/vBP46Yb/SwUtlHx0tRL6EdQy7naIlm+/ISOF16XyUWjmNYjevBc1ek6pSdnVcP5bl2zQ8SYXDTGCn69zO4nkn64EuVb0QuAQ4G1jjc4+dwDQRmer+MBFZJiKDIjK4b9++hF0PgZdz2Dmpl/D7Qfk5Rd1UEjBWusCIS1DyWtzENtslrilIKhwEcM2UjLrvq6pHiuoMqnoQuBm4VETaPO4x6tc3VV2pqnNUdc6MGTMSdj0EXqvOWbPK2/n9oPycol4ErdisdIERl6AKq6effrKWVxStyrTYpiCpcNgDdLjOdRTPBzEROAoc87hHB/AK8HLCvlWHKGYKv1h0d+ghBK/YrHRBZAa2D9B1RxcTPjuBrju6GNg+UO8uZYu4ZiHTYpuGpMLhQeA8EekGEJG5wJuBH4vI4yJyTvH8YhE5o/j/ScBtwFpVHQPWAktFpLg5Mh8F1pc0jVzjJ0iOHIlmB28yu3nSiX1g+wDLvruM3Yd2oyi7D+1m2XeXmYAokcQsZFps05BIOKjqIeAKYJWIPAncASwCpgCdQMlv0AZsEpH/AjYDvwf+T/HaGuAx4EkRGQROpyAgGo9GKAJXZdKY2JdvWs7hkcPjzh0eOczyTcvT7m4+SWIWMi02MnnVYiWvC/Q5c+bo4OBgvbsRjb4+uOceuO46uPvuevcmk3Td0cXuQ7vLzndO7WTXTbtC3WPCZyeglH+vBWHs0yF9QI3K0BDMnFnYHa5EWxu88AK87nX161eDUlrsOBcrU1qmsPLSlfR099SlTyLylKrOqdTOMqRrhUV4hOLFQy9GOu9Fx1S3Gyz4fFNhZqGakmct1oRDrbAIj1CkMbGvWLiCKS1Txp2b0jKFFQtXJOpbQ2BmoZrit6jZfWh35s1LJhxqgUV4hCaNib2nu4eVl66kc2ongtA5tbOuanymcAY39PbChAkFc2eDBjfUm6BFTdaDJMznUAv6+uC++8av2FpbYelS8z14MLB9gOWblvPioRfpmNrBioUrbGJPG6fvwXwOVaPve318bfBrvtej+NLSIqzPYVItOtP0mCofiZ7uHhMG1cbLzGkLlVQZ2D7A6mdWB7bZfWg3XXd0ZXIhZJqDYTQbYSKWhoZg8WJ44AHTKGLiF3nnRJBxkXWl151TO6smKCxayTAMb8JELAVlUFu+TijCRNi5Q65Lr7OQuNl8wsG+2EazU8nMWSns2iqyhiLIGS1IxffXO+S1+YSDfbGNZqdSOZagsGvL1wmNV+QdQHtbO2s/uJbOqZ0V7xElvydtmks42Be7IclreYK6UElzrhR2bfk6ofEKqV73wXXsv2U/Pd09LDpnUUUNop6Jm80lHOyL3XBYkb2IVNKcg/wRlq8TmZ7uHnbdtIuxT4+x66ZdJxzMpUgmrzIvJeqduNk8wsG+2A1JnssT1JwwmnOQP8JKb6SG1/cWYKJMzEziZvPkOQR9sS2+O7ekUYupaaiU2zA0dHIDIK/w1be+1fJ1UsLv+zmmY5kpDtk8moMlojUkVmQvJGE050ompybbV6Sa5OF72zzCwb7YDYkV2QtJJZOQBWvEJk5ARB6+t80jHIyGxIrshaSS5mzBGrGIGxCRh++tlc9IAys1YOQZ2wAoNmlsTlVralY+Q0QWiMjPRORZERkUkXkebV4rIveIyHMi8qSIPObYd7pdRF4Vka2O45+S9qumWGKdkWcsCik2jRwQkUg4iMgZwHrgelU9D/gY8KCIuNMCLwD+U1Xfoqpzge8AtxevTQc2q+o8x/GxJP2qKWarNfJOmsEaTVaeJg+O5bgk1RwuBp5X1S0AqvowMAQsdDZS1Y2qut5xaoiTYbTTgfNEZIuI/JeI3CUir03Yr9phtloj76S5AVCTadF5cCzHJZHPQURuBf5cVa92nPs28Kiq3unzntcCjwP/W1UfEZFTgFFVPS4ipwL/AFwEzFFX50RkGbAMoKOjY/bu3cHlcKuO2WqNRiLpBkBNuoFQ3janqpXPQYDjrnOjfvcVkXZgA/AZVX0EQFWPqerx4v9fBW4B3gy80f1+VV2pqnNUdc6MGTMSdj0F0rLVNpkqbmSUpFpwk2rRfiUy8k5S4bAHcBvXOornxyEirwc2Aber6rqAe0qxX39M2Lfqk5attslUcSODJC0vY+VpGo6kwuFBCv6CUuTRXAqr/h+LyOMick7xfCcFwdCvqv/ivIGIfKAoOBARoWBWelhV/5Cwb9Vn2zbYuxfmz4clS+LZas2hbWSBpFqwRTw1HImEg6oeAq4AVonIk8AdwCJgCtAJTC02vR14LfB/HeGqj5RuA6wXkUHgCeB/APnRy/r74bHHYN26eBN8k6riSQibkWqlvCOQVAu28jQNhyXBJcHLId3aCkuXhivmZw7tyJQyUp0VLae0TBmXXTqwfYAbN97IgSMHxr3X3c6oEpYUmmlsD+la4KVKDw/D174Gzz4b7/2mPQRSqUR3SXi4BYO7neFDGsER5kNrCEw4xMXtgHOiCh/+cOV7mCoeCqd5yKtUAZzMSPWrk+9uZ/gQNLGHERzmQ2sYTDjExWvV72THjso/DKsUWxF3YTM/ShmplSb/RshcrRqVJvYwGoH50BoGEw5x8Vr1O2lpsR9GCty48cZATQCgZULLiYzUoMm/UTJXq0bQxB5GI7Bw1obChENc3Kv+N71p/PXSD+OZZyzBLSYD2wc8fQdunBqFVzkDgPa2dnNGB1FpYu/vLwgMgNFR74WP+dAaChMOafD00/CrX5WfP34cenrMOReTsM7j0bFRbtx4I1DIVl1y/hImykSgsCdv75xe9t+y3wRDEEETe0lwjIwUzo+MeGsE5kNrKEw4pMFVV3mfHx4u+B7MOReLKM7jkoYxsH2A1c+s5nihIgvH9Tirn1ltOQ6VCJrYnVpDCS/twXxoDYUJh6Q8/TT84hfl5zdtKlS4bGkpvDb1OjJxnMeVQl0NH0oTeynjf2jo5MS+ZctJraHEyAisXm0LngbGhENS/LSGD37Q34ZrhfZC4ec/8KK9rR0I3nzFMqZD4BWRtGEDTJ48vt3EiXD4sC14GhgTDkn5zW+8zx865G/DtSShULj32S35Eby485JChXg/bWNa27RYe/02FX4RSX7+CFUzlzYwJhyS8pGPFArueeFlw33kEUsSioCzHPLqv1pdpkkIQu+c3hPOZr/NVwAzN1XCL5Q1KGzbzKUNiwmHpGzZ4p8MN2tWuXNu/nxLEoqJW5PonNrJ2g+u5avv+2pgm5WXruTgkYOe97SM6SJBoaxOR/PeveNNTJbL0LCYcEjKtm0Fx3Nra+F1a2uhbLdXlIYlCSUmzMYqzjYrFq5g+ablvtnVljFdJGyOguUyxCKP/i4TDkmJMuHbD6umOEtveOHMmM7jjzdVwuYoWC5DZNwlYPLi7zLhkJQoE779sGpKUBG+krmpp7sntz/eVAmbo2C5DJHJa3i17eeQlLe+tZDr4GbWLPvB1JkJn53ga07qnNp5YkP4V4Zf8SzT0d7Wzv5b9le7m0aD4/c9FISxTwcU76wStp9DrbCVVGbx8ycIMk5L8KvfdODIgebSHoyq4Pc9jOPvqqX504SD0bB4hbUKElj6203WVX8j+/iFV0etEFxr82di4SAiC0TkZyLyrIgMisg8jzYiIv0i8ryI7BCRdSJyaqVrhpEEr7DWKIIB8HVmNyWW2R8Lv/Dqkr8rrCZQa99FIp+DiJwB7ATer6pbROQi4AHgbFU97Gh3DfBR4J2qekRE7gdeVdUbgq4FfXZmfA5GbhjYPsDV66+OJCAmykRGPzVaxV7liL4+uOceuO66cHukG4GE2Q/dSVq+i1r5HC4GnlfVLQCq+jAwBCx0tbsSuEdVjxRf3wn8dYhrhpEafvkOgvi+p1Tdtemx7T9TJ6omkKbvIgxJhcNMCpqDk53F80HtdgLTRGRqhWvjEJFlRdPV4L59+xJ23Wg2/LKhFaVzaqfnNb/zTYdt/5k6QUUivUjLdxGWpMJBAPfSatTjvu52JT19QoVr41DVlao6R1XnzJgxI3anjebEb4XVObWz5j+8XGGZ/VUhqiYQ5LuoBkmFwx7A/SQdxfNB7TqAV4CXK1wzjNQIEgBRfnhNl01tmf1VIc6CJEz5mNRQ1dgHMBXYD3QXX88FXgLagceBc4rnrwUeBVqLr78CrK50LeiYPXu2Gsa6Z9dp55c6VT4j2vmlTl337LpU23u9f8qKKcpnOHFMWTEl8n1yxaxZXpk8hfNGIpJ+H+MADGqI+T1xhrSI/CXwj4BSMAndTEEb2AJcpqqDIjIR+AfgfcU2O4AbVPWPQdeCPteilYyo0R5p0HVHl2d4a+fUTnbdtKsqn2kYaRI2WsnKZ5QYGoLFi+GBB+B1r0vvvkbVqMdEnbVSCIYRFSufERXbnS13RI32SINahxMaRr0w4QAWw51T6jFRW1ST0SyYcIDKMdxDQzBvHrztbSY4MkQ9JupahxMaRr0wn8PQEMycCUePnjzX1gYvvHDS99DXB1/72sn/W+mAzDCwfYDlm5afKL9dCks1DMMbc0iHpa8P7rtv/CY8ra2wdGlBCAwNwdlnw7FjhWuTJ8Nvf2tOa8MwAsnqwsUc0mGptDtbfz+MjIy/Zk7rpifviXB573/WaYTdBU1zCMKtNZQw7aGpqUd+RZrkvf+1IsnKP8v5MKY5pIFbayhh2kNTk9c9gUvkvf+1IOnKvx5h1mljwiGILVvKa8pA4VzJ7GQ0HXn/4fv1c/eh3bkye1STpALUL5x6gkzIzRibcAjCb39o2yOk1FMXAAARJ0lEQVS6qcl7IlxQP/NmF49LJZ9L0gWAV5g1FPYHycsYm3AwjIjEya/IkgPYb+KC/JmX4oxrGJNR0pV/KR9mokwsu5aXMTbhYBgRiZoIl7XIlVL//ciLeSzuuIYxGaWx8u/p7mFMvett5WGMLVrJMKpMViNXstqvsETtfyn6yOs9UF48cWD7AEv+3xLPrWLDjlEWx9iilQwjI2TVgZ33OlFRxtWpZfjhNiWlsfLP8xibcDCMBISxeWfVgZ33OlFRxtXLlOTEb8JO+rfL8xibWckwYhI2mcySzqpDlHH124cDTu4h7rclbKP97cysZBhVJmwsfJ5Xj1kmyrj6rfRLtn+/v0Uz/+0SaQ4isgD4EjAJGKawvedWj3avpbAV6HzgT8AxoE9Vt4tIO/AisN3xlp+q6seCPts0B6Pe2K5w+SGpBpDVInpxqLrmICJnAOuB61X1POBjwIMi4hVAfQHwn6r6FlWdC3wHuL14bTqwWVXnOY5AwWAYWSCrvgSjnCQaQJiQ2SzlsaRFbM1BRK4EblTVtzvOPQ18UlW/W+G9HwaWqur/EpF3UBAyL1DQQLYCn1PVPwTdwzQHo940oj3aKKdSOGrevgepaQ4i0ioiW90H8GZgp6v5TmBmhfuVTEyfLZ4aBP6nqr4NuIiCeWqDiIjHe5eJyKCIDO7bt69S1w2jqsRdjTbiKjNPRB3/SiGzjVrIMInm8AngTap6jePcA8BWVf2Sz3vagR8AX1LVdT5tJgJ/BGap6q/9Pt80ByOP5G2V2Sg4E+AEGecrapnQwimTTuGV4VcAaG9r585L7jzx96ikOeTN91SLaKU9gNu42lE879Wh1wObgNv9BEOpabFff0zQN8OoOnE0gEZdZWYZdwKceyIfGRs5IRgADhw5wLXfufbE37NSIltQHaY8a4dJhMODwHki0g0gInMpmJp+KCLtIvK4iJxTvNZJQTD0q+q/OG8iIh8oCg6KpqR/AB6u5HMwjGpSaeKPW9cnq9nSjUylBDgvRsZGTgjsSubDoDpMWailFZekoax/CfwjoMAocLOqbhGRs4AtwGWqOigi/w78JeA0Ex1T1QUicinwCaAFGAOeBW5R1YNBn21mJaNahDH9xK2ZU4taO40UdpkGQQlwQUQxCznHfIJMSFSPqdqENStZhrRhuAgzgce1M3sJnpYJLZx+yukcPHIw8WSedZ9GPQSX39+zEnEn86z7ICxD2jBiEsb042dnntY2LfDebhNFe1s7IsKBIwdSMUHUy6cRxv9Sr9LlXmYfoSwY0vN9cWiU/BcTDobhIsyPe8XCFbRObC1r88djf6w42fV097Drpl2MfXqMP2v9M4aPD4+7nmQyr4dPI+ykXy/B5eUzuG7Odb4bHjnfF4c8V2J1YsLBMFyE+XH3dPdwWutpZe91OjLDkPZkHnbVmmauRdhJv57OeKdA3nXTLjb8ekOgk7pzamfZubDaUWk8SrvA5bUekwkHw3ARNrnt4BHvmIkXD70YevJNaoJwf86icxZVFGxpm3fCTvpxnrVaCYNBAslrlR+2hIYzZPa4Hj9xr7wJBjCHtGHExs/R2d7WzpHRI6GcwkkcyH7vXXL+Ejb8eoOv0zftiKmw94v6rNV0rvv1eaJMZPVfrS67f5hnzOKub16YQ9owqoyf+QkIbVtPUhDOz5yz4dcbxplQ3PdK27wT1sYe9Vmr6aPw67OXYIBwY9ZoOSyT6t0Bw8grpUnEHZp59fqrPdv7TRI93T2xVsJxJ6OOqR2eK9y40TR+4+D1TFGe1e854oSlevUDwvUZwo1Z2uNab0xzMIwEuB2dPd09NQtlDPqcIFt9NaJpvMYhKX7PJ0gqvocofQ4zZo0SpVTChINhpEzcSSKq89Xvcxads4hrv3PtOOeps1ZQXnY3W7FwhWc+gqI1r0UVZszyMq5hMYe0YVSBqJnAcZ2vXp9z48YbOXDkQFnb9rZ29t+yP9mD1Rj5rHeyWlayjfOIlc8wjByRZqSL34QKoJ/O1+89LxFAecKilQwjRwQ5X/Na8rlEklyFJHZ821QpGRatZBgZwC/SBTiRcAXhSjq0t7X7mpVqycD2gTITV9RniRpV5Pxsp5nO+blx7teMmFnJMDKAl8/BTVhTysD2Aa75zjWMjo2eODdpwiS+edk3azYJVnqeapcoL+365iZKgmKjYmYlw8gZbZPaAq9HSaZyR/mEqUKaJpU22EkrMcyvrIWfFnbgyAHbiS8kJhwMo86UJjgvU5CTsHkSyzctZ2RsZNy5kbERbtx4Y+w+RiVMIl4a+GVRl4rehSWvWczVxISDYdSZMNtYOp2wlRytfhPdgSMHkM9KTZyzQZN/3MQwr+f2e9ZS0Tv35/r5XfKaxVxNTDgYRp0JWrW6k6nCVAetNNHVYpMdv/0u4KQZJ8rn+z233+ZKpTFzJ6TdecmdDZXFXE2S7iG9APgShainYeAGVd3q0W428EPgV47T/6aqt4uIAP8AfAg4DvwM+FtVfTXos80hbTQKUWL5w7Qd2D7AVeuvqvi5fhVI02L6P04PNJVFcQSnUQG3RLPvsV11h7SInAGsB65X1fOAjwEPiojX9krTgW+r6jzHcXvx2hJgETBLVf8cGAG+GLdfhpE3osTyhym219PdEyps9bgeH6dBpJ0X4LffRYkojmC/5z545GDkkhXVqAPViCTJc7gYeF5VtwCo6sMiMgQsBL7rajsdeI+IPFF8/SPgC6r6J+BK4B5VPVK8diewCbghQd8MIzdEieUPW/nzzkvurBgaC+MnaL+8gLiTZ1DuRomwjuCg545b1dYIpqLmICKtIrLVfQBvBna6mu8EZnrcZj3QpaoXApcAZwNritdmuu6zE5gmIlM9+rJMRAZFZHDfvn0VH84w8kLY1WycvRMgOJT1xUMvVmXvBK++ugnrCG60iqd5oKJwUNVhlzlonqrOo2D+Oe5qPup1T1U9okXnhqoeBG4GLhWRNkBc9yll7njdZ6WqzlHVOTNmzAjzfIbRUESp/FkSOPppZe0H1/qGd3ZM7UhtoxqnaWr5puUsOX+Jr4CKMrk3WsXTPJDErLQHeLfrXAfw7yHeOxE4Chwr3se5fOgAXgFeTtA3w2hY4phRSu29Kr8GZRRHCfH0Klmx+pnV4yKtkjiCzXxUW5KEsj4InCci3QAiMpeCqemHItIuIo+LyDnFa4uLDmxEZBJwG7BWVceAtcBSESnFvX0UWK9JwqgMI4dUu1Bc0Oo7DbNNJdOUOYLzRWzNQVUPicgVwCoRUQrmoEWq+rKInAV0AiW/QRuwSUTGAAUeAT5VvLYGeCPwpIiMAjswZ7TRZAQViktzEvVbfcctcOckD3soN3sYaxSs8J5hZIBG2LcgzWeoxiQed0OlRsMK7xlGhnGbkPxCPrO06q5EWhFFYbLA41CNiKxGxoSDYdQYr8nPL9Q0TzV/0oooqtYkngezV5awzX4Mo8Z4TX6KIgjKSTNvluP4/cw+aUQUVWsSD5tAaBQwzcEwaozfJKdoLuL4q2X2KeE3WSedxC2RLhomHAyjxvhNciXHbb1DPSuF1PqZfa5af1UqIbjVmsQtkS4aZlYyjBqzYuEK32S0ehMmpDbIvJNGCG4aYbVB9zZhEA4LZTWMOpDVePsw4ahB0VVe7Y1sYaGshmFEJowzOExBvXpGAFU707xZMOFgGDWm2g7dJIRxBrsrvnrht0Nbtcny2OYNEw6GUWOynIwVpST4rpt2se6D62iZ0FJ2nz8N/6kuE3KWxzZvmHAwjBqT5WSsqBE9Pd09nH7K6WXnh48P12VCzvLY5g2LVjKMGpP1ZKyoET1+24HWY0LO+tjmCdMcDKPGNFoyVrWS1uLQaGNbT0w4GEaNabRkrCxNyI02tvXE8hwMw0hMVvM2jHLC5jmYcDAMw2giLAnOMAzDiE0i4SAiC0TkZyLyrIgMisg8n3bfF5GtjmObiPy2eG22iBx0Xb85Sb8Mw8g2lsWcfWKHsorIGcB64P2qukVELgIeFJGzVXVcFoqqvtf13r8Hzim+nA58W1X/Jm5fDMPID7XaL9tIRhLN4WLgeVXdAqCqDwNDwMKgN4lIG3ATcFvx1HTgPSLyRPFYISKnJeiXYRgZxrKY80FFzUFEWoFHPS5tBHa6zu0EZla4ZS+wQVV/V3y9HvgXVVURmQZ8BVgD/JVHX5YBywA6OiypxTDyiGUx54OKwkFVh4EyX4KIfAI47jo9SoA24tAa3um4/xHH/w8W/Q2/E5E257Xi9ZXASihEK1Xqu2EY2cOymPNBErPSHsD91+wonvejD9ioqkFLhInAUeBYgr4ZhpFRspQ0Z/iTRDg8CJwnIt0AIjIXeDPwQxFpF5HHRaTkdEZEplDQGj7vvImILC46txGRSRR8EWtVdSxB3wzDyCiWxZwPYkcrqeohEbkCWCUiSsGktEhVXxaRs4BOYKrjLX3A91XVrU+2AZtEZAxQ4BHgU3H7ZRhG9rHtOrOPZUgbhmE0EZYhbRiGYcTGhINhGIZRhgkHwzAMowwTDoZhGEYZuXVIi8g+oDyTprZMB/bXuQ9xyXPfId/9z3PfId/9t75Dp6rOqNQot8IhC4jIYBivfxbJc98h3/3Pc98h3/23vofHzEqGYRhGGSYcDMMwjDJMOCRjZb07kIA89x3y3f889x3y3X/re0jM52AYhmGUYZqDYRiGUYYJB8MwDKMMEw4REJEWEfmYiIyIyOKAdiIi/SLyvIjsEJF1InJqLfvq0acFIvIzEXlWRAZFpGwDp2K72SJyUES2Oo6bs9jfLI5ziZD9z8RYe/Sr4vc842Mfpv9ZHfu/FZFnit+ZZ0Wkz6fd9SLySxH5uYg8JCKvTb0zqmpHyINC2fGbgceAxQHtrgGeAtqKr+8HvlLHfp8BHADeVnx9EfAHYIpH24uBe+s8zqH6m7VxjtH/uo+1T/8rfs+zOvYR+p+5saew0dk/AX9WfH0mcAQ409XuIuBFYEbx9WeAh1LvT70HJI8H8HAF4bARWOZ4PQs4UMf+Xglsdp17GrjUo20PhczzJ4rHCuC0LPY3a+Mco/91H+sKz+H7Pc/q2Efof6bHvtjHycAhoMN1/mvA5x2vX0NhP52paX6+mZVciEirS9UsHa0RbjMT2Ol4vROYJiJTfdqngl/fKezQt9PVfGexn27WA12qeiFwCXA2sKaa/fbAPX7g3d+6jHMIwvY/C2Mdl6yOfVjyMPZ3AA9o+bbK48ZeVV+iIES60vzw2DvBNSqqOgx42uMjIMBxx+vR4r9VFcZ+fReRT7j6U+pTWX9U9Yjj/weLdtjfiUib81qVcY8fePe3LuMcglD9z8hYxyWrYx+KrI+9iHyOglnpcq/LhPw9JyEXf8gcsgfocLzuAF4BXq5Pd8r6Q/H1nhDvnQgcBY6l3akAwvY3a+NcIu5412Os45LVsY9LZsZeRP4JOBe4vLjgczNu7EVkCtBOuN9zaEw4pICItIvI4yJyTvHUWmCpwxT1UWC9Fg2EdeBB4DwR6QYQkbkUTE0/dPddRBaLyBnF/08CbgPWqupYBvr744yPc4lQ/c/IWIciB9/xQDL6PXf3cYKIfB04C7iiJBhEZKKIbBKRdxWbrgV6HCa864HHVXVfmv0xs1I6TAE6gdIfaw3wRuBJERkFdgA31KlvqOohEbkCWCUiSkEFXaSqL4vIWYzvexuwSUTGAAUeAT6Vhf6S8XEuEaH/dR/rCORi7APIw9gvAv4WGAR+KiKl85+j4BOZBqCqPxGRu4FHRGQE2Av4htbHxcpnGIZhGGWYWckwDMMow4SDYRiGUYYJB8MwDKMMEw6GYRhGGSYcDMMwjDJMOBiGYRhlmHAwDMMwyjDhYBiGYZRhwsEwDMMo4/8DPCgpAmZ2f1EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "y_pred_idx = y_pred.reshape(-1) # 열 벡터 대신 1차원 배열\n",
    "plt.plot(X_test[y_pred_idx, 1], X_test[y_pred_idx, 2], 'go', label=\"양성\")\n",
    "plt.plot(X_test[~y_pred_idx, 1], X_test[~y_pred_idx, 2], 'r^', label=\"음성\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "훨씬 더 좋아졌네요! 새로 추가한 특성이 확실히 도움이 많이 되었습니다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "텐서보드 서버를 시작해서 최근 실행을 찾아 학습 곡선을 확인해 보세요(즉, 에포크 횟수에 대해 테스트 세트로 평가한 손실이 얼마나 되는지):\n",
    "\n",
    "```\n",
    "$ tensorboard --logdir=tf_logs\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이제 하이퍼파라미터(가령, `batch_size`나 `learning_rate`)를 조정하면서 훈련을 여러번 실행해 보고 학습 곡선을 비교해 보겠습니다. 그리드 서치나 랜덤 서치를 구현해서 이 과정을 자동화할 수도 있습니다. 다음은 배치 크기와 학습률에 대한 간단한 랜덤 서치 구현입니다. 간단하게 하기 위해서 체크포인트 관리 부분은 제외했습니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "반복 0\n",
      "  logdir: tf_logs/logreg-run-20181109132747/\n",
      "  batch_size: 54\n",
      "  learning_rate: 0.004430375245218265\n",
      "  훈련: .....................\n",
      "  정밀도: 0.9797979797979798\n",
      "  재현율: 0.9797979797979798\n",
      "반복 1\n",
      "  logdir: tf_logs/logreg-run-20181109132935/\n",
      "  batch_size: 22\n",
      "  learning_rate: 0.0017826497151386947\n",
      "  훈련: .....................\n",
      "  정밀도: 0.9797979797979798\n",
      "  재현율: 0.9797979797979798\n",
      "반복 2\n",
      "  logdir: tf_logs/logreg-run-20181109133349/\n",
      "  batch_size: 74\n",
      "  learning_rate: 0.00203228544324115\n",
      "  훈련: .....................\n",
      "  정밀도: 0.9696969696969697\n",
      "  재현율: 0.9696969696969697\n",
      "반복 3\n",
      "  logdir: tf_logs/logreg-run-20181109133511/\n",
      "  batch_size: 58\n",
      "  learning_rate: 0.004491523825137997\n",
      "  훈련: .....................\n",
      "  정밀도: 0.9797979797979798\n",
      "  재현율: 0.9797979797979798\n",
      "반복 4\n",
      "  logdir: tf_logs/logreg-run-20181109133654/\n",
      "  batch_size: 61\n",
      "  learning_rate: 0.07963234721775589\n",
      "  훈련: .....................\n",
      "  정밀도: 0.9801980198019802\n",
      "  재현율: 1.0\n",
      "반복 5\n",
      "  logdir: tf_logs/logreg-run-20181109133832/\n",
      "  batch_size: 92\n",
      "  learning_rate: 0.0004634250583294876\n",
      "  훈련: .....................\n",
      "  정밀도: 0.912621359223301\n",
      "  재현율: 0.9494949494949495\n",
      "반복 6\n",
      "  logdir: tf_logs/logreg-run-20181109133938/\n",
      "  batch_size: 74\n",
      "  learning_rate: 0.047706818419354494\n",
      "  훈련: .....................\n",
      "  정밀도: 0.98\n",
      "  재현율: 0.98989898989899\n",
      "반복 7\n",
      "  logdir: tf_logs/logreg-run-20181109134059/\n",
      "  batch_size: 58\n",
      "  learning_rate: 0.0001694044709524274\n",
      "  훈련: .....................\n",
      "  정밀도: 0.9\n",
      "  재현율: 0.9090909090909091\n",
      "반복 8\n",
      "  logdir: tf_logs/logreg-run-20181109134243/\n",
      "  batch_size: 61\n",
      "  learning_rate: 0.04171461199412461\n",
      "  훈련: .....................\n",
      "  정밀도: 0.9801980198019802\n",
      "  재현율: 1.0\n",
      "반복 9\n",
      "  logdir: tf_logs/logreg-run-20181109134422/\n",
      "  batch_size: 92\n",
      "  learning_rate: 0.00010742922968438615\n",
      "  훈련: .....................\n",
      "  정밀도: 0.8823529411764706\n",
      "  재현율: 0.7575757575757576\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import reciprocal\n",
    "\n",
    "n_search_iterations = 10\n",
    "\n",
    "for search_iteration in range(n_search_iterations):\n",
    "    batch_size = np.random.randint(1, 100)\n",
    "    learning_rate = reciprocal(0.0001, 0.1).rvs(random_state=search_iteration)\n",
    "\n",
    "    n_inputs = 2 + 4\n",
    "    logdir = log_dir(\"logreg\")\n",
    "    \n",
    "    print(\"반복\", search_iteration)\n",
    "    print(\"  logdir:\", logdir)\n",
    "    print(\"  batch_size:\", batch_size)\n",
    "    print(\"  learning_rate:\", learning_rate)\n",
    "    print(\"  훈련: \", end=\"\")\n",
    "\n",
    "    reset_graph()\n",
    "\n",
    "    X = tf.placeholder(tf.float32, shape=(None, n_inputs + 1), name=\"X\")\n",
    "    y = tf.placeholder(tf.float32, shape=(None, 1), name=\"y\")\n",
    "\n",
    "    y_proba, loss, training_op, loss_summary, init, saver = logistic_regression(\n",
    "        X, y, learning_rate=learning_rate)\n",
    "\n",
    "    file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())\n",
    "\n",
    "    n_epochs = 10001\n",
    "    n_batches = int(np.ceil(m / batch_size))\n",
    "\n",
    "    final_model_path = \"./my_logreg_model_%d\" % search_iteration\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        sess.run(init)\n",
    "\n",
    "        for epoch in range(n_epochs):\n",
    "            for batch_index in range(n_batches):\n",
    "                X_batch, y_batch = random_batch(X_train_enhanced, y_train, batch_size)\n",
    "                sess.run(training_op, feed_dict={X: X_batch, y: y_batch})\n",
    "            loss_val, summary_str = sess.run([loss, loss_summary], feed_dict={X: X_test_enhanced, y: y_test})\n",
    "            file_writer.add_summary(summary_str, epoch)\n",
    "            if epoch % 500 == 0:\n",
    "                print(\".\", end=\"\")\n",
    "\n",
    "        saver.save(sess, final_model_path)\n",
    "\n",
    "        print()\n",
    "        y_proba_val = y_proba.eval(feed_dict={X: X_test_enhanced, y: y_test})\n",
    "        y_pred = (y_proba_val >= 0.5)\n",
    "        \n",
    "        print(\"  정밀도:\", precision_score(y_test, y_pred))\n",
    "        print(\"  재현율:\", recall_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "하이퍼파라미터의 적절한 스케일을 감잡을 수 없을 때 사이파이(SciPy)의 `stats` 모듈의 `reciprocal()` 함수를 사용하여 난수 분포를 얻을 수 있습니다. 좀 더 자세한 내용은 2장의 연습문제 해답을 보세요."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "nav_menu": {
   "height": "603px",
   "width": "616px"
  },
  "toc": {
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 6,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}