milnet_xling.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "milnet_xling.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zj7ViHva58R_",
        "colab_type": "text"
      },
      "source": [
        "### Package Preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QUz01gDzateB",
        "colab_type": "code",
        "outputId": "972f0211-adad-44ea-82f2-e6f2a4dfadb6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 258
        }
      },
      "source": [
        "!pip install tf_sentencepiece\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import tensorflow_hub as hub\n",
        "import tf_sentencepiece\n",
        "from random import shuffle, choice\n",
        "import re\n",
        "import os\n",
        "import datetime\n",
        "from functools import reduce\n",
        "from operator import itemgetter\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tf_sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/72/fe/363d78d29c556d0da642ffe285c2c7573b6a83239a9b0d08d83376c9fbac/tf_sentencepiece-0.1.82.1-py2.py3-none-manylinux1_x86_64.whl (2.8MB)\n",
            "\u001b[K     |████████████████████████████████| 2.8MB 2.8MB/s \n",
            "\u001b[?25hInstalling collected packages: tf-sentencepiece\n",
            "Successfully installed tf-sentencepiece-0.1.82.1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING: Logging before flag parsing goes to stderr.\n",
            "W0716 08:03:34.886610 140485658814336 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/tf_sentencepiece/sentencepiece_processor_ops.py:259: The name tf.NotDifferentiable is deprecated. Please use tf.no_gradient instead.\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KCsHRfC556Oo",
        "colab_type": "text"
      },
      "source": [
        "### Configurations"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i4zp8CgboUSO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "xling_encoding_len = 512\n",
        "max_seg = 10\n",
        "level_class_cnt = 3\n",
        "\n",
        "test_percentage = 0.1\n",
        "validation_percentage = 0.1\n",
        "\n",
        "dropout_rate = 0.5\n",
        "eta = 1e-4\n",
        "hidden_feature_dim = 100\n",
        "attention_key_dim = 100\n",
        "gru_feature_dim = 50\n",
        "\n",
        "batch_size = 512\n",
        "epochs = 8\n",
        "\n",
        "label_re = re.compile('(\\d+)\\.\\d+')\n",
        "sentence_re = re.compile('(?:\\.|!|\\?)\\s')\n",
        "\n",
        "input_path = '/content/gdrive/My Drive/data_source/milnet/raw_text/gourmet.txt'\n",
        "model_out_path = '/content/gdrive/My Drive/data_source/milnet/results/food_xling_c3.h5'\n",
        "log_out_dir = '/content/gdrive/My Drive/data_source/milnet/log/'\n",
        "\n",
        "sample_amount = 0\n",
        "with open(input_path) as in_file:\n",
        "    sample_amount = len(in_file.read().split('\\n\\n')) - 1\n",
        "sample_indices = [*range(sample_amount)]\n",
        "shuffle(sample_indices)\n",
        "train_samples = sample_indices[0:int(sample_amount * (1 - test_percentage - validation_percentage))]\n",
        "validation_samples = sample_indices[int(sample_amount * (1 - test_percentage - validation_percentage)): int(sample_amount * (1 - test_percentage))]\n",
        "test_samples = sample_indices[int(sample_amount * (1 - test_percentage)):]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3lzuDqbj4oOn",
        "colab_type": "text"
      },
      "source": [
        "### Data Preloading"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kvM3Gi7P4m8t",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "g = tf.Graph()\n",
        "with g.as_default():\n",
        "    text_input = tf.placeholder(dtype=tf.string, shape=[None])\n",
        "    en_de_embed = hub.Module(\"https://tfhub.dev/google/universal-sentence-encoder-xling/en-de/1\")\n",
        "    embedded_text = en_de_embed(text_input)\n",
        "    init_op = tf.group([tf.global_variables_initializer(), tf.tables_initializer()])\n",
        "g.finalize()\n",
        "session = tf.Session(graph=g)\n",
        "session.run(init_op)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Amu2Opkazbg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def __pad_doc_encoding(doc_encoding, max_seg):\n",
        "    if doc_encoding.shape[0] > max_seg:\n",
        "        return doc_encoding[:max_seg]\n",
        "    elif doc_encoding.shape[0] < max_seg:\n",
        "        topad_len = max_seg - doc_encoding.shape[0]\n",
        "        pad_width = [(0, 0) if i != 0 else (0, topad_len) for i in range(len(doc_encoding.shape))]\n",
        "        return np.pad(doc_encoding, pad_width, 'constant', constant_values=0)\n",
        "    else:\n",
        "        return doc_encoding\n",
        "\n",
        "def __label_map(raw_label):\n",
        "    if raw_label == 1 or raw_label == 2:\n",
        "        return 0\n",
        "    elif raw_label == 3:\n",
        "        return 1\n",
        "    else:\n",
        "        return 2\n",
        "\n",
        "def __balance_data(feature_array, label_array):\n",
        "    to_balance_indices = np.concatenate([np.where(label_array == 2)[0], np.where(label_array == 4)[0]])\n",
        "    return np.delete(feature_array, to_balance_indices, axis=0), np.delete(label_array, to_balance_indices, axis=0)\n",
        "\n",
        "def data_generator(sample_indices, input_path, segment_re, label_re, \n",
        "                   batch_size=batch_size, max_seg=max_seg, xling_len=xling_encoding_len, epochs=epochs, use_balance=True):\n",
        "    global session, embedded_text, text_input\n",
        "    with open(input_path) as in_file:\n",
        "        file_content = [*itemgetter(*sample_indices)(in_file.read().split('\\n\\n'))]\n",
        "        for _ in range(epochs):\n",
        "            shuffle(file_content)\n",
        "            feature_cache, label_cache = [], []\n",
        "            batch_index = 0\n",
        "            for sample in file_content:\n",
        "                label_cache.append(sample.split('\\n')[0])\n",
        "                feature_cache.append([*filter(lambda x: len(x) > 1, segment_re.split(' '.join(sample.split('\\n')[1:])))])\n",
        "                batch_index += 1\n",
        "                if batch_index == batch_size:\n",
        "                    len_lst = [*map(len, feature_cache)]\n",
        "                    batch_features = session.run(embedded_text, feed_dict={text_input: reduce(lambda x, y: x + y, feature_cache)})\n",
        "                    label_array = np.array([np.array([int(label_re.findall(l)[0])]) for l in label_cache])\n",
        "                    feature_array = np.zeros((batch_size, max_seg, xling_len))\n",
        "                    for index, length in enumerate(len_lst):\n",
        "                        feature_array[index] = __pad_doc_encoding(np.array(batch_features[:length]), max_seg)\n",
        "                        batch_features = batch_features[length:]\n",
        "                    feature_array = np.array(feature_array)\n",
        "                    if use_balance:\n",
        "                        feature_array, label_array = __balance_data(feature_array, label_array)\n",
        "                    yield feature_array, np.array([np.array([__label_map(l[0])]) for l in label_array])\n",
        "                    feature_cache, label_cache = [], []\n",
        "                    batch_index = 0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NJO9L0aAhUEG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "shared_sublayer_cache = {}\n",
        "\n",
        "def branch_execute(layer_in, sublayer, args={}):\n",
        "    instance_cnt = layer_in.shape[1]\n",
        "    sliced_inputs = [tf.keras.layers.Lambda(lambda x: x[:,i])(layer_in) \n",
        "                     for i in range(instance_cnt)]\n",
        "    branch_layers = [sublayer(**{**{'layer_in': sliced_inputs[i]}, **args}) \n",
        "                     for i in range(instance_cnt)]\n",
        "    expand_layer = tf.keras.layers.Lambda(lambda x: tf.keras.backend.expand_dims(x, axis=1))\n",
        "    expanded_layers = [expand_layer(branch_layers[i]) for i in range(instance_cnt)]\n",
        "    concated_layer = tf.keras.layers.Concatenate(axis=1)(expanded_layers)\n",
        "    return concated_layer\n",
        "\n",
        "def __seg_classifier_layer_share(layer_in, class_cnt, dropout_rate, eta):\n",
        "    global shared_sublayer_cache\n",
        "    if 'shared_seg_classifier_sublayers' not in shared_sublayer_cache:\n",
        "        shared_sublayer_cache['shared_seg_classifier_sublayers'] = {\n",
        "            'drop_out_layer': tf.keras.layers.Dropout(\n",
        "                dropout_rate\n",
        "            ),\n",
        "            'dense_layer': tf.keras.layers.Dense(\n",
        "                units=class_cnt,\n",
        "                activation='softmax',\n",
        "                kernel_regularizer=tf.keras.regularizers.l2(eta),\n",
        "                bias_regularizer=tf.keras.regularizers.l2(eta)\n",
        "            )\n",
        "        }\n",
        "    shared_layers = shared_sublayer_cache['shared_seg_classifier_sublayers']\n",
        "    drop_out_layer = shared_layers['drop_out_layer'](layer_in)\n",
        "    dense_layer = shared_layers['dense_layer'](drop_out_layer)\n",
        "    return dense_layer\n",
        "\n",
        "def __attention_layer_share(layer_in, attention_key_dim, dropout_rate, eta):\n",
        "    global shared_sublayer_cache\n",
        "    if 'shared_attention_sublayers' not in shared_sublayer_cache:\n",
        "        shared_sublayer_cache['shared_attention_sublayers'] = {\n",
        "            'drop_out_layer': tf.keras.layers.Dropout(\n",
        "                dropout_rate\n",
        "            ),\n",
        "            'dense_layer': tf.keras.layers.Dense(\n",
        "                units=attention_key_dim, \n",
        "                activation='tanh',\n",
        "                kernel_regularizer=tf.keras.regularizers.l2(eta),\n",
        "                bias_regularizer=tf.keras.regularizers.l2(eta)\n",
        "            ),\n",
        "            'nobias_dense_layer': tf.keras.layers.Dense(\n",
        "                units=1, \n",
        "                use_bias=False, \n",
        "                bias_regularizer=tf.keras.regularizers.l2(eta)\n",
        "            )\n",
        "        }\n",
        "    shared_layers = shared_sublayer_cache['shared_attention_sublayers']\n",
        "    drop_out_layer = shared_layers['drop_out_layer'](layer_in)\n",
        "    dense_layer = shared_layers['dense_layer'](drop_out_layer)\n",
        "    nobias_dense_layer = shared_layers['nobias_dense_layer'](dense_layer)\n",
        "    return nobias_dense_layer\n",
        "\n",
        "def bidirectional_gru_layer(layer_in, gru_feature_dim):\n",
        "    bidirectional_layer = tf.keras.layers.Bidirectional(\n",
        "        tf.keras.layers.GRU(gru_feature_dim, return_sequences=True)\n",
        "    )(layer_in)\n",
        "    return bidirectional_layer\n",
        "\n",
        "def merge_layer(layer_in, class_cnt, eta):\n",
        "    dot_layer = tf.keras.layers.Dot(axes=1)(layer_in)\n",
        "    flatten_layer = tf.keras.layers.Flatten()(dot_layer)\n",
        "    dense_layer = tf.keras.layers.Dense(\n",
        "        units=class_cnt, \n",
        "        activation='softmax',\n",
        "        kernel_regularizer=tf.keras.regularizers.l2(eta),\n",
        "        bias_regularizer=tf.keras.regularizers.l2(eta)\n",
        "    )(flatten_layer)\n",
        "    return dense_layer\n",
        "\n",
        "def performance_judge(model, generator, class_cnt):\n",
        "    eps = np.finfo(float).eps\n",
        "    accuracy, precisions, recalls, f1s = [], [], [], []\n",
        "    for i, (features, labels) in enumerate(generator):\n",
        "        predicted = model.predict(features)\n",
        "        precisions.append([])\n",
        "        recalls.append([])\n",
        "        f1s.append([])\n",
        "        contingency_table = np.zeros((class_cnt, class_cnt))\n",
        "        for index in range(features.shape[0]):\n",
        "            contingency_table[int(labels[index][0])][np.argmax(predicted[index])] += 1\n",
        "        accuracy.append(np.trace(contingency_table) / features.shape[0])\n",
        "        for index in range(class_cnt):\n",
        "            precisions[i].append(contingency_table[index][index] / (np.sum(contingency_table[:, index]) + eps))\n",
        "            recalls[i].append(contingency_table[index][index] / (np.sum(contingency_table[index, :]) + eps))\n",
        "            f1s[i].append(2 * precisions[i][-1] * recalls[i][-1] / ((precisions[i][-1] + recalls[i][-1]) + eps))\n",
        "    precisions = [float(sum(l))/len(l) for l in zip(*precisions)]\n",
        "    recalls = [float(sum(l))/len(l) for l in zip(*recalls)]\n",
        "    f1s = [float(sum(l))/len(l) for l in zip(*f1s)]\n",
        "    print('Accuracy:', round(reduce(lambda x, y: x + y, accuracy) / len(accuracy), 3))\n",
        "    for index in range(class_cnt):\n",
        "        print('_____ Class', index, '_____')\n",
        "        print('Precision\\t', round(precisions[index], 3))\n",
        "        print('Recall\\t\\t', round(recalls[index], 3))\n",
        "        print('F1 Score\\t', round(f1s[index], 3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5kSdKsCX9iwa",
        "colab_type": "code",
        "outputId": "4f62e343-fd70-4a4b-c49b-acc124b4d65c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "print('Constructing Model ...', end='')\n",
        "\n",
        "model_input = tf.keras.Input((max_seg, xling_encoding_len))\n",
        "\n",
        "biglu_layer = bidirectional_gru_layer(\n",
        "    model_input, \n",
        "    gru_feature_dim=50\n",
        ")\n",
        "\n",
        "attention_layer = branch_execute(\n",
        "    biglu_layer, \n",
        "    sublayer=__attention_layer_share, \n",
        "    args={\n",
        "        'attention_key_dim': 100,\n",
        "        'dropout_rate': 0.5,\n",
        "        'eta': 1e-4\n",
        "    }\n",
        ")\n",
        "\n",
        "softmaxed_attention_layer = tf.keras.layers.Softmax(\n",
        "    axis=1\n",
        ")(attention_layer)\n",
        "\n",
        "classification_layer = branch_execute(\n",
        "    model_input, \n",
        "    sublayer=__seg_classifier_layer_share, \n",
        "    args={\n",
        "        'class_cnt': level_class_cnt,\n",
        "        'dropout_rate': 0.5,\n",
        "        'eta': 1e-4\n",
        "    }\n",
        ")\n",
        "\n",
        "merge_layer = merge_layer(\n",
        "    [softmaxed_attention_layer, classification_layer],\n",
        "    class_cnt=level_class_cnt,\n",
        "    eta=1e-4\n",
        ")\n",
        "\n",
        "model = tf.keras.Model(model_input, merge_layer)\n",
        "\n",
        "print('\\rModel Constructed. Compiling ...', end='')\n",
        "\n",
        "model.compile(\n",
        "    optimizer=tf.keras.optimizers.Adam(clipvalue=0.5),\n",
        "    loss=tf.keras.losses.SparseCategoricalCrossentropy(),\n",
        "    metrics=['accuracy']\n",
        ")\n",
        "\n",
        "print('\\rModel Compiled.')\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0716 08:08:03.067019 140485658814336 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "W0716 08:08:03.093594 140485658814336 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling GlorotUniform.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "W0716 08:08:03.095370 140485658814336 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling Orthogonal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
            "W0716 08:08:03.101636 140485658814336 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:97: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Model Compiled.\n",
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 10, 512)]    0                                            \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional (Bidirectional)   (None, 10, 100)      168900      input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda (Lambda)                 (None, 100)          0           bidirectional[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_1 (Lambda)               (None, 100)          0           bidirectional[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_2 (Lambda)               (None, 100)          0           bidirectional[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_3 (Lambda)               (None, 100)          0           bidirectional[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_4 (Lambda)               (None, 100)          0           bidirectional[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_5 (Lambda)               (None, 100)          0           bidirectional[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_6 (Lambda)               (None, 100)          0           bidirectional[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_7 (Lambda)               (None, 100)          0           bidirectional[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_8 (Lambda)               (None, 100)          0           bidirectional[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "lambda_9 (Lambda)               (None, 100)          0           bidirectional[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dropout (Dropout)               (None, 100)          0           lambda[0][0]                     \n",
            "                                                                 lambda_1[0][0]                   \n",
            "                                                                 lambda_2[0][0]                   \n",
            "                                                                 lambda_3[0][0]                   \n",
            "                                                                 lambda_4[0][0]                   \n",
            "                                                                 lambda_5[0][0]                   \n",
            "                                                                 lambda_6[0][0]                   \n",
            "                                                                 lambda_7[0][0]                   \n",
            "                                                                 lambda_8[0][0]                   \n",
            "                                                                 lambda_9[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 100)          10100       dropout[0][0]                    \n",
            "                                                                 dropout[1][0]                    \n",
            "                                                                 dropout[2][0]                    \n",
            "                                                                 dropout[3][0]                    \n",
            "                                                                 dropout[4][0]                    \n",
            "                                                                 dropout[5][0]                    \n",
            "                                                                 dropout[6][0]                    \n",
            "                                                                 dropout[7][0]                    \n",
            "                                                                 dropout[8][0]                    \n",
            "                                                                 dropout[9][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_11 (Lambda)              (None, 512)          0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_12 (Lambda)              (None, 512)          0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_13 (Lambda)              (None, 512)          0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_14 (Lambda)              (None, 512)          0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_15 (Lambda)              (None, 512)          0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_16 (Lambda)              (None, 512)          0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_17 (Lambda)              (None, 512)          0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_18 (Lambda)              (None, 512)          0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_19 (Lambda)              (None, 512)          0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "lambda_20 (Lambda)              (None, 512)          0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 1)            100         dense[0][0]                      \n",
            "                                                                 dense[1][0]                      \n",
            "                                                                 dense[2][0]                      \n",
            "                                                                 dense[3][0]                      \n",
            "                                                                 dense[4][0]                      \n",
            "                                                                 dense[5][0]                      \n",
            "                                                                 dense[6][0]                      \n",
            "                                                                 dense[7][0]                      \n",
            "                                                                 dense[8][0]                      \n",
            "                                                                 dense[9][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 512)          0           lambda_11[0][0]                  \n",
            "                                                                 lambda_12[0][0]                  \n",
            "                                                                 lambda_13[0][0]                  \n",
            "                                                                 lambda_14[0][0]                  \n",
            "                                                                 lambda_15[0][0]                  \n",
            "                                                                 lambda_16[0][0]                  \n",
            "                                                                 lambda_17[0][0]                  \n",
            "                                                                 lambda_18[0][0]                  \n",
            "                                                                 lambda_19[0][0]                  \n",
            "                                                                 lambda_20[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "lambda_10 (Lambda)              (None, 1, 1)         0           dense_1[0][0]                    \n",
            "                                                                 dense_1[1][0]                    \n",
            "                                                                 dense_1[2][0]                    \n",
            "                                                                 dense_1[3][0]                    \n",
            "                                                                 dense_1[4][0]                    \n",
            "                                                                 dense_1[5][0]                    \n",
            "                                                                 dense_1[6][0]                    \n",
            "                                                                 dense_1[7][0]                    \n",
            "                                                                 dense_1[8][0]                    \n",
            "                                                                 dense_1[9][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 3)            1539        dropout_1[0][0]                  \n",
            "                                                                 dropout_1[1][0]                  \n",
            "                                                                 dropout_1[2][0]                  \n",
            "                                                                 dropout_1[3][0]                  \n",
            "                                                                 dropout_1[4][0]                  \n",
            "                                                                 dropout_1[5][0]                  \n",
            "                                                                 dropout_1[6][0]                  \n",
            "                                                                 dropout_1[7][0]                  \n",
            "                                                                 dropout_1[8][0]                  \n",
            "                                                                 dropout_1[9][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "concatenate (Concatenate)       (None, 10, 1)        0           lambda_10[0][0]                  \n",
            "                                                                 lambda_10[1][0]                  \n",
            "                                                                 lambda_10[2][0]                  \n",
            "                                                                 lambda_10[3][0]                  \n",
            "                                                                 lambda_10[4][0]                  \n",
            "                                                                 lambda_10[5][0]                  \n",
            "                                                                 lambda_10[6][0]                  \n",
            "                                                                 lambda_10[7][0]                  \n",
            "                                                                 lambda_10[8][0]                  \n",
            "                                                                 lambda_10[9][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "lambda_21 (Lambda)              (None, 1, 3)         0           dense_2[0][0]                    \n",
            "                                                                 dense_2[1][0]                    \n",
            "                                                                 dense_2[2][0]                    \n",
            "                                                                 dense_2[3][0]                    \n",
            "                                                                 dense_2[4][0]                    \n",
            "                                                                 dense_2[5][0]                    \n",
            "                                                                 dense_2[6][0]                    \n",
            "                                                                 dense_2[7][0]                    \n",
            "                                                                 dense_2[8][0]                    \n",
            "                                                                 dense_2[9][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "softmax (Softmax)               (None, 10, 1)        0           concatenate[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_1 (Concatenate)     (None, 10, 3)        0           lambda_21[0][0]                  \n",
            "                                                                 lambda_21[1][0]                  \n",
            "                                                                 lambda_21[2][0]                  \n",
            "                                                                 lambda_21[3][0]                  \n",
            "                                                                 lambda_21[4][0]                  \n",
            "                                                                 lambda_21[5][0]                  \n",
            "                                                                 lambda_21[6][0]                  \n",
            "                                                                 lambda_21[7][0]                  \n",
            "                                                                 lambda_21[8][0]                  \n",
            "                                                                 lambda_21[9][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dot (Dot)                       (None, 1, 3)         0           softmax[0][0]                    \n",
            "                                                                 concatenate_1[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "flatten (Flatten)               (None, 3)            0           dot[0][0]                        \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 3)            12          flatten[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 180,651\n",
            "Trainable params: 180,651\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MaTGynpcAi61",
        "colab_type": "code",
        "outputId": "26d61b72-d480-419d-e9b6-7acec2ba20b5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 887
        }
      },
      "source": [
        "logdir = os.path.join(log_out_dir, datetime.datetime.now().strftime(\"%Y%m%d_%H%M%S\"))\n",
        "tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=0)\n",
        "\n",
        "model.fit_generator(\n",
        "    data_generator(train_samples, input_path, sentence_re, label_re, use_balance=True),\n",
        "    validation_data=data_generator(validation_samples, input_path, sentence_re, label_re, use_balance=True),\n",
        "    steps_per_epoch=(sample_amount * (1 - test_percentage - validation_percentage) // batch_size) - 1,\n",
        "    validation_steps=(sample_amount * (validation_percentage) // batch_size) - 1,\n",
        "    validation_freq=2,\n",
        "    epochs=epochs,\n",
        "    callbacks=[tensorboard_callback]\n",
        ")\n",
        "\n",
        "model.save(model_out_path)\n",
        "\n",
        "print('########## Training Error ##########')\n",
        "performance_judge(model, data_generator(train_samples, input_path, sentence_re, label_re, epochs=1, use_balance=True), level_class_cnt)\n",
        "print('')\n",
        "print('############ Test Error ############')\n",
        "performance_judge(model, data_generator(train_samples, input_path, sentence_re, label_re, epochs=1, use_balance=True), level_class_cnt)\n",
        "\n",
        "print(logdir)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0716 08:08:46.489855 140485658814336 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "  2/701 [..............................] - ETA: 1:22:20 - loss: 1.1507 - acc: 0.3211"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0716 08:08:55.576535 140485658814336 callbacks.py:257] Method (on_train_batch_end) is slow compared to the batch update (0.105707). Check your callbacks.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "701/701 [==============================] - 1319s 2s/step - loss: 0.9158 - acc: 0.5672\n",
            "Epoch 2/8\n",
            "701/701 [==============================] - 1494s 2s/step - loss: 0.7858 - acc: 0.6160 - val_loss: 0.7611 - val_acc: 0.6254\n",
            "Epoch 3/8\n",
            "701/701 [==============================] - 1299s 2s/step - loss: 0.7436 - acc: 0.6513\n",
            "Epoch 4/8\n",
            "701/701 [==============================] - 1499s 2s/step - loss: 0.7115 - acc: 0.6830 - val_loss: 0.6971 - val_acc: 0.6904\n",
            "Epoch 5/8\n",
            "701/701 [==============================] - 1304s 2s/step - loss: 0.6808 - acc: 0.7168\n",
            "Epoch 6/8\n",
            "701/701 [==============================] - 1658s 2s/step - loss: 0.6563 - acc: 0.7401 - val_loss: 0.6610 - val_acc: 0.7288\n",
            "Epoch 7/8\n",
            "701/701 [==============================] - 1836s 3s/step - loss: 0.6365 - acc: 0.7556\n",
            "Epoch 8/8\n",
            "701/701 [==============================] - 2103s 3s/step - loss: 0.6214 - acc: 0.7643 - val_loss: 0.6190 - val_acc: 0.7673\n",
            "########## Training Error ##########\n",
            "Accuracy: 0.78\n",
            "_____ Class 0 _____\n",
            "Precision\t 0.792\n",
            "Recall\t\t 0.783\n",
            "F1 Score\t 0.787\n",
            "_____ Class 1 _____\n",
            "Precision\t 0.687\n",
            "Recall\t\t 0.699\n",
            "F1 Score\t 0.692\n",
            "_____ Class 2 _____\n",
            "Precision\t 0.864\n",
            "Recall\t\t 0.86\n",
            "F1 Score\t 0.861\n",
            "\n",
            "############ Test Error ############\n",
            "Accuracy: 0.78\n",
            "_____ Class 0 _____\n",
            "Precision\t 0.792\n",
            "Recall\t\t 0.783\n",
            "F1 Score\t 0.786\n",
            "_____ Class 1 _____\n",
            "Precision\t 0.687\n",
            "Recall\t\t 0.698\n",
            "F1 Score\t 0.692\n",
            "_____ Class 2 _____\n",
            "Precision\t 0.864\n",
            "Recall\t\t 0.859\n",
            "F1 Score\t 0.861\n",
            "/content/gdrive/My Drive/data_source/milnet/log/20190716_080839\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u6mviP8_f_4p",
        "colab_type": "code",
        "outputId": "4b08b781-637b-46e9-f9be-f8162708678e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "%load_ext tensorboard\n",
        "%tensorboard --logdir logs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GkM3V8yEVlTr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}