Keras Transfer Learning Bug.ipynb

keras-transfer-learning-bug.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Keras Transfer Learning Bug.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python2",
      "display_name": "Python 2"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/gist/kechan/9487fad4dfeaede212e3d9899fb21105/keras-transfer-learning-bug.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "aGiXoDI6qTVq",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "### This notebook is too highlight a potential Keras bug related to transfer learning"
      ]
    },
    {
      "metadata": {
        "id": "PAYiUC8MqebI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "1d377b12-0a09-405d-f1ea-9f7a584fa638"
      },
      "cell_type": "code",
      "source": [
        "ls -l"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 16\r\n",
            "drwxr-xr-x 4 root root 4096 May 21 00:26 \u001b[0m\u001b[01;34mdata\u001b[0m/\r\n",
            "drwxr-xr-x 1 root root 4096 May 20 18:39 \u001b[01;34mdatalab\u001b[0m/\r\n",
            "drwxr-xr-x 6 root root 4096 May 21 00:01 \u001b[01;34mexperiments\u001b[0m/\r\n",
            "drwxr-xr-x 6 root root 4096 May 21 00:03 \u001b[01;34mKerasVision\u001b[0m/\r\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bnBefs2dqowR",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### git clone, pip install and download data"
      ]
    },
    {
      "metadata": {
        "id": "Po4mfXH9qibF",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/kechan/KerasVision.git"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "KLGECfScq25a",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!pip install -U -q PyDrive\n",
        "!pip install tqdm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "niG7EpBiq4VT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def download_data_from_gdrive(ids, filenames):\n",
        "    \n",
        "    for id, filename in zip(ids, filenames):\n",
        "        uploaded = drive.CreateFile({'id': id})\n",
        "        uploaded.GetContentFile(filename)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2beEyY89q6Yd",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IS5FYvLFq86M",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "download_data_from_gdrive(['1Zdt10Q1Jn-hrq2o1mmvQ1j4DgBTxxGIq', '1FgVh2oGqH9Pr4Ze2NETyLnBTPtC0hTui', '1X6ijkgbWCzATPCJLx0rBCy5jtUkjo2KG'], \n",
        "                          ['train_224_224.hdf5.gz', 'validation_224_224.hdf5.gz', 'test_224_224.hdf5.gz'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "PWdjXDMLrAiJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!mkdir data\n",
        "!mkdir data/224x224_cropped_merged_heads_hdf5"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "XgGEMx23rQWu",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!gunzip train_224_224.hdf5.gz\n",
        "!mv train_224_224.hdf5 /content/data/224x224_cropped_merged_heads_hdf5\n",
        "\n",
        "!gunzip validation_224_224.hdf5.gz\n",
        "!mv validation_224_224.hdf5 /content/data/224x224_cropped_merged_heads_hdf5\n",
        "\n",
        "!gunzip test_224_224.hdf5.gz\n",
        "!mv test_224_224.hdf5 /content/data/224x224_cropped_merged_heads_hdf5"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "PpyEj5mDrSUA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "992bbe16-8df8-4a21-e28d-c2bf6e1ac667"
      },
      "cell_type": "code",
      "source": [
        "ls -l /content/data"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 8\r\n",
            "drwxr-xr-x 2 root root 4096 May 21 00:05 \u001b[0m\u001b[01;34m224x224_cropped_merged_heads_hdf5\u001b[0m/\r\n",
            "drwxr-xr-x 2 root root 4096 May 21 00:26 \u001b[01;34mMobileNet_224x224_cropped_merged_heads_hdf5\u001b[0m/\r\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "X-PQQHiira1Q",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### Switch dir"
      ]
    },
    {
      "metadata": {
        "id": "w0-tuwE6rVcw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "66139685-4162-442f-dbcd-6b489e3c4900"
      },
      "cell_type": "code",
      "source": [
        "cd KerasVision/"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/KerasVision\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ortDlcDvrfTQ",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### Imports"
      ]
    },
    {
      "metadata": {
        "id": "PDe-9F4rrdNe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "65267da2-0182-4115-d128-3a30aeeaf9b0"
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "from tqdm import tqdm\n",
        "import matplotlib.pyplot as plt\n",
        "from keras.preprocessing import image\n",
        "\n",
        "from keras.models import Sequential, Model\n",
        "from keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten, InputLayer\n",
        "from keras.layers import BatchNormalization, Activation\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from keras import optimizers\n",
        "from keras.utils import to_categorical, plot_model\n",
        "from keras.models import load_model\n",
        "from keras.applications import VGG16, MobileNet\n",
        "\n",
        "from data.data_util import *\n",
        "from data.load_data import from_splitted_hdf5\n",
        "from train import *\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import pickle\n",
        "\n",
        "%load_ext autoreload\n",
        "%autoreload 2\n",
        "\n",
        "%matplotlib inline\n",
        "\n",
        "from data.augmentation.CustomImageDataGenerator import * \n",
        "\n",
        "import h5py"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "metadata": {
        "id": "NYItIoGUriRQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "train_set_x, train_set_y, dev_set_x, dev_set_y, _, _, classes = from_splitted_hdf5('../data/224x224_cropped_merged_heads_hdf5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IksmUYk4rm4O",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7b5c9584-b9fc-4e3c-e150-67db14f3af09"
      },
      "cell_type": "code",
      "source": [
        "train_set_x.shape, train_set_x.dtype, train_set_x[0][0][100][2]"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((2978, 224, 224, 3), dtype('uint8'), 101)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "metadata": {
        "id": "DwmJTqCarzV1",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### Get the conv_base"
      ]
    },
    {
      "metadata": {
        "id": "8ibs7RsSrw1G",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "conv_base = MobileNet(weights='imagenet', include_top=False, input_shape=(224, 224, 3))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lvgG5kc7r9ev",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### Extract features at the last conv_base layer"
      ]
    },
    {
      "metadata": {
        "id": "PRqdKsSur6jv",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "datagen = ImageDataGenerator(rescale=1./255)\n",
        "batch_size = 32"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "3brTr6uRsGC_",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def extract_features(set_x, set_y, sample_count):\n",
        "    features = np.zeros(shape=(sample_count, 7, 7, 1024))\n",
        "    labels = np.zeros(shape=(sample_count,1))\n",
        "    \n",
        "    generator = datagen.flow(set_x, set_y, batch_size=batch_size)\n",
        "    \n",
        "    i = 0\n",
        "    for inputs_batch, labels_batch in tqdm(generator):\n",
        "        \n",
        "        features_batch = conv_base.predict(inputs_batch)\n",
        "        \n",
        "        #print(features_batch.shape)\n",
        "        #print(labels_batch.shape)\n",
        "        \n",
        "        features[i * batch_size : (i + 1) * batch_size] = features_batch\n",
        "        labels[i * batch_size : (i + 1) * batch_size] = labels_batch\n",
        "        i += 1\n",
        "        \n",
        "        if i * batch_size >= sample_count:\n",
        "            # Note that since generators yield data indefinitely in a loop,\n",
        "            # we must `break` after every image has been seen once.\n",
        "            break\n",
        "            \n",
        "    return features, labels"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "y2S5e_O_sIah",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "feature_train_set_x, feature_train_set_y = extract_features(train_set_x, train_set_y, len(train_set_x))\n",
        "feature_dev_set_x, feature_dev_set_y = extract_features(dev_set_x, dev_set_y, len(dev_set_x))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6NXoIy3SsMGx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# one-hot \n",
        "feature_train_set_y = to_categorical(feature_train_set_y)\n",
        "feature_dev_set_y = to_categorical(feature_dev_set_y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "g0dKYqyNs7Nl",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "#### Our own FC"
      ]
    },
    {
      "metadata": {
        "id": "1V2qxqmrs5UT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "fc = Sequential()\n",
        "\n",
        "fc.add(InputLayer(input_shape=(7, 7, 1024)))\n",
        "fc.add(Flatten())\n",
        "fc.add(Dense(1024))\n",
        "fc.add(BatchNormalization())\n",
        "fc.add(Activation('relu'))\n",
        "fc.add(Dropout(0.5))\n",
        "fc.add(Dense(7, activation='softmax'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "e-blVIFkvMo6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "outputId": "1e805809-b1de-44ad-a4ae-0a69c4ad5f98"
      },
      "cell_type": "code",
      "source": [
        "fc.summary()"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_3 (InputLayer)         (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "flatten_2 (Flatten)          (None, 50176)             0         \n",
            "_________________________________________________________________\n",
            "dense_3 (Dense)              (None, 1024)              51381248  \n",
            "_________________________________________________________________\n",
            "batch_normalization_2 (Batch (None, 1024)              4096      \n",
            "_________________________________________________________________\n",
            "activation_2 (Activation)    (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dropout_2 (Dropout)          (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dense_4 (Dense)              (None, 7)                 7175      \n",
            "=================================================================\n",
            "Total params: 51,392,519\n",
            "Trainable params: 51,390,471\n",
            "Non-trainable params: 2,048\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "mjOB2CfKtAMU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "fc.compile(optimizer=optimizers.Adam(lr=1e-5), loss='categorical_crossentropy', metrics=['accuracy'])\n",
        "all_history = {}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "o2WSs278tE5N",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 394
        },
        "outputId": "d15466ac-7088-45f2-e189-444db46fbc5e"
      },
      "cell_type": "code",
      "source": [
        "history = fc.fit(feature_train_set_x, feature_train_set_y, epochs=10, batch_size=batch_size,\n",
        "                 validation_data=(feature_dev_set_x, feature_dev_set_y)\n",
        "                )"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 2978 samples, validate on 600 samples\n",
            "Epoch 1/10\n",
            "2978/2978 [==============================] - 9s 3ms/step - loss: 1.1804 - acc: 0.5960 - val_loss: 0.9053 - val_acc: 0.6750\n",
            "Epoch 2/10\n",
            "2978/2978 [==============================] - 8s 3ms/step - loss: 0.4112 - acc: 0.8690 - val_loss: 0.7652 - val_acc: 0.7300\n",
            "Epoch 3/10\n",
            "2978/2978 [==============================] - 8s 3ms/step - loss: 0.2476 - acc: 0.9271 - val_loss: 0.7242 - val_acc: 0.7483\n",
            "Epoch 4/10\n",
            "1344/2978 [============>.................] - ETA: 4s - loss: 0.1693 - acc: 0.9501"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2978/2978 [==============================] - 8s 3ms/step - loss: 0.1581 - acc: 0.9594 - val_loss: 0.6829 - val_acc: 0.7633\n",
            "Epoch 5/10\n",
            "2978/2978 [==============================] - 8s 3ms/step - loss: 0.1163 - acc: 0.9728 - val_loss: 0.7083 - val_acc: 0.7583\n",
            "Epoch 6/10\n",
            "2978/2978 [==============================] - 8s 3ms/step - loss: 0.0947 - acc: 0.9809 - val_loss: 0.7157 - val_acc: 0.7600\n",
            "Epoch 7/10\n",
            "2816/2978 [===========================>..] - ETA: 0s - loss: 0.0648 - acc: 0.9908"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2978/2978 [==============================] - 8s 3ms/step - loss: 0.0657 - acc: 0.9909 - val_loss: 0.6672 - val_acc: 0.7783\n",
            "Epoch 8/10\n",
            "2978/2978 [==============================] - 8s 3ms/step - loss: 0.0635 - acc: 0.9882 - val_loss: 0.6361 - val_acc: 0.7950\n",
            "Epoch 9/10\n",
            "2978/2978 [==============================] - 8s 3ms/step - loss: 0.0439 - acc: 0.9950 - val_loss: 0.7124 - val_acc: 0.7717\n",
            "Epoch 10/10\n",
            "2978/2978 [==============================] - 8s 3ms/step - loss: 0.0340 - acc: 0.9980 - val_loss: 0.7569 - val_acc: 0.7583\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Y-pvKLS0tjk4",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "whole_model = Sequential()\n",
        "whole_model.add(conv_base)\n",
        "whole_model.add(Flatten())\n",
        "whole_model.add(Dense(1024))\n",
        "whole_model.add(BatchNormalization())\n",
        "whole_model.add(Activation('relu'))\n",
        "whole_model.add(Dropout(0.5))\n",
        "whole_model.add(Dense(7, activation='softmax'))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "6V2kRly0uQ0W",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "conv_base.trainable = False\n",
        "for layer in conv_base.layers:\n",
        "  layer.trainable = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Ergz12eBuYeH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 374
        },
        "outputId": "8623db1a-5b5d-4dc2-ad8b-45b50fd22054"
      },
      "cell_type": "code",
      "source": [
        "whole_model.summary()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "mobilenet_1.00_224 (Model)   (None, 7, 7, 1024)        3228864   \n",
            "_________________________________________________________________\n",
            "flatten_3 (Flatten)          (None, 50176)             0         \n",
            "_________________________________________________________________\n",
            "dense_5 (Dense)              (None, 1024)              51381248  \n",
            "_________________________________________________________________\n",
            "batch_normalization_3 (Batch (None, 1024)              4096      \n",
            "_________________________________________________________________\n",
            "activation_3 (Activation)    (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dropout_3 (Dropout)          (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dense_6 (Dense)              (None, 7)                 7175      \n",
            "=================================================================\n",
            "Total params: 54,621,383\n",
            "Trainable params: 51,390,471\n",
            "Non-trainable params: 3,230,912\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "egGs21pduaAU",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# normalize the X\n",
        "renorm_train_set_x = train_set_x.astype('float32')/255.\n",
        "renorm_dev_set_x = dev_set_x.astype('float32')/255.\n",
        "\n",
        "# one-hot the Y\n",
        "one_hot_train_set_y = to_categorical(train_set_y)\n",
        "one_hot_dev_set_y = to_categorical(dev_set_y)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "AyLX3FSwv1ha",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "whole_model.compile(optimizer=optimizers.Adam(lr=1e-5), loss='categorical_crossentropy', metrics=['accuracy'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "htGkeoa9uq-e",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 394
        },
        "outputId": "ff25668e-66d1-47b4-8bc2-a7a426bb0acc"
      },
      "cell_type": "code",
      "source": [
        "history2 = whole_model.fit(renorm_train_set_x, one_hot_train_set_y, epochs=10, batch_size=batch_size,\n",
        "                           validation_data=(renorm_dev_set_x, one_hot_dev_set_y)\n",
        "                          )"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train on 2978 samples, validate on 600 samples\n",
            "Epoch 1/10\n",
            "2978/2978 [==============================] - 22s 7ms/step - loss: 1.3852 - acc: 0.5299 - val_loss: 1.0660 - val_acc: 0.5783\n",
            "Epoch 2/10\n",
            "2978/2978 [==============================] - 20s 7ms/step - loss: 0.6148 - acc: 0.7952 - val_loss: 1.1765 - val_acc: 0.5417\n",
            "Epoch 3/10\n",
            "2978/2978 [==============================] - 20s 7ms/step - loss: 0.3436 - acc: 0.8912 - val_loss: 1.2330 - val_acc: 0.5350\n",
            "Epoch 4/10\n",
            "1312/2978 [============>.................] - ETA: 9s - loss: 0.2523 - acc: 0.9314"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2978/2978 [==============================] - 20s 7ms/step - loss: 0.2410 - acc: 0.9305 - val_loss: 1.1966 - val_acc: 0.5517\n",
            "Epoch 5/10\n",
            "2978/2978 [==============================] - 20s 7ms/step - loss: 0.1884 - acc: 0.9469 - val_loss: 1.1642 - val_acc: 0.5617\n",
            "Epoch 6/10\n",
            "2978/2978 [==============================] - 20s 7ms/step - loss: 0.1523 - acc: 0.9594 - val_loss: 1.1414 - val_acc: 0.5800\n",
            "Epoch 7/10\n",
            "2752/2978 [==========================>...] - ETA: 1s - loss: 0.1156 - acc: 0.9706"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "2978/2978 [==============================] - 20s 7ms/step - loss: 0.1188 - acc: 0.9701 - val_loss: 1.2988 - val_acc: 0.5467\n",
            "Epoch 8/10\n",
            "2978/2978 [==============================] - 20s 7ms/step - loss: 0.0934 - acc: 0.9809 - val_loss: 1.2739 - val_acc: 0.5483\n",
            "Epoch 9/10\n",
            "2978/2978 [==============================] - 20s 7ms/step - loss: 0.0880 - acc: 0.9795 - val_loss: 1.3413 - val_acc: 0.5400\n",
            "Epoch 10/10\n",
            "2976/2978 [============================>.] - ETA: 0s - loss: 0.0634 - acc: 0.9866"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r2978/2978 [==============================] - 20s 7ms/step - loss: 0.0635 - acc: 0.9866 - val_loss: 1.2878 - val_acc: 0.5483\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "WoF99nUPvq01",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}