two_stream_vgg_16_multi_loss.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load training, validation and testing data¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from data_helper import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "IMAGE_FOLDER_PATH = 'dataset/resized/frames/'\n",
    "LABEL_FOLDER_PATH = 'dataset/labels/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------------------------------------\n",
      "Load object recoginition inputs and labels...\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[Train (Head)] number of image paths: 12744\n",
      "[Train (Hand)] number of image paths: 12744\n",
      "[Train (Label)] number of labels: 12744\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[Validation (Head)] number of image paths: 2248\n",
      "[Validation (Hand)] number of image paths: 2248\n",
      "[Validation (Label)] number of labels: 2248\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[Test (Head)] number of image paths: 12776\n",
      "[Test (Hand)] number of image paths: 12776\n",
      "[Test (Label)] number of labels: 12776\n",
      "----------------------------------------------------------------------------------------------------\n",
      "Load gesture recoginition inputs and labels...\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[Train (Head)] number of image paths: 12744\n",
      "[Train (Hand)] number of image paths: 12744\n",
      "[Train (Label)] number of labels: 12744\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[Validation (Head)] number of image paths: 2248\n",
      "[Validation (Hand)] number of image paths: 2248\n",
      "[Validation (Label)] number of labels: 2248\n",
      "----------------------------------------------------------------------------------------------------\n",
      "[Test (Head)] number of image paths: 12776\n",
      "[Test (Hand)] number of image paths: 12776\n",
      "[Test (Label)] number of labels: 12776\n"
     ]
    }
   ],
   "source": [
    "# Load object recognition labels\n",
    "print('-'*100)\n",
    "print('Load object recoginition inputs and labels...')\n",
    "train_head_image_paths, train_hand_image_paths, train_obj_labels, \\\n",
    "val_head_image_paths, val_hand_image_paths, val_obj_labels, \\\n",
    "test_head_image_paths, test_hand_image_paths, test_obj_labels = load_dataset(image_folder_path=IMAGE_FOLDER_PATH,\n",
    "                                                                             label_folder_path=LABEL_FOLDER_PATH,\n",
    "                                                                             label_type='obj',\n",
    "                                                                             hand_types=['left', 'right'],\n",
    "                                                                             with_head=True,\n",
    "                                                                             validation_split_ratio=0.15)\n",
    "# Load gesture recognition labels\n",
    "# Since the splitting train/val set doesn't involve randomness,\n",
    "# the order of data is same as the ones loaded from the above `load_dataset(..., label_type='obj', ...)\n",
    "print('-'*100)\n",
    "print('Load gesture recoginition inputs and labels...')\n",
    "train_head_image_paths, train_hand_image_paths, train_ges_labels, \\\n",
    "val_head_image_paths, val_hand_image_paths, val_ges_labels, \\\n",
    "test_head_image_paths, test_hand_image_paths, test_ges_labels = load_dataset(image_folder_path=IMAGE_FOLDER_PATH,\n",
    "                                                                             label_folder_path=LABEL_FOLDER_PATH,\n",
    "                                                                             label_type='ges',\n",
    "                                                                             hand_types=['left', 'right'],\n",
    "                                                                             with_head=True,\n",
    "                                                                             validation_split_ratio=0.15)\n",
    "\n",
    "# Only take hand image paths for baseline\n",
    "train_image_paths =  train_hand_image_paths\n",
    "val_image_paths = val_hand_image_paths\n",
    "test_image_paths = test_hand_image_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Use Tensorflow to build computational graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "import tensorflow.contrib.slim.nets\n",
    "import vgg_preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Path configs and hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PRETRAINED_HAND_GESTURE_MODEL_PATH = 'model/hand_gesture_vgg_16/hand_gesture_vgg_16_model'\n",
    "PRETRAINED_HAND_OBJ_MODEL_PATH = 'model/hand_obj_vgg_16/hand_obj_vgg_16_model'\n",
    "MODEL_PATH = 'model/two_stream_vgg_16_multi_loss/two_stream_vgg_16_multi_loss'\n",
    "\n",
    "num_classes_obj = 24 # object classes\n",
    "num_classes_ges = 13 # gesture classes\n",
    "batch_size = 16 # batch_size=32 is not enough for GPU with only 11170 MiB memory when optimizing 2 VGG nets.\n",
    "num_workers = 20\n",
    "max_epochs1 = 30\n",
    "max_epochs2 = 30\n",
    "max_patience = 5 # For early stopping\n",
    "learning_rate1 = 1e-3\n",
    "learning_rate2 = 1e-5\n",
    "dropout_keep_prob = 0.5\n",
    "weight_decay = 5e-4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Building blocks of  two-stream CNN model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stream_vgg_16(inputs,\n",
    "                  is_training=True,\n",
    "                  dropout_keep_prob=0.5,\n",
    "                  spatial_squeeze=False,\n",
    "                  scope='stream_vgg_16',\n",
    "                  fc_conv_padding='VALID'):\n",
    "    \n",
    "    \"\"\" Reference from \"https://github.com/tensorflow/models/blob/master/research/slim/nets/vgg.py\"\n",
    "        A VGG16 net excluding 'fc7' and 'fc8' layers.\n",
    "        \n",
    "        Returns: \n",
    "            A shape=(?, 4096) deep features if spatial_squeeze == True, else shape=(?, 1, 1, 4096).\n",
    "            and end_points dict.\n",
    "    \"\"\"\n",
    "    \n",
    "    with tf.variable_scope(scope, 'stream_vgg_16', [inputs]) as sc:\n",
    "            end_points_collection = sc.name + '_end_points'\n",
    "            \n",
    "            with slim.arg_scope([slim.conv2d, slim.max_pool2d],\n",
    "                                outputs_collections=end_points_collection):\n",
    "                \n",
    "                net = slim.repeat(inputs, 2, slim.conv2d, 64, [3, 3], scope='conv1')\n",
    "                net = slim.max_pool2d(net, [2, 2], scope='pool1')\n",
    "                net = slim.repeat(net, 2, slim.conv2d, 128, [3, 3], scope='conv2')\n",
    "                net = slim.max_pool2d(net, [2, 2], scope='pool2')\n",
    "                net = slim.repeat(net, 3, slim.conv2d, 256, [3, 3], scope='conv3')\n",
    "                net = slim.max_pool2d(net, [2, 2], scope='pool3')\n",
    "                net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv4')\n",
    "                net = slim.max_pool2d(net, [2, 2], scope='pool4')\n",
    "                net = slim.repeat(net, 3, slim.conv2d, 512, [3, 3], scope='conv5')\n",
    "                net = slim.max_pool2d(net, [2, 2], scope='pool5')\n",
    "                # Use conv2d instead of fully_connected layers.\n",
    "                net = slim.conv2d(net, 4096, [7, 7], padding=fc_conv_padding, scope='fc6')\n",
    "                net = slim.dropout(net, dropout_keep_prob, is_training=is_training, scope='dropout6')\n",
    "                # Convert end_points_collection into a end_point dict.\n",
    "                end_points = slim.utils.convert_collection_to_dict(end_points_collection)\n",
    "                \n",
    "                if spatial_squeeze:\n",
    "                    net = tf.squeeze(net, [1, 2], name='fc6/squeezed')\n",
    "                    end_points[sc.name + '/fc6'] = net\n",
    "                return net, end_points\n",
    "            \n",
    "def muli_label_fusion_fc(inputs,\n",
    "              num_classes_ges=13,\n",
    "              num_classes_obj=24,\n",
    "              is_training=True,\n",
    "              dropout_keep_prob=0.5,\n",
    "              spatial_squeeze=True,\n",
    "              scope='muli_label_fusion_fc'):\n",
    "    \n",
    "    \"\"\" Reference from \"https://github.com/tensorflow/models/blob/master/research/slim/nets/vgg.py\"\n",
    "        A VGG16 net only include 'fc7' and 'fc8' layers.\n",
    "        \n",
    "        Args:\n",
    "            inputs: A list of tensor with shape like (?, 1, 1, 4096). (the 2, 3 axis must be \"1\")\n",
    "            \n",
    "        Returns:\n",
    "            The last op containing the log predictions and end_points dict.\n",
    "    \"\"\"\n",
    "    \n",
    "    with tf.variable_scope(scope, 'muli_label_fusion_fc', [inputs]) as sc:\n",
    "        end_points_collection = sc.name + '_end_points'\n",
    "                \n",
    "        with slim.arg_scope([slim.conv2d],\n",
    "                            outputs_collections=end_points_collection):\n",
    "            # [(?, 1, 1, 4096), (?, 1, 1, 4096)] => (?, 1, 1, 8192)\n",
    "            net = tf.concat(inputs, axis=3)\n",
    "            # (?, 1, 1, 8192) => (?, 1, 1, 4096)\n",
    "            net = slim.conv2d(net, 4096, [1, 1], scope='fc7')\n",
    "            net = slim.dropout(net, dropout_keep_prob, is_training=is_training, scope='dropout7')\n",
    "            # For gesture logits: (?, 1, 1, 4096) => (?, 1, 1, num_classes_ges)\n",
    "            ges_net = slim.conv2d(net, num_classes_ges, [1, 1],\n",
    "                                  activation_fn=None,\n",
    "                                  normalizer_fn=None,\n",
    "                                  scope='ges_fc8')\n",
    "            # For object logits: (?, 1, 1, 4096) => (?, 1, 1, num_classes_obj)\n",
    "            obj_net = slim.conv2d(net, num_classes_obj, [1, 1],\n",
    "                                  activation_fn=None,\n",
    "                                  normalizer_fn=None,\n",
    "                                  scope='obj_fc8')\n",
    "            # Convert end_points_collection into a end_point dict.\n",
    "            end_points = slim.utils.convert_collection_to_dict(end_points_collection)\n",
    "            \n",
    "            if spatial_squeeze:\n",
    "                # (?, 1, 1, num_classes) => (?, num_classes)\n",
    "                ges_net = tf.squeeze(ges_net, [1, 2], name='ges_fc8/squeezed')\n",
    "                end_points[sc.name + '/ges_fc8'] = ges_net\n",
    "                \n",
    "                obj_net = tf.squeeze(obj_net, [1, 2], name='obj_fc8/squeezed')\n",
    "                end_points[sc.name + '/obj_fc8'] = obj_net\n",
    "                \n",
    "            return ges_net, obj_net, end_points"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Build our two-stream CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.37 s, sys: 27.9 ms, total: 2.4 s\n",
      "Wall time: 2.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "def dataset_map_fn(image_path, ges_label, obj_label, is_training):\n",
    "    # Load image\n",
    "    image_string = tf.read_file(image_path)\n",
    "    image_decoded = tf.image.decode_png(image_string, channels=3)\n",
    "    image = tf.cast(image_decoded, tf.float32)\n",
    "    # Preprocess image\n",
    "    preprocessed_image = tf.cond(is_training,\n",
    "                                 true_fn=lambda: vgg_preprocessing.preprocess_image(image, 224, 224, is_training=True),\n",
    "                                 false_fn=lambda: vgg_preprocessing.preprocess_image(image, 224, 224, is_training=False))\n",
    "    return preprocessed_image, ges_label, obj_label\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "    # ---------------------------------------------------------------------\n",
    "    # Indicates whether we are in training or in test mode\n",
    "    # Since VGG16 has applied `dropout`, we need to disable it when testing.\n",
    "    is_training = tf.placeholder(dtype=tf.bool, name='is_training')\n",
    "    \n",
    "    # Training, validation, testing data to feed in.\n",
    "    image_paths = tf.placeholder(dtype=tf.string, shape=(None,), name='image_paths')\n",
    "    ges_labels = tf.placeholder(dtype=tf.int32, shape=(None,), name='ges_labels')\n",
    "    obj_labels = tf.placeholder(dtype=tf.int32, shape=(None,), name='obj_labels')\n",
    "    \n",
    "    # Use dataset API to automatically generate batch data by iterator.\n",
    "    dataset = tf.contrib.data.Dataset.from_tensor_slices((image_paths, ges_labels, obj_labels))\n",
    "    dataset = dataset.map(lambda image_path, ges_label, obj_label: dataset_map_fn(image_path, ges_label, obj_label, is_training))\n",
    "    dataset = dataset.shuffle(buffer_size=10000)\n",
    "    batched_dataset = dataset.batch(batch_size)\n",
    "    \n",
    "    # Now we define an iterator that can operator on dataset.\n",
    "    # The iterator can be reinitialized by calling:\n",
    "    # sess.run(dataset_init_op, feed_dict={image_paths=train_image_paths, labels=train_labels}) \n",
    "    # for 1 epoch on the training set.\n",
    "    \n",
    "    # Once this is done, we don't need to feed any value for images and labels\n",
    "    # as they are automatically pulled out from the iterator queues.\n",
    "\n",
    "    # A reinitializable iterator is defined by its structure. We could use the\n",
    "    # `output_types` and `output_shapes` properties of dataset.\n",
    "    # The dataset will be fed with training, validation or testing data.\n",
    "    iterator = tf.contrib.data.Iterator.from_structure(batched_dataset.output_types,\n",
    "                                                       batched_dataset.output_shapes)\n",
    "    \n",
    "    # A batch of data to feed into the networks.\n",
    "    batch_images, batch_ges_labels, batch_obj_labels = iterator.get_next()\n",
    "    dataset_init_op = iterator.make_initializer(batched_dataset)\n",
    "    \n",
    "    # =====================================================================\n",
    "    # Start to build our two-stream cnn model.\n",
    "    vgg = tf.contrib.slim.nets.vgg\n",
    "    # Apply L2 regularization with weight decay.\n",
    "    with slim.arg_scope(vgg.vgg_arg_scope(weight_decay=weight_decay)):\n",
    "        # Firstly, build our first stream cnn model -- pretrained hand gesture vgg16 net,\n",
    "        # excluding 'fc7' and'fc8' layers\n",
    "        hand_gesture_model_4096_features, _ = stream_vgg_16(batch_images,\n",
    "                                                            is_training=is_training,\n",
    "                                                            dropout_keep_prob=dropout_keep_prob,\n",
    "                                                            spatial_squeeze=False,\n",
    "                                                            scope='hand_gesture_vgg_16')\n",
    "\n",
    "        # Secondly, build our second stream cnn model -- pretrained hand obj vgg16 net,\n",
    "        # excluding 'fc7' and'fc8' layers\n",
    "        hand_obj_model_4096_features, _ = stream_vgg_16(batch_images,\n",
    "                                                        is_training=is_training,\n",
    "                                                        dropout_keep_prob=dropout_keep_prob,\n",
    "                                                        spatial_squeeze=False,\n",
    "                                                        scope='hand_obj_vgg_16')\n",
    "\n",
    "        # Finally, concatenate our 2 stream cnn models with fc layers architecture in vgg16 net.\n",
    "        ges_logits, obj_logits, _ = muli_label_fusion_fc(inputs=[hand_gesture_model_4096_features, \n",
    "                                                                 hand_obj_model_4096_features],\n",
    "                                                         num_classes_ges=num_classes_ges,\n",
    "                                                         num_classes_obj=num_classes_obj,\n",
    "                                                         is_training=is_training,\n",
    "                                                         dropout_keep_prob=dropout_keep_prob,\n",
    "                                                         spatial_squeeze=True,\n",
    "                                                         scope='muli_label_fusion_fc')\n",
    "        \n",
    "    # =====================================================================\n",
    "\n",
    "    # ---------------------------------------------------------------------\n",
    "    # Restore only the layers up to 'fc6' (included)\n",
    "    # Calling function `hand_gesture_model_init_fn(sess)` will load all the pretrained weights.\n",
    "    hand_gesture_model_variables = slim.get_variables(scope='hand_gesture_vgg_16')\n",
    "    # Since the variable scope name of checkpoint file is 'vgg_16',\n",
    "    # which is different from our new scope name 'hand_gesture_vgg_16',\n",
    "    # we need to re-map variable names in order to make `Saver` know which new variable name to restore.\n",
    "    # If we don't know the variable names of source checkpoint file, we can run the script `inspect_checkpoint.py`, \n",
    "    # For example:\n",
    "    # $ python inspect_checkpoint.py --file_name=model/hand_gesture_vgg_16/hand_gesture_vgg_16_model\n",
    "    # to inspect the variable names from our source checkpoint file.\n",
    "    hand_gesture_model_init_fn = tf.contrib.framework.assign_from_checkpoint_fn(\n",
    "        PRETRAINED_HAND_GESTURE_MODEL_PATH,\n",
    "        var_list={var.name.replace('hand_gesture_vgg_16', 'vgg_16').split(':')[0]: var for var in hand_gesture_model_variables}\n",
    "    )\n",
    "        \n",
    "    # Same as we done to `hand_gesture_model`.\n",
    "    hand_obj_model_variables = slim.get_variables(scope='hand_obj_vgg_16')\n",
    "    hand_obj_model_init_fn = tf.contrib.framework.assign_from_checkpoint_fn(\n",
    "        PRETRAINED_HAND_OBJ_MODEL_PATH,\n",
    "        var_list={var.name.replace('hand_obj_vgg_16', 'vgg_16').split(':')[0]: var for var in hand_obj_model_variables}\n",
    "    )\n",
    "    \n",
    "    # ---------------------------------------------------------------------\n",
    "    # Using tf.losses, any loss is added to the tf.GraphKeys.LOSSES collection\n",
    "    # We can then call the total loss easily\n",
    "    ges_loss = tf.losses.sparse_softmax_cross_entropy(labels=batch_ges_labels, logits=ges_logits)\n",
    "    obj_loss = tf.losses.sparse_softmax_cross_entropy(labels=batch_obj_labels, logits=obj_logits)\n",
    "    # Same as: ges_loss + obj_loss + regularization losses\n",
    "    # We will then jointly minimize this loss. (Joint training)\n",
    "    loss = tf.losses.get_total_loss()\n",
    "    \n",
    "    # First we want to train only the reinitialized last layer fc8 for a few epochs.\n",
    "    # We run minimize the loss only with respect to the `fusion_fc` variables (weight and bias).\n",
    "    fusion_fc_optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate1)\n",
    "    fusion_fc_train_op = fusion_fc_optimizer.minimize(loss, var_list=slim.get_variables(scope='muli_label_fusion_fc'))\n",
    "    \n",
    "    # Then we want to finetune the entire model for a few epochs.\n",
    "    # We run minimize the loss only with respect to all the variables.\n",
    "    full_optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate2)\n",
    "    full_train_op = full_optimizer.minimize(loss)\n",
    "    \n",
    "    # Evaluation metrics\n",
    "    ges_prediction = tf.to_int32(tf.argmax(ges_logits, 1))\n",
    "    obj_prediction = tf.to_int32(tf.argmax(obj_logits, 1))\n",
    "    \n",
    "    ges_correct_prediction = tf.equal(ges_prediction, batch_ges_labels)\n",
    "    obj_correct_prediction = tf.equal(obj_prediction, batch_obj_labels)\n",
    "    \n",
    "    ges_accuracy = tf.reduce_mean(tf.cast(ges_correct_prediction, tf.float32))\n",
    "    obj_accuracy = tf.reduce_mean(tf.cast(obj_correct_prediction, tf.float32))\n",
    "    \n",
    "    # Initialize the variables (i.e. assign their default value)\n",
    "    init_op = tf.global_variables_initializer()\n",
    "    \n",
    "    # 'Saver' op to save and restore all the variables\n",
    "    saver = tf.train.Saver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(sess, loss, correct_prediction, dataset_init_op, feed_dict):\n",
    "    \"\"\"\n",
    "        Evaluation in training loop.\n",
    "        Check the performance of the model on either train, val or test (depending on `dataset_init_op`)\n",
    "        Note: The arguments are tensorflow operators defined in the graph.\n",
    "    \"\"\"\n",
    "    \n",
    "    # Initialize the correct dataset.\n",
    "    sess.run(dataset_init_op, feed_dict=feed_dict)\n",
    "\n",
    "    data_loss = 0\n",
    "    num_correct = 0\n",
    "    num_samples = 0\n",
    "    \n",
    "    # Evaluate on every batch.\n",
    "    while True:\n",
    "        try:\n",
    "            # Disable `is_training` since we have `dropout` in VGG net.\n",
    "            _loss, _correct_prediction = sess.run([loss, correct_prediction], feed_dict={is_training: False})\n",
    "\n",
    "            data_loss += _loss\n",
    "            num_correct += _correct_prediction.sum() # e.g: [True, False, True].sum() = 2\n",
    "            num_samples += _correct_prediction.shape[0] # Batch size\n",
    "            \n",
    "        except tf.errors.OutOfRangeError:\n",
    "            break\n",
    "\n",
    "    data_loss = data_loss / num_samples\n",
    "    acc = num_correct / num_samples\n",
    "\n",
    "    return data_loss, acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# --------------------------------------------------------------------------\n",
    "# Now that we have built the graph and finalized it, we define the session.\n",
    "# The session is the interface to *run* the computational graph.\n",
    "# We can call our training operations with `sess.run(train_op)` for instance\n",
    "sess = tf.Session(graph=graph)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize variables or restore variables from checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restore variables from checkpoint...\n",
      "INFO:tensorflow:Restoring parameters from model/two_stream_vgg_16_multi_loss/two_stream_vgg_16_multi_loss\n",
      "Regaining max validation accuracy...\n",
      "CPU times: user 33.3 s, sys: 2.63 s, total: 36 s\n",
      "Wall time: 35.3 s\n",
      "Max validation accuracy: 0.5026690391459074\n"
     ]
    }
   ],
   "source": [
    "RESTORE = True\n",
    "max_acc = 0.0\n",
    "\n",
    "if RESTORE:\n",
    "    print('Restore variables from checkpoint...')\n",
    "    # If checkpoint exists, restore it to session.\n",
    "    saver.restore(sess, MODEL_PATH)\n",
    "    # Regain max validation accuracy from model\n",
    "    print('Regaining max validation accuracy...')\n",
    "    %time _, val_acc = evaluate(sess, loss, obj_correct_prediction, dataset_init_op, \\\n",
    "                                feed_dict={image_paths: val_image_paths, \\\n",
    "                                           ges_labels: val_ges_labels, \\\n",
    "                                           obj_labels: val_obj_labels, \\\n",
    "                                           is_training: False})\n",
    "    max_acc = val_acc\n",
    "    print('Max validation accuracy: {}'.format(max_acc))\n",
    "else:\n",
    "    print('Initialize variables from scratch...')\n",
    "    # Initialize all variables\n",
    "    sess.run(init_op)\n",
    "    # Load the pretrained weights for 2-stream model\n",
    "    hand_gesture_model_init_fn(sess) \n",
    "    hand_obj_model_init_fn(sess)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Only train `fusion_fc` layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/30 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 1/30\n",
      "[Train] loss: 0.2788430961025487 | accuracy: 0.6783584431889517\n",
      "[Validation] loss: 0.3968479046830079 | accuracy: 0.44795373665480426\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  3%|▎         | 1/30 [09:36<4:38:47, 576.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/two_stream_cnn_multi_loss/two_stream_cnn_multi_loss\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 2/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  7%|▋         | 2/30 [18:40<4:24:30, 566.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.28337458681551375 | accuracy: 0.6321406151914627\n",
      "[Validation] loss: 0.43667007319867823 | accuracy: 0.36610320284697506\n",
      "Model not improved at epoch 2/30. Patience: 1/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 3/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 3/30 [27:35<4:10:50, 557.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.2702017653609549 | accuracy: 0.7161016949152542\n",
      "[Validation] loss: 0.417505411490851 | accuracy: 0.4332740213523132\n",
      "Model not improved at epoch 3/30. Patience: 2/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 4/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 13%|█▎        | 4/30 [36:36<3:59:20, 552.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.28550542899444276 | accuracy: 0.689579409918393\n",
      "[Validation] loss: 0.46263894873581746 | accuracy: 0.4359430604982206\n",
      "Model not improved at epoch 4/30. Patience: 3/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 5/30\n",
      "[Train] loss: 0.27254550895149127 | accuracy: 0.7187696170747018\n",
      "[Validation] loss: 0.43909855393752506 | accuracy: 0.44973309608540923\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 17%|█▋        | 5/30 [46:03<3:51:59, 556.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/two_stream_cnn_multi_loss/two_stream_cnn_multi_loss\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 6/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 6/30 [55:11<3:41:37, 554.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.28126385108300594 | accuracy: 0.7239485247959824\n",
      "[Validation] loss: 0.42033609737280847 | accuracy: 0.44395017793594305\n",
      "Model not improved at epoch 6/30. Patience: 1/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 7/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 23%|██▎       | 7/30 [1:04:13<3:31:05, 550.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.275807647026341 | accuracy: 0.7295197740112994\n",
      "[Validation] loss: 0.42454573331778583 | accuracy: 0.4297153024911032\n",
      "Model not improved at epoch 7/30. Patience: 2/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 8/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 27%|██▋       | 8/30 [1:13:15<3:20:56, 548.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.28270302208728637 | accuracy: 0.7039391086001255\n",
      "[Validation] loss: 0.4525430399752172 | accuracy: 0.4199288256227758\n",
      "Model not improved at epoch 8/30. Patience: 3/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 9/30\n",
      "[Train] loss: 0.26118825236656945 | accuracy: 0.7387005649717514\n",
      "[Validation] loss: 0.42843583661042073 | accuracy: 0.4555160142348754\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 9/30 [1:22:46<3:14:09, 554.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/two_stream_cnn_multi_loss/two_stream_cnn_multi_loss\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 10/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|███▎      | 10/30 [1:31:46<3:03:31, 550.58s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.2575963624266058 | accuracy: 0.7289704959196485\n",
      "[Validation] loss: 0.4138512411999957 | accuracy: 0.4386120996441281\n",
      "Model not improved at epoch 10/30. Patience: 1/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 11/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 37%|███▋      | 11/30 [1:40:52<2:53:53, 549.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.24177933450173225 | accuracy: 0.7392498430634024\n",
      "[Validation] loss: 0.4235157073604679 | accuracy: 0.4470640569395018\n",
      "Model not improved at epoch 11/30. Patience: 2/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 12/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 12/30 [1:49:54<2:44:03, 546.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.24713817287449258 | accuracy: 0.71939736346516\n",
      "[Validation] loss: 0.4542589709427857 | accuracy: 0.4181494661921708\n",
      "Model not improved at epoch 12/30. Patience: 3/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 13/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 43%|████▎     | 13/30 [1:59:00<2:34:51, 546.56s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.23675097464317804 | accuracy: 0.7376020087884495\n",
      "[Validation] loss: 0.408826076899559 | accuracy: 0.45062277580071175\n",
      "Model not improved at epoch 13/30. Patience: 4/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 14/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 47%|████▋     | 14/30 [2:08:04<2:25:34, 545.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.23609283428033492 | accuracy: 0.7411330822347771\n",
      "[Validation] loss: 0.4025343847444473 | accuracy: 0.4354982206405694\n",
      "Model not improved at epoch 14/30. Patience: 5/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 15/30\n",
      "[Train] loss: 0.2397894070572024 | accuracy: 0.723556183301946\n",
      "[Validation] loss: 0.3837939893223637 | accuracy: 0.4372775800711744\n",
      "Model not improved at epoch 15/30. Patience: 6/5\n",
      "Max patience exceeded. Early stopping.\n"
     ]
    }
   ],
   "source": [
    "patience = 0\n",
    "\n",
    "# Update only the last layer for a few epochs.\n",
    "for epoch in tqdm(range(max_epochs1)):\n",
    "    # Run an epoch over the training data.\n",
    "    print('-'*110)\n",
    "    print('Starting epoch {}/{}'.format(epoch+1, max_epochs1))\n",
    "    # Here we initialize the iterator with the training set.\n",
    "    # This means that we can go through an entire epoch until the iterator becomes empty.\n",
    "    sess.run(dataset_init_op, feed_dict={image_paths: train_image_paths,\n",
    "                                         ges_labels: train_ges_labels,\n",
    "                                         obj_labels: train_obj_labels,\n",
    "                                         is_training: True})\n",
    "    while True:\n",
    "        try:\n",
    "            _ = sess.run(fusion_fc_train_op, feed_dict={is_training: True})\n",
    "        except tf.errors.OutOfRangeError:\n",
    "            break\n",
    "\n",
    "    # Check performance every epoch\n",
    "    train_loss, train_acc = evaluate(sess, loss, obj_correct_prediction, dataset_init_op,\n",
    "                                     feed_dict={image_paths: train_image_paths,\n",
    "                                                ges_labels: train_ges_labels,\n",
    "                                                obj_labels: train_obj_labels,\n",
    "                                                is_training: True})\n",
    "    \n",
    "    val_loss, val_acc = evaluate(sess, loss, obj_correct_prediction, dataset_init_op,\n",
    "                                 feed_dict={image_paths: val_image_paths,\n",
    "                                            ges_labels: val_ges_labels,\n",
    "                                            obj_labels: val_obj_labels,\n",
    "                                            is_training: False})\n",
    "    \n",
    "    print('[Train] loss: {} | accuracy: {}'.format(train_loss, train_acc))\n",
    "    print('[Validation] loss: {} | accuracy: {}'.format(val_loss, val_acc))\n",
    "    \n",
    "    # Save checkpoint\n",
    "    if val_acc > max_acc:\n",
    "        patience = 0\n",
    "        max_acc = val_acc\n",
    "        save_path = saver.save(sess, MODEL_PATH)\n",
    "        print(\"Model updated and saved in file: %s\" % save_path)\n",
    "    else:\n",
    "        patience += 1\n",
    "        print('Model not improved at epoch {}/{}. Patience: {}/{}'.format(epoch+1, max_epochs1, patience, max_patience))\n",
    "    # Early stopping.\n",
    "    if patience > max_patience:\n",
    "        print('Max patience exceeded. Early stopping.')\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train all layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/30 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 1/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  3%|▎         | 1/30 [12:54<6:14:28, 774.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.23845204673543788 | accuracy: 0.732736974262398\n",
      "[Validation] loss: 0.4419526137491138 | accuracy: 0.44261565836298933\n",
      "Model not improved at epoch 1/30. Patience: 1/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 2/30\n",
      "[Train] loss: 0.22054174677126825 | accuracy: 0.7616133082234777\n",
      "[Validation] loss: 0.3758830139645478 | accuracy: 0.46307829181494664\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  7%|▋         | 2/30 [26:19<6:05:41, 783.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/two_stream_cnn_multi_loss/two_stream_cnn_multi_loss\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 3/30\n",
      "[Train] loss: 0.21093015048164132 | accuracy: 0.7744036409290647\n",
      "[Validation] loss: 0.3608174811902844 | accuracy: 0.4870996441281139\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 10%|█         | 3/30 [39:48<5:56:07, 791.40s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/two_stream_cnn_multi_loss/two_stream_cnn_multi_loss\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 4/30\n",
      "[Train] loss: 0.20130126538399398 | accuracy: 0.7957470182046453\n",
      "[Validation] loss: 0.36882651021896307 | accuracy: 0.4919928825622776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 13%|█▎        | 4/30 [53:17<5:45:09, 796.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/two_stream_cnn_multi_loss/two_stream_cnn_multi_loss\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 5/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 17%|█▋        | 5/30 [1:06:08<5:28:45, 789.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.19344496074070067 | accuracy: 0.8064187068424357\n",
      "[Validation] loss: 0.4113207931408254 | accuracy: 0.46619217081850534\n",
      "Model not improved at epoch 5/30. Patience: 1/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 6/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 20%|██        | 6/30 [1:19:09<5:14:38, 786.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.19042110719062663 | accuracy: 0.8125392341494037\n",
      "[Validation] loss: 0.36272104238275954 | accuracy: 0.4697508896797153\n",
      "Model not improved at epoch 6/30. Patience: 2/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 7/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 23%|██▎       | 7/30 [1:32:02<5:00:00, 782.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.18490752223259385 | accuracy: 0.8247802887633396\n",
      "[Validation] loss: 0.3816025112871598 | accuracy: 0.4719750889679715\n",
      "Model not improved at epoch 7/30. Patience: 3/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 8/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 27%|██▋       | 8/30 [1:45:03<4:46:43, 781.98s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.1857325349936153 | accuracy: 0.8231324544883867\n",
      "[Validation] loss: 0.36322162488601384 | accuracy: 0.49110320284697506\n",
      "Model not improved at epoch 8/30. Patience: 4/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 9/30\n",
      "[Train] loss: 0.17731385222973498 | accuracy: 0.8434557438794726\n",
      "[Validation] loss: 0.4357446225300378 | accuracy: 0.5026690391459074\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 30%|███       | 9/30 [1:58:32<4:36:34, 790.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model updated and saved in file: model/two_stream_cnn_multi_loss/two_stream_cnn_multi_loss\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 10/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 33%|███▎      | 10/30 [2:11:33<4:22:25, 787.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.17575880532472746 | accuracy: 0.8462806026365348\n",
      "[Validation] loss: 0.37600329454683323 | accuracy: 0.5008896797153025\n",
      "Model not improved at epoch 10/30. Patience: 1/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 11/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 37%|███▋      | 11/30 [2:24:28<4:08:12, 783.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.17373526521022412 | accuracy: 0.8453389830508474\n",
      "[Validation] loss: 0.42352240047420897 | accuracy: 0.4724199288256228\n",
      "Model not improved at epoch 11/30. Patience: 2/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 12/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 40%|████      | 12/30 [2:37:29<3:54:51, 782.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.17099376196802898 | accuracy: 0.8557752667922159\n",
      "[Validation] loss: 0.40206928227719885 | accuracy: 0.4835409252669039\n",
      "Model not improved at epoch 12/30. Patience: 3/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 13/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 43%|████▎     | 13/30 [2:50:28<3:41:29, 781.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.1700003144880248 | accuracy: 0.8556183301946014\n",
      "[Validation] loss: 0.45908505053282633 | accuracy: 0.4893238434163701\n",
      "Model not improved at epoch 13/30. Patience: 4/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 14/30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 47%|████▋     | 14/30 [3:03:27<3:28:14, 780.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train] loss: 0.16696886105576328 | accuracy: 0.8620527306967984\n",
      "[Validation] loss: 0.3857013393339313 | accuracy: 0.498220640569395\n",
      "Model not improved at epoch 14/30. Patience: 5/5\n",
      "--------------------------------------------------------------------------------------------------------------\n",
      "Starting epoch 15/30\n",
      "[Train] loss: 0.16666279400173253 | accuracy: 0.8572661644695543\n",
      "[Validation] loss: 0.4500828131874261 | accuracy: 0.48087188612099646\n",
      "Model not improved at epoch 15/30. Patience: 6/5\n",
      "Max patience exceeded. Early stopping.\n"
     ]
    }
   ],
   "source": [
    "patience = 0\n",
    "\n",
    "# Train the entire model for a few more epochs, continuing with the *same* weights.\n",
    "for epoch in tqdm(range(max_epochs2)):\n",
    "    # Run an epoch over the training data.\n",
    "    print('-'*110)\n",
    "    print('Starting epoch {}/{}'.format(epoch+1, max_epochs2))\n",
    "    # Here we initialize the iterator with the training set.\n",
    "    # This means that we can go through an entire epoch until the iterator becomes empty.\n",
    "    sess.run(dataset_init_op, feed_dict={image_paths: train_image_paths,\n",
    "                                         ges_labels: train_ges_labels,\n",
    "                                         obj_labels: train_obj_labels,\n",
    "                                         is_training: True})\n",
    "    while True:\n",
    "        try:\n",
    "            _ = sess.run(full_train_op, feed_dict={is_training: True})    \n",
    "        except tf.errors.OutOfRangeError:\n",
    "            break\n",
    "\n",
    "    # Check performance every epoch\n",
    "    train_loss, train_acc = evaluate(sess, loss, obj_correct_prediction, dataset_init_op,\n",
    "                                     feed_dict={image_paths: train_image_paths,\n",
    "                                                ges_labels: train_ges_labels,\n",
    "                                                obj_labels: train_obj_labels,\n",
    "                                                is_training: True})\n",
    "    \n",
    "    val_loss, val_acc = evaluate(sess, loss, obj_correct_prediction, dataset_init_op,\n",
    "                                 feed_dict={image_paths: val_image_paths,\n",
    "                                            ges_labels: val_ges_labels,\n",
    "                                            obj_labels: val_obj_labels,\n",
    "                                            is_training: False})\n",
    "    \n",
    "    print('[Train] loss: {} | accuracy: {}'.format(train_loss, train_acc))\n",
    "    print('[Validation] loss: {} | accuracy: {}'.format(val_loss, val_acc))\n",
    "    \n",
    "    # Save checkpoint\n",
    "    if val_acc > max_acc:\n",
    "        patience = 0\n",
    "        max_acc = val_acc\n",
    "        save_path = saver.save(sess, MODEL_PATH)\n",
    "        print(\"Model updated and saved in file: %s\" % save_path)\n",
    "    else:\n",
    "        patience += 1\n",
    "        print('Model not improved at epoch {}/{}. Patience: {}/{}'.format(epoch+1, max_epochs1, patience, max_patience))\n",
    "    # Early stopping.\n",
    "    if patience > max_patience:\n",
    "        print('Max patience exceeded. Early stopping.')\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from model/two_stream_cnn_multi_loss/two_stream_cnn_multi_loss\n",
      "[Test] loss: 0.30849285870697174 | accuracy: 0.6404195366311835\n",
      "CPU times: user 3min 8s, sys: 20.6 s, total: 3min 29s\n",
      "Wall time: 3min 22s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "saver.restore(sess, MODEL_PATH)\n",
    "\n",
    "test_loss, test_acc = evaluate(sess, loss, obj_correct_prediction, dataset_init_op,\n",
    "                               feed_dict={image_paths: test_image_paths,\n",
    "                                          ges_labels: test_ges_labels,\n",
    "                                          obj_labels: test_obj_labels,\n",
    "                                          is_training: False})\n",
    "\n",
    "print('[Test] loss: {} | accuracy: {}'.format(test_loss, test_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Save estimated probabilities of decisions (logits) to plot precision-recall curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/799 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading testing dataset...\n",
      "CPU times: user 16.7 ms, sys: 16 µs, total: 16.7 ms\n",
      "Wall time: 14.7 ms\n",
      "Predicting on every batch...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 799/799 [03:10<00:00,  4.18it/s] \n"
     ]
    }
   ],
   "source": [
    "# Load testing dataset\n",
    "print('Loading testing dataset...')\n",
    "%time sess.run(dataset_init_op, feed_dict={ \\\n",
    "    image_paths: test_image_paths, \\\n",
    "    ges_labels: test_ges_labels, \\\n",
    "    obj_labels: test_obj_labels, \\\n",
    "    is_training: False \\\n",
    "})\n",
    "\n",
    "logits_list = []\n",
    "num_batches = len(test_image_paths) // batch_size + 1\n",
    "\n",
    "# Evaluate on every batch.\n",
    "print('Predicting on every batch...')\n",
    "with tqdm(total=num_batches) as progress:\n",
    "    while True:\n",
    "        try:\n",
    "            # Disable `is_training` since we have `dropout` in VGG net.\n",
    "            _logits = sess.run(obj_logits, feed_dict={is_training: False})\n",
    "            logits_list.append(_logits)\n",
    "            progress.update(1)\n",
    "        except tf.errors.OutOfRangeError:\n",
    "            break\n",
    "    logits_list = np.concatenate(logits_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('{}_logits.npy'.format(MODEL_PATH), 'wb') as file:\n",
    "    np.save(file, logits_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}