Capsule Network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "from torch.optim import Adam\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "USE_CUDA = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Mnist:\n",
    "    def __init__(self, batch_size):\n",
    "        dataset_transform = transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.1307,), (0.3081,))\n",
    "                   ])\n",
    "\n",
    "        train_dataset = datasets.MNIST('../data', train=True, download=True, transform=dataset_transform)\n",
    "        test_dataset = datasets.MNIST('../data', train=False, download=True, transform=dataset_transform)\n",
    "        \n",
    "        self.train_loader  = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "        self.test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=True)        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class ConvLayer(nn.Module):\n",
    "    def __init__(self, in_channels=1, out_channels=256, kernel_size=9):\n",
    "        super(ConvLayer, self).__init__()\n",
    "\n",
    "        self.conv = nn.Conv2d(in_channels=in_channels,\n",
    "                               out_channels=out_channels,\n",
    "                               kernel_size=kernel_size,\n",
    "                               stride=1\n",
    "                             )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return F.relu(self.conv(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class PrimaryCaps(nn.Module):\n",
    "    def __init__(self, num_capsules=8, in_channels=256, out_channels=32, kernel_size=9):\n",
    "        super(PrimaryCaps, self).__init__()\n",
    "\n",
    "        self.capsules = nn.ModuleList([\n",
    "            nn.Conv2d(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=2, padding=0) \n",
    "                          for _ in range(num_capsules)])\n",
    "    \n",
    "    def forward(self, x):\n",
    "        u = [capsule(x) for capsule in self.capsules]\n",
    "        u = torch.stack(u, dim=1)\n",
    "        u = u.view(x.size(0), 32 * 6 * 6, -1)\n",
    "        return self.squash(u)\n",
    "    \n",
    "    def squash(self, input_tensor):\n",
    "        squared_norm = (input_tensor ** 2).sum(-1, keepdim=True)\n",
    "        output_tensor = squared_norm *  input_tensor / ((1. + squared_norm) * torch.sqrt(squared_norm))\n",
    "        return output_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class DigitCaps(nn.Module):\n",
    "    def __init__(self, num_capsules=10, num_routes=32 * 6 * 6, in_channels=8, out_channels=16):\n",
    "        super(DigitCaps, self).__init__()\n",
    "\n",
    "        self.in_channels = in_channels\n",
    "        self.num_routes = num_routes\n",
    "        self.num_capsules = num_capsules\n",
    "\n",
    "        self.W = nn.Parameter(torch.randn(1, num_routes, num_capsules, out_channels, in_channels))\n",
    "\n",
    "    def forward(self, x):\n",
    "        batch_size = x.size(0)\n",
    "        x = torch.stack([x] * self.num_capsules, dim=2).unsqueeze(4)\n",
    "\n",
    "        W = torch.cat([self.W] * batch_size, dim=0)\n",
    "        u_hat = torch.matmul(W, x)\n",
    "\n",
    "        b_ij = Variable(torch.zeros(1, self.num_routes, self.num_capsules, 1))\n",
    "        if USE_CUDA:\n",
    "            b_ij = b_ij.cuda()\n",
    "\n",
    "        num_iterations = 3\n",
    "        for iteration in range(num_iterations):\n",
    "            c_ij = F.softmax(b_ij)\n",
    "            c_ij = torch.cat([c_ij] * batch_size, dim=0).unsqueeze(4)\n",
    "\n",
    "            s_j = (c_ij * u_hat).sum(dim=1, keepdim=True)\n",
    "            v_j = self.squash(s_j)\n",
    "            \n",
    "            if iteration < num_iterations - 1:\n",
    "                a_ij = torch.matmul(u_hat.transpose(3, 4), torch.cat([v_j] * self.num_routes, dim=1))\n",
    "                b_ij = b_ij + a_ij.squeeze(4).mean(dim=0, keepdim=True)\n",
    "\n",
    "        return v_j.squeeze(1)\n",
    "    \n",
    "    def squash(self, input_tensor):\n",
    "        squared_norm = (input_tensor ** 2).sum(-1, keepdim=True)\n",
    "        output_tensor = squared_norm *  input_tensor / ((1. + squared_norm) * torch.sqrt(squared_norm))\n",
    "        return output_tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(Decoder, self).__init__()\n",
    "        \n",
    "        self.reconstraction_layers = nn.Sequential(\n",
    "            nn.Linear(16 * 10, 512),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(512, 1024),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Linear(1024, 784),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "        \n",
    "    def forward(self, x, data):\n",
    "        classes = torch.sqrt((x ** 2).sum(2))\n",
    "        classes = F.softmax(classes)\n",
    "        \n",
    "        _, max_length_indices = classes.max(dim=1)\n",
    "        masked = Variable(torch.sparse.torch.eye(10))\n",
    "        if USE_CUDA:\n",
    "            masked = masked.cuda()\n",
    "        masked = masked.index_select(dim=0, index=max_length_indices.squeeze(1).data)\n",
    "        \n",
    "        reconstructions = self.reconstraction_layers((x * masked[:, :, None, None]).view(x.size(0), -1))\n",
    "        reconstructions = reconstructions.view(-1, 1, 28, 28)\n",
    "        \n",
    "        return reconstructions, masked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CapsNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CapsNet, self).__init__()\n",
    "        self.conv_layer = ConvLayer()\n",
    "        self.primary_capsules = PrimaryCaps()\n",
    "        self.digit_capsules = DigitCaps()\n",
    "        self.decoder = Decoder()\n",
    "        \n",
    "        self.mse_loss = nn.MSELoss()\n",
    "        \n",
    "    def forward(self, data):\n",
    "        output = self.digit_capsules(self.primary_capsules(self.conv_layer(data)))\n",
    "        reconstructions, masked = self.decoder(output, data)\n",
    "        return output, reconstructions, masked\n",
    "    \n",
    "    def loss(self, data, x, target, reconstructions):\n",
    "        return self.margin_loss(x, target) + self.reconstruction_loss(data, reconstructions)\n",
    "    \n",
    "    def margin_loss(self, x, labels, size_average=True):\n",
    "        batch_size = x.size(0)\n",
    "\n",
    "        v_c = torch.sqrt((x**2).sum(dim=2, keepdim=True))\n",
    "\n",
    "        left = F.relu(0.9 - v_c).view(batch_size, -1)\n",
    "        right = F.relu(v_c - 0.1).view(batch_size, -1)\n",
    "\n",
    "        loss = labels * left + 0.5 * (1.0 - labels) * right\n",
    "        loss = loss.sum(dim=1).mean()\n",
    "\n",
    "        return loss\n",
    "    \n",
    "    def reconstruction_loss(self, data, reconstructions):\n",
    "        loss = self.mse_loss(reconstructions.view(reconstructions.size(0), -1), data.view(reconstructions.size(0), -1))\n",
    "        return loss * 0.0005"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "capsule_net = CapsNet()\n",
    "if USE_CUDA:\n",
    "    capsule_net = capsule_net.cuda()\n",
    "optimizer = Adam(capsule_net.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train accuracy: 0.12\n",
      "train accuracy: 0.9\n",
      "train accuracy: 0.94\n",
      "train accuracy: 0.96\n",
      "train accuracy: 0.99\n",
      "train accuracy: 0.96\n",
      "0.229411779922\n",
      "test accuracy: 0.96\n",
      "0.0547490972094\n",
      "train accuracy: 0.98\n",
      "train accuracy: 0.98\n",
      "train accuracy: 0.99\n",
      "train accuracy: 0.99\n",
      "train accuracy: 1.0\n",
      "train accuracy: 0.99\n",
      "0.0456192491871\n",
      "test accuracy: 0.98\n",
      "0.0390225026663\n",
      "train accuracy: 0.99\n",
      "train accuracy: 0.99\n",
      "train accuracy: 1.0\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-133-f33ca0acefd3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0mtrain_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbatch_id\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m100\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "batch_size = 100\n",
    "mnist = Mnist(batch_size)\n",
    "\n",
    "n_epochs = 30\n",
    "\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    capsule_net.train()\n",
    "    train_loss = 0\n",
    "    for batch_id, (data, target) in enumerate(mnist.train_loader):\n",
    "\n",
    "        target = torch.sparse.torch.eye(10).index_select(dim=0, index=target)\n",
    "        data, target = Variable(data), Variable(target)\n",
    "\n",
    "        if USE_CUDA:\n",
    "            data, target = data.cuda(), target.cuda()\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        output, reconstructions, masked = capsule_net(data)\n",
    "        loss = capsule_net.loss(data, output, target, reconstructions)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        train_loss += loss.data[0]\n",
    "        \n",
    "        if batch_id % 100 == 0:\n",
    "            print \"train accuracy:\", sum(np.argmax(masked.data.cpu().numpy(), 1) == \n",
    "                                   np.argmax(target.data.cpu().numpy(), 1)) / float(batch_size)\n",
    "        \n",
    "    print train_loss / len(mnist.train_loader)\n",
    "        \n",
    "    capsule_net.eval()\n",
    "    test_loss = 0\n",
    "    for batch_id, (data, target) in enumerate(mnist.test_loader):\n",
    "\n",
    "        target = torch.sparse.torch.eye(10).index_select(dim=0, index=target)\n",
    "        data, target = Variable(data), Variable(target)\n",
    "\n",
    "        if USE_CUDA:\n",
    "            data, target = data.cuda(), target.cuda()\n",
    "\n",
    "        output, reconstructions, masked = capsule_net(data)\n",
    "        loss = capsule_net.loss(data, output, target, reconstructions)\n",
    "\n",
    "        test_loss += loss.data[0]\n",
    "        \n",
    "        if batch_id % 100 == 0:\n",
    "            print \"test accuracy:\", sum(np.argmax(masked.data.cpu().numpy(), 1) == \n",
    "                                   np.argmax(target.data.cpu().numpy(), 1)) / float(batch_size)\n",
    "    \n",
    "    print test_loss / len(mnist.test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def plot_images_separately(images):\n",
    "    \"Plot the six MNIST images separately.\"\n",
    "    fig = plt.figure()\n",
    "    for j in xrange(1, 7):\n",
    "        ax = fig.add_subplot(1, 6, j)\n",
    "        ax.matshow(images[j-1], cmap = matplotlib.cm.binary)\n",
    "        plt.xticks(np.array([]))\n",
    "        plt.yticks(np.array([]))\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWQAAABFCAYAAAB9nJwHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADTFJREFUeJzt3XuQ1WMcx/H3hkRKUiFd3RlEUtJFIVkzFV1YqmkIIdGW\nS0m5pCmkaHMJbSGXEmI2ZDCTu9g1IwyjVOs2ad1SZNu1/jjzfX7n7J7ddju35+z5vP7Z/Pbsen57\nznnO93me7/N9sioqKhARkdRrkOoGiIhIiDpkERFPqEMWEfGEOmQREU+oQxYR8YQ6ZBERT6hDFhHx\nhDpkERFPqEMWEfHEnnV5cIsWLSo6dOiQoKYkXmFhYUlFRUXLmh6je/Rfbe4RMuM+M+EeIXPus04d\ncocOHfj00093v1UplpWVtWlXj9E9+q829wiZcZ+ZcI+QOfepKQsREU+oQxYR8YQ6ZBERT9RpDjkV\ntm/fDsC///7rrjVt2hSAPff0vvkiIrWmCFlExBNehph//PEHkydPBuDdd98F4KuvvnLfv/nmmwHI\nzc0FoFWrVkluYXzZIQGPP/44ALNmzeK7774D4OqrrwYgLy+PPfbYIzUNlIz1zz//ADB79mzuu+8+\nAN5++20AOnfunLJ21VeKkEVEPOFVhFxSUgLAwIED+eijj6p93N133w3AN998A8CyZcvScj75k08+\nAWDq1KkArFq1CoCGDRuy9957A/Dwww8DMHToUM4888wUtHL3lZaWMnfuXCC4t/HjxwOh51j89/nn\nnwMwbdo0d+3bb78FMi9CXr9+Pa+99lqtH3/ttdfW+f/hVS+2cuVKgBo743ArVqwA4JZbbuGee+5J\nWLsS4fXXX2fo0KFAsHA5YMAAIDQ9se+++wLQvn17AH766acUtDI248ePdx8o5r333gPgueeeY/Dg\nwaloVtz89ddfQGg6LTs7G4DDDz8cIOLD057Xbt26AbDXXnsls5lSg59//hmAU089FYBJkyZV6Ujt\ng2fTpk38/vvvVX5H48aNAdz7ORaashAR8YRXEXLHjh0BaNSoEUcddRQQDHFPPvlkAIqLi7nssssA\n+PXXX4HQgkO6RMhvvfUWAMOGDaO0tBQI7tEWKdu1a+cef+SRRya5hfFjw91wBx10EBCKKtMxQt6y\nZQvr168H4KKLLgJCr0lj23vDt/naa/Oqq64CYO7cuTRq1Cgp7Y2Vvcfqq7Vr1wKh5xWgdevWbirU\nnl8bwZ5xxhmMHTsWgDZt2rjfYdOlNjqKhSJkERFPeBUh9+7dG4B169ax3377AcEmENOpUycOO+ww\nIL0+vcvKyoAgtW3btm3ceuutAEyfPr3K4y16tnnKdPLDDz8A8OOPP1b5Xn5+PgD9+vVLaptitXPn\nTgB69OjhFrWiOfjgg4HIUc66desAeOSRR4DQYvQzzzwT8XhfPfvss+7fzZo1A6Bv376pak7cvfDC\nCwBu8Xnw4MHMnDkTgObNmwNwySWXAHDTTTclvD1edcimdevW1X5v9erVFBUVRVzr0aNHopsUk7Ky\nMiZOnAiEFrMAxowZw5QpU6r9mVdffRWADRs2AHDEEUckuJXxY8M/+xru448/BtKnQ16+fDkADz30\nEEC1nbHli0+YMAGIfL6++OILAAoLCwG4//77ueCCCwBYsGABACeeeGK8mx4Ty423QAKge/fuQPrn\n/Zvi4mLeeOMNAM477zx3/eKLLwbgxhtvBJK7I1hTFiIinvAyQo7GhowrV66kvLw84nvnnHNOKppU\na/Pnz2fevHkR14499tgaF3YsMrNIsmvXrolrYJzde++9QLAYEs4iSd/ZrrTZs2cDsHnz5iqPOemk\nkwBYvHgxRx99NEDU5/T444+P+Dpo0CAXEVsO+ssvvxzP5sfMRgI2ooMgQq4vDjjgAJdI8P3337vr\nqSyErwhZRMQTaRMhv/POO0AQsQD06dMHSM5k++6wRZypU6e6pHGbl6ppznDJkiUsW7YMgKeffhqA\nBg38/+y0OePKc/wA48aNA4KFEh/ZLqwZM2a4XZS2uBouJycHgAceeACo+5xqs2bN3Fz6nDlzdru9\niRQeGRtLRa0vmjRp4kYtW7duBUKL6E2aNElZm/x/l4uIZIikR8hWxay2W4HffPNNABYuXOiutWjR\nAoDbb78dwNV98IVFxhYNjx492qXS7LPPPtX+3N9//w3AY489xogRIwDSavPExo0bgaDGSDib58/K\nykpmk+rEUhLLy8ujRsYA2dnZrtpgeGRs6xoffvghEGwRnzBhAg0bNqzyew455BAgmG/3zciRIwG4\n7bbb3LX+/funqjkJM2nSJACXSrtz586Ie062hHbIlo+6cOFC17FaGlcstRkeffRRIMhb9oVNq1hH\nfO655wIwc+bMGjtiYyVHi4qKXAGldCm5WVZWxowZM6pct9xVnxeE/vzzTyA4BCG8lorVKbC0KJtK\nCrdjxw63ezQ8bxdCz/1LL70EBOmZvgUQ0Tz11FOpbkJSWIrtK6+8AkSfbksmTVmIiHgi7hFySUkJ\n8+fPB4Jphmg7tnZXt27dvBw6rVmzxlWJOvDAA4HQBgCoeZoCQot4EJTazMvL47TTTktUUxOipKQk\naupW5b+Jb5YuXepSz6Jt+rjhhhuAYHoMgkjaalrMmjWrSmRstm7dyllnnQWEKvxBegz9n3/++Yj/\n7tixY9Spl/rCRtsffPCBSxawTSPJvG9FyCIinohbhGxzw126dIlaM7Qyi5hGjhzp5uVqM6/82Wef\nuUUjS8z3wZNPPulSZywSOvTQQ2v8GauGlpeXBwQLKZdffnmimhl3//33HwB33nlnle/17NnTzYv7\nqqioKGpkPHz4cCA4Lsw2Ji1YsIDffvsNIKWLP4m0du1a9342/fr1czW66yPb0FNaWsrq1asB3Ndk\nbvOPuUO2BZELL7wQIKIztkLc3bt35+yzzwZwwzebTM/OznZFosPZgti2bduAYNW6tLTUvTl8YO3K\nz893+dDHHHPMLn9u+/btruym1QuwTIx0WciDoMZD5UL0EMo5Ttc3seWi2pvR8qgnTpzoFipry0qo\npstuyy+//LLKLsvRo0enqDXJZVkXECQlJJOmLEREPBFzhGxRnVWygqAIueVYjhgxwkXSL774IoBL\nEwrPWbWSm/3792fp0qVA8CmVyv3l0VjeqZ011q5du1oNzy3yyMnJcX8zO7oqHato2fA9mt05UyzZ\nbKdWZZZLbt5//333719++SXie9nZ2a5Mqo2Ywl133XVAqHZCOrD3Xjo75ZRTgNDuy7q8r5o2bepG\nR/E4kqmuFCGLiHgi5gi5crQAQY1XK9Kdn5/v9v3bkSnhbI7NikSH1yaN9vsLCgqA4GDCVLCFSIuI\nlixZUmPCf3hkDKF7sHnXnj17JrKpCRVeL9fYPJytF/hs+PDhLsq3Of2aNG7cmCuvvBIIdlF27drV\n1UGOFiH7mvJXnfA61vYes4jTd5bCaAvmkydPdpUWbZNPbVVe1EvGhh5FyCIinog5Ql60aBEQWaPA\nIg2rbRDNcccdB8A111zjKvRHm2NbvHhxlWvVzfslS3l5uRsF2PEulmUSzcaNG918qs0Xz5s3jyuu\nuCLBLU28aNXKTj/9dCA9KtQ1aNDAzfGagoIC9/ps2bIlENxTr169XPZQuOpek1lZWUk9cSLe7DlM\nl8yfu+66Cwj6o0WLFrmMrRNOOAGIngVla0KFhYWuponVlrFo214LiRTzK2XUqFFAKA/XROuI7Y9g\n6UPDhg0DgkJBtdWqVSt3AnWqLF++3A1n7MMkGstHnjZtGl9//TUQ7F4cNWpU2rzIo7FhbfhhAXYS\nb+fOnVPSpt1lb97rr78+4mtdrFmzJur1tm3bute6JJ4Fg1Y+dPPmze70aFvcsw/XcDb1VlBQ4I6v\nstz6ZHTExv8QRkQkQ8QcIQ8YMAAI9veHp8wMGTIEgClTprgFvroWKLfdUPap1qdPnxoPQU2G4uJi\nlxoTnjBvu5usroGdJt22bVtX8SsdFrpqww4K2LFjh7tmOwx3tUOxPrKym5UNHDgwyS3JbJYYMGbM\nGCBUGfLBBx8EggSBFStWuMfvv//+ET/fpk0bN+1R0+g3URQhi4h4IuYI2aLg888/Hwi20kKw/TSW\nRQ2bv7HFMx9s2LDBjQguvfRSIFT5yzZ6WHFz+5SePn16UuehkqFydbPmzZszduzYFLUm9Wye8o47\n7gCC17yVDJDksjWrOXPm0KVLFyD6JiYbedtCrS38pUrcln9tgSpddiPFIicnxw17rHQmhHZsQbDS\na4uPPp+SES/jxo2r8wJtfVJ5J6kVqxk0aFAKWhObIUOGuOCiU6dOKW5N7HwK5nZFUxYiIp5I3wTJ\nFOrdu3dMR1DVB0888QQQjAbCq2RlIpu6s6OPqkuDSwe5ubnk5uamuhkZSRGyiIgnFCHLbunbt2/E\n10xnC9h21JONHETqQhGyiIgnFCGLxFGvXr0AWLVqVYpbIulIEbKIiCfUIYuIeCLLKhvV6sFZWVuA\nTYlrTsK1r6ioqHHLnO4xLezyHiEz7jMT7hEy6D7r0iGLiEjiaMpCRMQT6pBFRDyhDllExBPqkEVE\nPKEOWUTEE+qQRUQ8oQ5ZRMQT6pBFRDyhDllExBP/AwL3rM94Bza/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f21403b5050>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_images_separately(data[:6,0].data.cpu().numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWQAAABFCAYAAAB9nJwHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAD6RJREFUeJztnVtsVFUXgL/TltIC/UuhlGstiqJiRQSKKNRLtEYUFRRp\nURPUoJCIF9D4YBQxGKUxPggxSoJIMCGiVSKoCIqgooACCiq0CCIXqxYFBIqCc/kfJmufoZ2ZzrRz\n2bXrexktu6d7zz5nnXXfjt/vR1EURUk9aamegKIoihJABbKiKIolqEBWFEWxBBXIiqIolqACWVEU\nxRJUICuKoliCCmRFURRLUIGsKIpiCSqQFUVRLCEjlsH5+fn+vn37JmgqiWfz5s1/+P3+bpHG6Brt\nJ5o1QttYZ1tYI7SddcYkkPv27cumTZuaP6sU4zjO3qbG6BrtJ5o1QttYZ1tYI7SddarLQlEUxRJU\nICuKoliCCmRFURRLUIGsKIpiCSqQFUVRLCGmLItEMH36dH777TcADh48CMCxY8c4deoUABdffDEA\n999/PwCDBw9OwSxbht/vp76+HoAdO3YAsGfPHrPu4uJiAM4991wAevXqheM4KZiporQtGh7Q4fP5\nzM8yMpIvHlVDVhRFsYSkvwImTpwIwGeffQbAL7/8wr///ht2/DfffAPABx98AMDLL7/M1VdfDUBO\nTk4ip9pi9u4NpB5WVlby4YcfArB//34APB5Po/EDBw4EYObMmYwYMQKAgoKCZEw1LmzduhWAw4cP\nA/DXX38Zi6CmpgaA7OxsAEpKSrj77rtTMEslGr7//nsAjhw5AgSs1u3btwPwww8/AJCVlQW0vr08\nfvw4ABs2bGDLli0AdO3aFQjcsz/++CMQkE0A559/PgA9e/Zk6NChAIwcOTIhc0uaQL799tsBWL16\nNRBYOBBRGAcj5v2cOXOMiW+bQBZT58033wRg4cKFAKxcubKRaRSKbdu2AVBeXm6+lz///BOAzp07\nk5Zmr0Hj8/k4evQoAFdddVXYcYWFhQB07949KfOKN+vWreP5558HYNmyZWHHvfLKKwBUVFTQqVMn\nALN/trujfD6fcbGVlpaGHdda9tLr9QKwZMkSAD7++GMgIJBFYYjE8uXLAcjNzTWyZ/z48QBMnTo1\nrs+lvU+4oihKGyMpGvLatWvx+XwAtG/fHmi+lrB582befvttAB5++OHTrplK/H4/b7zxBgBPP/00\n4JrpsRJsNYgptX//fvr06dPCWSYOj8fD5Zdf3uQ4cdmcccYZiZ5Si6itrQXcPXzqqacA+Pzzz6P6\n/SlTpgBQVVXF3LlzATjnnHMASE9Pj+tc443X62X48OFNjpO9LCoqSvSUmo3X6zVa8IYNGwBYsWIF\n4Frd0XLixAl27doFYFyQq1at4r333ovXdFVDVhRFsYWkaMjdu3fn2muvBdygzr59+wD45JNPYrqW\n1+tl9+7dAPz8888A9O/fP+V+uV27dvHOO+8AzdeMI7FkyRIeeeSRuF83XsRqpUyYMCFBM2k5R48e\nZf369QCMGzeuRdeqrq6mqqoKgDvuuAOAPn360K5du5ZNMoFkZmbGNL6ioiJBM2k5fr+fb7/9FoCv\nv/4acOMysVJYWEiXLl0A9zs666yz4jBLl6QI5Ly8PGN6S5DgwIEDAJSVlZlc4xMnTgCwceNG3n33\nXSBwQwfj8/lMFF+EcCqFsQTrVq9ebR68SEgAoF27diYoIqaf5F6HCgDu3r2b33//HbAziFJfX8/s\n2bMBmDVrVthxr7/+OuBG6G1CvvdNmzaZoE0kxFTPzMw0e3jy5MnTrvXHH38YM1f+zWZhDIHn8Lnn\nngMi7+XixYsBO/dSOHDggMnQElkS7BIU2RH8zN14440ADBkyBIB+/foBgX2T3OTevXsDcMEFF5jf\njYccUpeFoiiKJSRFQ+7WrRujRo0C3BQU0RZOnTpltMZff/0VCJgUeXl5Ya8nQREb0t6+++47AObN\nmxfVeNE4Hn/8cfOzOXPmAPDss88CGE0Y3Lfu0qVLTa6njRpyeno6X331VZPjrr/++iTMpnnI915Z\nWWmC0KGQoK3c04cPH2b69OmAm6MreDweY/n17Nkz7nNOBOnp6SYAFglZv43I/lVXVxsXp6RlBtNQ\nu509ezbnnXceAIMGDQJcN2tubq7RrsVlEW9rRzVkRVEUS0haYYi8SUS7lU+/32/+rUOHDkDgTSRB\nlYacPHnSVLSlsopNqn0kkCeBg3BIL45gzVh44IEHTrvmrFmzGvkijxw5YqoWS0pKWjr9uLNjxw5W\nrlzZ5DgJitiI+ETXrFkTdsyECRMoLy8H3LV4vV5++umnkOM9Ho+JFeTm5sZzugmjpqaGjz76qMlx\nnTt3TsJsmocUtqxfv95YsZGsnrKyMiAQF5CUTLHA5dNxHKNJJyoOoBqyoiiKJSRFQ05LS2uUOSC+\nmOzsbFNGLZHqUP4r8TP369fPdEVLZYK91LsvWLCgybHTpk3jnnvuAdy3dKhyy44dOwLQo0cPkxYo\neDwe6urqWjTnRCJFD60ZyYYIVc4v/QwmT57MsWPHALfPyqJFi/j7779DXtNxHJNF1FqQrILWiMgZ\nycSqrq4OuzfBrFq1Cgjss1iqgsibZHR/S4pAdhynkUA+dOgQEPjipJHJxo0bATfIFYri4mKGDRuW\noJlGjwhkeYlEoqysjAEDBgCuIPb7/eY7kUZD8mJq3769GSdBUMdxou77kQqam9tpAyJgpTovLS2t\nkXkrJvDixYvNf3/xxRcAjV6ewTiOE1XVm01IQ6HWjDxLsVa3vvjiiyZoPnbsWCDwEoZAden//ve/\nOM6yMeqyUBRFsYSkBfWkqk60EDnSu6amxmga4nwPhWjFF110UcrT3Y4cOcJLL70U9fgrrrgiZAK6\nIBqa9EsIRn4vLS0tJQ2zoyWSlmg7YqJKMCczM5N//vnntDFiCS1YsMDsSTQWy8SJE40rqrUgbSdb\nI7I3UrQzaNAgY21GCuoFI9ae9Ka59NJLAairqzNyKFEBTdWQFUVRLCEpKteaNWtMWtiiRYsAtwF2\nqEbtoRDNul+/flH1Fk4k1dXVUSXOC+3atTNv5+Byb1mHNHYPhYzp2LEjF154YXOnnHBuvfXWVE+h\n2Ui65WWXXQYEeh7L/Srff0N/f7SMHDky5t4Qqeamm25K9RRajFjRw4YN46GHHgLcQy6kQMTn8xnr\nKLgbpVhHMm7SpEkAPPPMM3Tr1g1wi0bi3bYhoQJ53bp1QKDB89KlSwG3Gi/WG1u+uPT0dBNUEbMh\n2b0s9u/fb/pOhEI2UNouZmRkmDnKxnu9XhMJFpdNKMTcKisrs/I8QTFvbc4AaQpxKUjw7dFHH2X+\n/PkAfPnllwCn7Xc0CoHs2+DBg1uNy0IqFWNtS2kj8rz179/fuAKnTZsGuHt54sQJdu7cCbjP4Nat\nW03AXmSV1DvU1taaYGE8+1cEoy4LRVEUS0iIhizuCDn6pKqqKqr0sEiIVrxw4ULTE0C0kJycnKQG\n+g4dOhQySCfzEU0reExw+hoEAkJynNWMGTMa/Q25lqy1pKQk4Sk3zSHa5typdjNFQoKlUlE3atQo\nc+zSfffdB7idCDt06GDcbtLoPBTBzfptOEAhGqTpelPYvJeCzNHn85m9lE/BcRzTt6JXr15A4B54\n6623ADeQK8/izp07zb0i8qhTp05x1ZJVQ1YURbGEuGvIu3btMn1H5WRp6X3cEkTD3L59u3mTS5pc\nSUlJUpPvwzWgl7ey+OIknS0nJ8f0ppCqoeXLl3PvvfeG/RtShTh69GgALrnkEqt6B0iam3SoC0es\nsYJUIt95Xl6e6Ykr85f9q6+vN6cvR9KQJQBbVFSU8sMTmkJ8pTNnzow4Tp5Bm5FnUJ69tLS0RkHV\n4FRSWZP4+evq6kxQr2F649GjRxulzqkPWVEU5T9K3DXkffv28emnnwKuZpyRkdHsst/gtxkENBbp\nrCVvq2T3UcjMzAzpR5OfiQ9KtOEBAwaYnqwy99deey3i35C0GulCZUuHN1mjnOgSqSBk0qRJRrOU\n1LLWgtx34jMU7fngwYNs2bIl7O9JJoxoyDZ3eJO9lDiAFG+FYvLkySY7wdYTQvx+v8maECu6ffv2\nJpvp5ptvBtwufcePHzfHOomsWrNmjUlDlXtWrKQuXbqY+yJRcYG4C+SCggKTvycLaYlaLwuXByI/\nP984588880wArrvuumZfvzmMHj2aysrKsP8uOaxNteQMR1ZWlnFnSHDIlsCQNKGXwFYk5s+fb9LH\nWkMgKBLy8t+2bRvvv/9+2HHShEjOWgvVRMoWpDnSq6++2uTYefPmmUMYbN5LeeaWLVsGwJ49e4z8\nESVIXH9+v99UDIdCFCppUN+7d28jpBOVW27v3aIoitLGiLuG3KNHD/P2yM/PBwIHPcaKtLw7++yz\nAbe+/MorrzRayJ133tni+TaHIUOGmPr2cI30m4Oktc2dO9ec0i2VQTbg8XiMOypaF1TDxu2JSqhP\nNE01JJf0xHifQpwoPB4Pa9euBdyAZVPs3bv3tP+3cS8bHu9WW1sb9frCId3fhg8fblyJiUI1ZEVR\nFEuIu4acn5/Pgw8+CLhvqaKiIuM8Fwd7JA1r+fLljbRs8cXl5eWZTk6pIjs72/hQ5WimtWvXRiyn\njsQNN9wAwGOPPQbA0KFDrQyCpaenm7LaaLSOJ598stEhAjZpU9HQsFl5KAYOHGiOhe/fvz/gWni2\nkp6ebgJZDTvbhWLGjBmN/OG27aXjOKaHsaTy1dfXmwKsWH3fYgVXVFQAMG7cuIQfipGQSj2JML/w\nwgtAoHZc3BaSHyiCubCwkK5duwKhHeVyE9i2+eJKkeDBihUrqKqqAjBBn1CNvuU07dtuu40xY8YA\nATcMuMEDW3Ecx5xnKAFbCTZ26dLFmHalpaUAjBgxwrS0bK3IQ3zLLbcAAReMuJZkvQUFBfTo0QNw\nq/1ibYyebBzHobi4GHB7O4hbpmvXrsZVJkHl0tJS69cErsti6tSpQGAfZJ+WLFkCRM6Nz8rKYsqU\nKYCrKF1zzTUJm29D1GWhKIpiCQnt9hacsycmXTR4vd6UnpcXC6Ihjhkzxmi8DVs2+v3+Ruaex+NJ\n2Mm1ieSuu+467TMS0TYEtxnZt/HjxwOBvGux8oSsrKyQp4nbjpzzKJ+RaG17KbJn7Nixxo3xxBNP\nAJhKy7q6OvOM9u3bFwho1Klsc6sasqIoiiVYeSZQa9GOwxHchD4crVE7jhWbiyKiRYqQysvLUzyT\n1PJf2Evp7CafNtL6v2VFUZT/CCqQFUVRLEEFsqIoiiWoQFYURbEEJ5bqFcdxDgJ7mxxoL0V+vz9i\ncwhdY6ugyTVC21hnW1gjtKF12txKT1EUpS2hLgtFURRLUIGsKIpiCSqQFUVRLEEFsqIoiiWoQFYU\nRbEEFciKoiiWoAJZURTFElQgK4qiWIIKZEVRFEv4P2LsVpWGtLqLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f21401841d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_images_separately(reconstructions[:6,0].data.cpu().numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}