lstm/Word embedding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "'''This script loads pre-trained word embeddings (GloVe embeddings)\n",
    "into a frozen Keras Embedding layer, and uses it to\n",
    "train a text classification model on the 20 Newsgroup dataset\n",
    "(classication of newsgroup messages into 20 different categories).\n",
    "GloVe embedding data can be found at:\n",
    "http://nlp.stanford.edu/data/glove.6B.zip\n",
    "(source page: http://nlp.stanford.edu/projects/glove/)\n",
    "20 Newsgroup data can be found at:\n",
    "http://www.cs.cmu.edu/afs/cs.cmu.edu/project/theo-20/www/data/news20.html\n",
    "'''\n",
    "\n",
    "from __future__ import print_function\n",
    "import os\n",
    "import numpy as np\n",
    "np.random.seed(1337)\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.layers import Dense, Input, Flatten\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation, Embedding\n",
    "from keras.layers import Conv1D, MaxPooling1D, Embedding\n",
    "from keras.models import Model\n",
    "from keras.layers import LSTM, SimpleRNN, GRU\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing word vectors.\n",
      "Found 400000 word vectors.\n",
      "Processing text dataset\n",
      "Found 19997 texts.\n"
     ]
    }
   ],
   "source": [
    "BASE_DIR = '/home/lich/Workspace/Learning'\n",
    "GLOVE_DIR = BASE_DIR + '/glove.6B/'\n",
    "TEXT_DATA_DIR = BASE_DIR + '/20_newsgroup/'\n",
    "MAX_SEQUENCE_LENGTH = 1000\n",
    "MAX_NB_WORDS = 20000\n",
    "EMBEDDING_DIM = 100\n",
    "VALIDATION_SPLIT = 0.2\n",
    "batch_size = 32\n",
    "\n",
    "# first, build index mapping words in the embeddings set\n",
    "# to their embedding vector\n",
    "\n",
    "print('Indexing word vectors.')\n",
    "\n",
    "embeddings_index = {}\n",
    "f = open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt'))\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "\n",
    "print('Found %s word vectors.' % len(embeddings_index))\n",
    "\n",
    "# second, prepare text samples and their labels\n",
    "print('Processing text dataset')\n",
    "\n",
    "texts = []  # list of text samples\n",
    "labels_index = {}  # dictionary mapping label name to numeric id\n",
    "labels = []  # list of label ids\n",
    "for name in sorted(os.listdir(TEXT_DATA_DIR)):\n",
    "    path = os.path.join(TEXT_DATA_DIR, name)\n",
    "    if os.path.isdir(path):\n",
    "        label_id = len(labels_index)\n",
    "        labels_index[name] = label_id\n",
    "        for fname in sorted(os.listdir(path)):\n",
    "            if fname.isdigit():\n",
    "                fpath = os.path.join(path, fname)\n",
    "                if sys.version_info < (3,):\n",
    "                    f = open(fpath)\n",
    "                else:\n",
    "                    f = open(fpath, encoding='latin-1')\n",
    "                texts.append(f.read())\n",
    "                f.close()\n",
    "                labels.append(label_id)\n",
    "\n",
    "print('Found %s texts.' % len(texts))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 214909 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "# finally, vectorize the text samples into a 2D integer tensor\n",
    "tokenizer = Tokenizer(nb_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(texts)\n",
    "sequences = tokenizer.texts_to_sequences(texts)\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1654\n"
     ]
    }
   ],
   "source": [
    "print(len(sequences[0]))\n",
    "# print(texts[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(word_index[\"cantaloupe\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of data tensor: (19997, 1000)\n",
      "Shape of label tensor: (19997, 20)\n",
      "Preparing embedding matrix.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)\n",
    "\n",
    "labels = to_categorical(np.asarray(labels))\n",
    "print('Shape of data tensor:', data.shape)\n",
    "print('Shape of label tensor:', labels.shape)\n",
    "\n",
    "# split the data into a training set and a validation set\n",
    "indices = np.arange(data.shape[0])\n",
    "np.random.shuffle(indices)\n",
    "data = data[indices]\n",
    "labels = labels[indices]\n",
    "nb_validation_samples = int(VALIDATION_SPLIT * data.shape[0])\n",
    "\n",
    "x_train = data[:-nb_validation_samples]\n",
    "y_train = labels[:-nb_validation_samples]\n",
    "x_val = data[-nb_validation_samples:]\n",
    "y_val = labels[-nb_validation_samples:]\n",
    "\n",
    "print('Preparing embedding matrix.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3999\n"
     ]
    }
   ],
   "source": [
    "print(nb_validation_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20001, 100)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "# prepare embedding matrix\n",
    "nb_words = min(MAX_NB_WORDS, len(word_index))\n",
    "embedding_matrix = np.zeros((nb_words + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index.items():\n",
    "    if i > MAX_NB_WORDS:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "\n",
    "print(embedding_matrix.shape)\n",
    "                \n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model.\n",
      "(15998, 1000)\n"
     ]
    }
   ],
   "source": [
    "# load pre-trained word embeddings into an Embedding layer\n",
    "# note that we set trainable = False so as to keep the embeddings fixed\n",
    "embedding_layer = Embedding(nb_words + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False)\n",
    "\n",
    "print('Training model.')\n",
    "\n",
    "# train a 1D convnet with global maxpooling\n",
    "sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "x = Conv1D(128, 5, activation='relu')(embedded_sequences)\n",
    "x = MaxPooling1D(5)(x)\n",
    "x = Conv1D(128, 5, activation='relu')(x)\n",
    "x = MaxPooling1D(5)(x)\n",
    "x = Conv1D(128, 5, activation='relu')(x)\n",
    "x = MaxPooling1D(35)(x)\n",
    "x = Flatten()(x)\n",
    "x = Dense(128, activation='relu')(x)\n",
    "\n",
    "preds = Dense(len(labels_index), activation='softmax')(x)\n",
    "\n",
    "model = Model(sequence_input, preds)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='rmsprop',\n",
    "              metrics=['acc'])\n",
    "print(x_train.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20000 1000\n",
      "(15998, 1000) (15998, 20) (3999, 1000) (3999, 20)\n",
      "[ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.\n",
      "  0.  0.]\n"
     ]
    }
   ],
   "source": [
    "print(nb_words, MAX_SEQUENCE_LENGTH)\n",
    "print(x_train.shape, y_train.shape, x_val.shape, y_val.shape)\n",
    "print(y_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Build model...\n",
      "Train...\n",
      "Train on 15998 samples, validate on 3999 samples\n",
      "Epoch 1/5\n",
      "15998/15998 [==============================] - 881s - loss: 0.1980 - acc: 0.9500 - val_loss: 0.1962 - val_acc: 0.9500\n",
      "Epoch 2/5\n",
      "15998/15998 [==============================] - 866s - loss: 0.1967 - acc: 0.9500 - val_loss: 0.1951 - val_acc: 0.9500\n",
      "Epoch 3/5\n",
      "15998/15998 [==============================] - 831s - loss: 0.1953 - acc: 0.9500 - val_loss: 0.1938 - val_acc: 0.9500\n",
      "Epoch 4/5\n",
      "15998/15998 [==============================] - 813s - loss: 0.1941 - acc: 0.9500 - val_loss: 0.1925 - val_acc: 0.9500\n",
      "Epoch 5/5\n",
      "15998/15998 [==============================] - 812s - loss: 0.1939 - acc: 0.9500 - val_loss: 0.1922 - val_acc: 0.9500\n",
      "3999/3999 [==============================] - 63s    \n",
      "Test score: 0.192154919208\n",
      "Test accuracy: 0.949999988079\n"
     ]
    }
   ],
   "source": [
    "embedding_layer = Embedding(nb_words + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH,\n",
    "                            trainable=False,\n",
    "                            dropout=0.2)\n",
    "batch_size = 32\n",
    "\n",
    "print('Build model...')\n",
    "# sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "# embedded_sequences = embedding_layer()\n",
    "model = Sequential()\n",
    "model.add(embedding_layer)\n",
    "model.add(LSTM(100, dropout_W=0.2, dropout_U=0.2))  # try using a GRU instead, for fun\n",
    "model.add(Dense(1))\n",
    "model.add(Activation('sigmoid'))\n",
    "model.add(Dense(len(labels_index), activation='softmax'))\n",
    "\n",
    "# try using different optimizers and different optimizer configs\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "print('Train...')\n",
    "model.fit(x_train, y_train, batch_size=batch_size, nb_epoch=5,\n",
    "          validation_data=(x_val, y_val))\n",
    "score, acc = model.evaluate(x_val, y_val,\n",
    "                            batch_size=batch_size)\n",
    "print('Test score:', score)\n",
    "print('Test accuracy:', acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 15998 samples, validate on 3999 samples\n",
      "Epoch 1/2\n",
      "15998/15998 [==============================] - 368s - loss: 1.4332 - acc: 0.5437 - val_loss: 0.1743 - val_acc: 0.9417\n",
      "Epoch 2/2\n",
      "15998/15998 [==============================] - 378s - loss: 0.1929 - acc: 0.9364 - val_loss: 0.1477 - val_acc: 0.9487\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fdaae4a28d0>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# happy learning!\n",
    "model.fit(x_train, y_train, validation_data=(x_val, y_val),\n",
    "          nb_epoch=2, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}