Deep-Learning-for-NLP-CS224d/skipgram.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def softmax(x):\n",
    "\t\t\t\t\t\t\t\t\t\t\t#x=np.array([[1001,1002],[3,4]])\n",
    "\tif len(x.shape) > 1:\t\t\t\t\t#x.shape=(2, 2)  len(x.shape)=2\t\t\t\t\n",
    "\t\ttmp = np.max(x, axis = 1)\t\t\t#np.max(x, axis = 1)=array([1002,  4])， max in each row\n",
    "\t\tx -= tmp.reshape((x.shape[0], 1))\t#tmp.reshape((x.shape[0], 1))， tmp becomes 2row1column\n",
    "\t\tx = np.exp(x)\t\t\t\t\t\t#xi - max this row, then exp\n",
    "\t\ttmp = np.sum(x, axis = 1)\t\t\t#array([ 1.36787944,  1.36787944])，sum of each row\n",
    "\t\tx /= tmp.reshape((x.shape[0], 1))\t#xi / sum this row\n",
    "\t\n",
    "\telse:\t\t\t\t\t\t\t\t\t#x=[1,2]   x.shape=(2,)   len(x.shape)=1\n",
    "\t\ttmp = np.max(x)\n",
    "\t\tx -= tmp\n",
    "\t\tx = np.exp(x)\n",
    "\t\ttmp = np.sum(x)\n",
    "\t\tx /= tmp\n",
    "\t\n",
    "\treturn x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gradcheck"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Function: \n",
    "# for each element in x\n",
    "# compare derivative calculated by formular and calculus\n",
    "# f: 1st parameter is cost function, 2nd parameter is gradient\n",
    "def gradcheck_naive(f, x):\n",
    "\t\n",
    "\t#Return an object capturing the current internal state of the generator\n",
    "\trndstate = random.getstate()\t\t\t#why use state??????\n",
    "\trandom.setstate(rndstate)\n",
    "\tfx, grad = f(x)\t\t\t\t\t\t\t#fx=np.sum(x ** 2), grad=x * 2 \n",
    "\th = 1e-4\n",
    "\t\n",
    "\t#Efficient multi-dimensional iterator object to iterate over arrays\n",
    "\t# Iterate over all indexes in x\n",
    "\tit = np.nditer(x, flags=['multi_index'], op_flags=['readwrite'])\t\n",
    "\t\n",
    "\twhile not it.finished:\n",
    "\t\tix = it.multi_index\t\t\t\t\t#starts from (0, 0) then (0, 1)\n",
    "\t\t\n",
    "\t\tx[ix] += h\t\t\t\t\t\t\t#To calculate [f(xi+h)-f(xi-h)] / 2h\n",
    "\t\trandom.setstate(rndstate)\n",
    "\t\tfxh, _ = f(x)\n",
    "\t\tx[ix] -= 2*h\n",
    "\t\trandom.setstate(rndstate)\n",
    "\t\tfxnh, _ = f(x)\n",
    "\t\tx[ix] += h\n",
    "\t\tnumgrad = (fxh - fxnh) / 2 / h\n",
    "\t\t\t\t\t\t\t\t\t\t\t#To compare gradient calculated by formular and calculus\n",
    "\t\treldiff = abs(numgrad - grad[ix]) / max(1, abs(numgrad), abs(grad[ix]))\n",
    "\t\tif reldiff > 1e-5:\n",
    "\t\t\tprint \"Gradient check failed.\"\n",
    "\t\t\tprint \"First gradient error found at index %s\" % str(ix)\n",
    "\t\t\tprint \"Your gradient: %f \\t Numerical gradient: %f\" % (grad[ix], numgrad)\n",
    "\t\t\treturn\n",
    "\t\t\n",
    "\t\tit.iternext()\n",
    "\t\t\n",
    "\tprint \"Gradient check passed\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def normalizeRows(x):\n",
    "\n",
    "\tN = x.shape[0]\n",
    "\tx /= np.sqrt(np.sum(x**2, axis=1)).reshape((N,1)) + 1e-30\n",
    "\t\n",
    "\treturn x\n",
    "\n",
    "\n",
    "def test_normalize_rows():\n",
    "\tprint \"Testing normalizeRows...\"\n",
    "\tx = normalizeRows(np.array([[3.0, 4.0],[1, 2]]))\n",
    "\tprint x\n",
    "\tassert (np.amax(np.fabs(x - np.array([[0.6,0.8],[0.4472136,0.89442719]]))) <= 1e-6)\n",
    "\tprint \"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### \"\"\" Softmax cost function for word2vec models \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def softmaxCostAndGradient(predicted, target, outputVectors, dataset):\n",
    "\t\"\"\" Softmax cost function for word2vec models \"\"\"\n",
    "\t\n",
    "\tprobabilities = softmax(predicted.dot(outputVectors.T))\t\t\t\n",
    "\tcost = -np.log(probabilities[target])\n",
    "\t\n",
    "\tdelta = probabilities\n",
    "\tdelta[target] -= 1\n",
    "\t\n",
    "\tN = delta.shape[0]\t\t\t\t\t\t\t\t\t\t\t\t#delta.shape = (5,)\n",
    "\tD = predicted.shape[0]\t\t\t\t\t\t\t\t\t\t\t#predicted.shape = (3,)\n",
    "\tgrad = delta.reshape((N, 1)) * predicted.reshape((1, D))\n",
    "\tgradPred = (delta.reshape((1, N)).dot(outputVectors)).flatten()\n",
    "\t\n",
    "\treturn cost, gradPred, grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### \"\"\" Skip-gram model in word2vec \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def skipgram(currentWord, C, contextWords, tokens, inputVectors, outputVectors,\n",
    "\tdataset, word2vecCostAndGradient = softmaxCostAndGradient):\n",
    "\t\"\"\" Skip-gram model in word2vec \"\"\"\n",
    "\t\n",
    "\tcurrentI = tokens[currentWord]\t\t\t\t\t\t#the order of this center word in the whole vocabulary\n",
    "\tpredicted = inputVectors[currentI, :]\t\t\t\t#turn this word to vector representation\n",
    "\t\n",
    "\tcost = 0.0\n",
    "\tgradIn = np.zeros(inputVectors.shape)\n",
    "\tgradOut = np.zeros(outputVectors.shape)\n",
    "\tfor cwd in contextWords:\t\t\t\t\t\t\t#contextWords is of 2C length\n",
    "\t\tidx = tokens[cwd]\n",
    "\t\tcc, gp, gg = word2vecCostAndGradient(predicted, idx, outputVectors, dataset)\n",
    "\t\tcost += cc\t\t\t\t\t\t\t\t\t\t#final cost/gradient is the 'sum' of result calculated by each word in context\n",
    "\t\tgradOut += gg\n",
    "\t\tgradIn[currentI, :] += gp\n",
    "\t\n",
    "\treturn cost, gradIn, gradOut"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  word2vec_sgd_wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def word2vec_sgd_wrapper(word2vecModel, tokens, wordVectors, dataset, C, word2vecCostAndGradient = softmaxCostAndGradient):\n",
    "\tbatchsize = 50\n",
    "\tcost = 0.0\n",
    "\tgrad = np.zeros(wordVectors.shape)   #each element in wordVectors has a gradient\n",
    "\tN = wordVectors.shape[0]\n",
    "\tinputVectors = wordVectors[:N/2, :]\n",
    "\toutputVectors = wordVectors[N/2:, :]\n",
    "\tfor i in xrange(batchsize):\t\t\t\t\t\t\t\t\t#train word2vecModel for 50 times\n",
    "\t\tC1 = random.randint(1, C)\n",
    "\t\tcenterword, context = dataset.getRandomContext(C1)\t\t#randomly choose 1 word, and generate a context of it\n",
    "\t\t\n",
    "\t\tif word2vecModel == skipgram:\n",
    "\t\t\tdenom = 1\n",
    "\t\telse:\n",
    "\t\t\tdenom = 1\n",
    "\t\t\n",
    "\t\tc, gin, gout = word2vecModel(centerword, C1, context, tokens, inputVectors, outputVectors, dataset, word2vecCostAndGradient)\n",
    "\t\tcost += c / batchsize / denom\t\t\t\t\t\t\t#calculate the average\n",
    "\t\tgrad[:N/2, :] += gin / batchsize / denom\n",
    "\t\tgrad[N/2:, :] += gout / batchsize / denom\n",
    "\t\n",
    "\treturn cost, grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## test_word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==== Gradient check for skip-gram ====\n",
      "Gradient check passed\n",
      "\n",
      "=== Results ===\n",
      "(11.166109001533981, array([[ 0.        ,  0.        ,  0.        ],\n",
      "       [ 0.        ,  0.        ,  0.        ],\n",
      "       [-1.26947339, -1.36873189,  2.45158957],\n",
      "       [ 0.        ,  0.        ,  0.        ],\n",
      "       [ 0.        ,  0.        ,  0.        ]]), array([[-0.41045956,  0.18834851,  1.43272264],\n",
      "       [ 0.38202831, -0.17530219, -1.33348241],\n",
      "       [ 0.07009355, -0.03216399, -0.24466386],\n",
      "       [ 0.09472154, -0.04346509, -0.33062865],\n",
      "       [-0.13638384,  0.06258276,  0.47605228]]))\n"
     ]
    }
   ],
   "source": [
    "def test_word2vec():\n",
    "    # Interface to the dataset for negative sampling\n",
    "    dataset = type('dummy', (), {})()\n",
    "    def dummySampleTokenIdx():\n",
    "        return random.randint(0, 4)\n",
    "\n",
    "    def getRandomContext(C):\n",
    "        tokens = [\"a\", \"b\", \"c\", \"d\", \"e\"]\n",
    "        return tokens[random.randint(0,4)], [tokens[random.randint(0,4)] \\\n",
    "           for i in xrange(2*C)]\n",
    "    dataset.sampleTokenIdx = dummySampleTokenIdx\n",
    "    dataset.getRandomContext = getRandomContext\n",
    "\n",
    "    random.seed(31415)\n",
    "    np.random.seed(9265)\n",
    "    dummy_vectors = normalizeRows(np.random.randn(10,3))\n",
    "    dummy_tokens = dict([(\"a\",0), (\"b\",1), (\"c\",2),(\"d\",3),(\"e\",4)])\n",
    "\n",
    "    print \"==== Gradient check for skip-gram ====\"\n",
    "    gradcheck_naive(lambda vec: word2vec_sgd_wrapper(skipgram, dummy_tokens, vec, dataset, 5), dummy_vectors)\n",
    "\n",
    "    print \"\\n=== Results ===\"\n",
    "    print skipgram(\"c\", 3, [\"a\", \"b\", \"e\", \"d\", \"b\", \"c\"], dummy_tokens, dummy_vectors[:5,:], dummy_vectors[5:,:], dataset)\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    test_word2vec()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}