src/gensim-doc-similarity.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['你好', '好的', '不错', '笨蛋'],\n",
      " ['笨蛋', '傻瓜', '傻子', '哈哈'],\n",
      " ['好的', '不错', '你好', '哈哈'],\n",
      " ['有趣', '可以', '好的', '不错', '还行'],\n",
      " ['傻瓜', '傻子', '二货', '还行'],\n",
      " ['可以', '好的', '不错', '哈哈'],\n",
      " ['有趣', '有趣', '哈哈', '哈哈']]\n"
     ]
    }
   ],
   "source": [
    "from gensim import corpora\n",
    "from pprint import pprint\n",
    "documents = [\"你好 好的 不错 笨蛋\",\n",
    "             \"笨蛋 傻瓜 傻子 哈哈\",\n",
    "             \"好的 不错 你好 哈哈\",\n",
    "             \"有趣 可以 好的 不错 还行\",\n",
    "             \"傻瓜 傻子 二货 还行\",\n",
    "             \"可以 好的 不错 哈哈\",\n",
    "             \"有趣 有趣 哈哈 哈哈\"]\n",
    "texts = [[word for word in document.split()] for document in documents]\n",
    "pprint(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['你好', '好的', '不错', '笨蛋'],\n",
      " ['笨蛋', '傻瓜', '傻子', '哈哈'],\n",
      " ['好的', '不错', '你好', '哈哈'],\n",
      " ['有趣', '可以', '好的', '不错', '还行'],\n",
      " ['傻瓜', '傻子', '还行'],\n",
      " ['可以', '好的', '不错', '哈哈'],\n",
      " ['有趣', '有趣', '哈哈', '哈哈']]\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "frequency = defaultdict(int)\n",
    "for text in texts:\n",
    "    for token in text:\n",
    "        frequency[token] += 1\n",
    "texts = [[token for token in text if frequency[token] > 1] for text in texts]\n",
    "pprint(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(10 unique tokens: ['不错', '你好', '好的', '笨蛋', '傻子']...)\n"
     ]
    }
   ],
   "source": [
    "dictionary = corpora.Dictionary(texts)\n",
    "dictionary.save('sample.dict')\n",
    "print(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[(0, 1), (1, 1), (2, 1), (3, 1)],\n",
      " [(3, 1), (4, 1), (5, 1), (6, 1)],\n",
      " [(0, 1), (1, 1), (2, 1), (6, 1)],\n",
      " [(0, 1), (2, 1), (7, 1), (8, 1), (9, 1)],\n",
      " [(4, 1), (5, 1), (9, 1)],\n",
      " [(0, 1), (2, 1), (6, 1), (7, 1)],\n",
      " [(6, 2), (8, 2)]]\n"
     ]
    }
   ],
   "source": [
    "corpus = [dictionary.doc2bow(text) for text in texts]\n",
    "corpora.MmCorpus.serialize('sample.mm', corpus)\n",
    "pprint(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.3011997233053068), (5, 0.6742695034927825), (7, 0.6742695034927825)]\n"
     ]
    }
   ],
   "source": [
    "from gensim import models, similarities\n",
    "tf_idf = models.TfidfModel(corpus)\n",
    "vec = [(0, 1), (5, 1), (7, 1)]\n",
    "print(tf_idf[vec])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.08686648 0.37695488 0.10641449 0.43870124 0.38928968 0.63969857\n",
      " 0.        ]\n"
     ]
    }
   ],
   "source": [
    "index = similarities.SparseMatrixSimilarity(tf_idf[corpus], num_features=10)\n",
    "sims = index[tf_idf[vec]]\n",
    "print(sims)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1.         0.3609379  0.71441036 0.13975498 0.         0.20378576\n",
      "  0.        ]\n",
      " [0.3609379  1.0000001  0.08823138 0.         0.64554304 0.08823138\n",
      "  0.10185669]\n",
      " [0.71441036 0.08823138 1.0000001  0.17120475 0.         0.37446705\n",
      "  0.1440983 ]\n",
      " [0.13975498 0.         0.17120475 1.         0.31315398 0.60019135\n",
      "  0.4952337 ]\n",
      " [0.         0.64554304 0.         0.31315398 0.99999994 0.\n",
      "  0.        ]\n",
      " [0.20378576 0.08823138 0.37446705 0.60019135 0.         1.\n",
      "  0.1440983 ]\n",
      " [0.         0.10185669 0.1440983  0.4952337  0.         0.1440983\n",
      "  0.99999994]]\n"
     ]
    }
   ],
   "source": [
    "print(index[tf_idf[corpus]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}