{ "cells": [ { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "## =======================================================\n", "## IMPORTING\n", "## =======================================================\n", "import os\n", "def get_data_from_files(path):\n", " directory = os.listdir(path)\n", " results = []\n", " for file in directory:\n", " f=open(path+file, encoding = \"ISO-8859-1\")\n", " results.append(f.read())\n", " f.close()\n", " return results\n", "\n", "\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from nltk.tokenize.casual import casual_tokenize\n", "tfidf_model = TfidfVectorizer(tokenizer = casual_tokenize)\n" ] }, { "cell_type": "code", "execution_count": 31, "metadata": {}, "outputs": [], "source": [ "# HW2\n", "# neg = get_data_from_files('../neg_cornell/')\n", "# pos = get_data_from_files('../pos_cornell/')\n", "\n", "neg = get_data_from_files('../neg_hw4/')\n", "pos = get_data_from_files('../pos_hw4/')\n", "\n", "import pandas as pd\n", "import numpy as np\n", "neg_df = pd.DataFrame(neg)\n", "pos_df = pd.DataFrame(pos)\n", "pos_df['PoN'] = 1\n", "neg_df['PoN'] = 0\n", "all_df = neg_df.append(pos_df)\n", "all_df\n", "all_df.rename(columns={0:'text'}, inplace=True)\n", "\n", "# # kaggle\n", "all_df = pd.read_csv('kaggle_pos1_neg0.csv')\n", "# all_df['0']" ] }, { "cell_type": "code", "execution_count": 32, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "(76478, 16053)\n" ] } ], "source": [ "tfidf_docs = tfidf_model.fit_transform(raw_documents = all_df['text']).toarray()\n", "print(tfidf_docs.shape)\n", "# sms.spam.sum()" ] }, { "cell_type": "code", "execution_count": 33, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([False, True, True, ..., False, False, True])" ] }, "execution_count": 33, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mask = all_df.PoN.astype(bool).values\n", "mask" ] }, { "cell_type": "code", "execution_count": 34, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([8.61273538e-04, 5.52010603e-05, 1.16105499e-04, ...,\n", " 1.99336919e-05, 1.27842975e-05, 2.65665717e-05])" ] }, "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ "n_centroid = tfidf_docs[mask].mean(axis=0)\n", "n_centroid" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([2.94149033e-04, 2.05401564e-05, 2.34525098e-04, ...,\n", " 0.00000000e+00, 0.00000000e+00, 0.00000000e+00])" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "p_centroid = tfidf_docs[~mask].mean(axis=0)\n", "p_centroid" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([ 0.01, 0.01, 0.01, ..., 0.01, -0. , -0. ])" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "negativity_score = tfidf_docs.dot(n_centroid - p_centroid)\n", "ns = negativity_score.round(2)\n", "ns\n" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [], "source": [ "from sklearn.preprocessing import MinMaxScaler\n", "all_df['lda_score'] = MinMaxScaler().fit_transform(negativity_score.reshape(-1,1))\n", "all_df['lda_predict'] = (all_df.lda_score > .5).astype(int)" ] }, { "cell_type": "code", "execution_count": 38, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
textPoNlda_scorelda_predict
0A series of escapades demonstrating the adage ...00.9451841
1good for the goose10.9223231
2good10.9289871
3the gander , some of which occasionally amuses...00.9281671
4amuses10.8996111
...............
76473quietly suggesting the sadness and obsession b...00.9139491
76474sadness and obsession00.9148231
76475sadness and00.9220261
76476forced avuncular chortles00.8986061
76477avuncular chortles10.8995151
\n", "

76478 rows × 4 columns

\n", "
" ], "text/plain": [ " text PoN lda_score \\\n", "0 A series of escapades demonstrating the adage ... 0 0.945184 \n", "1 good for the goose 1 0.922323 \n", "2 good 1 0.928987 \n", "3 the gander , some of which occasionally amuses... 0 0.928167 \n", "4 amuses 1 0.899611 \n", "... ... ... ... \n", "76473 quietly suggesting the sadness and obsession b... 0 0.913949 \n", "76474 sadness and obsession 0 0.914823 \n", "76475 sadness and 0 0.922026 \n", "76476 forced avuncular chortles 0 0.898606 \n", "76477 avuncular chortles 1 0.899515 \n", "\n", " lda_predict \n", "0 1 \n", "1 1 \n", "2 1 \n", "3 1 \n", "4 1 \n", "... ... \n", "76473 1 \n", "76474 1 \n", "76475 1 \n", "76476 1 \n", "76477 1 \n", "\n", "[76478 rows x 4 columns]" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "all_df" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [], "source": [ "# from pugnlp.stats import Confusion\n", "# Confusion(all_df['PoN lda_predict'.split()])" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.643" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "(1. - (all_df.PoN - all_df.lda_predict).abs().sum() / len(all_df)).round(3)\n" ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [], "source": [ "from sklearn.metrics import confusion_matrix\n", "cm=confusion_matrix(all_df['PoN'], all_df['lda_predict'])" ] }, { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 7072, 27273],\n", " [ 0, 42133]])" ] }, "execution_count": 42, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cm" ] }, { "cell_type": "code", "execution_count": 43, "metadata": {}, "outputs": [], "source": [ "cd = pd.read_csv('cats_and_dogs_sorted.csv')\n", "\n", "VOCABULARY = vocabulary='cat dog apple lion NYC love'.lower().split()\n", "DOCS = cd['0'].values\n", "def docs_to_tdm(docs=DOCS, vocabulary=VOCABULARY, verbosity=0):\n", " tfidfer = TfidfVectorizer(min_df=1, max_df=.99, stop_words=None, token_pattern=r'(?u)\\b\\w+\\b',\n", " vocabulary=vocabulary)\n", " tfidf_dense = pd.DataFrame(tfidfer.fit_transform(docs).todense())\n", " id_words = [(i, w) for (w, i) in tfidfer.vocabulary_.items()]\n", " tfidf_dense.columns = list(zip(*sorted(id_words)))[1]\n", "\n", " tfidfer.use_idf = False\n", " tfidfer.norm = None\n", " bow_dense = pd.DataFrame(tfidfer.fit_transform(docs).todense())\n", " bow_dense.columns = list(zip(*sorted(id_words)))[1]\n", " bow_dense = bow_dense.astype(int)\n", " tfidfer.use_idf = True\n", " tfidfer.norm = 'l2'\n", " if verbosity:\n", " print(tfidf_dense.T)\n", " return bow_dense.T, tfidf_dense.T, tfidfer\n", "\n" ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0123456789...253254255256257258259260261262
cat0000001110...0000000000
dog0000000000...0000000000
apple1101110000...0000000000
lion0000000100...0000000000
nyc1111100001...0000000000
love0010000011...0000000000
\n", "

6 rows × 263 columns

\n", "
" ], "text/plain": [ " 0 1 2 3 4 5 6 7 8 9 ... 253 254 255 \\\n", "cat 0 0 0 0 0 0 1 1 1 0 ... 0 0 0 \n", "dog 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 \n", "apple 1 1 0 1 1 1 0 0 0 0 ... 0 0 0 \n", "lion 0 0 0 0 0 0 0 1 0 0 ... 0 0 0 \n", "nyc 1 1 1 1 1 0 0 0 0 1 ... 0 0 0 \n", "love 0 0 1 0 0 0 0 0 1 1 ... 0 0 0 \n", "\n", " 256 257 258 259 260 261 262 \n", "cat 0 0 0 0 0 0 0 \n", "dog 0 0 0 0 0 0 0 \n", "apple 0 0 0 0 0 0 0 \n", "lion 0 0 0 0 0 0 0 \n", "nyc 0 0 0 0 0 0 0 \n", "love 0 0 0 0 0 0 0 \n", "\n", "[6 rows x 263 columns]" ] }, "execution_count": 44, "metadata": {}, "output_type": "execute_result" } ], "source": [ "a, b, c = docs_to_tdm()\n", "a\n" ] }, { "cell_type": "code", "execution_count": 45, "metadata": {}, "outputs": [], "source": [ "def lsa_models(vocabulary='cat dog apple lion NYC love'.lower().split(), docs=11, verbosity=0):\n", " # vocabulary = 'cat dog apple lion NYC love big small bright'.lower().split()\n", " if isinstance(docs, int):\n", " docs = DOCS\n", " tdm, tfidfdm, tfidfer = docs_to_tdm(docs=docs, vocabulary=vocabulary)\n", " lsa_bow_model = lsa(tdm) # (tdm - tdm.mean(axis=1)) # SVD fails to converge if you center, like PCA does\n", " lsa_bow_model['vocabulary'] = tdm.index.values\n", " lsa_bow_model['docs'] = docs\n", " err = accuracy_study(verbosity=verbosity, **lsa_bow_model)\n", " lsa_bow_model['err'] = err\n", " lsa_bow_model['accuracy'] = list(1. - np.array(err))\n", " \n", " lsa_tfidf_model = lsa(tdm=tfidfdm)\n", " lsa_bow_model['vocabulary'] = tfidfdm.index.values\n", " lsa_tfidf_model['docs'] = docs\n", " err = accuracy_study(verbosity=verbosity, **lsa_tfidf_model)\n", " lsa_tfidf_model['err'] = err\n", " lsa_tfidf_model['accuracy'] = list(1. - np.array(err))\n", "\n", " return lsa_bow_model, lsa_tfidf_model" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [], "source": [ "def lsa(tdm, verbosity=0):\n", " if verbosity:\n", " print(tdm)\n", " # 0 1 2 3 4 5 6 7 8 9 10\n", " # cat 0 0 0 0 0 0 1 1 1 0 1\n", " # dog 0 0 0 0 0 0 0 0 0 0 1\n", " # apple 1 1 0 1 1 1 0 0 0 0 0\n", " # lion 0 0 0 0 0 0 0 1 0 0 0\n", " # love 0 0 1 0 0 0 0 0 1 1 0\n", " # nyc 1 1 1 1 1 0 0 0 0 1 0\n", "\n", " u, s, vt = np.linalg.svd(tdm)\n", "\n", " u = pd.DataFrame(u, index=tdm.index)\n", " if verbosity:\n", " print('U')\n", " print(u.round(2))\n", " # U\n", " # 0 1 2 3 4 5\n", " # cat -0.04 0.83 -0.38 -0.00 0.11 0.38\n", " # dog -0.00 0.21 -0.18 -0.71 -0.39 -0.52\n", " # apple -0.62 -0.21 -0.51 0.00 0.49 -0.27\n", " # lion -0.00 0.21 -0.18 0.71 -0.39 -0.52\n", " # love -0.22 0.42 0.69 0.00 0.41 -0.37\n", " # nyc -0.75 0.00 0.24 -0.00 -0.52 0.32\n", "\n", " vt = pd.DataFrame(vt, index=['d{}'.format(i) for i in range(len(vt))])\n", " if verbosity:\n", " print('VT')\n", " print(vt.round(2))\n", " # VT\n", " # 0 1 2 3 4 5 6 7 8 9 10\n", " # d0 -0.44 -0.44 -0.31 -0.44 -0.44 -0.20 -0.01 -0.01 -0.08 -0.31 -0.01\n", " # d1 -0.09 -0.09 0.19 -0.09 -0.09 -0.09 0.37 0.47 0.56 0.19 0.47\n", " # d2 -0.16 -0.16 0.52 -0.16 -0.16 -0.29 -0.22 -0.32 0.17 0.52 -0.32\n", " # d3 0.00 -0.00 0.00 0.00 0.00 0.00 -0.00 0.71 0.00 0.00 -0.71\n", " # d4 -0.04 -0.04 -0.14 -0.04 -0.04 0.58 0.13 -0.33 0.62 -0.14 -0.33\n", " # d5 0.09 0.09 -0.10 0.09 0.09 -0.51 0.73 -0.27 0.01 -0.10 -0.27\n", " # d6 -0.55 0.24 0.15 0.36 -0.38 0.32 0.32 0.00 -0.32 0.17 0.00\n", " # d7 -0.32 0.46 0.23 -0.64 0.41 0.09 0.09 0.00 -0.09 -0.14 0.00\n", " # d8 -0.52 0.27 -0.24 0.39 0.22 -0.36 -0.36 -0.00 0.36 -0.12 0.00\n", " # d9 -0.14 -0.14 -0.58 -0.14 0.32 0.10 0.10 -0.00 -0.10 0.68 -0.00\n", " # d10 -0.27 -0.63 0.31 0.23 0.55 0.12 0.12 -0.00 -0.12 -0.19 -0.00\n", "\n", " # Reconstruct the original term-document matrix.\n", " # The sum of the squares of the error is 0.\n", "\n", " return {'u': u, 's': s, 'vt': vt, 'tdm': tdm}" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [], "source": [ "def accuracy_study(tdm=None, u=None, s=None, vt=None, verbosity=0, **kwargs):\n", " \"\"\" Reconstruct the term-document matrix and measure error as SVD terms are truncated\n", " \"\"\"\n", " smat = np.zeros((len(u), len(vt)))\n", " np.fill_diagonal(smat, s)\n", " smat = pd.DataFrame(smat, columns=vt.index, index=u.index)\n", " if verbosity:\n", " print()\n", " print('Sigma:')\n", " print(smat.round(2))\n", " print()\n", " print('Sigma without zeroing any dim:')\n", " print(np.diag(smat.round(2)))\n", " tdm_prime = u.values.dot(smat.values).dot(vt.values)\n", " if verbosity:\n", " print()\n", " print('Reconstructed Term-Document Matrix')\n", " print(tdm_prime.round(2))\n", "\n", " err = [np.sqrt(((tdm_prime - tdm).values.flatten() ** 2).sum() / np.product(tdm.shape))]\n", " if verbosity:\n", " print()\n", " print('Error without reducing dimensions:')\n", " print(err[-1])\n", " # 2.3481474529927113e-15\n", "\n", " smat2 = smat.copy()\n", " for numdim in range(len(s) - 1, 0, -1):\n", " smat2.iloc[numdim, numdim] = 0\n", " if verbosity:\n", " print('Sigma after zeroing out dim {}'.format(numdim))\n", " print(np.diag(smat2.round(2)))\n", " # d0 d1 d2 d3 d4 d5\n", " # ship 2.16 0.00 0.0 0.0 0.0 0.0\n", " # boat 0.00 1.59 0.0 0.0 0.0 0.0\n", " # ocean 0.00 0.00 0.0 0.0 0.0 0.0\n", " # voyage 0.00 0.00 0.0 0.0 0.0 0.0\n", " # trip 0.00 0.00 0.0 0.0 0.0 0.0\n", "\n", " tdm_prime2 = u.values.dot(smat2.values).dot(vt.values)\n", " err += [np.sqrt(((tdm_prime2 - tdm).values.flatten() ** 2).sum() / np.product(tdm.shape))]\n", " if verbosity:\n", " print('Error after zeroing out dim {}'.format(numdim))\n", " print(err[-1])\n", " return err" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [], "source": [ "lsa_bow, lsa_tdif_bow = lsa_models()" ] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'u': 0 1 2 3 4 5\n", " cat -0.963953 0.256998 0.039468 0.022509 0.051557 -0.004949\n", " dog -0.255938 -0.965615 0.043947 -0.007358 -0.003217 -0.009401\n", " apple -0.004551 -0.004413 -0.294570 0.179873 -0.006208 -0.938507\n", " lion -0.048551 0.016315 0.006249 0.012370 -0.998566 0.007174\n", " nyc -0.045182 -0.034756 -0.930156 0.163649 0.000160 0.323695\n", " love -0.029525 0.006817 -0.210973 -0.969614 -0.012643 -0.119421,\n", " 's': array([6.6864976 , 6.04182871, 4.19889332, 2.93925665, 2.21285739,\n", " 1.90273045]),\n", " 'vt': 0 1 2 3 4 5 6 \\\n", " d0 -0.007438 -0.007438 -0.011173 -0.007438 -0.007438 -0.000681 -0.144164 \n", " d1 -0.006483 -0.006483 -0.004624 -0.006483 -0.006483 -0.000730 0.042537 \n", " d2 -0.291678 -0.291678 -0.271769 -0.291678 -0.291678 -0.070154 0.009400 \n", " d3 0.116874 0.116874 -0.274207 0.116874 0.116874 0.061197 0.007658 \n", " d4 -0.002733 -0.002733 -0.005641 -0.002733 -0.002733 -0.002806 0.023299 \n", " ... ... ... ... ... ... ... ... \n", " d258 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 \n", " d259 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 \n", " d260 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 \n", " d261 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 \n", " d262 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 \n", " \n", " 7 8 9 ... 253 254 255 256 257 258 259 \\\n", " d0 -0.151425 -0.148580 -0.011173 ... -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 -0.0 \n", " d1 0.045237 0.043665 -0.004624 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", " d2 0.010888 -0.040845 -0.271769 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", " d3 0.011867 -0.322226 -0.274207 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", " d4 -0.427957 0.017585 -0.005641 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", " ... ... ... ... ... ... ... ... ... ... ... ... \n", " d258 0.000000 0.000000 0.000000 ... 0.0 0.0 0.0 0.0 0.0 1.0 0.0 \n", " d259 0.000000 0.000000 0.000000 ... 0.0 0.0 0.0 0.0 0.0 0.0 1.0 \n", " d260 0.000000 0.000000 0.000000 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", " d261 0.000000 0.000000 0.000000 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", " d262 0.000000 0.000000 0.000000 ... 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", " \n", " 260 261 262 \n", " d0 -0.0 -0.0 -0.0 \n", " d1 0.0 0.0 0.0 \n", " d2 0.0 0.0 0.0 \n", " d3 0.0 0.0 0.0 \n", " d4 0.0 0.0 0.0 \n", " ... ... ... ... \n", " d258 0.0 0.0 0.0 \n", " d259 0.0 0.0 0.0 \n", " d260 1.0 0.0 0.0 \n", " d261 0.0 1.0 0.0 \n", " d262 0.0 0.0 1.0 \n", " \n", " [263 rows x 263 columns],\n", " 'tdm': 0 1 2 3 4 5 6 7 8 9 ... 253 254 255 \\\n", " cat 0 0 0 0 0 0 1 1 1 0 ... 0 0 0 \n", " dog 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 \n", " apple 1 1 0 1 1 1 0 0 0 0 ... 0 0 0 \n", " lion 0 0 0 0 0 0 0 1 0 0 ... 0 0 0 \n", " nyc 1 1 1 1 1 0 0 0 0 1 ... 0 0 0 \n", " love 0 0 1 0 0 0 0 0 1 1 ... 0 0 0 \n", " \n", " 256 257 258 259 260 261 262 \n", " cat 0 0 0 0 0 0 0 \n", " dog 0 0 0 0 0 0 0 \n", " apple 0 0 0 0 0 0 0 \n", " lion 0 0 0 0 0 0 0 \n", " nyc 0 0 0 0 0 0 0 \n", " love 0 0 0 0 0 0 0 \n", " \n", " [6 rows x 263 columns],\n", " 'vocabulary': array(['cat', 'dog', 'apple', 'lion', 'nyc', 'love'], dtype=object),\n", " 'docs': array(['NYC is the Big Apple.', 'NYC is known as the Big Apple.',\n", " 'I love NYC!', 'I wore a hat to the Big Apple party in NYC.',\n", " 'Come to NYC. See the Big Apple!',\n", " 'Manhattan is called the Big Apple.',\n", " 'New York is a big city for a small cat.',\n", " 'The lion, a big cat, is the king of the jungle.',\n", " 'I love my pet cat.', 'I love New York City (NYC).',\n", " 'Your dog chased my cat.', 'Bright lights, big city?',\n", " \"Simba, in Lion King, was inspired by Bambi who wasn't even a cat.\",\n", " 'Does your dog have a dog house?',\n", " 'The cat steered clear of the dog house.', 'I love turtles.',\n", " 'Bengi was a small stray dog with a fluffy tan spotted coat.',\n", " 'The woman flew to NYC with her cat.',\n", " 'That dog is a big animal. He must eat a lot.',\n", " 'How big is New York? Is it a big city?',\n", " 'The dog ran through Central Park in NYC.', 'Where is NYC?',\n", " \"The Cat's Meow\", 'The dog sat on the floor.',\n", " 'The cat chased a mouse.', 'Dogs and cats love raw meat.',\n", " 'The cat never made eye contact.',\n", " 'The cat chased my laser pointer.', 'I pet the cat.',\n", " 'The cat died.', 'A dog chased the car, barking.',\n", " 'Mom and Mormor both love turtles.',\n", " 'The cat ate the bearded dragon.', 'A cat burglar stole my pets.',\n", " 'I chased your dog.',\n", " 'The lion tamer rode on top of the clown car into the ring carrying a chair.',\n", " 'NYC is a city that never sleeps.',\n", " 'I was in the dog house last night.', 'The cat in the window',\n", " 'He refused to sleep in the dog house.',\n", " 'Dogs love to smell the air rushing by in a car.',\n", " 'Your cat is cute.', 'A cat meowed on the hot tin roof',\n", " 'A dog chased my bike and barked loudly.',\n", " 'The dog ate my orchids.', 'The dog dropped the ball at my feet.',\n", " 'Dogs love to chase cars, trucks, and bikes.',\n", " 'So I went to NYC to be born again.',\n", " 'The man raised the lion from a cub and they still frolic in the jungle.',\n", " 'Wolves, dogs, and puppies love to play chase.',\n", " 'The dog rescued me from a hairy . ', 'He makes a good guard dog.',\n", " 'The dog walked up and sniffed my leg.',\n", " 'The cat hated getting in the car.', \"The cat licked it's fur.\",\n", " 'The black cat crossed my path.', 'The post man likes our dog.',\n", " 'The dog likes a scratch behind his ear.',\n", " 'He pet the dog on the head.', 'Mom loves to walk through NYC.',\n", " 'Rascal was a tabby cat.',\n", " 'NYC is the only city where you can hardly find a typical American.',\n", " 'My cat has long hair.', 'She was an alley cat.',\n", " 'Can your dog do tricks?', 'Fido chased the cat up the alley.',\n", " 'The cat chased a speedy rat.',\n", " \"Dogs don't have much room to run in a big city.\",\n", " 'The woman took her dog on the plane.',\n", " 'Is NYC a city or a way of life?',\n", " 'My dog is a good boy most of the time.',\n", " 'The cat coughed up a hair ball.', 'The cat died at the vet.',\n", " 'The fireman rescued the cat in the tree.',\n", " \"Cat, you ruined mom's dress!\",\n", " 'The dog whined until I pet its head.',\n", " 'The dog chased the ball and caught it.',\n", " 'The dog jumped up on the bed.',\n", " 'I ran from the dog and jumped the bench.',\n", " 'Ashley had a wiener dog that she took boating up the river.',\n", " 'He took his life in his hands, j-walking in NYC.',\n", " 'A stray cat played with the injured frog.',\n", " 'The lion opened his mouth wide as she put her head into his mouth.',\n", " 'Bengi was a movie about the adventures of a lovable stray dog.',\n", " 'The tabby cat had a fluffy tail.',\n", " 'The cat likes a scratch under her chin.',\n", " 'The dog flew down the street after the bike.',\n", " 'The man had a cat in his carry-on.',\n", " 'The cat crossed the lane and then the sidewalk.',\n", " 'Moon got mauled in a fight with an alley cat.',\n", " 'A black cat crossed the sidewalk in front of me.',\n", " \"The cat held the lizard's tail in its mouth.\",\n", " 'The raccoons ate all the cat food in the garage.',\n", " 'The dog wash was just a hose and they hated it.',\n", " 'There are no lions in NYC, but there are lots of house cats.',\n", " 'The cat meowed and I pet it until it purred.',\n", " 'Rascal was an alley cat before she became a Lane pet.',\n", " 'A car struck the cat and we took it to the vet.',\n", " 'A cat pounced at the lizard but came away with only its tail.',\n", " \"An old dog can learn new tricks if there's food involved.\",\n", " 'Our Bengi was a mottled tan dog that loved to run around the yard.',\n", " \"Animals don't drive cars, but my pet dog likes to stick his head out the window.\",\n", " 'The Cat in the Hat is not about an animal or a hat.',\n", " 'Ursa was smart and deceptive.',\n", " 'Are there fish in your fountain?',\n", " 'Australian sheep dogs are smarter than the sheep.',\n", " 'Are there fish in the pond?', 'Are there turtles in your pond?',\n", " 'How many pets do you have at home?',\n", " 'Where do you keep a turtle in your house?',\n", " 'Carnivore cunning and cooperation makes them smarter than herbivores.',\n", " 'No lone wolf would dare attack a lone moose or adult caribou.',\n", " 'I loved frogs and the color green.',\n", " 'Wolf puppies play with crows.',\n", " 'Crows help wolves track down prey and wolves share the kill.',\n", " 'Toxoplasmosis will change your mind.',\n", " 'America is littered with Toxoplasmosis.',\n", " 'Billy never had any pets at his house.',\n", " 'Ursa would inch her way into the dining room sheepishly.',\n", " 'Walter and Moon taught me how to crack video games.',\n", " 'Mice get attracted to the smell of cats when the have Toxoplasmosis.',\n", " \"No man is an island, unless he's a lone wolf like Walter Anderson.\",\n", " 'Alligators and wolves compete for food on Horn Island.',\n", " 'He use duck tape to keep its mouth closed.',\n", " 'The ranger dragged the alligator over the seawall.',\n", " 'What about frogs in the pond?',\n", " 'Clinton helped Clayton catch the alligator.',\n", " 'The Inner Harbor had an alligator, some turtles and lots of fish.',\n", " 'An alligator ate several pets and ducks before Clayton caught him.',\n", " 'Can lizards swim under water?',\n", " 'Char drooled with Pavlovian delight at the hotdog in my hand.',\n", " \"Rascal hated the car because it's associated with the vet.\",\n", " 'She caught a frog with her paw.',\n", " 'She bit the frog with her teeth.',\n", " 'Berk, the vet, has ideas about sports games for people.',\n", " \"You don't get a fever from Toxoplasmosis, you just get aggressive.\",\n", " 'Bear loved to hang his head out the truck window.',\n", " 'Wild cats chase bikes and runners but not cars or trucks.',\n", " 'Ursa used to chase her tail when she was young.',\n", " 'Lizards, turtles, and alligators are kind-of green and slimy.',\n", " \"The dogs licked my plate so I didn't have to wash it.\",\n", " 'Dogs wag their tail when they are happy.',\n", " 'Ursa, a black lab, would beat her tail against the wall until it was raw.',\n", " 'Bear lapped water from the hose with his tongue.',\n", " 'Rascal lapped milk from her bowl, curling her tongue.',\n", " 'Bear was bloody and panting after mauling the goats and sheep.',\n", " 'Will cuddled with Moon and Zoe on the couch.',\n", " 'Char and Ursa played on the green grass in the back yard.',\n", " 'Rascal was a stray when we found her in a tree in the back yard.',\n", " 'A black kitten crossed the road dodging cars like Frogger.',\n", " 'Goats and sheep make great lawn mowers for a boat yard.',\n", " 'Bear lapped out of the truck window.',\n", " 'The Inner Harbor was our playground.',\n", " 'The dogs were not allowed on the couch or in the dining room.',\n", " 'Lane rescued Moon with Will power.',\n", " 'Men become more gullible once they get Toxoplasmosis.',\n", " 'Wolves eat deer and stay away from sheep if they smell humans.',\n", " 'Ants get a virus that makes crawl to the tip of a blade of grass.',\n", " 'Brian wanted to start an alligator farm.',\n", " 'Humans harbor infectious diseases from domestic pets.',\n", " 'Our neighbors raised baby alligators.',\n", " 'Early humans slept in the barn with domesticated animals.',\n", " 'Sheep and deer eat grass.', 'My brother had an aquarium.',\n", " 'Some lizards can grow a new tail.',\n", " 'Mormor loved turtles and had them all over her house.',\n", " 'The litter box smells.', 'The snapping turtle won the race.',\n", " 'Jupiter and Moon each had their own food bowl.',\n", " 'A turtle beat the rabbit in a race.',\n", " 'Cats and dogs sleeping together.',\n", " 'Women become more trusting once they get Toxoplasmosis.',\n", " \"It's Berk a vet?\", 'I froze as he sniffed.',\n", " 'I rode my bike home.', 'Mom loves to walk around Manhattan.',\n", " 'Give me such shows — give me the streets of Manhattan!',\n", " 'She loves dogs.', 'Is that a pet rat in your carry-on?',\n", " 'He put a hat on his head.', 'He put his hat in the overhead bin.',\n", " 'The car is in the garage.', 'Dogs like to chase cars.',\n", " 'The car had a bike rack.',\n", " 'Marc steered my bike into a parked car.',\n", " 'A cute kitten played with its mother.',\n", " 'Where do you live little guy?', 'Where did you come from?',\n", " 'You sure are cute.', 'Go lie down on your bed.',\n", " 'Be a good boy and go on home now.', \"That's a good boy.\",\n", " 'It rained cats and dogs.', 'She keeps a clean house.',\n", " 'She took the train into the city to see the ball drop.',\n", " 'The snake chased the rat.', 'Rascal taught me empathy and care.',\n", " \"Snakes aren't usually considered pets.\",\n", " \"She doesn't like hats in the car but I do.\",\n", " 'She rode her bike though central park wearing a hat.',\n", " \"Animals, including pets, don't like riding in cars.\",\n", " \"Cats don't like riding into the city in a car.\",\n", " 'The rat ran into a hole in the back.',\n", " 'The rat ate a hole in his hat.', 'How many dogs are in the city?',\n", " 'Where is Soho? In New York City?',\n", " 'Look at me! Look at me! Look at me NOW!',\n", " 'It is fun to have fun. But you have to know how.',\n", " 'Honey, it was ruined when she bought it.',\n", " '\"He should not be here,\" said the fish in the pot. ',\n", " '\"He should not be here when your mother is not.” ',\n", " 'Speedy was too fast for Sylvester.', 'You are an animal.',\n", " \"You're an animal!\", 'Animals are not allowed on this flight.',\n", " 'Some flights allow animals in carry-ons.',\n", " \"Snakes aren't usually allowed on planes.\",\n", " 'The litter box is in the back of the house.',\n", " 'Do all dogs go to heaven?', 'Cats and dogs playing together.',\n", " 'Kittens are cute.', \"Jupiter's hair stood on end.\",\n", " 'The lizard aquarium was moist.', 'Turtles need water.',\n", " 'Cats hat water.', 'I chased the ferret with a water pistol.',\n", " 'The ferret got struck by lightening.', 'We have a car carrier.',\n", " 'Algernon lost his mind.',\n", " 'Flowers for Algernon is my favorite book.',\n", " 'The puppy played in the flower bed.', 'She brought me flowers.',\n", " 'I kept the compost full of worms and the flowers bloomed.',\n", " 'Her orchids and my Amaryllis bloomed the same day.',\n", " 'Char are the flowers.', 'Marc chased rascal with a squirt gun.',\n", " 'Rascal hid in the Cypress tree.',\n", " 'I cried at the end of Algernon.',\n", " 'Algernon taught me about animal consciousness, smarts.',\n", " 'Books taught me how to read people.', 'Moon leapt into my lap.',\n", " 'I want to be reborn as a Lane pet.', 'The giving tree gave out.',\n", " 'Do you have a pet?', \"That's a cute kitten.\",\n", " 'Old dogs can learn tricks.', 'Sit Ubu, sit.', 'Sit Char, sit.',\n", " 'Sit Bear, sit.', 'I flew a kite.', 'It rained cats and dogs.',\n", " 'Do dogs go to heaven?', 'What kind of pet do you have?',\n", " 'Ursa ran a squirrel up the tree.', 'The catbird seat',\n", " 'Lindstrom pets are spoiled.',\n", " \"I painted Turtle's shell with nail polish.\",\n", " 'I named my pet rock Rocky.', 'Are you a vet?',\n", " 'My flowers are blooming.',\n", " \"A single flower grew in Benji's grave.\",\n", " 'Char chased the squirrel.',\n", " 'I gnawed the frog legs with my teeth.'], dtype=object),\n", " 'err': [8.84399591290972e-16,\n", " 0.04789870465549901,\n", " 0.07346710202233811,\n", " 0.10426990005625925,\n", " 0.14847568482532236,\n", " 0.21255097031038958],\n", " 'accuracy': [0.9999999999999991,\n", " 0.9521012953445009,\n", " 0.9265328979776619,\n", " 0.8957300999437408,\n", " 0.8515243151746776,\n", " 0.7874490296896104]}" ] }, "execution_count": 49, "metadata": {}, "output_type": "execute_result" } ], "source": [ "lsa_bow" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" } }, "nbformat": 4, "nbformat_minor": 2 }