{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# HW8 HW7 V4" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from nltk.tokenize.casual import casual_tokenize\n", "tfidf_model = TfidfVectorizer(tokenizer = casual_tokenize)\n", "from sklearn.preprocessing import MinMaxScaler" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "Counter({False: 34320, True: 121740})" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "from collections import Counter\n", "import pandas as pd\n", "train=pd.read_csv(\"../WK7/kaggle-sentiment/train.tsv\", delimiter='\\t')\n", "y=train['Sentiment'].values\n", "X=train['Phrase'].values\n", "train.to_csv('kaggle_csv.csv')\n", "\n", "df = pd.DataFrame()\n", "df['Phrase'] = train['Phrase']\n", "df['S0'] = train['Sentiment']\n", "\n", "def get_lda(df):\n", " tfidf_docs = tfidf_model.fit_transform(raw_documents = df['Phrase']).toarray()\n", " mask = df.PoN.astype(bool).values\n", " n_centroid = tfidf_docs[mask].mean(axis=0)\n", " p_centroid = tfidf_docs[~mask].mean(axis=0)\n", " negativity_score = tfidf_docs.dot(n_centroid - p_centroid)\n", " df['lda_score'] = MinMaxScaler().fit_transform(negativity_score.reshape(-1,1))\n", " df['lda_predict'] = (df.lda_score > .5).astype(int)\n", " score = (1. - (df.PoN - df.lda_predict).abs().sum() / len(df)).round(3) \n", " return score, df, negativity_score, n_centroid, p_centroid\n", "\n", "def get_lda_submission(df, n_centroid, p_centroid):\n", " tfidf_docs = tfidf_model.fit_transform(raw_documents = df['Phrase']).toarray()\n", " negativity_score = tfidf_docs.dot(n_centroid - p_centroid)\n", " df['lda_score'] = MinMaxScaler().fit_transform(negativity_score.reshape(-1,1))\n", " df['lda_predict'] = (df.lda_score > .5).astype(int)\n", " return df\n", "\n", "df['PoN'] = [0 if x < 2 else 1 for x in df['S0']]\n", "len(df[df['PoN'] == 1])\n", "lda_score, df, neg_score, n_cent, p_cent = get_lda(df)\n", "\n", "columns = ['Phrase']\n", "new_df = pd.DataFrame(df, columns = columns)\n", "predicted_df = get_lda_submission(new_df, n_cent, p_cent)\n", "\n", "predicted_df['actual'] = df['S0']\n", "predicted_df['bool'] = [0 if x < 2 else 1 for x in predicted_df['actual']]\n", "predicted_df['check'] = predicted_df.apply(lambda x: (x['lda_predict'] == x['bool']), axis=1)\n", "\n", "Counter(predicted_df['check'])" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [ { "ename": "ValueError", "evalue": "shapes (66292,9970) and (16394,) not aligned: 9970 (dim 1) != 16394 (dim 0)", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpredicted_df_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_lda_submission\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_cent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_cent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", "\u001b[0;32m\u001b[0m in \u001b[0;36mget_lda_submission\u001b[0;34m(df, n_centroid, p_centroid)\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_lda_submission\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_centroid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mp_centroid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0mtfidf_docs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtfidf_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mraw_documents\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Phrase'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mnegativity_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtfidf_docs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_centroid\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mp_centroid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lda_score'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMinMaxScaler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnegativity_score\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lda_predict'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlda_score\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m.5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mValueError\u001b[0m: shapes (66292,9970) and (16394,) not aligned: 9970 (dim 1) != 16394 (dim 0)" ] } ], "source": [ "predicted_df_test = get_lda_submission(test, n_cent, p_cent)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "16394\n", "16394\n", "66292\n" ] } ], "source": [ "print(len(n_cent))\n", "print(len(p_cent))\n", "print(len(test))" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
PhraseIdSentenceIdPhrase
01560618545An intermittently pleasing but mostly routine ...
11560628545An intermittently pleasing but mostly routine ...
21560638545An
31560648545intermittently pleasing but mostly routine effort
41560658545intermittently pleasing but mostly routine
............
6628722234811855A long-winded , predictable scenario .
6628822234911855A long-winded , predictable scenario
6628922235011855A long-winded ,
6629022235111855A long-winded
6629122235211855predictable scenario
\n", "

66292 rows × 3 columns

\n", "
" ], "text/plain": [ " PhraseId SentenceId Phrase\n", "0 156061 8545 An intermittently pleasing but mostly routine ...\n", "1 156062 8545 An intermittently pleasing but mostly routine ...\n", "2 156063 8545 An\n", "3 156064 8545 intermittently pleasing but mostly routine effort\n", "4 156065 8545 intermittently pleasing but mostly routine\n", "... ... ... ...\n", "66287 222348 11855 A long-winded , predictable scenario .\n", "66288 222349 11855 A long-winded , predictable scenario\n", "66289 222350 11855 A long-winded ,\n", "66290 222351 11855 A long-winded\n", "66291 222352 11855 predictable scenario\n", "\n", "[66292 rows x 3 columns]" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test=pd.read_csv(\"../WK7/kaggle-sentiment/test.tsv\", delimiter='\\t')\n", "test\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "training_encoded = pd.get_dummies(train, columns=['Phrase'])\n", "test_encoded = pd.get_dummies(test, columns=['Phrase'])\n", "test_encoded_for_model = test_encoded.reindex(columns = training_encoded.columns, \n", " fill_value=0)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.4" } }, "nbformat": 4, "nbformat_minor": 2 }