HW2xHW4

VECTORIZATION (Pandas style!)

STEP 1: Import ALL the things

Import libraries

In [1]:
##########################################
# NOTE: I'm toying with the idea of requiring the library just above 
# when I use it so it makes more sense in context
##########################################
# import os
# import pandas as pd
# from nltk.tokenize import word_tokenize, sent_tokenize
# from nltk.sentiment import SentimentAnalyzer
# from nltk.sentiment.util import *
# from nltk.probability import FreqDist
# from nltk.sentiment.vader import SentimentIntensityAnalyzer
# sid = SentimentIntensityAnalyzer()

Import data from files

In [2]:
import os
def get_data_from_files(path):
    directory = os.listdir(path)
    results = []
    for file in directory:
        f=open(path+file)
        results.append(f.read())
        f.close()
    return results

# neg = get_data_from_files('../neg_cornell/')
# pos = get_data_from_files('../pos_cornell/')

neg = get_data_from_files('../neg_hw4/')
pos = get_data_from_files('../pos_hw4/')

STEP 2: Prep Data

STEP 2a: Turn that fresh text into a pandas DF

In [3]:
import pandas as pd
neg_df = pd.DataFrame(neg)
pos_df = pd.DataFrame(pos)

STEP 2b: Label it

In [4]:
pos_df['PoN'] = 'P'
neg_df['PoN'] = 'N'

STEP 2c: Combine the dfs

In [5]:
all_df = neg_df.append(pos_df)
In [6]:
all_df[:3]
Out[6]:
0 PoN
0 I went to XYZ restaurant last week and I was v... N
1 In each of the diner dish there are at least o... N
2 This is the last place you would want to dine ... N

STEP 3: TOKENIZE (and clean)!!

In [7]:
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import *
In [8]:
## Came back and added sentences for tokinization for "Summary experiment"
def get_sentence_tokens(review):
    return sent_tokenize(review)
    
all_df['sentences'] = all_df.apply(lambda x: get_sentence_tokens(x[0]), axis=1)
all_df['num_sentences'] = all_df.apply(lambda x: len(x['sentences']), axis=1)
In [9]:
def get_tokens(sentence):
    tokens = word_tokenize(sentence)
    clean_tokens = [word.lower() for word in tokens if word.isalpha()]
    return clean_tokens

all_df['tokens'] = all_df.apply(lambda x: get_tokens(x[0]), axis=1)
all_df['num_tokens'] = all_df.apply(lambda x: len(x['tokens']), axis=1)
In [10]:
all_df[:3]
Out[10]:
0 PoN sentences num_sentences tokens num_tokens
0 I went to XYZ restaurant last week and I was v... N [I went to XYZ restaurant last week and I was ... 3 [i, went, to, xyz, restaurant, last, week, and... 50
1 In each of the diner dish there are at least o... N [In each of the diner dish there are at least ... 4 [in, each, of, the, diner, dish, there, are, a... 78
2 This is the last place you would want to dine ... N [This is the last place you would want to dine... 7 [this, is, the, last, place, you, would, want,... 151

STEP 4: Remove Stopwords

In [11]:
from nltk.corpus import stopwords
stop_words = set(stopwords.words("english"))
def remove_stopwords(sentence):
    filtered_text = []
    for word in sentence:
        if word not in stop_words:
            filtered_text.append(word)
    return filtered_text
all_df['no_sw'] = all_df.apply(lambda x: remove_stopwords(x['tokens']),axis=1)
all_df['num_no_sw'] = all_df.apply(lambda x: len(x['no_sw']),axis=1)
In [12]:
all_df[:5]
Out[12]:
0 PoN sentences num_sentences tokens num_tokens no_sw num_no_sw
0 I went to XYZ restaurant last week and I was v... N [I went to XYZ restaurant last week and I was ... 3 [i, went, to, xyz, restaurant, last, week, and... 50 [went, xyz, restaurant, last, week, disappoint... 25
1 In each of the diner dish there are at least o... N [In each of the diner dish there are at least ... 4 [in, each, of, the, diner, dish, there, are, a... 78 [diner, dish, least, one, fly, waiting, hour, ... 31
2 This is the last place you would want to dine ... N [This is the last place you would want to dine... 7 [this, is, the, last, place, you, would, want,... 151 [last, place, would, want, dine, price, expens... 61
3 I went to this restaurant where I had ordered ... N [I went to this restaurant where I had ordered... 6 [i, went, to, this, restaurant, where, i, had,... 75 [went, restaurant, ordered, complimentary, sal... 33
4 I went there with two friends at 6pm. Long que... N [I went there with two friends at 6pm., Long q... 10 [i, went, there, with, two, friends, at, long,... 73 [went, two, friends, long, queue, didnt, take,... 38

STEP 5: Create a Frequency Distribution

In [13]:
from nltk.probability import FreqDist
def get_most_common(tokens):
    fdist = FreqDist(tokens)
    return fdist.most_common(12)
all_df['topwords_unfil'] = all_df.apply(lambda x: get_most_common(x['tokens']),axis=1)
In [14]:
def get_most_common(tokens):
    fdist = FreqDist(tokens)
    return fdist.most_common(12)
all_df['topwords_fil'] = all_df.apply(lambda x: get_most_common(x['no_sw']),axis=1)
In [15]:
def get_fdist(tokens):
    return (FreqDist(tokens))
    
all_df['freq_dist'] = all_df.apply(lambda x: get_fdist(x['no_sw']),axis=1)
all_df['freq_dist_unfil'] = all_df.apply(lambda x: get_fdist(x['tokens']),axis=1)
In [16]:
all_df[:3]
Out[16]:
0 PoN sentences num_sentences tokens num_tokens no_sw num_no_sw topwords_unfil topwords_fil freq_dist freq_dist_unfil
0 I went to XYZ restaurant last week and I was v... N [I went to XYZ restaurant last week and I was ... 3 [i, went, to, xyz, restaurant, last, week, and... 50 [went, xyz, restaurant, last, week, disappoint... 25 [(was, 4), (to, 3), (i, 2), (and, 2), (the, 2)... [(went, 1), (xyz, 1), (restaurant, 1), (last, ... {'went': 1, 'xyz': 1, 'restaurant': 1, 'last':... {'i': 2, 'went': 1, 'to': 3, 'xyz': 1, 'restau...
1 In each of the diner dish there are at least o... N [In each of the diner dish there are at least ... 4 [in, each, of, the, diner, dish, there, are, a... 78 [diner, dish, least, one, fly, waiting, hour, ... 31 [(the, 6), (in, 4), (to, 4), (of, 3), (that, 3... [(want, 3), (dish, 2), (diner, 1), (least, 1),... {'diner': 1, 'dish': 2, 'least': 1, 'one': 1, ... {'in': 4, 'each': 1, 'of': 3, 'the': 6, 'diner...
2 This is the last place you would want to dine ... N [This is the last place you would want to dine... 7 [this, is, the, last, place, you, would, want,... 151 [last, place, would, want, dine, price, expens... 61 [(to, 10), (the, 9), (and, 7), (we, 5), (is, 4... [(minutes, 3), (place, 2), (price, 2), (servic... {'last': 1, 'place': 2, 'would': 1, 'want': 1,... {'this': 1, 'is': 4, 'the': 9, 'last': 1, 'pla...

STEP 6: Try Different Sentiment Analysis Tools

VADER

In [25]:
from nltk.sentiment.vader import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()
def get_vader_score(review):
    return sid.polarity_scores(review)

all_df['vader_all'] = all_df.apply(lambda x: get_vader_score(x[0]),axis=1)
In [26]:
def separate_vader_score(vader_score, key):
    return vader_score[key]

all_df['v_compound'] = all_df.apply(lambda x: separate_vader_score(x['vader_all'], 'compound'),axis=1)
all_df['v_neg'] = all_df.apply(lambda x: separate_vader_score(x['vader_all'], 'neg'),axis=1)
all_df['v_neu'] = all_df.apply(lambda x: separate_vader_score(x['vader_all'], 'neu'),axis=1)
all_df['v_pos'] = all_df.apply(lambda x: separate_vader_score(x['vader_all'], 'pos'),axis=1)

DIY SUMMARY

In [27]:
all_df[0][17]
Out[27]:
17    Mikes Pizza High Point NY Service was very slo...
17                                                    ?
Name: 0, dtype: object
In [28]:
def get_weighted_freq_dist(review, freq_dist):
    try:
        max_freq = max(freq_dist.values())
        for word in freq_dist.keys():
            freq_dist[word] = (freq_dist[word]/max_freq)
        return freq_dist
    except:
        return 'nope'

all_df['weighted_freq_dist'] = all_df.apply(lambda x: get_weighted_freq_dist(x['sentences'], x['freq_dist']),axis=1)
In [29]:
def get_sentence_score(review, freq_dist):
    sentence_scores = {}
    for sent in review:
        for word in nltk.word_tokenize(sent.lower()):
            if word in freq_dist.keys():
                if len(sent.split(' ')) < 30:
                    if sent not in sentence_scores.keys():
                        sentence_scores[sent] = freq_dist[word]
                    else:
                        sentence_scores[sent] += freq_dist[word]
    return sentence_scores

all_df['sentence_scores'] = all_df.apply(lambda x: get_sentence_score(x['sentences'], x['freq_dist']),axis=1)
In [30]:
def get_summary_sentences(sentence_scores):
    sorted_sentences = sorted(sentence_scores.items(), key=lambda kv: kv[1], reverse=True)
    return ''.join(sent[0] for sent in sorted_sentences[:5])

all_df['summary_sentences'] = all_df.apply(lambda x: get_summary_sentences(x['sentence_scores']), axis=1)
In [31]:
summaries = all_df['summary_sentences'].tolist()
In [32]:
summaries[3]
Out[32]:
'Once I had finished eating the food the waiter actually came back and said why did you order the salad when you couldnt finish it.The waiter made a bad face when I asked him for the salad.I went to this restaurant where I had ordered for the complimentary salad.I would never ever go back to that restaurant.I was aghast and asked him to bill me for it.'

Doing VADER on the Summary Section

In [33]:
all_df['vader_sum_all'] = all_df.apply(lambda x: get_vader_score(x['summary_sentences']),axis=1)
In [34]:
all_df['v_compound_sum'] = all_df.apply(lambda x: separate_vader_score(x['vader_sum_all'], 'compound'),axis=1)
all_df['v_neg_sum'] = all_df.apply(lambda x: separate_vader_score(x['vader_sum_all'], 'neg'),axis=1)
all_df['v_neu_sum'] = all_df.apply(lambda x: separate_vader_score(x['vader_sum_all'], 'neu'),axis=1)
all_df['v_pos_sum'] = all_df.apply(lambda x: separate_vader_score(x['vader_sum_all'], 'pos'),axis=1)

Doing VADER on the Most Frequent Words

In [35]:
def get_freq_words(freq_dist):
    sorted_words = sorted(freq_dist.items(), key=lambda kv: kv[1], reverse=True)
    return ' '.join(word[0] for word in sorted_words[:50])

all_df['v_freq_words'] = all_df.apply(lambda x: get_freq_words(x['freq_dist']), axis=1)

all_df['vader_fq_all'] = all_df.apply(lambda x: get_vader_score(x['v_freq_words']),axis=1)
all_df['v_compound_fd'] = all_df.apply(lambda x: separate_vader_score(x['vader_fq_all'], 'compound'),axis=1)
all_df['v_neg_fd'] = all_df.apply(lambda x: separate_vader_score(x['vader_fq_all'], 'neg'),axis=1)
all_df['v_neu_fd'] = all_df.apply(lambda x: separate_vader_score(x['vader_fq_all'], 'neu'),axis=1)
all_df['v_pos_fd'] = all_df.apply(lambda x: separate_vader_score(x['vader_fq_all'], 'pos'),axis=1)

STEP 7: Test Step 6 with Machine Learning!!

Naive Bayes

In [36]:
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB

def get_NB(small_df, labels):
    x_train, x_test, y_train, y_test = train_test_split(small_df.values, labels, test_size=0.3, random_state = 109)

    gnb = GaussianNB()
    gnb.fit(x_train, y_train)
    y_pred = gnb.predict(x_test)
    from sklearn import metrics
    print("Accuracy:", metrics.accuracy_score(y_test, y_pred))

TEST 1: Vader Scores (Original)

In [37]:
small_df = all_df.filter(['v_compound','v_pos', 'v_neg', 'v_neu']) # 0.645
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

TEST 2: Vader Scores (from Summary)

In [38]:
small_df = all_df.filter(['v_compound_sum','v_pos_sum', 'v_neg_sum', 'v_neu_sum']) # 0.59
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8214285714285714

TEST 3: Vader Scores (original) AND Vader Scores (summary)

In [39]:
small_df = all_df.filter(['v_compound_sum','v_pos_sum', 'v_neg_sum', 'v_neu_sum', 
                          'v_compound','v_pos', 'v_neg', 'v_neu']) # 0.618
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

TEST 4: Vader Scores (50 most frequent -- filtered -- words)

In [40]:
small_df = all_df.filter(['v_compound_fd','v_pos_fd', 'v_neu_fd', 'v_neg_fd']) # 0.598
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

TEST 5: All compound Vader Scores

In [41]:
small_df = all_df.filter(['v_compound_fd','v_compound_sum', 'v_compound']) # 0.615
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

TEST 6: ALL THE NUMBERS!!

In [42]:
small_df = all_df.filter(['v_compound_sum','v_pos_sum', 'v_neg_sum', 'v_neu_sum', 
                          'v_compound_fd','v_pos_fd', 'v_neg_fd', 'v_neu_fd', 
                          'v_compound','v_pos', 'v_neg', 'v_neu']) # 0.613
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

TEST 7: Test UNFILTERED most frequent words

In [43]:
def get_freq_words(freq_dist):
    sorted_words = sorted(freq_dist.items(), key=lambda kv: kv[1], reverse=True)
    return ' '.join(word[0] for word in sorted_words[:50])

all_df['v_freq_words_unfil'] = all_df.apply(lambda x: get_freq_words(x['freq_dist_unfil']), axis=1)

all_df['vader_fd_all_unfil'] = all_df.apply(lambda x: get_vader_score(x['v_freq_words_unfil']),axis=1)

all_df['v_compound_fd_uf'] = all_df.apply(lambda x: separate_vader_score(x['vader_fd_all_unfil'], 'compound'),axis=1)
all_df['v_neg_fd_uf'] = all_df.apply(lambda x: separate_vader_score(x['vader_fd_all_unfil'], 'neg'),axis=1)
all_df['v_neu_fd_uf'] = all_df.apply(lambda x: separate_vader_score(x['vader_fd_all_unfil'], 'neu'),axis=1)
all_df['v_pos_fd_uf'] = all_df.apply(lambda x: separate_vader_score(x['vader_fd_all_unfil'], 'pos'),axis=1)
In [44]:
small_df = all_df.filter(['v_compound_sum','v_pos_sum', 'v_neg_sum', 'v_neu_sum', 
                          'v_compound_fd','v_pos_fd', 'v_neg_fd', 'v_neu_fd', 
                          'v_compound_fd_uf','v_pos_fd_uf', 'v_neg_fd_uf', 'v_neu_fd_uf',
                          'v_compound','v_pos', 'v_neg', 'v_neu']) # 0.618
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571
In [45]:
small_df = all_df.filter(['v_compound_fd_uf','v_pos_fd_uf', 'v_neg_fd_uf', 'v_neu_fd_uf']) # 0.603
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571
In [46]:
summaries_pos = all_df[all_df['PoN'] == 'P']
summaries_neg = all_df[all_df['PoN'] == 'N']
In [47]:
summaries_pos_list = summaries_pos['summary_sentences'].tolist()
summaries_neg_list = summaries_neg['summary_sentences'].tolist()
In [48]:
summaries_pos_list[:1]
Out[48]:
['Everything is great great just great!!!I mean the food is great and people are great.This restaurant ROCKS!I love it.I like it.']
In [49]:
summaries_neg_list[:1]
Out[49]:
['The food was bland service was terrible and the place was so cramped that we had to keep getting up to let people pass.Overall not a good idea for big groups looking for a fun night out.I went to XYZ restaurant last week and I was very disappointed.']
In [51]:
### VERSION 1
#     all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
#     unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg)
#     sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
#     training_set = sentim_analyzer.apply_features(training_docs)
#     test_set = sentim_analyzer.apply_features(testing_docs)
sentim_analyzer = SentimentAnalyzer()

def get_nltk_negs(tokens):
    all_words_neg = sentim_analyzer.all_words([mark_negation(tokens)])
    return all_words_neg

def get_unigram_feats(neg_tokens):
    unigram_feats = sentim_analyzer.unigram_word_feats(neg_tokens)
    return unigram_feats
    
all_df['nltk_negs'] = all_df.apply(lambda x: get_nltk_negs(x['tokens']), axis=1)
all_df['unigram_feats'] = all_df.apply(lambda x: get_unigram_feats(x['nltk_negs']), axis=1)
# all_df['nltk_unfil'] = all_df.apply(lambda x: get_nltk_data(x['tokens']), axis=1)
In [69]:
### VERSION 2
#     all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
#     unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg)
#     sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
#     training_set = sentim_analyzer.apply_features(training_docs)
#     test_set = sentim_analyzer.apply_features(testing_docs)
sentim_analyzer = SentimentAnalyzer()

def get_nltk_data(tokens):
#     print(tokens)
    neg_tokens = sentim_analyzer.all_words([mark_negation(tokens)])
    unigram_feats = sentim_analyzer.unigram_word_feats(neg_tokens)
    sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
#     print(sentim_analyzer.apply_features(tokens))
    return sentim_analyzer.apply_features(tokens)


# def get_unigram_feats(neg_tokens):
    
#     return unigram_feats
nltk_df = pd.DataFrame()
nltk_df['nltk_data'] = all_df.apply(lambda x: get_nltk_data(x['tokens']), axis=1)

# all_df['nltk']
# all_df['unigram_feats'] = all_df.apply(lambda x: get_unigram_feats(x['nltk_negs']), axis=1)
# all_df['nltk_unfil'] = all_df.apply(lambda x: get_nltk_data(x['tokens']), axis=1)
In [53]:
# all_df['nltk_all'] = 0
In [70]:
nltk_df
Out[70]:
nltk_data
0 ({'contains(i)': True, 'contains(this)': False...
1 ({'contains(i)': True, 'contains(this)': False...
2 ({'contains(i)': True, 'contains(this)': False...
3 ({'contains(i)': True, 'contains(this)': False...
4 ({'contains(i)': True, 'contains(this)': False...
... ...
41 ({'contains(i)': True, 'contains(this)': False...
42 ({'contains(i)': False, 'contains(this)': Fals...
43 ({'contains(i)': True, 'contains(this)': False...
44 ({'contains(i)': True, 'contains(this)': False...
45 ({'contains(i)': True, 'contains(this)': False...

92 rows × 1 columns

In [65]:
all_df['nltk_negs']
Out[65]:
0     [i, went, to, xyz, restaurant, last, week, and...
1     [in, each, of, the, diner, dish, there, are, a...
2     [this, is, the, last, place, you, would, want,...
3     [i, went, to, this, restaurant, where, i, had,...
4     [i, went, there, with, two, friends, at, long,...
                            ...                        
41    [this, place, was, one, of, the, best, restaur...
42    [the, best, experience, i, ever, had, happened...
43    [this, japanese, restaurant, is, so, popular, ...
44    [hibachi, the, grill, is, one, of, my, favorit...
45    [i, went, to, this, restaurant, in, downtown, ...
Name: nltk_negs, Length: 92, dtype: object
In [56]:
from nltk.tokenize import casual_tokenize
from collections import Counter
all_df['bow_nosw'] = all_df.apply(lambda x: Counter(casual_tokenize(x[0])), axis=1)
In [59]:
all_df[:3]
Out[59]:
0 PoN sentences num_sentences tokens num_tokens no_sw num_no_sw topwords_unfil topwords_fil ... v_freq_words_unfil vader_fd_all_unfil v_compound_fd_uf v_neg_fd_uf v_neu_fd_uf v_pos_fd_uf nltk_negs unigram_feats nltk_all bow_nosw
0 I went to XYZ restaurant last week and I was v... N [I went to XYZ restaurant last week and I was ... 3 [i, went, to, xyz, restaurant, last, week, and... 50 [went, xyz, restaurant, last, week, disappoint... 25 [(was, 4), (to, 3), (i, 2), (and, 2), (the, 2)... [(went, 1), (xyz, 1), (restaurant, 1), (last, ... ... was to i and the a for went xyz restaurant las... {'neg': 0.193, 'neu': 0.736, 'pos': 0.071, 'co... -0.6807 0.193 0.736 0.071 [i, went, to, xyz, restaurant, last, week, and... [was, to, i, and, the, a_NEG, for_NEG, went, x... 0 {'I': 2, 'went': 1, 'to': 3, 'XYZ': 1, 'restau...
1 In each of the diner dish there are at least o... N [In each of the diner dish there are at least ... 4 [in, each, of, the, diner, dish, there, are, a... 78 [diner, dish, least, one, fly, waiting, hour, ... 31 [(the, 6), (in, 4), (to, 4), (of, 3), (that, 3... [(want, 3), (dish, 2), (diner, 1), (least, 1),... ... the in to of that i want dish are it for a is ... {'neg': 0.069, 'neu': 0.906, 'pos': 0.026, 'co... -0.4939 0.069 0.906 0.026 [in, each, of, the, diner, dish, there, are, a... [to_NEG, the, want_NEG, the_NEG, in, of, dish,... 0 {'In': 1, 'each': 1, 'of': 3, 'the': 4, 'diner...
2 This is the last place you would want to dine ... N [This is the last place you would want to dine... 7 [this, is, the, last, place, you, would, want,... 151 [last, place, would, want, dine, price, expens... 61 [(to, 10), (the, 9), (and, 7), (we, 5), (is, 4... [(minutes, 3), (place, 2), (price, 2), (servic... ... to the and we is had for them not that but whe... {'neg': 0.07, 'neu': 0.828, 'pos': 0.101, 'com... 0.1901 0.070 0.828 0.101 [this, is, the, last, place, you, would, want,... [to_NEG, the_NEG, and_NEG, we_NEG, had_NEG, fo... 0 {'This': 1, 'is': 4, 'the': 7, 'last': 1, 'pla...

3 rows × 41 columns