HW2: VECTORIZATION (Pandas style!)¶

STEP 1: Import ALL the things¶

Import libraries¶

##########################################
# NOTE: I'm toying with the idea of requiring the library just above 
# when I use it so it makes more sense in context
##########################################
# import os
# import pandas as pd
# from nltk.tokenize import word_tokenize, sent_tokenize
# from nltk.sentiment import SentimentAnalyzer
# from nltk.sentiment.util import *
# from nltk.probability import FreqDist
# from nltk.sentiment.vader import SentimentIntensityAnalyzer
# sid = SentimentIntensityAnalyzer()

Import data from files¶

import os
def get_data_from_files(path):
    directory = os.listdir(path)
    results = []
    for file in directory:
        f=open(path+file)
        results.append(f.read())
        f.close()
    return results

# neg = get_data_from_files('../neg_cornell/')
# pos = get_data_from_files('../pos_cornell/')

neg = get_data_from_files('../neg_hw4/')
pos = get_data_from_files('../pos_hw4/')

STEP 2: Prep Data¶

STEP 2a: Turn that fresh text into a pandas DF¶

import pandas as pd
neg_df = pd.DataFrame(neg)
pos_df = pd.DataFrame(pos)

STEP 2b: Label it¶

pos_df['PoN'] = 'P'
neg_df['PoN'] = 'N'

STEP 2c: Combine the dfs¶

all_df = neg_df.append(pos_df)

all_df

STEP 3: TOKENIZE (and clean)!!¶

from nltk.tokenize import word_tokenize, sent_tokenize
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import *

## Came back and added sentences for tokinization for "Summary experiment"
def get_sentence_tokens(review):
    return sent_tokenize(review)
    
all_df['sentences'] = all_df.apply(lambda x: get_sentence_tokens(x[0]), axis=1)
all_df['num_sentences'] = all_df.apply(lambda x: len(x['sentences']), axis=1)

def get_tokens(sentence):
    tokens = word_tokenize(sentence)
    clean_tokens = [word.lower() for word in tokens if word.isalpha()]
    return clean_tokens

all_df['tokens'] = all_df.apply(lambda x: get_tokens(x[0]), axis=1)
all_df['num_tokens'] = all_df.apply(lambda x: len(x['tokens']), axis=1)

all_df

STEP 4: Remove Stopwords¶

from nltk.corpus import stopwords
stop_words = set(stopwords.words("english"))
def remove_stopwords(sentence):
    filtered_text = []
    for word in sentence:
        if word not in stop_words:
            filtered_text.append(word)
    return filtered_text
all_df['no_sw'] = all_df.apply(lambda x: remove_stopwords(x['tokens']),axis=1)
all_df['num_no_sw'] = all_df.apply(lambda x: len(x['no_sw']),axis=1)

all_df

STEP 5: Create a Frequency Distribution¶

from nltk.probability import FreqDist
def get_most_common(tokens):
    fdist = FreqDist(tokens)
    return fdist.most_common(12)
all_df['topwords_unfil'] = all_df.apply(lambda x: get_most_common(x['tokens']),axis=1)

def get_most_common(tokens):
    fdist = FreqDist(tokens)
    return fdist.most_common(12)
all_df['topwords_fil'] = all_df.apply(lambda x: get_most_common(x['no_sw']),axis=1)

def get_fdist(tokens):
    return (FreqDist(tokens))
    
all_df['freq_dist'] = all_df.apply(lambda x: get_fdist(x['no_sw']),axis=1)
all_df['freq_dist_unfil'] = all_df.apply(lambda x: get_fdist(x['tokens']),axis=1)

all_df

STEP 6: Try Different Sentiment Analysis Tools¶

VADER¶

from nltk.sentiment.vader import SentimentIntensityAnalyzer
sid = SentimentIntensityAnalyzer()
def get_vader_score(review):
    return sid.polarity_scores(review)

all_df['vader_all'] = all_df.apply(lambda x: get_vader_score(x[0]),axis=1)

def separate_vader_score(vader_score, key):
    return vader_score[key]

all_df['v_compound'] = all_df.apply(lambda x: separate_vader_score(x['vader_all'], 'compound'),axis=1)
all_df['v_neg'] = all_df.apply(lambda x: separate_vader_score(x['vader_all'], 'neg'),axis=1)
all_df['v_neu'] = all_df.apply(lambda x: separate_vader_score(x['vader_all'], 'neu'),axis=1)
all_df['v_pos'] = all_df.apply(lambda x: separate_vader_score(x['vader_all'], 'pos'),axis=1)

DIY SUMMARY¶

all_df[0][17]

17    Mikes Pizza High Point NY Service was very slo...
17                                                    ?
Name: 0, dtype: object

def get_weighted_freq_dist(review, freq_dist):
    try:
        max_freq = max(freq_dist.values())
        for word in freq_dist.keys():
            freq_dist[word] = (freq_dist[word]/max_freq)
        return freq_dist
    except:
        return 'nope'

all_df['weighted_freq_dist'] = all_df.apply(lambda x: get_weighted_freq_dist(x['sentences'], x['freq_dist']),axis=1)

def get_sentence_score(review, freq_dist):
    sentence_scores = {}
    for sent in review:
        for word in nltk.word_tokenize(sent.lower()):
            if word in freq_dist.keys():
                if len(sent.split(' ')) < 30:
                    if sent not in sentence_scores.keys():
                        sentence_scores[sent] = freq_dist[word]
                    else:
                        sentence_scores[sent] += freq_dist[word]
    return sentence_scores

all_df['sentence_scores'] = all_df.apply(lambda x: get_sentence_score(x['sentences'], x['freq_dist']),axis=1)

def get_summary_sentences(sentence_scores):
    sorted_sentences = sorted(sentence_scores.items(), key=lambda kv: kv[1], reverse=True)
    return ''.join(sent[0] for sent in sorted_sentences[:5])

all_df['summary_sentences'] = all_df.apply(lambda x: get_summary_sentences(x['sentence_scores']), axis=1)

summaries = all_df['summary_sentences'].tolist()

summaries[3]

'Once I had finished eating the food the waiter actually came back and said why did you order the salad when you couldnt finish it.The waiter made a bad face when I asked him for the salad.I went to this restaurant where I had ordered for the complimentary salad.I would never ever go back to that restaurant.I was aghast and asked him to bill me for it.'

Doing VADER on the Summary Section¶

all_df['vader_sum_all'] = all_df.apply(lambda x: get_vader_score(x['summary_sentences']),axis=1)

all_df['v_compound_sum'] = all_df.apply(lambda x: separate_vader_score(x['vader_sum_all'], 'compound'),axis=1)
all_df['v_neg_sum'] = all_df.apply(lambda x: separate_vader_score(x['vader_sum_all'], 'neg'),axis=1)
all_df['v_neu_sum'] = all_df.apply(lambda x: separate_vader_score(x['vader_sum_all'], 'neu'),axis=1)
all_df['v_pos_sum'] = all_df.apply(lambda x: separate_vader_score(x['vader_sum_all'], 'pos'),axis=1)

Doing VADER on the Most Frequent Words¶

def get_freq_words(freq_dist):
    sorted_words = sorted(freq_dist.items(), key=lambda kv: kv[1], reverse=True)
    return ' '.join(word[0] for word in sorted_words[:50])

all_df['v_freq_words'] = all_df.apply(lambda x: get_freq_words(x['freq_dist']), axis=1)

all_df['vader_fq_all'] = all_df.apply(lambda x: get_vader_score(x['v_freq_words']),axis=1)
all_df['v_compound_fd'] = all_df.apply(lambda x: separate_vader_score(x['vader_fq_all'], 'compound'),axis=1)
all_df['v_neg_fd'] = all_df.apply(lambda x: separate_vader_score(x['vader_fq_all'], 'neg'),axis=1)
all_df['v_neu_fd'] = all_df.apply(lambda x: separate_vader_score(x['vader_fq_all'], 'neu'),axis=1)
all_df['v_pos_fd'] = all_df.apply(lambda x: separate_vader_score(x['vader_fq_all'], 'pos'),axis=1)

STEP 7: Test `Step 6` with Machine Learning!!¶

Naive Bayes¶

from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB

def get_NB(small_df, labels):
    x_train, x_test, y_train, y_test = train_test_split(small_df.values, labels, test_size=0.3, random_state = 109)

    gnb = GaussianNB()
    gnb.fit(x_train, y_train)
    y_pred = gnb.predict(x_test)
    from sklearn import metrics
    print("Accuracy:", metrics.accuracy_score(y_test, y_pred))

TEST 1: Vader Scores (Original)¶

small_df = all_df.filter(['v_compound','v_pos', 'v_neg', 'v_neu']) # 0.645
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

TEST 2: Vader Scores (from Summary)¶

small_df = all_df.filter(['v_compound_sum','v_pos_sum', 'v_neg_sum', 'v_neu_sum']) # 0.59
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8214285714285714

TEST 3: Vader Scores (original) AND Vader Scores (summary)¶

small_df = all_df.filter(['v_compound_sum','v_pos_sum', 'v_neg_sum', 'v_neu_sum', 
                          'v_compound','v_pos', 'v_neg', 'v_neu']) # 0.618
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

TEST 4: Vader Scores (50 most frequent -- filtered -- words)¶

small_df = all_df.filter(['v_compound_fd','v_pos_fd', 'v_neu_fd', 'v_neg_fd']) # 0.598
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

TEST 5: All `compound` Vader Scores¶

small_df = all_df.filter(['v_compound_fd','v_compound_sum', 'v_compound']) # 0.615
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

TEST 6: ALL THE NUMBERS!!¶

small_df = all_df.filter(['v_compound_sum','v_pos_sum', 'v_neg_sum', 'v_neu_sum', 
                          'v_compound_fd','v_pos_fd', 'v_neg_fd', 'v_neu_fd', 
                          'v_compound','v_pos', 'v_neg', 'v_neu']) # 0.613
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

TEST 7: Test UNFILTERED most frequent words¶

def get_freq_words(freq_dist):
    sorted_words = sorted(freq_dist.items(), key=lambda kv: kv[1], reverse=True)
    return ' '.join(word[0] for word in sorted_words[:50])

all_df['v_freq_words_unfil'] = all_df.apply(lambda x: get_freq_words(x['freq_dist_unfil']), axis=1)

all_df['vader_fd_all_unfil'] = all_df.apply(lambda x: get_vader_score(x['v_freq_words_unfil']),axis=1)

all_df['v_compound_fd_uf'] = all_df.apply(lambda x: separate_vader_score(x['vader_fd_all_unfil'], 'compound'),axis=1)
all_df['v_neg_fd_uf'] = all_df.apply(lambda x: separate_vader_score(x['vader_fd_all_unfil'], 'neg'),axis=1)
all_df['v_neu_fd_uf'] = all_df.apply(lambda x: separate_vader_score(x['vader_fd_all_unfil'], 'neu'),axis=1)
all_df['v_pos_fd_uf'] = all_df.apply(lambda x: separate_vader_score(x['vader_fd_all_unfil'], 'pos'),axis=1)

small_df = all_df.filter(['v_compound_sum','v_pos_sum', 'v_neg_sum', 'v_neu_sum', 
                          'v_compound_fd','v_pos_fd', 'v_neg_fd', 'v_neu_fd', 
                          'v_compound_fd_uf','v_pos_fd_uf', 'v_neg_fd_uf', 'v_neu_fd_uf',
                          'v_compound','v_pos', 'v_neg', 'v_neu']) # 0.618
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

small_df = all_df.filter(['v_compound_fd_uf','v_pos_fd_uf', 'v_neg_fd_uf', 'v_neu_fd_uf']) # 0.603
get_NB(small_df, all_df['PoN'])

Accuracy: 0.8571428571428571

summaries_pos = all_df[all_df['PoN'] == 'P']
summaries_neg = all_df[all_df['PoN'] == 'N']

summaries_pos_list = summaries_pos['summary_sentences'].tolist()
summaries_neg_list = summaries_neg['summary_sentences'].tolist()

STEP 8: Test NLTK: Naive Bayes from HW1¶

from nltk.classify import NaiveBayesClassifier
from nltk.tokenize import word_tokenize
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import *

def get_tokens(sentence):
    tokens = word_tokenize(sentence)
    clean_tokens = [word.lower() for word in tokens if word.isalpha()]
    return clean_tokens

def get_nltk_train_test(array, label, num_train):
    tokens = [get_tokens(sentence) for sentence in array]
    docs = [(sent, label) for sent in tokens]
    train_docs = docs[:num_train]
    test_docs = docs[num_train:len(array)]
    return [train_docs, test_docs]


def get_nltk_NB(NEG_DATA, POS_DATA, num_train):
    train_neg, test_neg = get_nltk_train_test(NEG_DATA, 'neg', num_train)
    train_pos, test_pos = get_nltk_train_test(POS_DATA, 'pos', num_train)

    training_docs = train_neg + train_pos
    testing_docs = test_neg + test_pos

    sentim_analyzer = SentimentAnalyzer()
    all_words_neg = sentim_analyzer.all_words([mark_negation(doc) for doc in training_docs])
    unigram_feats = sentim_analyzer.unigram_word_feats(all_words_neg)
    sentim_analyzer.add_feat_extractor(extract_unigram_feats, unigrams=unigram_feats)
    training_set = sentim_analyzer.apply_features(training_docs)
    test_set = sentim_analyzer.apply_features(testing_docs)

    trainer = NaiveBayesClassifier.train
    classifier = sentim_analyzer.train(trainer, training_set)
    
    results = []
    for key,value in sorted(sentim_analyzer.evaluate(test_set).items()):
        print('{0}: {1}'.format(key,value))

neg_df = all_df[all_df['PoN'] == 'N']
neg_df_list = neg_df[0].tolist()

pos_df = all_df[all_df['PoN'] == 'P']
pos_df_list = pos_df[0].tolist()

get_nltk_NB(neg_df_list, pos_df_list, 32)

Training classifier
Evaluating NaiveBayesClassifier results...
Accuracy: 0.75
F-measure [neg]: 0.6956521739130435
F-measure [pos]: 0.787878787878788
Precision [neg]: 0.8888888888888888
Precision [pos]: 0.6842105263157895
Recall [neg]: 0.5714285714285714
Recall [pos]: 0.9285714285714286

	0	PoN
0	I went to XYZ restaurant last week and I was v...	N
1	In each of the diner dish there are at least o...	N
2	This is the last place you would want to dine ...	N
3	I went to this restaurant where I had ordered ...	N
4	I went there with two friends at 6pm. Long que...	N
...	...	...
41	This place was one of the best restaurant I ha...	P
42	The best experience I ever had happened in Lon...	P
43	This Japanese restaurant is so popular recentl...	P
44	Hibachi the grill is one of my favorite restau...	P
45	I went to this ultra-luxurious restaurant in D...	P

	0	PoN	sentences	num_sentences	tokens	num_tokens
0	I went to XYZ restaurant last week and I was v...	N	[I went to XYZ restaurant last week and I was ...	3	[i, went, to, xyz, restaurant, last, week, and...	50
1	In each of the diner dish there are at least o...	N	[In each of the diner dish there are at least ...	4	[in, each, of, the, diner, dish, there, are, a...	78
2	This is the last place you would want to dine ...	N	[This is the last place you would want to dine...	7	[this, is, the, last, place, you, would, want,...	151
3	I went to this restaurant where I had ordered ...	N	[I went to this restaurant where I had ordered...	6	[i, went, to, this, restaurant, where, i, had,...	75
4	I went there with two friends at 6pm. Long que...	N	[I went there with two friends at 6pm., Long q...	10	[i, went, there, with, two, friends, at, long,...	73
...	...	...	...	...	...	...
41	This place was one of the best restaurant I ha...	P	[This place was one of the best restaurant I h...	6	[this, place, was, one, of, the, best, restaur...	70
42	The best experience I ever had happened in Lon...	P	[The best experience I ever had happened in Lo...	3	[the, best, experience, i, ever, had, happened...	42
43	This Japanese restaurant is so popular recentl...	P	[This Japanese restaurant is so popular recent...	12	[this, japanese, restaurant, is, so, popular, ...	88
44	Hibachi the grill is one of my favorite restau...	P	[Hibachi the grill is one of my favorite resta...	5	[hibachi, the, grill, is, one, of, my, favorit...	65
45	I went to this ultra-luxurious restaurant in D...	P	[I went to this ultra-luxurious restaurant in ...	5	[i, went, to, this, restaurant, in, downtown, ...	63

	0	PoN	sentences	num_sentences	tokens	num_tokens	no_sw	num_no_sw
0	I went to XYZ restaurant last week and I was v...	N	[I went to XYZ restaurant last week and I was ...	3	[i, went, to, xyz, restaurant, last, week, and...	50	[went, xyz, restaurant, last, week, disappoint...	25
1	In each of the diner dish there are at least o...	N	[In each of the diner dish there are at least ...	4	[in, each, of, the, diner, dish, there, are, a...	78	[diner, dish, least, one, fly, waiting, hour, ...	31
2	This is the last place you would want to dine ...	N	[This is the last place you would want to dine...	7	[this, is, the, last, place, you, would, want,...	151	[last, place, would, want, dine, price, expens...	61
3	I went to this restaurant where I had ordered ...	N	[I went to this restaurant where I had ordered...	6	[i, went, to, this, restaurant, where, i, had,...	75	[went, restaurant, ordered, complimentary, sal...	33
4	I went there with two friends at 6pm. Long que...	N	[I went there with two friends at 6pm., Long q...	10	[i, went, there, with, two, friends, at, long,...	73	[went, two, friends, long, queue, didnt, take,...	38
...	...	...	...	...	...	...	...	...
41	This place was one of the best restaurant I ha...	P	[This place was one of the best restaurant I h...	6	[this, place, was, one, of, the, best, restaur...	70	[place, one, best, restaurant, price, little, ...	32
42	The best experience I ever had happened in Lon...	P	[The best experience I ever had happened in Lo...	3	[the, best, experience, i, ever, had, happened...	42	[best, experience, ever, happened, london, bri...	21
43	This Japanese restaurant is so popular recentl...	P	[This Japanese restaurant is so popular recent...	12	[this, japanese, restaurant, is, so, popular, ...	88	[japanese, restaurant, popular, recently, japa...	49
44	Hibachi the grill is one of my favorite restau...	P	[Hibachi the grill is one of my favorite resta...	5	[hibachi, the, grill, is, one, of, my, favorit...	65	[hibachi, grill, one, favorite, restaurants, l...	30
45	I went to this ultra-luxurious restaurant in D...	P	[I went to this ultra-luxurious restaurant in ...	5	[i, went, to, this, restaurant, in, downtown, ...	63	[went, restaurant, downtown, new, york, known,...	35

	0	PoN	sentences	num_sentences	tokens	num_tokens	no_sw	num_no_sw	topwords_unfil	topwords_fil	freq_dist	freq_dist_unfil
0	I went to XYZ restaurant last week and I was v...	N	[I went to XYZ restaurant last week and I was ...	3	[i, went, to, xyz, restaurant, last, week, and...	50	[went, xyz, restaurant, last, week, disappoint...	25	[(was, 4), (to, 3), (i, 2), (and, 2), (the, 2)...	[(went, 1), (xyz, 1), (restaurant, 1), (last, ...	{'went': 1, 'xyz': 1, 'restaurant': 1, 'last':...	{'i': 2, 'went': 1, 'to': 3, 'xyz': 1, 'restau...
1	In each of the diner dish there are at least o...	N	[In each of the diner dish there are at least ...	4	[in, each, of, the, diner, dish, there, are, a...	78	[diner, dish, least, one, fly, waiting, hour, ...	31	[(the, 6), (in, 4), (to, 4), (of, 3), (that, 3...	[(want, 3), (dish, 2), (diner, 1), (least, 1),...	{'diner': 1, 'dish': 2, 'least': 1, 'one': 1, ...	{'in': 4, 'each': 1, 'of': 3, 'the': 6, 'diner...
2	This is the last place you would want to dine ...	N	[This is the last place you would want to dine...	7	[this, is, the, last, place, you, would, want,...	151	[last, place, would, want, dine, price, expens...	61	[(to, 10), (the, 9), (and, 7), (we, 5), (is, 4...	[(minutes, 3), (place, 2), (price, 2), (servic...	{'last': 1, 'place': 2, 'would': 1, 'want': 1,...	{'this': 1, 'is': 4, 'the': 9, 'last': 1, 'pla...
3	I went to this restaurant where I had ordered ...	N	[I went to this restaurant where I had ordered...	6	[i, went, to, this, restaurant, where, i, had,...	75	[went, restaurant, ordered, complimentary, sal...	33	[(i, 6), (the, 6), (to, 3), (for, 3), (salad, ...	[(salad, 3), (restaurant, 2), (waiter, 2), (as...	{'went': 1, 'restaurant': 2, 'ordered': 1, 'co...	{'i': 6, 'went': 1, 'to': 3, 'this': 1, 'resta...
4	I went there with two friends at 6pm. Long que...	N	[I went there with two friends at 6pm., Long q...	10	[i, went, there, with, two, friends, at, long,...	73	[went, two, friends, long, queue, didnt, take,...	38	[(there, 3), (but, 3), (it, 3), (a, 3), (i, 2)...	[(two, 2), (friends, 2), (long, 2), (didnt, 2)...	{'went': 1, 'two': 2, 'friends': 2, 'long': 2,...	{'i': 2, 'went': 1, 'there': 3, 'with': 1, 'tw...
...	...	...	...	...	...	...	...	...	...	...	...	...
41	This place was one of the best restaurant I ha...	P	[This place was one of the best restaurant I h...	6	[this, place, was, one, of, the, best, restaur...	70	[place, one, best, restaurant, price, little, ...	32	[(the, 5), (i, 3), (and, 3), (this, 2), (best,...	[(best, 2), (area, 2), (place, 1), (one, 1), (...	{'place': 1, 'one': 1, 'best': 2, 'restaurant'...	{'this': 2, 'place': 1, 'was': 1, 'one': 1, 'o...
42	The best experience I ever had happened in Lon...	P	[The best experience I ever had happened in Lo...	3	[the, best, experience, i, ever, had, happened...	42	[best, experience, ever, happened, london, bri...	21	[(the, 3), (in, 3), (food, 2), (a, 2), (was, 2...	[(food, 2), (best, 1), (experience, 1), (ever,...	{'best': 1, 'experience': 1, 'ever': 1, 'happe...	{'the': 3, 'best': 1, 'experience': 1, 'i': 1,...
43	This Japanese restaurant is so popular recentl...	P	[This Japanese restaurant is so popular recent...	12	[this, japanese, restaurant, is, so, popular, ...	88	[japanese, restaurant, popular, recently, japa...	49	[(is, 4), (the, 4), (japanese, 2), (a, 2), (fo...	[(japanese, 2), (food, 2), (right, 2), (restau...	{'japanese': 2, 'restaurant': 1, 'popular': 1,...	{'this': 1, 'japanese': 2, 'restaurant': 1, 'i...
44	Hibachi the grill is one of my favorite restau...	P	[Hibachi the grill is one of my favorite resta...	5	[hibachi, the, grill, is, one, of, my, favorit...	65	[hibachi, grill, one, favorite, restaurants, l...	30	[(the, 8), (is, 6), (it, 3), (hibachi, 2), (gr...	[(hibachi, 2), (grill, 2), (restaurants, 2), (...	{'hibachi': 2, 'grill': 2, 'one': 1, 'favorite...	{'hibachi': 2, 'the': 8, 'grill': 2, 'is': 6, ...
45	I went to this ultra-luxurious restaurant in D...	P	[I went to this ultra-luxurious restaurant in ...	5	[i, went, to, this, restaurant, in, downtown, ...	63	[went, restaurant, downtown, new, york, known,...	35	[(i, 4), (this, 3), (and, 3), (restaurant, 2),...	[(restaurant, 2), (expensive, 2), (went, 1), (...	{'went': 1, 'restaurant': 2, 'downtown': 1, 'n...	{'i': 4, 'went': 1, 'to': 1, 'this': 3, 'resta...

HW2: VECTORIZATION (Pandas style!)¶

STEP 1: Import ALL the things¶

Import libraries¶

Import data from files¶

STEP 2: Prep Data¶

STEP 2a: Turn that fresh text into a pandas DF¶

STEP 2b: Label it¶

STEP 2c: Combine the dfs¶

STEP 3: TOKENIZE (and clean)!!¶

STEP 4: Remove Stopwords¶

STEP 5: Create a Frequency Distribution¶

STEP 6: Try Different Sentiment Analysis Tools¶

VADER¶

DIY SUMMARY¶

Doing VADER on the Summary Section¶

Doing VADER on the Most Frequent Words¶

STEP 7: Test Step 6 with Machine Learning!!¶

Naive Bayes¶

TEST 1: Vader Scores (Original)¶

TEST 2: Vader Scores (from Summary)¶

TEST 3: Vader Scores (original) AND Vader Scores (summary)¶

TEST 4: Vader Scores (50 most frequent -- filtered -- words)¶

TEST 5: All compound Vader Scores¶

TEST 6: ALL THE NUMBERS!!¶

TEST 7: Test UNFILTERED most frequent words¶

STEP 8: Test NLTK: Naive Bayes from HW1¶

STEP 7: Test `Step 6` with Machine Learning!!¶

TEST 5: All `compound` Vader Scores¶