IST 718 | LAB 3¶
01-23-20
V1 DATA SOURCE: Coaches9¶
O: OBTAIN¶
In [32]:
import pandas as pd
data = pd.read_csv("Coaches9.csv")
In [33]:
data.head()
Out[33]:
S: SCRUB¶
- Change the columns with dollar amounts as strings to floats
- Inspect and possibly replace missing values
In [34]:
attrs = list(data[data.columns[3:]].columns)
data[attrs] = data[attrs].replace({'\$':'', ',': '', '--':None}, regex=True).astype('float')
In [35]:
data.dtypes
Out[35]:
In [36]:
data.isnull().sum()
Out[36]:
Replacing Missing Values¶
Four options:
- Drop the schools with missing values
- Drop the attributes with missing values (e.g. 'BonusPaid')
- Replace the missing values with some value (e.g. mean, 0)
- Find additional data
For V1, we are going to
- Drop the schools without
SchoolPay - Drop the attribute
BonusPaid - Fill
Bonus&Buyoutwith the attribute mean*
For V2, we are going to use #4 and a more advanced version of #3* taking within conference mean
In [37]:
data.dropna(subset=['SchoolPay'], inplace=True)
In [38]:
data.drop('BonusPaid', axis=1, inplace=True)
In [39]:
median_bonus = data['Bonus'].median()
median_buyout = data['Buyout'].median()
data["Bonus"].fillna(median_bonus, inplace = True)
data["Buyout"].fillna(median_buyout, inplace = True)
In [40]:
data.isnull().sum()
Out[40]:
E: EXPLORE¶
In [41]:
%matplotlib inline
import matplotlib.pyplot as plt
In [42]:
from pandas.plotting import scatter_matrix
scatter_matrix(data, figsize=(12,8))
plt.show()
From these two visualizations, it's clear we need to:
- Drop either
SchoolPayorTotalPay - Remove (or supplement)
AssistantPay
In [43]:
data.drop(['TotalPay','AssistantPay'], axis=1, inplace=True)
scatter_matrix(data, figsize=(12,8))
plt.show()
In [44]:
data.head()
Out[44]:
Add More Data¶
We've lost a lot of data at this point and we need to add more. To do so we:
- O: Obtain data
- S: Scrub data
- M: MERGE data! Add a "key" to both columns for merging
This sub OSM occured in an attached auxiliary file
In [52]:
data['school_key'] = data.apply(lambda x: ''.join(x['School'].lower().split(' '))[:8], axis=1)
In [53]:
data
Out[53]:
In [55]:
winloss = pd.read_csv('winloss_withkey.csv')
In [56]:
test = data.copy()
In [59]:
data_wl = test.merge(winloss, left_on="school_key", right_on="team_key")
In [60]:
data_wl
Out[60]:
In [66]:
data_for_m = data_wl.drop(['school_key','Team','team_key'], axis=1)
In [67]:
columns = ['school','conf','coach','school_pay', 'bonus', 'buyout', 'rank', 'W','L', 'T', 'ratio']
data_for_m.columns = columns
data_for_m
Out[67]:
In [68]:
scatter_matrix(data_for_m, figsize=(12,8))
plt.show()
In [69]:
data_for_m.drop('T', axis=1, inplace=True)
In [71]:
data = data_for_m.copy()
In [72]:
from sklearn.model_selection import train_test_split
train_set, test_set = train_test_split(data, test_size=0.2, random_state=42)
data = train_set.drop('school_pay', axis=1)
data_labels = train_set['school_pay'].copy()
In [73]:
from sklearn.compose import ColumnTransformer
data_num = data.drop(['school', 'conf', 'coach'],axis =1)
num_attribs = list(data_num)
cat_attribs = ['school','conf','coach']
In [74]:
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
num_pipeline = Pipeline([
('imputer', SimpleImputer(strategy="median")),
# ('attribs_addr', CombinedAttributesAdder()),
('std_scaler', StandardScaler()),
])
In [75]:
full_pipeline = ColumnTransformer([
('num', num_pipeline, num_attribs),
('cat', OneHotEncoder(), cat_attribs)
])
In [76]:
data_prepared = full_pipeline.fit_transform(data)
LINEAR REGRESSION¶
In [77]:
from sklearn.linear_model import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit(data_prepared, data_labels)
Out[77]:
In [78]:
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import cross_val_score
import numpy as np
def display_scores(scores):
print("Scores:", scores)
print("Mean:", scores.mean())
print("Stardard Deviation:", scores.std())
In [79]:
scores = cross_val_score(lin_reg, data_prepared, data_labels, scoring="neg_mean_squared_error", cv=10)
lin_reg_rmse_scores = np.sqrt(-scores)
display_scores(lin_reg_rmse_scores)
DECISION TREE¶
In [80]:
from sklearn.tree import DecisionTreeRegressor
tree_reg = DecisionTreeRegressor()
tree_reg.fit(data_prepared, data_labels)
Out[80]:
In [81]:
scores = cross_val_score(tree_reg, data_prepared, data_labels, scoring="neg_mean_squared_error", cv=10)
tree_reg_rmse_scores = np.sqrt(-scores)
display_scores(tree_reg_rmse_scores)
RANDOM FOREST¶
In [82]:
from sklearn.ensemble import RandomForestRegressor
forest_reg = RandomForestRegressor()
forest_reg.fit(data_prepared, data_labels)
Out[82]:
In [83]:
scores = cross_val_score(forest_reg, data_prepared, data_labels, scoring="neg_mean_squared_error", cv=10)
forest_reg_rmse_scores = np.sqrt(-scores)
display_scores(forest_reg_rmse_scores)
In [ ]: