1. Look at the Big Picture¶
- Frame the problem
- Select a performance measure
- Check assumptions
2. Get the Data¶
In [80]:
import os
import tarfile
import urllib
In [81]:
DOWNLOAD_ROOT = "https://raw.githubusercontent.com/ageron/handson-ml2/master/"
HOUSING_PATH = os.path.join("datasets", "housing")
HOUSING_URL = DOWNLOAD_ROOT + "datasets/housing/housing.tgz"
In [82]:
def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
os.makedirs(housing_path, exist_ok=True)
tgz_path = os.path.join(housing_path, "housing.tgz")
urllib.request.urlretrieve(housing_url, tgz_path)
housing_tgz = tarfile.open(tgz_path)
housing_tgz.extractall(path=housing_path)
housing_tgz.close()
In [83]:
fetch_housing_data()
In [84]:
import pandas as pd
def load_housing_data(housing_path=HOUSING_PATH):
csv_path = os.path.join(housing_path, "housing.csv")
return pd.read_csv(csv_path)
In [85]:
housing = load_housing_data()
-- 2: Take a Quick Look at the Data Structure¶
In [86]:
housing.head()
Out[86]:
In [87]:
housing.info()
In [88]:
housing['ocean_proximity'].value_counts()
Out[88]:
In [89]:
%matplotlib inline
import matplotlib.pyplot as plt
housing.hist(bins=50, figsize=(20,15))
plt.show()
-- 2. Create a Test Set¶
In [90]:
import numpy as np
def split_train_test(data, test_ratio):
shuffled_indices = np.random.permutation(len(data))
test_set_size = int(len(data) * test_ratio)
test_indices = shuffled_indices[:test_set_size]
train_indices = shuffled_indices[test_set_size:]
return data.iloc[train_indices], data.iloc[test_indices]
In [91]:
train_set, test_set = split_train_test(housing, 0.2)
In [92]:
len(train_set)
Out[92]:
In [93]:
len(test_set)
Out[93]:
In [94]:
from sklearn.model_selection import train_test_split
train_set, test_set = train_test_split(housing, test_size=0.2, random_state=42)
In [95]:
# Insure that there is a proportional number of each cagetory in the test set
housing["income_cat"] = pd.cut(housing["median_income"],
bins=[0.,1.5,3.0,4.5,6., np.inf],
labels=[1,2,3,4,5])
housing["income_cat"].hist()
Out[95]:
In [96]:
from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(housing, housing["income_cat"]):
strat_train_set = housing.loc[train_index]
strat_test_set = housing.loc[test_index]
In [97]:
strat_test_set["income_cat"].value_counts() / len(strat_test_set)
Out[97]:
In [98]:
# Remove the `income cat` attribute so data is bad to original
for set_ in (strat_train_set, strat_test_set):
set_.drop("income_cat", axis=1, inplace=True)
In [99]:
housing.plot(kind="scatter", x="longitude", y="latitude")
Out[99]:
In [100]:
housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.1)
Out[100]:
In [101]:
housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.4,
s=housing["population"]/100, label="population", figsize=(10,7),
c="median_house_value", cmap=plt.get_cmap("jet"), colorbar=True,)
plt.legend()
Out[101]:
-- 3. Looking For Correlations¶
NOTES:
- A clustering algo could be useful for detecting the main cluster and for adding new features that measure the proximity to the cluster events
- Correlation Coefficient ranges from -1 to 1
- When it is close to 1, strong positive linear relationshiop (EX: Median house value increases as median income increases)
- When it is close to -1, strong negative linear relationship
- Correlation Coefficient only measures linear relationships
In [102]:
corr_matrix = housing.corr()
In [103]:
corr_matrix
Out[103]:
In [104]:
corr_matrix["median_house_value"].sort_values(ascending=False)
Out[104]:
In [105]:
from pandas.plotting import scatter_matrix
attributes = ["median_house_value", "median_income", "total_rooms", "housing_median_age"]
scatter_matrix(housing[attributes], figsize=(12,8))
Out[105]:
NOTE: This plots every numerical atribute against every ther numerical attribute, plus a nistogram of each numerical attribute
In [106]:
housing.plot(kind="scatter", x="median_income", y="median_house_value", alpha=0.1)
Out[106]:
-- 3. Experimenting with Attribute Combinations¶
In [107]:
housing["rooms_per_household"] = housing["total_rooms"]/housing["households"]
housing["bedrooms_per_room"] = housing["total_bedrooms"]/housing["total_rooms"]
housing["population_per_household"] = housing["population"]/housing["households"]
In [108]:
corr_matrix = housing.corr()
In [109]:
corr_matrix["median_house_value"].sort_values(ascending=False)
Out[109]:
4. Prepare the Data for Machine Learning Algorithms¶
NOTES:
- write functions for this
In [111]:
housing = strat_train_set.drop("median_house_value", axis=1)
housing_labels = strat_train_set["median_house_value"].copy()
-- 4. Data Cleaning¶
What do we do with missing features? Either:
- Get rid of the districts
- Get rid of the whole attribute (bedrooms)
- Replace the value with some value (zero, mean, median)
In [113]:
housing.dropna(subset=["total_bedrooms"]) # option 1
housing.drop("total_bedrooms", axis=1) # option 2
median = housing["total_bedrooms"].median() # option 3
housing["total_bedrooms"].fillna(median, inplace=True)
In [115]:
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy="median")
# NOTE: median can only be computed with numerical attributes so we have to drop "ocean proximity"
housing_num = housing.drop("ocean_proximity", axis=1)
In [116]:
imputer.fit(housing_num)
Out[116]:
In [117]:
imputer.statistics_
Out[117]:
In [118]:
housing_num.median().values
Out[118]:
In [119]:
X = imputer.transform(housing_num)
housing_tr = pd.DataFrame(X, columns=housing_num.columns, index=housing_num.index)
-- 4. Handling Text and Categorical Attributes¶
In [120]:
housing_cat = housing[["ocean_proximity"]]
housing_cat.head()
Out[120]:
In [ ]: