In [1]:
import os
import tarfile
import urllib
In [2]:
DOWNLOAD_ROOT = "https://raw.githubusercontent.com/ageron/handson-ml2/master/"
HOUSING_PATH = os.path.join("datasets", "housing")
HOUSING_URL = DOWNLOAD_ROOT + "datasets/housing/housing.tgz"
In [5]:
def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
os.makedirs(housing_path, exist_ok=True)
tgz_path = os.path.join(housing_path, "housing.tgz")
urllib.request.urlretrieve(housing_url, tgz_path)
housing_tgz = tarfile.open(tgz_path)
housing_tgz.extractall(path=housing_path)
housing_tgz.close()
In [6]:
fetch_housing_data()
In [9]:
import pandas as pd
def load_housing_data(housing_path=HOUSING_PATH):
csv_path = os.path.join(housing_path, "housing.csv")
return pd.read_csv(csv_path)
In [10]:
housing = load_housing_data()
In [11]:
housing.head()
Out[11]:
In [12]:
housing.info()
In [13]:
housing['ocean_proximity'].value_counts()
Out[13]:
In [15]:
%matplotlib inline
import matplotlib.pyplot as plt
housing.hist(bins=50, figsize=(20,15))
plt.show()
In [19]:
import numpy as np
def split_train_test(data, test_ratio):
shuffled_indices = np.random.permutation(len(data))
test_set_size = int(len(data) * test_ratio)
test_indices = shuffled_indices[:test_set_size]
train_indices = shuffled_indices[test_set_size:]
return data.iloc[train_indices], data.iloc[test_indices]
In [21]:
train_set, test_set = split_train_test(housing, 0.2)
In [22]:
len(train_set)
Out[22]:
In [23]:
len(test_set)
Out[23]:
In [ ]:
In [24]:
housing.plot(kind="scatter", x="longitude", y="latitude")
Out[24]:
In [27]:
housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.1)
Out[27]:
In [28]:
housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.4,
s=housing["population"]/100, label="population", figsize=(10,7),
c="median_house_value", cmap=plt.get_cmap("jet"), colorbar=True,)
plt.legend()
Out[28]:
Looking For Correlations¶
NOTES:
- A clustering algo could be useful for detecting the main cluster and for adding new features that measure the proximity to the cluster events
- Correlation Coefficient ranges from -1 to 1
- When it is close to 1, strong positive linear relationshiop (EX: Median house value increases as median income increases)
- When it is close to -1, strong negative linear relationship
- Correlation Coefficient only measures linear relationships
In [31]:
corr_matrix = housing.corr()
In [33]:
corr_matrix
Out[33]:
In [34]:
corr_matrix["median_house_value"].sort_values(ascending=False)
Out[34]:
In [35]:
from pandas.plotting import scatter_matrix
attributes = ["median_house_value", "median_income", "total_rooms", "housing_median_age"]
scatter_matrix(housing[attributes], figsize=(12,8))
Out[35]:
NOTE: This plots every numerical atribute against every ther numerical attribute, plus a nistogram of each numerical attribute
In [36]:
housing.plot(kind="scatter", x="median_income", y="median_house_value", alpha=0.1)
Out[36]:
Experimenting with Attribute Combinations¶
In [38]:
housing["rooms_per_household"] = housing["total_rooms"]/housing["households"]
housing["bedrooms_per_room"] = housing["total_bedrooms"]/housing["total_rooms"]
housing["population_per_household"] = housing["population"]/housing["households"]
In [39]:
corr_matrix = housing.corr()
In [40]:
corr_matrix["median_house_value"].sort_values(ascending=False)
Out[40]:
Prepare the Data for Machine Learning Algorithms¶
NOTES:
- write functions for this
In [ ]: