STEP 1: Import ALL the things!¶
In [1]:
import numpy as np
import pandas as pd
## dealing with categorical variables
from sklearn.preprocessing import LabelEncoder
import os
import warnings
warnings.filterwarnings('ignore')
import matplotlib.pyplot as plt
import seaborn as sns
In [2]:
app_train = pd.read_csv('application_train.csv')
app_train.shape
Out[2]:
In [3]:
app_test = pd.read_csv('application_test.csv')
In [4]:
app_train['TARGET'].value_counts()
Out[4]:
2b. Are there missing values?¶
In [5]:
df_na = pd.DataFrame(app_train.isna().sum())
df_na['percent'] = (df_na[0] / app_train.shape[0]) *100
df_na.sort_values(by="percent", ascending = False)
Out[5]:
2bi -- Options for handling missing data¶
- imputation
- XGBoost
2c. How will we handle our categorical variables?¶
- One-hot encoding?
- Labels?
2d. What should we do with outliers?¶
STEP 3: Find relationships!¶
In [6]:
correlations = app_train.corr()['TARGET'].sort_values()
In [7]:
correlations.tail(20)
Out[7]:
Negative Correlations¶
In [8]:
correlations.head(20)
Out[8]:
In [9]:
app_train['DAYS_BIRTH'] = abs(app_train['DAYS_BIRTH'])
plt.style.use('fivethirtyeight')
plt.hist(app_train['DAYS_BIRTH'] / 365, edgecolor = 'k', bins = 25)
plt.title('Age of Client'); plt.xlabel('Age (years)'); plt.ylabel('Count');
In [10]:
app_train['DAYS_LAST_PHONE_CHANGE'] = abs(app_train['DAYS_LAST_PHONE_CHANGE'])
plt.style.use('fivethirtyeight')
plt.hist(app_train['DAYS_LAST_PHONE_CHANGE'] / 365, edgecolor = 'k', bins = 25)
plt.title('Days Since Phone Change'); plt.xlabel('Days'); plt.ylabel('Count');
In [11]:
type(correlations.tail(20))
Out[11]:
In [12]:
high_corr = list(correlations.tail(20).axes[0])
high_corr
Out[12]:
In [13]:
high_corr_for_graphs = ['CNT_FAM_MEMBERS',
'CNT_CHILDREN',
'AMT_REQ_CREDIT_BUREAU_YEAR',
'OWN_CAR_AGE',
'DAYS_REGISTRATION',
'DAYS_ID_PUBLISH',
'DAYS_LAST_PHONE_CHANGE',
'DAYS_BIRTH']
for col in high_corr_for_graphs:
# print(app_train[col].value_counts())
# print(app_train[col].dtype)
df = app_train.copy()
df[col] = abs(df[col])
plt.style.use('fivethirtyeight')
plt.hist(df[col] / 365, edgecolor = 'k', bins = 25)
plt.title(col); plt.xlabel('x'); plt.ylabel('Count');
plt.show()
3b. A closer look at age¶
(using a Kernel Density Estimation Plot!!)
In [14]:
plt.figure(figsize = (10, 8))
sns.kdeplot(app_train.loc[app_train['TARGET'] == 0, 'DAYS_BIRTH'] / 365, label = 'target == 0')
# KDE plot of loans which were not repaid on time
sns.kdeplot(app_train.loc[app_train['TARGET'] == 1, 'DAYS_BIRTH'] / 365, label = 'target == 1')
# Labeling of plot
plt.xlabel('Age (years)'); plt.ylabel('Density'); plt.title('Distribution of Ages');
Bin the age data¶
In [15]:
age_data = app_train[['TARGET', 'DAYS_BIRTH']]
age_data['YEARS_BIRTH'] = age_data['DAYS_BIRTH'] / 360
In [17]:
## This helps us get 20-25, 25-30 etc
age_data['YEARS_BINNED'] = pd.cut(age_data['YEARS_BIRTH'], bins = np.linspace(20,70, num = 11))
age_data.head(10)
Out[17]:
Group by the bin and get averages¶
In [18]:
age_groups = age_data.groupby('YEARS_BINNED').mean()
age_groups
Out[18]:
Plot our newly binned data¶
In [19]:
plt.figure(figsize=(8,8))
plt.bar(age_groups.index.astype(str), 100*age_groups['TARGET'])
plt.xticks(rotation = 75); plt.xlabel('Age Group (years)'); plt.ylabel('Failure to Repay (%)')
plt.title('Failure to Repay by Age Group');
FIRST BIT OF INFORMATION FOR STAKEHOLDERS:¶
"It appears that younger applicants are more likely to not repay their loans. Helping younger applicants with financial planning and guidance might help mitigate this" "
Exploring Negative Correlations¶
In [20]:
ext_data = app_train[['TARGET', 'EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3', 'DAYS_BIRTH']]
ext_data_corrs = ext_data.corr()
ext_data_corrs
Out[20]:
In [21]:
plt.figure(figsize = (8, 6))
sns.heatmap(ext_data_corrs, cmap = plt.cm.RdYlBu_r, vmin = -0.25, annot = True, vmax = 0.6)
plt.title('Correlation Heatmap');
In [ ]: