In [1]:
Out[1]:
In [2]:
collisions.cache()
Out[2]:
In [3]:
# STEP 3 - Print Schema
print("Records: {}".format(collisions.count()))
collisions.printSchema()
In [4]:
# STEP 4 - Load visualization packages
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
%matplotlib inline
sns.set_style("white")
plt.rcParams['figure.figsize'] = (12, 12)
In [5]:
# STEP 5 - SCRUB
collisions_df = collisions
collisions_pd = collisions_df[collisions_df['LATITUDE'] != 0][['LATITUDE',
'LONGITUDE',
'CRASH DATE',
'CRASH TIME',
'BOROUGH',
'ON STREET NAME',
'CROSS STREET NAME',
'NUMBER OF PERSONS INJURED',
'NUMBER OF PERSONS KILLED',
'CONTRIBUTING FACTOR VEHICLE 1']].toPandas()
collisions_pd.columns = ['Latitude', 'Longitude', 'Date', 'Time', 'Borough', 'On Street',
'Cross Street', 'Persons Injured', 'Persons Killed', 'Contributing Factor']
collisions_pd['Latitude'] = collisions_pd['Latitude'].astype(float)
collisions_pd['Longitude'] = collisions_pd['Longitude'].astype(float)
collisions_pd['Persons Killed'] = collisions_pd['Persons Killed'].astype(float)
collisions_pd['Persons Injured'] = collisions_pd['Persons Injured'].astype(float)
#divide dataset in accidents which are: fatal, non-lethal but with person damage, non of the above
killed_pd = collisions_pd[collisions_pd['Persons Killed']!=0]
injured_pd = collisions_pd[np.logical_and(collisions_pd['Persons Injured']!=0, collisions_pd['Persons Killed']==0)]
nothing_pd = collisions_pd[np.logical_and(collisions_pd['Persons Killed']==0, collisions_pd['Persons Injured']==0)]
In [6]:
# STEP 6 - EXPLORE
#create scatterplots
plt.figure(figsize=(20,20))
plt.scatter(collisions_pd.Longitude, collisions_pd.Latitude, alpha=0.05, s=4, color='darkseagreen')
#adjust more settings
plt.title('Motor Vehicle Collisions in New York City', size=25)
plt.xlim((-74.26,-73.7))
plt.ylim((40.5,40.92))
plt.xlabel('Longitude',size=20)
plt.ylabel('Latitude',size=20)
plt.show()
In [43]:
borough = collisions_df.groupby('BOROUGH').count().sort('count').toPandas().iloc[1:,:]
borough['BOROUGH'] = [u'STATEN ISLAND', u'BRONX', u'MANHATTAN', u'QUEENS',u'BROOKLYN',u'None']
colors = ['g','0.75','y','k','b','r']
borough.sort_values(by='count', ascending=True)['count'].plot.barh(color=colors)
plt.xlabel('Collisions')
plt.ylabel('Borough')
plt.title('Total Number of Collisions by Borough', size=15)
plt.yticks(range(1,6), borough['BOROUGH'])
plt.tight_layout()
plt.show()
In [10]:
## Because I love playing around, I wanted to "automate" this instead of typing it out for each Borough
manhattan = collisions_pd[collisions_pd['Borough'] == 'MANHATTAN']
all_boroughs = collisions_pd.Borough.unique()
## FIRST ATTEMPT
## creating a dictionary
# d = {"{}".format(val): collisions_pd[collisions_pd['Borough'] == val] for val in all_boroughs}
# for b in all_boroughs:
# if b != None:
# print(d[b].head())
## And iteratively assigning variables... by copying and printing print statments
# for b in all_boroughs:
# if b != None:
# subset = "= d['"+b+"']"
# print(b.lower(), subset)
In [114]:
plt.figure(figsize=(20,20))
plt.scatter(nothing_pd.Longitude, nothing_pd.Latitude, alpha=0.04, s=1, color = 'blue')
plt.scatter(injured_pd.Longitude, injured_pd.Latitude, alpha=0.1, s=1, color = 'yellow')
plt.scatter(killed_pd.Longitude, killed_pd.Latitude,s=5, color = 'red')
blue_patch = mpatches.Patch( label= 'car body damage', alpha = 0.2, color = 'blue')
yellow_patch = mpatches.Patch( label= 'personal injury', alpha = 0.5, color = 'yellow')
red_patch = mpatches.Patch( label= 'lethal accidents', color = 'red')
plt.legend([blue_patch, yellow_patch, red_patch], ('car body damage', 'personal injury', 'fatal accidents'),
loc='upper left', prop={'size':20})
plt.title('Severity of Motor Vehicle Collisions in New York City', size=20)
plt.xlim((-74.26,-73.7))
plt.ylim((40.5,40.92))
plt.xlabel('Longitude', size=20)
plt.ylabel('Latitude', size=20)
plt.savefig('NYCaccidents.png')
plt.show()
In [85]:
## What are the other things we need to look at now regarding traffic?
## Should we condition it on population data
## How much is population density and how much is population movement?
In [99]:
## SECOND ATTEMPT
# I realized that I could just print out the statements using string interpolation (DUH)
# And then copy paste those statements below (much faster)
all_boroughs
Out[99]:
In [107]:
## CREATE SUBSET STATEMENTS
for b in all_boroughs:
if b != None:
subset = "= collisions_pd[collisions_pd['Borough'] == '"+b+"']"
print(b.lower().replace(' ',''), subset)
In [116]:
## CREATE SCATTER PLOT STATEMENTS
colors = "white blue black red green yellow"
colors = colors.split()
for borough, c in zip(all_boroughs, colors):
if borough != None:
b = borough.lower().replace(' ', '')
statement = 'plt.scatter('+b+'.Longitude,'+b+'.Latitude, s=1, color="'+c+'", marker=".")'
print(statement)
In [117]:
## CREATE LEGEND STATEMENTS
for borough, c in zip(all_boroughs, colors):
if borough != None:
b = borough.title()
statement = c+"_patch = mpatches.Patch(color='"+c+"', label='"+b+"')"
print(statement)
In [120]:
plt.figure(figsize=(20,20))
brooklyn = collisions_pd[collisions_pd['Borough'] == 'BROOKLYN']
manhattan = collisions_pd[collisions_pd['Borough'] == 'MANHATTAN']
queens = collisions_pd[collisions_pd['Borough'] == 'QUEENS']
bronx = collisions_pd[collisions_pd['Borough'] == 'BRONX']
statenisland = collisions_pd[collisions_pd['Borough'] == 'STATEN ISLAND']
plt.scatter(brooklyn.Longitude,brooklyn.Latitude, s=1, color="blue", marker=".")
plt.scatter(manhattan.Longitude,manhattan.Latitude, s=1, color="black", marker=".")
plt.scatter(queens.Longitude,queens.Latitude, s=1, color="red", marker=".")
plt.scatter(bronx.Longitude,bronx.Latitude, s=1, color="green", marker=".")
plt.scatter(statenisland.Longitude,statenisland.Latitude, s=1, color="yellow", marker=".")
blue_patch = mpatches.Patch(color='blue', label='Brooklyn')
black_patch = mpatches.Patch(color='black', label='Manhattan')
red_patch = mpatches.Patch(color='red', label='Queens')
green_patch = mpatches.Patch(color='green', label='Bronx')
yellow_patch = mpatches.Patch(color='yellow', label='Staten Island')
plt.legend([blue_patch, black_patch, red_patch, green_patch, yellow_patch], ([b.title() for b in all_boroughs if b != None]),
loc='upper left', prop={'size':20})
plt.title('Severity of Motor Vehicle Collisions by NYC Borough', size=20)
plt.xlim((-74.26,-73.7))
plt.ylim((40.5,40.92))
plt.xlabel('Longitude', size=20)
plt.ylabel('Latitude', size=20)
plt.savefig('NYCaccidentsByBorough.png')
plt.show()
In [ ]:
In [ ]: