In [ ]:
In [1]:
import urllib.request, json
import pandas as pd
all_jobs = []
def get_data(i):
url = "https://www.amazon.jobs/en/search.json?radius=24km&facets[]=location&facets[]=business_category&facets[]=category&facets[]=schedule_type_id&facets[]=employee_class&facets[]=normalized_location&facets[]=job_function_id&offset={}&result_limit=10&sort=relevant&latitude=&longitude=&loc_group_id=&loc_query=&base_query=data%20scientist&city=&country=®ion=&county=&query_options=&".format(i)
with urllib.request.urlopen(url) as url:
data = json.loads(url.read().decode())
return data
def add_data_to_df(data):
for job in data['jobs']:
all_jobs.append(job)
def do_the_thing():
i = 10
# while i < 21: for testing lololol
while i < 3071:
data = get_data(i)
add_data_to_df(data)
i += 10
do_the_thing()
df = pd.DataFrame(all_jobs)
In [2]:
def get_label(team):
try:
return team['label']
except:
return 'no label'
df['team-label'] = df.apply(lambda x: get_label(x['team']), axis=1)
df.to_csv('aws_jobs_20210316.csv')
Remove anything outside the US¶
In [5]:
print('BEFORE', len(df))
df_us = df[df['country_code'] == 'US']
print('AFTER', len(df_us))
Keep only jobs that include "data science" or "data scientist" in the title¶
In [49]:
df = df_us.copy()
def get_ds(title):
if 'data scien' in title.lower():
return True
else:
return False
df['is_ds'] = df.apply(lambda x: get_ds(x['title']), axis=1)
df_ds = df[df['is_ds'] == True]
In [50]:
print('BEFORE', len(df_us))
print('AFTER', len(df_ds))
Remove "Senior" or "Sr" jobs¶
In [70]:
df = df_ds.copy()
def no_sr(title):
if 'sr.' in title.lower() or 'senior' in title.lower() or 'sr ' in title.lower():
return True
else:
return False
df['is_sr'] = df.apply(lambda x: no_sr(x['title']), axis=1)
df_sr = df[df['is_sr'] == False]
In [71]:
print('BEFORE', len(df_ds))
print('AFTER', len(df_sr))
Remove "Principal" or "Manager"¶
In [72]:
df = df_sr.copy()
def no_sr(title):
if 'principal' in title.lower() or 'manager' in title.lower():
return True
else:
return False
df['is_pm'] = df.apply(lambda x: no_sr(x['title']), axis=1)
df_pm = df[df['is_pm'] == False]
In [73]:
print('BEFORE', len(df_sr))
print('AFTER', len(df_pm))
How many of these 226 have unique "basic qualifications"?¶
In [75]:
df = df_pm.copy()
len(df['basic_qualifications'].value_counts())
Out[75]:
How many of these 226 have the same job title?¶
In [76]:
len(df['title'].value_counts())
Out[76]:
What's the job distribution for business category?¶
In [79]:
df['business_category'].value_counts()
Out[79]:
In [85]:
df['business_category'].value_counts().plot(kind = 'barh')
Out[85]:
In [86]:
len(df['team-label'].value_counts())
Out[86]:
In [87]:
df['title'].value_counts()
Out[87]:
In [88]:
len(df['title'].value_counts())
Out[88]:
In [39]:
# df = df_sr.copy()
# df_grouped = pd.DataFrame(df.groupby('basic_qualifications')['title'])
In [40]:
df
Out[40]:
In [ ]:
In [41]:
import numpy as np
table = pd.pivot_table(df, values=['business_category'], index=['title', 'basic_qualifications'],
aggfunc={'business_category': np.sum})
In [42]:
len(table)
Out[42]:
In [43]:
table
Out[43]:
In [45]:
len(df)
Out[45]:
In [47]:
len(df['title'].value_counts())
Out[47]:
In [ ]: