In [ ]:
# GETTING THE IDS FROM IMDB:
from bs4 import BeautifulSoup
import pandas as pd
import requests
import time
import re
def get_movie_url(year):
url = ('https://www.imdb.com/search/title/?title_type=feature&boxoffice_gross_us=1,&release_date='+str(year)+'-01-01,'+str(year)+'-12-31&countries=us&view=simple&count=250')
url2 = ('https://www.imdb.com/search/title/?title_type=feature&boxoffice_gross_us=1,&&release_date='+str(year)+'-01-01,'+str(year)+'-12-31&countries=us&view=simple&count=250&start=251')
headers = {'Accept-Language': 'en-US'}
movies_html = requests.get(url.format(), headers=headers).content
movies2_html = requests.get(url2.format(), headers=headers).content
soup = BeautifulSoup(movies_html, 'html.parser')
soup2 = BeautifulSoup(movies2_html, 'html.parser')
soup_main = soup.find("div", {"id": "main"})
soup_main2 = soup2.find("div", {"id": "main"})
movies = soup_main.findAll('a', {'href': re.compile('/title/tt')})
movies2 = soup_main2.findAll('a', {'href': re.compile('/title/tt')})
# create an array to house all **UNIQUE** ids for each movie
ids_for_movies_in_year = []
for movie in movies:
# Getting the movie attributes, splitting them on '/'
title_id = movie.attrs['href'].split('/')[2]
# Here is where we are saying "does this ID already exist in our array? No? OK add it!"
if title_id not in ids_for_movies_in_year:
ids_for_movies_in_year.append(title_id)
# Testing
# print(movie.attrs['href'].split('/')[2])
for movie in movies2:
# Getting the movie attributes, splitting them on '/'
title_id = movie.attrs['href'].split('/')[2]
# Here is where we are saying "does this ID already exist in our array? No? OK add it!"
if title_id not in ids_for_movies_in_year:
ids_for_movies_in_year.append(title_id)
# Testing
# print(movie.attrs['href'].split('/')[2])
return(ids_for_movies_in_year)
all_the_ids = []
for year in range(1970, 2020):
all_the_ids.append((get_movie_url(year)))
all_the_ids_df = pd.DataFrame(all_the_ids)
all_the_ids_df.to_csv(r'imdb_ids.csv')
all_the_2018_ids_df = pd.DataFrame(all_the_ids[48])
all_the_2018_ids_df.to_csv(r'imdb_ids_2018.csv')
In [ ]:
# GETTING THE DATA FROM THE TMDB API (THE JSON!):
# import requests
import json
import csv
from urllib.parse import quote
def write_csv(data):
df = pd.DataFrame(data)
df.to_csv('2018_movies.csv', index=False)
headers = {'Accept-Language': 'en-US'}
payload = "{}"
api_key = open("tmdb_api_key.txt")
api_key = api_key.read()
all_movie_data = []
with open('imdb_ids_2018.csv', encoding='utf-8') as csvfile:
movies = csv.reader(csvfile)
for movie in movies:
try:
url = "https://api.themoviedb.org/3/movie/"
thing_looking_for = movie[1]
my_api_key = "?api_key=" + api_key
full_url = url + thing_looking_for + my_api_key
res = requests.get(full_url, payload, headers=headers)
data = res.content.decode('UTF-8')
jdata = json.loads(data)
try:
title = jdata['title']
budget = jdata['budget']
genres = jdata['genres']
production_companies = jdata['production_companies']
release_date = jdata['release_date']
revenue = jdata['revenue']
profit = revenue - budget
popularity = jdata['popularity']
vote_average = jdata['vote_average']
vote_count = jdata['vote_count']
except KeyError:
title = 'NA'
budget = 'NA'
genres = 'NA'
production_companies = 'NA'
release_date = 'NA'
revenue = 'NA'
profit = 'NA'
popularity = 'NA'
vote_average = 'NA'
vote_count = 'NA'
movie_data = {
'release_date': release_date,
'title': title,
'budget': budget,
'genres': genres,
'production_companies': production_companies,
'revenue': revenue,
'profit': profit,
'popularity': popularity,
'vote_average': vote_average,
'vote_count': vote_count
}
all_movie_data.append(movie_data)
except UnicodeDecodeError:
director_data = {}
all_movie_data_df = pd.DataFrame(all_movie_data)
print(all_movie_data_df)
write_csv(all_movie_data)
In [40]:
# ANALYZING THE JSON DATA
#Packages that will be used in this notebook
# import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
#For this assignment, we are using the movies data set that we got from tmdb using their JSON api.
#Reading in the csv file using pandas' read.csv function
movies = pd.read_csv('2018_movies.csv')
#Pandas was able to recognize that the first row of data was the column names, this was helpful because when we downloaded
#the csv after getting the information, our csv did not include proper column names, instead the column names were in
#the first row of our data. pd.read_csv correctly noticed that the first row of our file contained the column names.
In [41]:
#Looking at the movies df...
movies
# Pandas recognized that the first row was the column names and moved the first row to column names,
#however it left the first row blank... We need to remove that row
Out[41]:
In [42]:
#first getting the length of our df
movies.shape
#We have 503 rows and 10 columns
Out[42]:
In [43]:
#removing the first row using the drop function. We are specifying that we are
#removing row 0 and leaving columns blank to show that we are removing all columns from row 0.
movies = movies.drop([0,])
In [44]:
#Checking to make sure that this worked... If it did our new row count will be 502.
movies.shape
#It did!
Out[44]:
In [45]:
#removing rows with missing values to do this we are using the dropna function in pandas and setting inplace = True
#which means to replace the original movies df with the movies df that has the rows with missing values removed
movies.dropna(inplace = True)
In [46]:
movies.shape
#We can see that 4 rows had missing values and were dropped
Out[46]:
In [47]:
movies
Out[47]:
In [48]:
#Looking at the second entry in our df, we can see the Movie Night Hunter does not have any information about budget,
#revenue and profit... This is important information for our analysis so we are going to drop any movies that have a 0 in
#the budget column
#To do this we are going to get a list of index names for movies that have a budget equal to 0
index_names = movies[movies["budget"] == 0].index
In [49]:
#We can see that 325 movies in our df have a budget of 0 dollars... We have to drop these movies
index_names
Out[49]:
In [50]:
#We are using the drop function and droping all rows that are stored in index_names, inplace = True to replace the df with the new df
movies.drop(index_names, inplace = True)
In [51]:
#We can see that we now have 173 movies
movies.shape
Out[51]:
In [52]:
movies
Out[52]:
In [53]:
#We also need to remove movies that have a 0 revenue, as we believe that this data just has not be collected, and therefore is
#not accurate. We are using the same method that we used to drop the entries that had a 0 in the budget column
index_names = movies[movies["revenue"] == 0].index
In [54]:
#We are using the drop function and droping all rows that are stored in index_names, inplace = True to replace the df with the new df
movies.drop(index_names, inplace = True)
In [55]:
#Getting the rows and dimensions of the updated df
movies.shape
Out[55]:
In [56]:
#Checking to the data type for the release_date column
movies.release_date.dtype
#It shows that it is saved as an object, we want to convert this to date format
Out[56]:
In [57]:
#Changing the data type to date by using the pd_to_datetime function, this will allow us to extract each element of the date
movies["release_date"] = pd.to_datetime(movies["release_date"])
In [58]:
#now we want to extract the month, day, and year and create new columns named month, day and year
movies["month"], movies["day"], movies["year"] = movies["release_date"].dt.month, movies["release_date"].dt.day, movies["release_date"].dt.year
In [59]:
movies.head()
#This shows that we were successful in extracting the day month and year from the release_date. We now have 3 new columns
#month, day, and year
Out[59]:
In [60]:
#Getting summary statistics for our df
movies.describe()
Out[60]:
In [61]:
'''It is interesting to note that the lowest budget for a movie in our df is only $258,157 and the maximum budget is
$300,000,000. The lowest revenue is $4,537. The maximum revenue is $2,046,240,000. This is a massive revenue. We
aggregated a profit column and the minimum profit is - $60,477,350 and maximum profit is $1,746,240,000. This is a
massive profit. Their is a large range in popularity scores. The maximum popularity score is 71.54 and the minimum is
3.54. The mean popularity is 18.4 and 75% of the movies have a popularity score less than 21.2. This makes us question
if the maximum popularity score might be an error, or it might correspond to the movie with the highest profit.
We should investigate what movie this score references. The vote_average column has a range of 4 - 8.4, with an
average of 6.5. The vote_count has a maximum of 14,913. This might reference the same movie that had the largest
popularity. The max value in this column, also appears to be an outlier, as 75% of the movies have less than
2151 votes and the average vote count is 1701'''
Out[61]:
In [62]:
#We have decided that we want to aggregate a percent profit column as well, in an attempt to normalize the data
#To do this we are diving the profit column by the budget column and multiply the result by 100 and saving it in a
#new column named percent_profit
movies["percent_profit"] = movies["profit"]/movies["budget"]*100
In [64]:
#Looking at the movies df again
movies
#We have successfully added a new column named percent_profit.
# Saving a column as-is for future use
movies_original_df = movies
In [31]:
#We feel like we will want to have a discretized df as well to answer some of our questions...
#The first column we are discretizing is the budget column
#Looking at the breakdown of budget
movies.budget.describe()
Out[31]:
In [32]:
'''First, we are going to discretize the budget column.
We are discretizing the budget column into four groups: extremely_low, low, high and extremely_high. To do this we first
need to create a list of the categories'''
categories = ["extremely_low", "low", "high", "extremely_high"]
'''Now we need to show where we want to insert the breaks. We have decided that extremely low budgets are budgets less
than $13,000,000, low have budgets between $13,000,000 and $30,000,000, high have budgets between $30,000,000 and
$62,192,550, and extremely_high have budgets between $62,192,550 and $300,000,000. We chose the values based on the
quartiles.
'''
#Saving the movies df as movies_discretized_df
movies_discretized_df = movies
#Discretizing the budget columns using the cut function from pandas
movies_discretized_df["budget"] = pd.cut(movies_discretized_df["budget"], [0, 13000000, 30000000, 62192550, 300000001], labels = categories)
In [33]:
#checking to see if it worked
movies_discretized_df.budget.dtype
#It did!
Out[33]:
In [28]:
#Now we are going to repeat the steps to discretize the revenue column
movies_discretized_df.revenue.describe()
Out[28]:
In [29]:
#We are using the same categories as above
'''extremely_low revenue are revenues less than $21,458,200, low revenues are revenues between $21,458,200 and
$62,954,020, high revenues are revenues between $62,954,020 and $187,976,900, and extremely_high revenues between
$187,976,900 and $2,046,240,000.
'''
movies_discretized_df["revenue"] = pd.cut(movies_discretized_df["revenue"], [0, 21458200, 62954020, 187976900, 2046240001], labels = categories)
In [30]:
movies_discretized_df.revenue.dtype
Out[30]:
In [31]:
#Now we are going to repeat the steps to discretized the profit column
movies_discretized_df.profit.describe()
Out[31]:
In [32]:
'''negative profit are profits less than $0, low profits are profits between $0 and
$29,314,900, high profits are profits between $29,314,900 and $140,784,100, and extremely_high profits between
$140,784,100 and $1,746,240,001.
'''
categories = ["negative", "low", "high", "extremely_high"]
movies_discretized_df["profit"] = pd.cut(movies_discretized_df["profit"], [-60477351, 0, 29314900, 140784100, 1746240001], labels = categories)
In [33]:
movies_discretized_df.profit.dtype
Out[33]:
In [34]:
#Now we are going to repeat the steps to discretize the popularity column
movies_discretized_df.popularity.describe()
Out[34]:
In [35]:
categories = ["extremely_low", "low", "high", "extremely_high"]
'''extremely_low popularity are popularities less than 12.442, low popularities are popularities between 12.442 and
15.7405, high popularity are popularities between 15.7405 and 21.23025 and extremely_high popularity between 21.23025
and 71.538'''
movies_discretized_df["popularity"] = pd.cut(movies_discretized_df["popularity"], [0, 12.442, 15.7405, 21.23025, 71.538], labels = categories)
In [36]:
movies_discretized_df["popularity"]
Out[36]:
In [37]:
#Now we are going to repeat the steps to discretize the vote avg
movies_discretized_df.vote_average.describe()
Out[37]:
In [38]:
#We are using the same categories as above
'''extremely_low vote_average are vote averages less than 6, low are between 6 to 6.5, high between 6.5 and 7 and
extremely_high 7 and 8.5'''
movies_discretized_df["vote_average"] = pd.cut(movies_discretized_df["vote_average"], [0, 6, 6.5, 7, 8.5], labels = categories)
In [39]:
movies_discretized_df["vote_average"]
Out[39]:
In [40]:
#Now we are going to repeat the steps to discretize the vote count
movies_discretized_df.vote_count.describe()
Out[40]:
In [41]:
#We are using the same categories as above
'''extremely_low vote counts are vote counts less than 440, low are between 440 and 1151, high are between 1151 and 2522
and extremely_high are between 2522 and 14913'''
movies_discretized_df["vote_count"] = pd.cut(movies_discretized_df["vote_count"], [0, 440, 1151, 2522, 14914], labels = categories)
In [42]:
movies_discretized_df["vote_count"]
Out[42]:
In [43]:
movies_discretized_df.percent_profit.describe()
Out[43]:
In [44]:
'''extremely_low are percent profits between -100 and 0, low between 6.5 and 108, high between 108 and 436 and
extremely_high between 436 and 6527'''
categories = ["negative", "low", "high", "extremely_high"]
movies_discretized_df["percent_profit"] = pd.cut(movies_discretized_df["percent_profit"], [-100, 0, 108, 436, 6528], labels = categories )
In [45]:
movies_discretized_df["percent_profit"]
Out[45]:
In [46]:
movies_discretized_df.day.describe()
Out[46]:
In [47]:
#We are setting new categories for the day column by creating a new column for week
'''week_1 is the first 7 days of the month, week_2 is days 8 - 14, week_3 is days 15 - 21, and week_4 are the
rest of the days'''
categories = ["week_1", "week_2", "week_3", "week_4"]
movies_discretized_df["week"] = pd.cut(movies_discretized_df["day"], [0, 8, 15, 22, 32], labels = categories)
In [48]:
movies_discretized_df.head()
Out[48]:
In [49]:
#We have successfully discretized the df, now we can remove the day and release_date column
movies_discretized_df.drop(columns=['day', 'release_date'], inplace = True)
In [50]:
#Checking to make sure that it worked
movies_discretized_df.head()
Out[50]:
In [51]:
#Question 1:
#How are the amounts of percent_profits distributed across budget levels?
'''We want to compare the budget category percentage make up for each percent_profit level. To do this we need to
get the count for each budget level, the count for each percent_profit level by budget level and then divide
the count of the percent_profit/count of budget level and multiply by 100. We have to do this for each
budget level and level of percent_profits. We think that we could potentially answer this question by group bys.'''
movies_discretized_count = movies_discretized_df.groupby(["budget", "percent_profit"])["budget"].count()
'''Taking the output from the line above and converting it to a data frame. We are using pandas, which we important as pd.
First, we call the package we are using then the function from that package and then what we want to run the function on.
pd.function(item to use). We are using the DataFrame function from the pandas package on the series created by our group by'''
movies_discretized_count_df = pd.DataFrame(movies_discretized_count)
#Checking to see what our df looks like.
movies_discretized_count_df
#Changing the column name from budget to counts
movies_discretized_count_df.columns = ["counts"]
#Checking to see what our df looks like.
movies_discretized_count_df
Out[51]:
In [52]:
#We want to get a total count for the number of percent_profit counts for each budget level. We will experiment to see how this is possible
'''This shows that we have 2 indexes budget and percent_profit... We want to create columns from each index
We are creating a new column named budget by extracting the values from the first index (0) which is the budget
index'''
movies_discretized_count_df["budget_category"]=movies_discretized_count_df.index.get_level_values(0)
#We are creating a new column named total_donations by extracting the values from the second index (1) which is total_donations
movies_discretized_count_df["percent_profit_category"] = movies_discretized_count_df.index.get_level_values(1)
#Checking to make sure it worked...
movies_discretized_count_df
#It did!
Out[52]:
In [53]:
#Now we want to remove the indexes so, we can create a new group by to get the sum of the counts for each group
#To do this we are using the reset_index(drop = True) This will drop our group by indexes and allow us to create a new one.
movies_discretized_count_df = movies_discretized_count_df.reset_index(drop = True)
In [54]:
movies_discretized_count_df
Out[54]:
In [55]:
#Now we are getting the sum of each budget category.
budget_discretized_count_df = movies_discretized_count_df.groupby(["budget_category"])["counts"].sum()
In [56]:
#Checking the results
budget_discretized_count_df
Out[56]:
In [57]:
'''We ultimately want a column that contains the total counts for each budget group. We are probably doing this in
a roundabout way, but as I am extremely new to python this is the best way I can think of doing it. We are going to create
a new column that replicates the income_level called income_level_count and then we will use the replace function to
replace the 1s with their total count, the 2s with their total count... '''
#First, replicating the income level column in a column named budget_category_count
movies_discretized_count_df["budget_category_count"] = movies_discretized_count_df["budget_category"]
#Now replacing the income level with the total count for each income level
movies_discretized_count_df["budget_category_count"] = movies_discretized_count_df["budget_category_count"].replace(["extremely_low"], 35)
movies_discretized_count_df["budget_category_count"] = movies_discretized_count_df["budget_category_count"].replace(["low"], 35)
movies_discretized_count_df["budget_category_count"] = movies_discretized_count_df["budget_category_count"].replace(["high"], 32)
movies_discretized_count_df["budget_category_count"] = movies_discretized_count_df["budget_category_count"].replace(["extremely_high"], 34)
In [58]:
#Checking to see if that worked:
movies_discretized_count_df
Out[58]:
In [59]:
#Okay, we are one step closer... Now, we need to create a column that takes the counts/budget_category_counts * 100
movies_discretized_count_df["percent"] = movies_discretized_count_df["counts"]/movies_discretized_count_df["budget_category_count"] *100
In [60]:
#Looking at our data frame... It worked!!!
movies_discretized_count_df
Out[60]:
In [61]:
#We no longer need the count columns
movies_discretized_count_df.drop(["counts", "budget_category_count"], axis = 1, inplace = True )
In [62]:
'''Attempting to graph this data using a grouped bar chart:
formula: df.pivot(columns, group, values).plot(kind = "type of graph", color = ["color to use, can be a list of colors"],
title = "you can set the title of your graph here")'''
graph = movies_discretized_count_df.pivot("budget_category", "percent_profit_category",
"percent").plot(kind="bar", color = ["crimson", "salmon", "palegreen", "darkgreen"],
title = "Percent of Percent Profit to Budget Category")
#Changing the y label of our graph to Percent
plt.ylabel("Percent")
#Changing the x axis label of our graph to Budget Category
plt.xlabel("Budget Category")
#Making it so the tick labels are not angled
plt.xticks(rotation = 0)
#How to change the tick labels (we ended up not needing this, but want to keep for future reference)
#plt.Axes.set_xticklabels(graph, labels = ['extremely low', 'low', 'high', 'extremely high'])
#moving the legend position to underneath the graph, also setting it to have 4 columns so the legend is in a
#straight single line and adding a legend title
plt.legend( loc = "lower center", bbox_to_anchor = (.5, -.4), ncol = 4, title = "Percent Profit Category")
Out[62]:
In [63]:
'''This graph proved very interesting. Movies with an extremely low budget have the highest percentage make-up of making an extremely
high percent profit. Movies with an extremely high budget are the most likely to be profitable overall, being that they
are the least likely to have a negative profit, with only 5.9% of the movies classified as having an extremely high
budget in our dataset made a negative profit. Movies with an low or high budget only make an extremely high
percent profit less than 17.1% and 15.6% of the time respectively. They also have the highest chance of making a low or
negative profit out of all of the budget categories. Based, on this analysis, percent profits are not uniformally
distributed across budget levels. Movies with an extremely high budget are the least likely to have a negative percent
profit. Movies with an extremely low budget are the most likely to have an extremely high percent profit. Our
recommendation to studios, would be to either have a extremely low or extremely high budget and to veer away from
productions with an extremely low or high budget. Further analysis for tighter recommendatios is needed.
'''
Out[63]:
In [76]:
#Question: Do big name production companies impact the percent profit?
#To answer this question we are first going to create a for loop that will loop through the production_companies column
#in order movies_discretized_df and store the production company in a list called production_company. The only issue
#with this method, is that if a movie has more than one production company this will not be shown and only the last
#company included in the for loop will be given credit.
production_company = []
for movie in movies_discretized_df['production_companies']:
if "Universal" in movie:
production_company.append("Universal")
elif "Sony" in movie:
production_company.append("Sony")
elif "Fox" in movie:
production_company.append("Fox")
elif "DreamWorks" in movie:
production_company.append("DW")
elif "MGM" in movie:
production_company.append("MGM")
elif "Paramount" in movie:
production_company.append("Paramount")
elif "Disney" in movie:
production_company.append("Disney")
elif "Warner Bros" in movie:
production_company.append("WB")
else:
production_company.append("None")
In [77]:
#Saving the list created by our for loop in a new column in our moview_discretized_df called main_production_co
movies_discretized_df["main_production_co"] = production_company
In [78]:
movies_discretized_df
Out[78]:
In [81]:
#Now we want to group by production company and percent profit
'''We want to compare the production company percentage make up for each percent_profit level. To do this we need to
get the count for each production company, the count for each percent_profit level by production company and then divide
the count of the percent_profit/count of production company and multiply by 100. We have to do this for each
production company and level of percent_profits. We think that we could potentially answer this question by group bys.'''
movies_discretized_count_q2 = movies_discretized_df.groupby(["main_production_co", "percent_profit"])["main_production_co"].count()
'''Taking the output from the line above and converting it to a data frame. We are using pandas, which we important as pd.
First, we call the package we are using then the function from that package and then what we want to run the function on.
pd.function(item to use). We are using the DataFrame function from the pandas package on the series created by our group by'''
movies_discretized_count_df_q2 = pd.DataFrame(movies_discretized_count_q2)
#Checking to see what our df looks like.
movies_discretized_count_df_q2
#Changing the column name from budget to counts
movies_discretized_count_df_q2.columns = ["counts"]
#Checking to see what our df looks like.
movies_discretized_count_df_q2
Out[81]:
In [82]:
#We want to get a total count for the number of percent_profit counts for each production company.
'''This shows that we have 2 indexes budget and percent_profit... We want to create columns from each index
We are creating a new column named budget by extracting the values from the first index (0) which is the budget
index'''
movies_discretized_count_df_q2["production_company"]=movies_discretized_count_df_q2.index.get_level_values(0)
#We are creating a new column named total_donations by extracting the values from the second index (1) which is total_donations
movies_discretized_count_df_q2["percent_profit_category"] = movies_discretized_count_df_q2.index.get_level_values(1)
#Checking to make sure it worked...
movies_discretized_count_df_q2
#It did!
Out[82]:
In [84]:
#Now we want to remove the indexes so, we can create a new group by to get the sum of the counts for each group
#To do this we are using the reset_index(drop = True) This will drop our group by indexes and allow us to create a new one.
movies_discretized_count_df_q2 = movies_discretized_count_df_q2.reset_index(drop = True)
In [85]:
#Now we are getting the sum of each production company category.
production_company_discretized_count_df_q2 = movies_discretized_count_df_q2.groupby(["production_company"])["counts"].sum()
In [87]:
#Checking the results
production_company_discretized_count_df_q2
Out[87]:
In [89]:
'''We ultimately want a column that contains the total counts for each production company. We are going to create
a new column that replicates the production company called production_company_count and then we will use the replace function to
replace the 1s with their total count, the 2s with their total count... '''
#First, replicating the income level column in a column named budget_category_count
movies_discretized_count_df_q2["production_company_count"] = movies_discretized_count_df_q2["production_company"]
#Now replacing the income level with the total count for each income level
movies_discretized_count_df_q2["production_company_count"] = movies_discretized_count_df_q2["production_company_count"].replace(["DW"], 1)
movies_discretized_count_df_q2["production_company_count"] = movies_discretized_count_df_q2["production_company_count"].replace(["Disney"], 8)
movies_discretized_count_df_q2["production_company_count"] = movies_discretized_count_df_q2["production_company_count"].replace(["Fox"], 11)
movies_discretized_count_df_q2["production_company_count"] = movies_discretized_count_df_q2["production_company_count"].replace(["None"], 66)
movies_discretized_count_df_q2["production_company_count"] = movies_discretized_count_df_q2["production_company_count"].replace(["Paramount"], 9)
movies_discretized_count_df_q2["production_company_count"] = movies_discretized_count_df_q2["production_company_count"].replace(["Sony"], 8)
movies_discretized_count_df_q2["production_company_count"] = movies_discretized_count_df_q2["production_company_count"].replace(["Universal"], 20)
movies_discretized_count_df_q2["production_company_count"] = movies_discretized_count_df_q2["production_company_count"].replace(["WB"], 13)
In [90]:
movies_discretized_count_df_q2
Out[90]:
In [91]:
#Okay, we are one step closer... Now, we need to create a column that takes the counts/budget_category_counts * 100
movies_discretized_count_df_q2["percent"] = movies_discretized_count_df_q2["counts"]/movies_discretized_count_df_q2["production_company_count"] *100
In [92]:
#Looking at our data frame... It worked!!!
movies_discretized_count_df_q2
Out[92]:
In [93]:
#We no longer need the count columns
movies_discretized_count_df_q2.drop(["counts", "production_company_count"], axis = 1, inplace = True )
In [94]:
'''Attempting to graph this data using a grouped bar chart:
formula: df.pivot(columns, group, values).plot(kind = "type of graph", color = ["color to use, can be a list of colors"],
title = "you can set the title of your graph here")'''
graph = movies_discretized_count_df_q2.pivot("production_company", "percent_profit_category",
"percent").plot(kind="bar", color = ["crimson", "salmon", "palegreen", "darkgreen"],
title = "Percent of Percent Profit to Production Company")
#Changing the y label of our graph to Percent
plt.ylabel("Percent")
#Changing the x axis label of our graph to Budget Category
plt.xlabel("Production Company")
#Making it so the tick labels are not angled
plt.xticks(rotation = 0)
#How to change the tick labels (we ended up not needing this, but want to keep for future reference)
#plt.Axes.set_xticklabels(graph, labels = ['extremely low', 'low', 'high', 'extremely high'])
#moving the legend position to underneath the graph, also setting it to have 4 columns so the legend is in a
#straight single line and adding a legend title
plt.legend( loc = "lower center", bbox_to_anchor = (.5, -.4), ncol = 4, title = "Percent Profit Category")
Out[94]:
In [ ]:
'''This graph provides some insights, however, most of our movies have more than one main production company and only
one production company is being shown. For example, DreamWorks and Universal had a movie named First Man and it was
profitable. However, based on the way that we assigned a main production company, only Universal was given credit for
that movie.'''
In [108]:
# Question:
#Does time of the month the movie is released affect percent profit?
'''We want to compare the percent_profit level percentage make up for each time of month. To do this we need to
get the count for each time of month, the count for each percent_profit level by time of month company and then divide
the count of the percent_profit/count of time of month and multiply by 100. We have to do this for each
time of month and level of percent_profits.'''
movies_discretized_count_q3 = movies_discretized_df.groupby(["week", "percent_profit"])["week"].count()
'''Taking the output from the line above and converting it to a data frame. We are using pandas, which we important as pd.
First, we call the package we are using then the function from that package and then what we want to run the function on.
pd.function(item to use). We are using the DataFrame function from the pandas package on the series created by our group by'''
movies_discretized_count_df_q3 = pd.DataFrame(movies_discretized_count_q3)
#Checking to see what our df looks like.
movies_discretized_count_df_q3
#Changing the column name from week to counts
movies_discretized_count_df_q3.columns = ["counts"]
#Checking to see what our df looks like.
movies_discretized_count_df_q3
Out[108]:
In [109]:
#We want to get a total count for the number of percent_profit counts for each week.
'''This shows that we have 2 indexes week and percent_profit... We want to create columns from each index
We are creating a new column named week by extracting the values from the first index (0) which is the week
index'''
movies_discretized_count_df_q3["week"]=movies_discretized_count_df_q3.index.get_level_values(0)
#We are creating a new column named total_donations by extracting the values from the second index (1) which is percent_profit
movies_discretized_count_df_q3["percent_profit_category"] = movies_discretized_count_df_q3.index.get_level_values(1)
#Checking to make sure it worked...
movies_discretized_count_df_q3
#It did!
Out[109]:
In [110]:
#Now we want to remove the indexes so, we can create a new group by to get the sum of the counts for each group
#To do this we are using the reset_index(drop = True) This will drop our group by indexes and allow us to create a new one.
movies_discretized_count_df_q3 = movies_discretized_count_df_q3.reset_index(drop = True)
In [107]:
Out[107]:
In [111]:
#Now we are getting the sum of each production company category.
week_discretized_count_df_q3 = movies_discretized_count_df_q3.groupby(["week"])["counts"].sum()
In [113]:
#Checking the results
week_discretized_count_df_q3
Out[113]:
In [114]:
'''We ultimately want a column that contains the total counts for each week. We are going to create
a new column that replicates the week called week_count and then we will use the replace function to
replace the 1s with their total count, the 2s with their total count... '''
#First, replicating the income level column in a column named budget_category_count
movies_discretized_count_df_q3["week_count"] = movies_discretized_count_df_q3["week"]
#Now replacing the income level with the total count for each income level
movies_discretized_count_df_q3["week_count"] = movies_discretized_count_df_q3["week_count"].replace(["week_1"], 37)
movies_discretized_count_df_q3["week_count"] = movies_discretized_count_df_q3["week_count"].replace(["week_2"], 34)
movies_discretized_count_df_q3["week_count"] = movies_discretized_count_df_q3["week_count"].replace(["week_3"], 33)
movies_discretized_count_df_q3["week_count"] = movies_discretized_count_df_q3["week_count"].replace(["week_4"], 32)
In [115]:
movies_discretized_count_df_q3
Out[115]:
In [116]:
#Okay, we are one step closer... Now, we need to create a column that takes the counts/week_count * 100
movies_discretized_count_df_q3["percent"] = movies_discretized_count_df_q3["counts"]/movies_discretized_count_df_q3["week_count"] *100
In [117]:
#Looking at our data frame... It worked!!!
movies_discretized_count_df_q3
Out[117]:
In [118]:
#We no longer need the count columns
movies_discretized_count_df_q3.drop(["counts", "week_count"], axis = 1, inplace = True )
In [119]:
'''Attempting to graph this data using a grouped bar chart:
formula: df.pivot(columns, group, values).plot(kind = "type of graph", color = ["color to use, can be a list of colors"],
title = "you can set the title of your graph here")'''
graph = movies_discretized_count_df_q3.pivot("week", "percent_profit_category",
"percent").plot(kind="bar", color = ["crimson", "salmon", "palegreen", "darkgreen"],
title = "Percent of Percent Profit to Week")
#Changing the y label of our graph to Percent
plt.ylabel("Percent")
#Changing the x axis label of our graph to Budget Category
plt.xlabel("Week")
#Making it so the tick labels are not angled
plt.xticks(rotation = 0)
#How to change the tick labels (we ended up not needing this, but want to keep for future reference)
#plt.Axes.set_xticklabels(graph, labels = ['extremely low', 'low', 'high', 'extremely high'])
#moving the legend position to underneath the graph, also setting it to have 4 columns so the legend is in a
#straight single line and adding a legend title
plt.legend( loc = "lower center", bbox_to_anchor = (.5, -.4), ncol = 4, title = "Percent Profit Category")
Out[119]:
In [2]:
'''This is interesting in that it shows that movies released within the first two weeks of the month tend to be more
profitable. We would like to look at a breakdown of month to percent profit for further analysis'''
Out[2]:
In [4]:
# Taking a brief detour back to our non-discretized df
movies_original_df
Out[4]:
In [34]:
# QUESTION: Do "Good" Movies Make Money? -- We're defining "Good" as vote average
plt.plot(movies_original_df.profit, movies_original_df.vote_average, 'o')
plt.title('Do "Good" Movies Make Money?')
plt.xlabel('Profit')
plt.ylabel('Vote Average')
plt.show()
In [38]:
# QUESTION: Does Popularity = Profit?
plt.plot(movies_original_df.profit, movies_original_df.popularity, 'o')
plt.title('Does Popularity = Profits?')
plt.xlabel('Profit')
plt.ylabel('Popularity')
plt.show()
In [65]:
# QUESTION: How does budget impact vote average?
plt.plot(movies_original_df.budget, movies_original_df.vote_average, 'o')
plt.title('How does Budget Impact Vote Average?')
plt.xlabel('Budget')
plt.ylabel('Vote Average')
plt.show()
In [66]:
# QUESTION: How does budget impact popularity?
plt.plot(movies_original_df.budget, movies_original_df.popularity, 'o')
plt.title('How does Budget Impact Popularity?')
plt.xlabel('Budget')
plt.ylabel('Popularity')
plt.show()
In [69]:
# QUESTION: Is there a relationship between "Above Average Movies" and Budget/Price?
below_avg = movies_original_df[movies_original_df.vote_average < 6.5]
above_avg = movies_original_df[movies_original_df.vote_average >= 6.5]
plt.plot(below_avg.budget, below_avg.profit, 'o', label="below average")
plt.plot(above_avg.budget, above_avg.profit, 'o', label="above average")
plt.title('BUDGET vs PROFIT by AVERAGE VOTE!')
plt.xlabel('BUDGET')
plt.ylabel('PROFIT')
plt.legend()
plt.show()
In [70]:
# QUESTION: Is there a relationship between "Above Average Movies" and Budget/Price?
below_avg = movies_original_df[movies_original_df.vote_average < 6.5]
above_avg = movies_original_df[movies_original_df.vote_average >= 6.5]
plt.plot(below_avg.budget, below_avg.percent_profit, 'o', label="below average")
plt.plot(above_avg.budget, above_avg.percent_profit, 'o', label="above average")
plt.title('BUDGET vs PERCENT PROFIT by AVERAGE VOTE!')
plt.xlabel('BUDGET')
plt.ylabel('PROFIT')
plt.legend()
plt.show()
In [71]:
# BIG QUESTION: What role do production companies play in the entertainment industry?
# Is there a relationship between production studio and average vote?
# Production studio and budget?
# Production studio and percent profit?
In [72]:
# Adding the BIG EIGHT Production Studios to the DF
# WARNER BROS
wb = []
for movie in movies_original_df['production_companies']:
if "Warner Bros" in movie:
wb.append(True)
else:
wb.append(False)
movies_original_df['wb'] = wb
# MGM
mgm = []
for movie in movies_original_df['production_companies']:
if "MGM" in movie:
mgm.append(True)
else:
mgm.append(False)
movies_original_df['mgm'] = mgm
# DREAMWORKS
dw = []
for movie in movies_original_df['production_companies']:
if "DreamWorks" in movie:
dw.append(True)
else:
dw.append(False)
movies_original_df['dw'] = dw
# SONY
sony = []
for movie in movies_original_df['production_companies']:
if "Sony" in movie:
sony.append(True)
else:
sony.append(False)
movies_original_df['sony'] = sony
# DISNEY
disney = []
for movie in movies_original_df['production_companies']:
if "Disney" in movie:
disney.append(True)
else:
disney.append(False)
movies_original_df['disney'] = disney
#FOX
fox = []
for movie in movies_original_df['production_companies']:
if "Century Fox" in movie:
fox.append(True)
else:
fox.append(False)
movies_original_df['fox'] = fox
# PARAMOUNT
paramount = []
for movie in movies_original_df['production_companies']:
if "Paramount" in movie:
paramount.append(True)
else:
paramount.append(False)
movies_original_df['paramount'] = paramount
#UNIVERSAL
universal = []
for movie in movies_original_df['production_companies']:
if "Universal" in movie:
universal.append(True)
else:
universal.append(False)
movies_original_df['universal'] = universal
In [73]:
sony = movies_original_df[movies_original_df.sony == True]
wb = movies_original_df[movies_original_df.wb == True]
disney = movies_original_df[movies_original_df.disney == True]
fox = movies_original_df[movies_original_df.fox == True]
universal = movies_original_df[movies_original_df.universal == True]
paramount = movies_original_df[movies_original_df.paramount == True]
dw = movies_original_df[movies_original_df.dw == True]
mgm = movies_original_df[movies_original_df.mgm == True]
plt.plot(sony.budget, sony.revenue, 'o', label="Sony")
plt.plot(wb.budget, wb.revenue, 'o', label="Warner Bros.")
plt.plot(disney.budget, disney.revenue, 'o', label="Disney")
plt.plot(fox.budget, fox.revenue, 'o', label="Fox")
plt.plot(universal.budget, universal.revenue, 'o', label="Universal")
plt.plot(paramount.budget, paramount.revenue, 'o', label="Paramount")
plt.plot(dw.budget, dw.revenue, 'o', label="DreamWorks")
plt.plot(mgm.budget, mgm.revenue, 'o', label="MGM")
plt.title('BUDGET vs REVENUE by PRODUCTION COMPANY')
plt.xlabel('BUDGET')
plt.ylabel('REVENUE')
plt.legend( loc = "lower center", bbox_to_anchor = (.5, -.4), ncol = 4, title = "Production Company")
plt.show()
In [74]:
sony = movies_original_df[movies_original_df.sony == True]
wb = movies_original_df[movies_original_df.wb == True]
disney = movies_original_df[movies_original_df.disney == True]
fox = movies_original_df[movies_original_df.fox == True]
universal = movies_original_df[movies_original_df.universal == True]
paramount = movies_original_df[movies_original_df.paramount == True]
dw = movies_original_df[movies_original_df.dw == True]
mgm = movies_original_df[movies_original_df.mgm == True]
plt.plot(sony.budget, sony.percent_profit, 'o', label="Sony")
plt.plot(wb.budget, wb.percent_profit, 'o', label="Warner Bros.")
plt.plot(disney.budget, disney.percent_profit, 'o', label="Disney")
plt.plot(fox.budget, fox.percent_profit, 'o', label="Fox")
plt.plot(universal.budget, universal.percent_profit, 'o', label="Universal")
plt.plot(paramount.budget, paramount.percent_profit, 'o', label="Paramount")
plt.plot(dw.budget, dw.percent_profit, 'o', label="DreamWorks")
plt.plot(mgm.budget, mgm.percent_profit, 'o', label="MGM")
plt.title('BUDGET vs PERCENT PROFIT by PRODUCTION COMPANY')
plt.xlabel('BUDGET')
plt.ylabel('PERCENT PROFIT')
plt.legend( loc = "lower center", bbox_to_anchor = (.5, -.4), ncol = 4, title = "Production Company")
plt.show()
In [ ]: