Getting the information about executed prisoners from the Texas Department of Criminal Justice¶

https://www.tdcj.texas.gov/death_row/dr_executed_offenders.html ¶

import requests
import lxml.html as lh
import pandas as pd
from bs4 import BeautifulSoup as bs

url="https://www.tdcj.texas.gov/death_row/dr_executed_offenders.html"
#Create a handle, page, to handle the contents of the website
page = requests.get(url)

print(page.status_code) #It should show 200 to verify that I collected the data from the website

200

#Creating the soup 
#Using beautiful soups prettify() function to print the data in a readable html format 
soup= bs(page.content, "html.parser")
# print(soup.prettify())

# The table we want is in the tag table and class "tdcj_table indent"
table = soup.find("table", class_="tdcj_table indent")

info = table.find("tr")
headers = info.text.split('\n')[1:-1]
headers

['Execution',
 'Link',
 'Link',
 'Last Name',
 'First Name',
 'TDCJNumber',
 'Age',
 'Date',
 'Race',
 'County']

all_info = table.find_all("tr")
all_dics = []
for info in all_info:
    row = info.find_all("td")
    row_obj = {}
    for i,column_value in enumerate(row):
        if i == 1:
            row_obj.update({column_value.find('a').text : column_value.find('a').get('href')})
        elif i == 2:
            row_obj.update({column_value.find('a').text : column_value.find('a').get('href')})
        else:
            row_obj.update({headers[i] : column_value.text})
    all_dics.append(row_obj)
# all_dics

df = pd.DataFrame(all_dics)

df

clean_df = df.copy()[1:]
clean_df = clean_df[clean_df.columns[:-3]]
clean_df

clean_df = df.copy()[1:]
clean_df = clean_df[clean_df.columns[:-3]]
clean_df

# for testing
# clean_df = clean_df[:5]
# for link in clean_df['Last Statement'].tolist():
    
def get_last_statement(link):
    try:
        url="https://www.tdcj.texas.gov/death_row/" + link
        page = requests.get(url)
        soup= bs(page.content, "html.parser")
        p_tags = soup.find_all("p")
        for p in p_tags:
            print
            try:
                if 'Last Statement' in p.text:
    #                 print(p)
                    return p.next_sibling.next_sibling.text
            except:
                return 'no statement'
    except:
        return 'no link'
            
    
clean_df['Last Statement Text'] = clean_df.apply(lambda x: get_last_statement(x['Last Statement']), axis=1)

clean_df

clean_df.to_csv('V1_with_statements.csv')

test_df = clean_df.copy()
def get_last_statement(link):
    try:
        url="https://www.tdcj.texas.gov/death_row/" + link
        page = requests.get(url)
        soup= bs(page.content, "html.parser")
        tr_tags = soup.find_all("tr")
        offender_obj = {}
        for t in tr_tags:
#             print(t)
            info = t.find_all('td')
            row_array = []
            for i in info:
                string = i.text.replace(u'\xa0', u'')
                if len(string) > 0:
                    row_array.append(i.text)
            print(row_array)
            try:
                if len(row_array) > 1:
                    offender_obj.update({row_array[0]: row_array[1]})
            except:
                return 'no field'
        return offender_obj
    except:
        return 'no link'
            
    
test_df['Offender Info'] = test_df.apply(lambda x: get_last_statement(x['Offender Information']), axis=1)

['Name', 'Hall, Justen Grant']
['TDCJ Number', '999497']
['Date of Birth', '06/16/1981']
['Date Received', '05/11/2005']
['Age (when Received)', '23']
['Education Level (Highest Grade Completed)', '9']
['Date of Offense', '10/28/2002']
['Age (at the time of Offense)', '21']
['County', 'El Paso']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Brown']
['Height (in Feet and Inches)', '6′5″']
['Weight (in Pounds)', '197']
['Eye Color', 'Hazel']
['Native County', 'El Paso']
['Native State', 'Texas']
['Name', 'Sparks, Robert']
['TDCJ Number', '999542']
['Date of Birth', '02/13/1974']
['Date Received', '01/08/2009']
['Age (when Received)', '34']
['Education Level (Highest Grade Completed)', '8']
['Date of Offense', '09/15/2007']
['Age (at the time of Offense)', '33']
['County', 'Dallas']
['Race', 'Black']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 7″']
['Weight (in Pounds)', '247']
['Eye Color', 'Brown']
['Native County', 'Dallas']
['Native State', 'Texas']
['Name', 'Soliz, Mark Anthony']
['TDCJ Number', '999571']
['Date of Birth', '01/27/1982']
['Date Received', '03/28/2012']
['Age (when Received)', '30']
['Education Level (Highest Grade Completed)', '8']
['Date of Offense', '06/29/2010']
['Age (at the time of Offense)', '28']
['County', 'Johnson']
['Race', 'Hispanic']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 5″']
['Weight (in Pounds)', '177']
['Eye Color', 'Brown']
['Native County', 'Tarrant']
['Native State', 'Texas']
['Name', 'Crutsinger, Billy Jack']
['TDCJ Number', '999459']
['Date of Birth', '10/05/1954']
['Date Received', '10/08/2003']
['Age (when Received)', '49']
['Education Level (Highest Grade Completed)', '11']
['Date of Offense', '04/06/2003']
['Age (at the time of Offense)', '48']
['County', 'Tarrant']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Gray']
['Height (in Feet and Inches)', '5′ 9″']
['Weight (in Pounds)', '217']
['Eye Color', 'Green']
['Native County', 'Tarrant']
['Native State', 'Texas']
['Name', 'Larry Ray Swearingen']
['TDCJ Number', '999361']
['Date of Birth', '5/21/1971']
['Date Received', '7/12/2000']
['Age (when    Received)', '29']
['Education Level (Highest Grade Completed)', '11']
['Date of Offense', '12/8/1998']
['Age (at the time of Offense)', '27']
['County', 'Montgomery']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Brown']
['Height (in Feet and Inches)', '5′ 10″']
['Weight (in Pounds)', '194']
['Eye Color', 'Blue']
['Native County', 'Montgomery']
['Native State', 'Texas']
[]
['Name', 'King, John William']
['TDCJ Number', '999295']
['Date of Birth', '11/03/1974']
['Date Received', '02/25/1999']
['Age (when Received)', '24']
['Education Level (Highest Grade Completed)', '10']
['Date of Offense', '06/07/1998']
['Age (at the time of Offense)', '23']
['County', 'Jasper']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Brown']
['Height (in Feet and Inches)', '5′ 9″']
['Weight (in Pounds)', '198']
['Eye Color', 'Brown']
['Native County', 'Atlanta']
['Native State', 'Georgia']

Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.
Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.

['Name', 'Braziel, Alvin Avon Jr.']
['TDCJ Number', '999393']
['Date of Birth', '3/16/1975']
['Date Received', '8/9/2001']
['Age (when Received)', '26']
['Education Level (Highest Grade Completed)', '8']
['Date of Offense', '9/21/1993']
['Age (at the time of Offense)', '18']
['County', 'Dallas']
['Race', 'Black']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 6″']
['Weight (in Pounds)', '166']
['Eye Color', 'Brown']
['Native County', 'Dallas']
['Native State', 'Texas']
['Name', 'Garcia, Joseph Christopher']
['TDCJ Number', '999441']
['Date of Birth', '11/06/1971']
['Date Received', '02/14/2003']
['Age (when Received)', '31']
['Education Level (Highest Grade Completed)', '12']
['Date of Offense', '12/24/2000']
['Age (at the time of Offense)', '29']
['County', 'Dallas']
['Race', 'Hispanic']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 9″']
['Weight (in Pounds)', '202']
['Eye Color', 'Brown']
['Native County', 'Bexar']
['Native State', 'Texas']

Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.

['Name', 'Acker, Daniel Clate']
['TDCJ Number', '999381']
['Date of Birth', '10/09/1971']
['Date Received', '04/02/2001']
['Age (when Received)', '29']
['Education Level (Highest Grade Completed)', '8']
['Date of Offense', '03/12/2000']
['Age (at the time of Offense)', '28']
['County', 'Hopkins']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Brown']
['Height (in Feet and Inches)', '6′ 0″']
['Weight (in Pounds)', '295']
['Eye Color', 'Brown']
['Native County', 'Hunt']
['Native State', 'Texas']
['Name', 'Clark, Troy']
['TDCJ Number', '999351']
['Date of Birth', '9/2/1967']
['Date Received', '3/31/2000']
['Age (when Received)', '32']
['Education Level (Highest Grade Completed)', '6 years']
['Date of Offense', '5/19/1998']
['Age (at the time of Offense)', '30']
['County', 'Smith']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Brown']
['Height (in Feet and Inches)', '5′ 8″']
['Weight (in Pounds)', '159']
['Eye Color', 'Blue']
['Native County', 'Bexar County']
['Native State', 'Texas']
['Name', 'Young, Christopher Anthony']
['TDCJ Number', '999508']
['Date of Birth', '09/24/1983']
['Date Received', '03/31/2006']
['Age (when Received)', '22']
['Education Level (Highest Grade Completed)', '09']
['Date of Offense', '11/21/2004']
['Age (at the time of Offense)', '21']
['County', 'Bexar']
['Race', 'Black']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '6′ 1″']
['Weight (in Pounds)', '216']
['Eye Color', 'Brown']
['Native County', 'Bexar']
['Native State', 'Texas']
['Name', 'Bible, Danny Paul']
['TDCJ Number', '999455']
['Date of Birth', '08/28/1951']
['Date Received', '07/17/2003']
['Age (when Received)', '51']
['Education Level (Highest Grade Completed)', '12']
['Date of Offense', '05/27/1979']
['Age (at the time of Offense)', '27']
['County', 'Harris']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Gray']
['Height (in Feet and Inches)', '5′ 7″']
['Weight (in Pounds)', '194']
['Eye Color', 'Blue']
['Native County', 'Brazoria']
['Native State', 'Texas']
['Name', 'Castillo, Juan Edward']
['TDCJ Number', '999502']
['Date of Birth', '04/03/1981']
['Date Received', '09/27/2005']
['Age (when Received)', '24']
['Education Level (Highest Grade Completed)', '10']
['Date of Offense', '12/03/2003']
['Age (at the time of Offense)', '22']
['County', 'Bexar']
['Race', 'Hispanic']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 11″']
['Weight (in Pounds)', '180']
['Eye Color', 'Brown']
['Native County', 'Bexar']
['Native State', 'Texas']
['Name', 'Davila, Erick Daniel']
['TDCJ Number', '999545']
['Date of Birth', '04/04/1987']
['Date Received', '02/27/2009']
['Age (when Received)', '21']
['Education Level (Highest Grade Completed)', '11th Grade']
['Date of Offense', '04/06/2008']
['Age (at the time of Offense)', '21']
['County', 'Tarrant']
['Race', 'Black']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 11″']
['Weight (in Pounds)', '161']
['Eye Color', 'Brown']
['Native County', 'Tarrant']
['Native State', 'Texas']
['Name', 'Rodriguez III, Rosendo']
['TDCJ Number', '999534']
['Date of Birth', '03/26/1980']
['Date Received', '05/14/2008']
['Age (when    Received)', '28']
['Education Level (Highest Grade Completed)', '12th Grade']
['Date of Offense', '09/13/2005']
['Age (at the time of Offense)', '25']
['County', 'Lubbock']
['Race', 'Hispanic']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 8″']
['Weight (in Pounds)', '198']
['Eye Color', 'Brown']
['Native County', 'Wichita']
['Native State', 'TX']
['Name', 'Battaglia, John David']
['TDCJ Number', '999412']
['Date of Birth', '08/02/1955']
['Date Received', '05/01/2002']
['Age (when    Received)', '46']
['Education Level (Highest Grade Completed)', '12']
['Date of Offense', '05/02/2001']
['Age (at the time of Offense)', '45']
['County', 'Dallas']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Grey']
['Height (in Feet and Inches)', '6′ 0″']
['Weight (in Pounds)', '188']
['Eye Color', 'Green']
['Native County', 'Dallas']
['Native State', 'Texas']
['Name', 'Rayford, William Earl']
['TDCJ Number', '999371']
['Date of Birth', '5/21/1953']
['Date Received', '1/4/2001']
['Age (when    Received)', '47']
['Education Level (Highest Grade Completed)', '12']
['Date of Offense', '11/16/99']
['Age (at the time of Offense)', '46']
['County', 'Dallas']
['Race', 'Black']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '6′ 4″']
['Weight (in Pounds)', '179']
['Eye Color', 'Brown']
['Native County', 'Dallas']
['Native State', 'Texas']
['Name', 'Shore, Anthony']
['TDCJ Number', '999488']
['Date of Birth', '06/25/1962']
['Date Received', '11/15/2004']
['Age (when Received)', '42']
['Education Level (Highest Grade Completed)', '12']
['Date of Offense', '04/16/1992']
['Age (at the time of Offense)', '29']
['County', 'Harris']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Gray']
['Height (in Feet and Inches)', '5′ 10″']
['Weight (in Pounds)', '198']
['Eye Color', 'Brown']
['Native County', 'Pennington']
['Native State', 'South Dakota']
['Name', 'Cardenas, Ruben Ramirez']
['TDCJ Number', '999275']
['Date of Birth', '04/07/1970']
['Date Received', '07/29/1998']
['Age (when    Received)', '28']
['Education Level (Highest Grade Completed)', '11']
['Date of Offense', '02/22/1997']
['Age (at the time of Offense)', '26']
['County', 'Hidalgo']
['Race', 'Hispanic']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 6″']
['Weight (in Pounds)', '204']
['Eye Color', 'Brown']
['Native County', 'Guanajuato']
['Native State', 'Mexico']
['Name', 'Preyor, Taichin']
['TDCJ Number', '999494']
['Date of Birth', '09/02/1970']
['Date Received', '03/30/2005']
['Age (when Received)', '34']
['Education Level (Highest Grade Completed)', '10']
['Date of Offense', '02/26/2004']
['Age (at the time of Offense)', '33']
['County', 'Bexar']
['Race', 'Black']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '6′ 2″']
['Weight (in Pounds)', '219']
['Eye Color', 'Brown']
['Native County', 'Bexar']
['Native State', 'Texas']

Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.
Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.

['Name', 'Edwards, Terry']
['TDCJ Number', '999463']
['Date of Birth', '08/10/1973']
['Date Received', '12/04/2003']
['Age (when Received)', '30']
['Education Level (Highest Grade Completed)', '12th grade']
['Date of Offense', '07/08/2002']
['Age (at the time of Offense)', '28']
['County', 'Dallas']
['Race', 'Black']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 7″']
['Weight (in Pounds)', '157']
['Eye Color', 'Brown']
['Native County', 'Dallas']
['Native State', 'Texas']
['Name', 'Wilkins, Christopher']
['TDCJ Number', '999533']
['Date of Birth', '09/29/1968']
['Date Received', '03/12/2008']
['Age (when Received)', '39']
['Education Level (Highest Grade Completed)', '12th Grade']
['Date of Offense', '10/28/2005']
['Age (at the time of Offense)', '37']
['County', 'Tarrant']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Brown']
['Height (in Feet and Inches)', '6′ 0″']
['Weight (in Pounds)', '196 lbs']
['Eye Color', 'Green']
['Native County', 'Harris County']
['Native State', 'Texas']
['Name', 'Fuller Jr., Barney Ronald']
['TDCJ Number', '999481']
['Date of Birth', '08/12/1958']
['Date Received', '07/21/2004']
['Age (when Received)', '45']
['Education Level (Highest Grade Completed)', '09']
['Date of Offense', '05/14/2003']
['Age (at the time of Offense)', '44']
['County', 'Houston']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Brown']
['Height (in Feet and Inches)', '5′ 8″']
['Weight (in Pounds)', '225']
['Eye Color', 'Brown']
['Native County', 'Summit']
['Native State', 'Ohio']
['Name', 'Vasquez, Pablo Lucio']
['TDCJ Number', '999297']
['Date of Birth', '08/11/1977']
['Date Received', '03/30/1999']
['Age (when Received)', '21']
['Education Level (Highest Grade Completed)', '8']
['Date of Offense', '04/18/1998']
['Age (at the time of Offense)', '20']
['County', 'Hidalgo']
['Race', 'Hispanic']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 6″']
['Weight (in Pounds)', '174']
['Eye Color', 'Brown']
['Native County', 'Dawson']
['Native State', 'Texas']
['Name', 'Ward, Adam Kelly']
['TDCJ Number', '999525']
['Date of Birth', '08/11/1982']
['Date Received', '06/27/2007']
['Age (when Received)', '24']
['Education Level (Highest Grade Completed)', 'Not available.']
['Date of Offense', '06/13/2005']
['Age (at the time of Offense)', '22']
['County', 'Hunt']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Brown']
['Height (in Feet and Inches)', '5′ 6″']
['Weight (in Pounds)', '149 lbs.']
['Eye Color', 'Hazel']
['Native County', 'Hunt County']
['Native State', 'Texas']
['Name', 'Wesbrook, Coy Wayne']
['TDCJ Number', '999281']
['Date of Birth', '2/1/58']
['Date Received', '9/2/98']
['Age (when Received)', '40']
['Education Level (Highest Grade Completed)', '8 years']
['Date of Offense', '11/13/97']
['Age (at the time of Offense)', '39']
['County', 'Harris']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Brown']
['Height (in Feet and Inches)', '5′ 10″']
['Weight (in Pounds)', '296']
['Eye Color', 'Brown']
['Native County', 'Harris']
['Native State', 'Texas']

Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.

['Name', 'Freeman, James']
['TDCJ Number', '999539']
['Date of Birth', '11/12/1980']
['Date Received', '11/10/2008']
['Age (when Received)', '27']
['Education Level (Highest Grade Completed)', '12th Grade']
['Date of Offense', '03/17/2007']
['Age (at the time of Offense)', '26']
['County', 'Wharton']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Brown']
['Height (in Feet and Inches)', '5′ 6″']
['Weight (in Pounds)', '155']
['Eye Color', 'Brown']
['Native County', 'Wharton County']
['Native State', 'Texas']
['Name', 'Masterson, Richard Allen']
['TDCJ Number', '999414']
['Date of Birth', '03/05/1972']
['Date Received', '05/15/2002']
['Age (when Received)', '30']
['Education Level (Highest Grade Completed)', '6']
['Date of Offense', '02/09/2001']
['Age (at the time of Offense)', '28']
['County', 'Harris']
['Race', 'White']
['Gender', 'Male']
['Hair Color', 'Brown']
['Height (in Feet and Inches)', '5′ 11″']
['Weight (in Pounds)', '233']
['Eye Color', 'Brown']
['Native County', 'Harris']
['Native State', 'Texas']
['Name', 'Holiday, Raphael Deon']
['TDCJ Number', '999419']
['Date of Birth', '07/20/1979']
['Date Received', '06/21/2002']
['Age (when Received)', '22']
['Education Level (Highest Grade Completed)', '9']
['Date of Offense', '09/06/2000']
['Age (at the time of Offense)', '21']
['County', 'Walker, on change of venue from Madison County']
['Race', 'Black']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 10″']
['Weight (in Pounds)', '194']
['Eye Color', 'Brown']
['Native County', 'Grimes']
['Native State', 'Texas']
['Name', 'Escamilla, Licho']
['TDCJ Number', '999432']
['Date of Birth', '07/03/1982']
['Date Received', '11/01/2002']
['Age (when Received)', '20']
['Education Level (Highest Grade Completed)', '8']
['Date of Offense', '11/25/2001']
['Age (at the time of Offense)', '20']
['County', 'Dallas']
['Race', 'Hispanic']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 6″']
['Weight (in Pounds)', '156']
['Eye Color', 'Brown']
['Native County', 'Dallas']
['Native State', 'Texas']
['Name', 'Garcia, Juan Martin']
['TDCJ Number', '999360']
['Date of Birth', '2/18/1980']
['Date Received', '6/21/2000']
['Age (when Received)', '20']
['Education Level (Highest Grade Completed)', '8']
['Date of Offense', '9/17/1998']
['Age (at the time of Offense)', '18']
['County', 'Harris']
['Race', 'Hispanic']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 5″']
['Weight (in Pounds)', '183']
['Eye Color', 'Brown']
['Native County', 'Harris']
['Native State', 'Texas']
['Name', 'Lopez, Daniel']
['TDCJ Number', '999555']
['Date of Birth', '09/15/1987']
['Date Received', '03/16/2010']
['Age (when    Received)', '22']
['Education Level (Highest Grade Completed)', '10th grade']
['Date of Offense', '03/11/2009']
['Age (at the time of Offense)', '21']
['County', 'Nueces']
['Race', 'Hispanic']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 7″']
['Weight (in Pounds)', '140 lbs.']
['Eye Color', 'Brown']
['Native County', 'Nueces County']
['Native State', 'Texas']
['Name', 'Russeau, Gregory Lynn']
['TDCJ Number', '999430']
['Date of Birth', '10/11/1969']
['Date Received', '10/15/2002']
['Age (when    Received)', '33']
['Education Level (Highest Grade Completed)', '11']
['Date of Offense', '05/30/2001']
['Age (at the time of Offense)', '31']
['County', 'Harris']
['Race', 'Black']
['Gender', 'Male']
['Hair Color', 'Black']
['Height (in Feet and Inches)', '5′ 11″']
['Weight (in Pounds)', '299']
['Eye Color', 'Brown']
['Native County', 'Smith']
['Native State', 'Texas']

Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.

import requests
import lxml.html as lh
import pandas as pd
from bs4 import BeautifulSoup as bs

import pandas as pd
df = pd.read_csv('V2_final_statements.csv')

test_df = df.copy()

def get_summary(link):
    try:
        url="https://www.tdcj.texas.gov/death_row/" + link
        page = requests.get(url)
        soup= bs(page.content, "html.parser")
#         print(soup)
        p_tags = soup.find_all("p")
#         for tag in p_tags:
#             print('---')
#             print(tag.text)
        
#         print(tr_tags)
        offender_obj = {}
        for t in p_tags:
            offender_obj.update({ t.span.text: t.span.next_sibling.next_sibling.strip() })

#             print(t.span.text)
#             print(t.span.next_sibling.next_sibling.strip())
# #             info = t.find_all('td')
#             row_array = []
#             for i in info:
#                 string = i.text.replace(u'\xa0', u'')
#                 if len(string) > 0:
#                     row_array.append(i.text)
#             print(row_array)
#             try:
#                 if len(row_array) > 1:
#                     offender_obj.update({row_array[0]: row_array[1]})
#             except:
#                 return 'no field'
        print(offender_obj)
        return offender_obj
    except:
        return 'no link'
            
# get_summary("dr_info/halljusten.html")
test_df['Summary'] = test_df.apply(lambda x: get_summary(x['Offender Information']), axis=1)

{'Prior Occupation': 'Laborer', 'Prior Prison Record': 'TDCJ#  914053 on a 2 year sentence from El    Paso County  for one count of Burglary of Habitat.', 'Summary of Incident': 'On  October 28, 2002, in El Paso County,   Texas, Hall fatally strangled a  female (race and age unknown) with a black electrical cord.', 'Co-Defendants': 'None', 'Race and Gender of Victim': 'Unknown  Female'}
{'Prior Occupation': 'Machine  Operator', 'Prior Prison Record': 'TDCJ  #592340 on a twelve year sentence for aggravated robbery with a deadly weapon  out of Dallas  county.', 'Summary of Incident': 'On  September 15, 2007, in Dallas, Texas, the subject fatally stabbed a 30-year-old black female and two 9-year-old black males. He then called 911 and  confessed to the crimes.', 'Co-Defendants': 'None', 'Race and Gender of Victim': 'Black  female; two black males.'}
{'Prior Occupation': 'Cabinet maker', 'Prior Prison Record': '#925766 - Theft; #1060195 - Unauthorized Use of a Motor Vehicle; #1225186 - Theft; #1354704 - Theft; #1410287 - Theft of Vehicle', 'Summary of Incident': "Subject and codefendant knocked on the victim's door and  pulled a 9 millimeter handgun when she answered.\xa0 They pushed her inside the house and removed  several electronics.\xa0 Before leaving the  residence, the subject shot the victim once in the back of the head causing her  death.", 'Co-Defendants': 'Jose Claremont Ramos, Jr.', 'Race and Gender of Victim': 'White female'}
{'Prior Occupation': 'Laborer', 'Prior Prison Record': '#1014350 - On a one-year sentence from Cochran County for Reckless Injury to Elderly with Bodily Injury', 'Summary of Incident': 'On April 6, 2003, in Fort Worth, Texas, Crutsinger entered the residence of a 71-year-old white female and an 88-year-old white female and stabbed both victims multiple times, resulting in their deaths.', 'Co-Defendants': 'None', 'Race and Gender of Victim': 'White female; White female'}
{'Prior Occupation': 'electrician,  mechanic, laborer', 'Prior Prison Record': '#758150 – On a two-year sentence for one count of Burglary of a Building (originally sentenced to probation, but revoked and sentenced to the TDCJ-ID when arrested for current offense)', 'Summary of Incident': 'On December 8, 1998, Swearingen kidnapped and strangled a 19-year-old white female.', 'Co-Defendants': 'None', 'Race and Gender of Victim': 'White  female'}
{'Prior Occupation': 'carpenter,  laborer', 'Prior Prison Record': 'TDCJ-ID  #624420, 10-year sentence for one count of Burglary; 7/28/97 released on Parole  to Orange County', 'Summary of Incident': 'On  06/07/98, during the nighttime hours, the subject and co-defendants, Lawrence  Brewer and Shawn Allen Berry, murdered James Byrd Jr., a 49-year old black  male, by dragging the victim behind their 1982 gray Ford pickup truck, located  on Huff Creek Road, in Jasper, Texas. The subject and the co-defendants picked  the victim up while he was hitchhiking in Jasper.', 'Co-Defendants': 'Berry, Shawn; Brewer,  Lawrence Russell', 'Race and Gender of Victim': 'Black  male'}

Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.

{}

Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.

{}
{'Prior Occupation': 'Laborer', 'Prior Prison Record': '#792374  on a 5 year sentence from Dallas   County for 1 count of  sexual assault of a child. (Current offense was committed prior to the offender  being incarcerated for the sexual assault conviction.)', 'Summary of Incident': 'On  9/21/1993 at 9:00 p.m. in Mesquite,  Braziel approached a newlywed couple walking on a jogging trail of a community  college. Braziel demanded money. When it was discovered that neither of the two  had any money in their possession, Braziel shot the 27 year old white male,  resulting in his death. Braziel then sexually assaulted the 23 year old white  female. Braziel linked to the crime in January 2001 when his DNA was found to  match the DNA taken from the female victim.', 'Co-Defendants': 'None', 'Race and Gender of Victim': 'White  male'}
{'Prior Occupation': 'maintenance,  laborer', 'Prior Prison Record': '#774391  on a 50 year sentence from Bexar   County for murder with a  deadly weapon; 12/13/2000 escaped from custody', 'Summary of Incident': 'On  12/24/2000, in Irving, Texas, Garcia and six co-defendants fatally  shot a 31 year old white male police officer while on escape from the TDCJ  Connally Unit.', 'Co-Defendants': 'Patrick  Murphy, George Rivas, Randy Halprin, Larry Harper, Michael Rodriguez, Donald  Newbury', 'Race and Gender of Victim': 'white  male'}

Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.

{'Prior Occupation': "electrician's  helper, plumber's helper, laborer", 'Prior Prison Record': 'Received by TDCJ on 05/04/1993 on a ten year sentence four Burglary of a Habitation out of Delta County. Released on Parole on 10/23/1995. Returned from Parole without new convictions on 04/21/1997. Released on Mandatory Supervision on 09/17/1999.', 'Summary of Incident': "On 03/12/2000, Acker caused the death of a 32 year old female. Acker kidnapped the victim, then murdered her by strangulation and blunt force trauma. The victim's  body was found along side a county road.", 'Co-Defendants': 'None', 'Race and Gender of Victim': 'Unknown female'}
{'Prior Occupation': 'Laborer', 'Prior Prison Record': 'TDCJ  #454580, 6/24/87 from Harris County on a 2-year sentence for 1-count Possession  of a Controlled Substance Cocaine, released 8/27/87 on Parole to Harris County;  TDC #630186, 1/8/93 from Gregg County on an 18-year sentence for 2 counts  Possession of a Controlled Substance, released on Parole 2/23/96 to Smith  County.', 'Summary of Incident': 'Convicted  in the May 1, 1998 drowning death of a 20-year-old white female. Clark and a  co-defendant took the victim to their residence, where they drowned her in the  bathtub. They then took the victim to a remote area where they dumped the body.  The victim was found 5 months later in a ditch by Tyler police.', 'Co-Defendants': 'Tory  Gene Bush', 'Race and Gender of Victim': 'White  female'}
{'Prior Occupation': 'Laborer', 'Prior Prison Record': 'None', 'Summary of Incident': 'On  11/21/2004 in Bexar   County, Young, while in  the course of a robbery, fatally shot a fifty five year old Asian male.', 'Co-Defendants': 'None', 'Race and Gender of Victim': 'Asian Male'}
{'Prior Occupation': 'Cook,  Laborer', 'Prior Prison Record': 'TDCJ#  1066210 on a two year sentence for Deadly Conduct with a Firearm from Bexar County.', 'Summary of Incident': 'On  12/03/2003 in Bexar   County, Castillo and 3  co-defendants fatally shot a 19 year old Hispanic male after demanding his  money.', 'Co-Defendants': 'Francisco  Gonzales, Debra Espinosa and Teresa Quintero', 'Race and Gender of Victim': 'Hispanic  Male'}
{'Prior Occupation': 'None', 'Prior Prison Record': 'TDCJ  #1369222 - Burglary Habitation', 'Summary of Incident': 'On  April 26, 2008, the subject and codefendant drove to a birthday party where the  codefendant dropped off the subject. The subject went to the back side of the  car, grabbed a model SKSS (762mm caliber) semiautomatic rifle out of the trunk  and walked toward the apartment. Once the subject got to the apartment where  the party was located, he went inside and opened fire. The shots fired resulted  in the death of an adult black female and a five year old black female. Several  other victims at the birthday party were injured.', 'Co-Defendants': 'Garfield Thompson', 'Race and Gender of Victim': 'One (1) black adult female and one (1) five year old black female.'}
{'Prior Occupation': 'Food Service, Office Clerk.', 'Prior Prison Record': 'None', 'Summary of Incident': "On 09/13/2005 in Lubbock County, Texas, a 29 year old white female body was found deceased in a suitcase. The subject had sexually assaulted the victim and then caused death by striking the victim with or against a hard object and then choking the victim. This also resulted in the death of the victim's 5 week old fetus. The subject was arrested by the Lubbock Police Department.", 'Co-Defendants': 'None', 'Race and Gender of Victim': '29 year old white female.'}
{'Prior Occupation': 'computer  software, accounting, laborer', 'Prior Prison Record': 'None', 'Summary of Incident': 'On  May 12, 2001, Battaglia shot and killed his two daughters, ages 6 and 9. The  mother of the children was on the phone with the 9 year old daughter at the  time of the offense and heard the gunshots.', 'Co-Defendants': 'None', 'Race and Gender of Victim': 'two  white females'}
{'Prior Occupation': 'Laborer', 'Prior Prison Record': '#437754  TDCJ-ID on a 23 year sentence from Dallas  County for 1 count of murder with a  deadly weapon; received in 1986 and released on mandatory supervision to Dallas County  on 12/5/1994.', 'Summary of Incident': "On  11/16/1999 in Dallas,  Rayford murdered a 44 year old black female by strangulation, stabbing, and  blunt force. Rayford took the victim's body and threw it into a nearby creek.  The murder took place in the home of the victim and was witnessed by her 11 and  14 year old sons.", 'Co-Defendants': 'None', 'Race and Gender of Victim': 'Black  female'}
{'Prior Occupation': 'Laborer', 'Prior Prison Record': 'None', 'Summary of Incident': "On  02/22/97, in Edinburg,  Cardenas and a co-defendant committed the offense of capital murder against a  16-year old Hispanic female. Cardenas  entered the victim's residence through a window, tied her up with duct tape,  then put her in a vehicle with the co-defendant and drove to a remote location.  Cardenas raped  the victim, beat her severely with his fists, and strangled her, causing her  death. Cardenas  dumped her body into a nearby canal.", 'Co-Defendants': 'Castillo,  Tony', 'Race and Gender of Victim': 'Hispanic  female'}
{}
{'Prior Occupation': 'Truck  Driver/Laborer', 'Prior Prison Record': 'None', 'Summary of Incident': 'On  2/26/2004 in Bexar County, Texas, Preyor entered the apartment of his  girlfriend, a twenty year old white female, and fatally stabbed her. As Preyor  fled the scene, he allegedly stabbed an adult Hispanic male that was in the  apartment.', 'Co-Defendants': 'None', 'Race and Gender of Victim': 'White  Female'}

Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.

{}

Some characters could not be decoded, and were replaced with REPLACEMENT CHARACTER.

test_df.to_csv('V3_final_statements.csv')

import pandas as pd
df = pd.read_csv('V2_final_statements.csv')

df

	Execution	Offender Information	Last Statement	Last Name	First Name	TDCJNumber	Age	Date	Race	County	Offender Information	Last Statement	Last Statemen
0	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
1	566	dr_info/halljusten.html	dr_info/halljustenlast.html	Hall	Justen	999497	38	11/6/2019	White	El Paso	NaN	NaN	NaN
2	565	dr_info/sparksrobert.html	dr_info/sparksrobertlast.html	Sparks	Robert	999542	45	9/25/2019	Black	Dallas	NaN	NaN	NaN
3	564	dr_info/solizmarkanthony.html	dr_info/solizmarkanthonylast.html	Soliz	Mark	999571	37	9/10/2019	Hispanic	Johnson	NaN	NaN	NaN
4	563	dr_info/crutsingerbilly.html	dr_info/crutsingerbillylast.html	Crutsinger	Billy	999459	64	9/4/2019	White	Tarrant	NaN	NaN	NaN
...	...	...	...	...	...	...	...	...	...	...	...	...	...
562	5	dr_info/skillerndoyle.jpg	dr_info/skillerndoylelast.html	Skillern	Doyle	518	49	01/16/1985	White	Lubbock	NaN	NaN	NaN
563	4	dr_info/barefootthomas.jpg	dr_info/barefootthomaslast.html	Barefoot	Thomas	621	39	10/30/1984	White	Bell	NaN	NaN	NaN
564	3	dr_info/obryanronald.jpg	dr_info/obryanronaldlast.html	O'Bryan	Ronald	529	39	03/31/1984	White	Harris	NaN	NaN	NaN
565	2	dr_info/autryjames.html	dr_info/no_last_statement.html	Autry	James	670	29	03/14/1984	White	Jefferson	NaN	NaN	NaN
566	1	dr_info/brookscharlie.html	dr_info/brookscharlielast.html	Brooks, Jr.	Charlie	592	40	12/07/1982	Black	Tarrant	NaN	NaN	NaN

	Execution	Offender Information	Last Statement	Last Name	First Name	TDCJNumber	Age	Date	Race	County
1	566	dr_info/halljusten.html	dr_info/halljustenlast.html	Hall	Justen	999497	38	11/6/2019	White	El Paso
2	565	dr_info/sparksrobert.html	dr_info/sparksrobertlast.html	Sparks	Robert	999542	45	9/25/2019	Black	Dallas
3	564	dr_info/solizmarkanthony.html	dr_info/solizmarkanthonylast.html	Soliz	Mark	999571	37	9/10/2019	Hispanic	Johnson
4	563	dr_info/crutsingerbilly.html	dr_info/crutsingerbillylast.html	Crutsinger	Billy	999459	64	9/4/2019	White	Tarrant
5	562	dr_info/swearingenlarry.html	dr_info/swearingenlarrylast.html	Swearingen	Larry	999361	48	8/21/2019	White	Montgomery
...	...	...	...	...	...	...	...	...	...	...
562	5	dr_info/skillerndoyle.jpg	dr_info/skillerndoylelast.html	Skillern	Doyle	518	49	01/16/1985	White	Lubbock
563	4	dr_info/barefootthomas.jpg	dr_info/barefootthomaslast.html	Barefoot	Thomas	621	39	10/30/1984	White	Bell
564	3	dr_info/obryanronald.jpg	dr_info/obryanronaldlast.html	O'Bryan	Ronald	529	39	03/31/1984	White	Harris
565	2	dr_info/autryjames.html	dr_info/no_last_statement.html	Autry	James	670	29	03/14/1984	White	Jefferson
566	1	dr_info/brookscharlie.html	dr_info/brookscharlielast.html	Brooks, Jr.	Charlie	592	40	12/07/1982	Black	Tarrant

	Execution	Offender Information	Last Statement	Last Name	First Name	TDCJNumber	Age	Date	Race	County
1	566	dr_info/halljusten.html	dr_info/halljustenlast.html	Hall	Justen	999497	38	11/6/2019	White	El Paso
2	565	dr_info/sparksrobert.html	dr_info/sparksrobertlast.html	Sparks	Robert	999542	45	9/25/2019	Black	Dallas
3	564	dr_info/solizmarkanthony.html	dr_info/solizmarkanthonylast.html	Soliz	Mark	999571	37	9/10/2019	Hispanic	Johnson
4	563	dr_info/crutsingerbilly.html	dr_info/crutsingerbillylast.html	Crutsinger	Billy	999459	64	9/4/2019	White	Tarrant
5	562	dr_info/swearingenlarry.html	dr_info/swearingenlarrylast.html	Swearingen	Larry	999361	48	8/21/2019	White	Montgomery
...	...	...	...	...	...	...	...	...	...	...
562	5	dr_info/skillerndoyle.jpg	dr_info/skillerndoylelast.html	Skillern	Doyle	518	49	01/16/1985	White	Lubbock
563	4	dr_info/barefootthomas.jpg	dr_info/barefootthomaslast.html	Barefoot	Thomas	621	39	10/30/1984	White	Bell
564	3	dr_info/obryanronald.jpg	dr_info/obryanronaldlast.html	O'Bryan	Ronald	529	39	03/31/1984	White	Harris
565	2	dr_info/autryjames.html	dr_info/no_last_statement.html	Autry	James	670	29	03/14/1984	White	Jefferson
566	1	dr_info/brookscharlie.html	dr_info/brookscharlielast.html	Brooks, Jr.	Charlie	592	40	12/07/1982	Black	Tarrant

	Unnamed: 0	Execution	Offender Information	Last Statement	Last Name	First Name	TDCJNumber	Age	Date	Race	County	Last Statement Text	Offender Info
0	1	566	dr_info/halljusten.html	dr_info/halljustenlast.html	Hall	Justen	999497	38	11/6/2019	White	El Paso	Yeah, I want to address the Roundtree family ...	{'Name': 'Hall, Justen Grant', 'TDCJ Number': ...
1	2	565	dr_info/sparksrobert.html	dr_info/sparksrobertlast.html	Sparks	Robert	999542	45	9/25/2019	Black	Dallas	Umm, Pamela can you hear me Stephanie, Hardy,...	{'Name': 'Sparks, Robert', 'TDCJ Number': '999...
2	3	564	dr_info/solizmarkanthony.html	dr_info/solizmarkanthonylast.html	Soliz	Mark	999571	37	9/10/2019	Hispanic	Johnson	It’s 6:09 on September 10th, Kayla and David,...	{'Name': 'Soliz, Mark Anthony', 'TDCJ Number':...
3	4	563	dr_info/crutsingerbilly.html	dr_info/crutsingerbillylast.html	Crutsinger	Billy	999459	64	9/4/2019	White	Tarrant	Hi ladies I wanted to tell ya’ll how much I l...	{'Name': 'Crutsinger, Billy Jack', 'TDCJ Numbe...
4	5	562	dr_info/swearingenlarry.html	dr_info/swearingenlarrylast.html	Swearingen	Larry	999361	48	8/21/2019	White	Montgomery	Lord forgive them. They don’t know what they ...	{'Name': 'Larry Ray Swearingen', 'TDCJ Number'...
...	...	...	...	...	...	...	...	...	...	...	...	...	...
561	562	5	dr_info/skillerndoyle.jpg	dr_info/skillerndoylelast.html	Skillern	Doyle	518	49	01/16/1985	White	Lubbock	I pray that my family will rejoice and will fo...	{}
562	563	4	dr_info/barefootthomas.jpg	dr_info/barefootthomaslast.html	Barefoot	Thomas	621	39	10/30/1984	White	Bell	When asked if he had a last statement, he rep...	{}
563	564	3	dr_info/obryanronald.jpg	dr_info/obryanronaldlast.html	O'Bryan	Ronald	529	39	03/31/1984	White	Harris	What is about to transpire in a few moments is...	{}
564	565	2	dr_info/autryjames.html	dr_info/no_last_statement.html	Autry	James	670	29	03/14/1984	White	Jefferson	This offender declined to make a last statemen...	{'Name': 'Autry, James David', 'TDCJ Number': ...
565	566	1	dr_info/brookscharlie.html	dr_info/brookscharlielast.html	Brooks, Jr.	Charlie	592	40	12/07/1982	Black	Tarrant	Statement to the Media: I, at this very moment...	{'Photo not available': 'Name', 'TDCJ Number':...

Getting the information about executed prisoners from the Texas Department of Criminal Justice¶

https://www.tdcj.texas.gov/death_row/dr_executed_offenders.html¶

https://www.tdcj.texas.gov/death_row/dr_executed_offenders.html ¶