+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ [ ]:
# Project Shark _ Data Frame Team
[3]:
import numpy as np
[4]:
!mamba install pandas
mambajs 0.21.1 + +Specs: xeus-python, numpy, matplotlib, pillow, ipywidgets>=8.1.6, ipyleaflet, scipy, pandas +Channels: emscripten-forge-4x, conda-forge + +Solving environment... +Solving took 1.3468000000044704 seconds + Name Version Build Channel +-------------------------------------------------------------------- ++ pandas 3.0.3 np23py313h1e705a5_0 emscripten-forge-4x ++ python-tzdata 2026.2 pyhd8ed1ab_0 conda-forge +- pip 26.1.2 pyh145f28c_0 conda-forge +
[5]:
import pandas as pd
[ ]:
# Analyze Shark Attacks in the world
[6]:
import os
print(os.getcwd())
/drive/notebooks +
[7]:
import platform
print(platform.platform())
Emscripten-4.0.9-wasm32-32bit +
[8]:
import os
print(os.getcwd())
/drive/notebooks +
[9]:
!pip install xlrd
mambajs 0.21.1 + +Process pip requirements ... + +Successfully installed xlrd-2.0.2 +
[14]:
import pandas as pd
df_shark = pd.read_excel("GSAF5.xls")
print(df_shark.head())
Date Year Type Country State \ +0 24th May 2026.0 Unprovoked Australia Queensland +1 17th May 2026.0 Questionable USA Maryland +2 16th May 2026.0 Unprovoked Australia Western Australia +3 14th April 2026.0 UNprovoked Maldives Gaafu Alif Atoll +4 3rd April 2026.0 Unprovoked Australia South Australia + + Location Activity \ +0 Kennedy Shoal Spearfishing +1 Assateague State Park Surfing +2 Horseshoe Reef Rottnest Island Skindiving +3 Kooddoo Swimming +4 Middleton Beach Fleurieu Peninsula Adelaide Surfing + + Name Sex Age ... \ +0 Michael Jensz M 39 ... +1 Brendan Oster M ? ... +2 Steven Mattaboni M 38 ... +3 Not stated - on honeymoon M ? ... +4 Oliver Tokic-Bensley M 16 ... + + Species \ +0 Undetermined Bull shark most likely +1 Blue fish bite most probable +2 Great White Shark +3 Unknown +4 Bronze Whaler + + Source pdf href formula href \ +0 Simon de Marchi: 9 News: 7 News: ABC News: NaN NaN NaN +1 Keithe Cowley: Delmarva Now: NaN NaN NaN +2 ABC News: 9 News: 7 News: Simon De Marchi NaN NaN NaN +3 The U.S. Sun: Simon De Marchi NaN NaN NaN +4 ABC News: The Guardian: Andrew Currie and Bob... NaN NaN NaN + + Case Number Case Number.1 original order Unnamed: 21 Unnamed: 22 +0 NaN NaN NaN NaN NaN +1 NaN NaN NaN NaN NaN +2 NaN NaN NaN NaN NaN +3 NaN NaN NaN NaN NaN +4 NaN NaN NaN NaN NaN + +[5 rows x 23 columns] +
[16]:
print(type(df_shark))
print(df_shark.columns.tolist())
<class 'pandas.DataFrame'> +['Date', 'Year', 'Type', 'Country', 'State', 'Location', 'Activity', 'Name', 'Sex', 'Age', 'Injury', 'Fatal Y/N', 'Time', 'Species ', 'Source', 'pdf', 'href formula', 'href', 'Case Number', 'Case Number.1', 'original order', 'Unnamed: 21', 'Unnamed: 22'] +
[17]:
# we clean the columns we selected
cols = ["Country", "State", "Location"]
for col in cols :
df_shark[col] = df_shark[col].astype(str).str.strip() # suppress the unecessary spaces in the text
df_shark[col] = df_shark[col].str.replace(r"[.,]+$", " ",regex=True) #suppress the comas or points at the end of the text
df_shark[col] = df_shark[col].replace(["nan", "NAN", "none", " "],np.nan) # Convert the nan
[19]:
df_shark[["Country", "State", "Location"]].head(20) # we verify the cleaning for the 20 first rows
[17]:
+
+
+
+
+
| + | Country | +State | +Location | +
|---|---|---|---|
| 0 | +Australia | +Queensland | +Kennedy Shoal | +
| 1 | +USA | +Maryland | +Assateague State Park | +
| 2 | +Australia | +Western Australia | +Horseshoe Reef Rottnest Island | +
| 3 | +Maldives | +Gaafu Alif Atoll | +Kooddoo | +
| 4 | +Australia | +South Australia | +Middleton Beach Fleurieu Peninsula Adelaide | +
| 5 | +Bahamas | +Andros Island | +Fresh Creek | +
| 6 | +Australia | +South Australia | +Cape Jaffa Limestone Coast | +
| 7 | +Australia | +NSW | +Little Avalon Beach | +
| 8 | +USA | +California | +Big River Beach Mendocino County | +
| 9 | +Australia | +Western Australia | +Exmouth | +
| 10 | +Australia | +Western Australia | +Exmouth | +
| 11 | +Australia | +Queensland | +Lady Elliott Island | +
| 12 | +New Caledonia | +Noumea | +Anse Vata near Point Magnin | +
| 13 | +Cayman Islands | +Little Cayman Island | +Caribbean Sea | +
| 14 | +Brazil | +Recife | +Del Chifre Beach in Olinda | +
| 15 | +Australia | +NSW | +Angels Beach East Ballina | +
| 16 | +Australia | +Tasmania | +Cooee Beach west of Burnie | +
| 17 | +Australia | +NSW | +Point Plomber North of Port Macquarie | +
| 18 | +Australia | +NSW | +Dee Why | +
| 19 | +Australia | +NSW | +North Steyne | +
[21]:
# For the Country column we will upper the text
df_shark["Country"] = df_shark["Country"].str.upper()
[22]:
# we amend the variation
country_mapping = {
"USA" : "UNITED STATES",
"U.S.A" : "UNITED STATES",
"UNITED STATES OF AMERICA" : "UNITED STATES",
"AUSTRALIA" : "AUSTRALIA",
"ENGLAND" : "UNITED KINGDOM",
"SCOTLAND" : "UNITED KINGDOM",
"WALES" : "UNITED KINGDOM",
}
df_shark["Country"] = df_shark["Country"].replace(country_mapping)
[23]:
df_shark["Country"].value_counts().head(20)
[21]:
Country +UNITED STATES 2581 +AUSTRALIA 1531 +SOUTH AFRICA 599 +NEW ZEALAND 146 +BAHAMAS 142 +PAPUA NEW GUINEA 136 +BRAZIL 123 +MEXICO 107 +ITALY 72 +FIJI 70 +NEW CALEDONIA 68 +PHILIPPINES 65 +REUNION 60 +EGYPT 53 +MOZAMBIQUE 52 +CUBA 49 +SPAIN 47 +UNITED KINGDOM 44 +INDIA 41 +FRENCH POLYNESIA 40 +Name: count, dtype: int64
[26]:
# We want to analyze the top 10 countries.
top_countries = (
df_shark["Country"]
.value_counts()
.head(10)
)
[27]:
top_countries.plot(
kind="bar"
) # Question : In which countries do most attacks occur?
[25]:
<Axes: xlabel='Country'>
[29]:
# for the State column cleansing we start with harmonazing the format
df_shark["State"] = df_shark["State"].str.title()
[30]:
# We correct now any possible abbreviations.
state_mapping = {
"Nsw" : "New South Wales",
"Qld" : "Queensland",
"Wa" : "Western Australia",
"Sa" : "South Australia"
}
df_shark["State"] = df_shark["State"].replace(state_mapping)
[31]:
df_shark["State"].value_counts().head(20)
[29]:
State +Florida 1193 +New South Wales 539 +Queensland 356 +Hawaii 345 +California 328 +Western Australia 247 +Kwazulu-Natal 219 +Western Cape Province 197 +South Carolina 176 +Eastern Cape Province 168 +South Australia 122 +North Carolina 122 +Victoria 98 +Texas 83 +Pernambuco 80 +North Island 76 +Torres Strait 72 +New Jersey 57 +New York 48 +Tasmania 45 +Name: count, dtype: int64
[35]:
import matplotlib.pyplot as plt
plt.figure(figsize=(12,6))
state_counts.sort_values().plot(
kind="barh"
)
plt.title("top 20 States with Most Shark Attacks")
plt.xlabel("Number of Attacks")
plt.show()
[36]:
# Analyze the percentage represented for each region :
state_percent = (
df_shark["State"]
.value_counts(normalize=True)
.mul(100)
.head(20)
)
state_percent.round(2)
[34]:
State +Florida 18.07 +New South Wales 8.16 +Queensland 5.39 +Hawaii 5.22 +California 4.97 +Western Australia 3.74 +Kwazulu-Natal 3.32 +Western Cape Province 2.98 +South Carolina 2.67 +Eastern Cape Province 2.54 +South Australia 1.85 +North Carolina 1.85 +Victoria 1.48 +Texas 1.26 +Pernambuco 1.21 +North Island 1.15 +Torres Strait 1.09 +New Jersey 0.86 +New York 0.73 +Tasmania 0.68 +Name: proportion, dtype: float64
[ ]:
# Analyze which region has the most fatal attack.
fatal_states = (
df_shark[df_shark["Fatal Y/N]=="Y"]
["State"]
[ ]:
# Cleaning the Location column
[76]:
print(df["Location"].head(20))
0 Kennedy Shoal +1 Assateague State Park +2 Horseshoe Reef Rottnest Island +3 Kooddoo +4 Middleton Beach Fleurieu Peninsula Adelaide +5 Fresh Creek +6 Cape Jaffa Limestone Coast +7 Little Avalon Beach +8 Big River Beach Mendocino County +9 Exmouth +10 Exmouth +11 Lady Elliott Island +12 Anse Vata near Point Magnin +13 Caribbean Sea +14 Del Chifre Beach in Olinda +15 Angels Beach East Ballina +16 Cooee Beach west of Burnie +17 Point Plomber North of Port Macquarie +18 Dee Why +19 North Steyne +Name: Location, dtype: str +
[77]:
print(df["Location"].nunique()) # check the total of unique values
4588 +
[78]:
print(df["Location"].isna().sum()) #check the sum of na
567 +
[79]:
df["Location"] = (
df["Location"]
.astype(str) # define the type as string.
.str.strip() #suppress the non needed spaces.
.str.replace(r"\s+", " ", regex=True)
.str.replace(r"[.,]+$", " ", regex=True) #suppress punctuation at the end of the text
.str.title()
)
[80]:
# We restore the actual missing values.
import numpy as np
df["Location"] = df["Location"].replace(
["Nan", "None", " ", "Unkown"],
np.nan
)
[81]:
# As the dataset conatains distances in miles or kms we will suppress them to keep only the location name.
# So that we have: before==> 10 miles north of Durban, after ==> North of Durban.
df["Location"] = df["Location"].str.replace(
r"\b\d+(\.\d+)?\s*(mile|miles|km|kilometer|kilometers|yard|yards|meter|meters|m)\b",
" ",
regex=True,
case=False
)
[82]:
# We strip out the remaining numbers.
df["Location"] = df["Location"].str.replace(
r"\b\d+\b",
" ",
regex=True
)
[83]:
# We clean the remained spaces that have been created by the cleansing before.
df["Location"] = (
df["Location"]
.str.replace(r"\s+", " ", regex=True)
.str.strip()
)
[ ]:
[ ]:
# Now we drop generic location entries as they offer no exploitable information.(offshore, Ocean, At sea, Off coast...)
[87]:
generic_locations = [
"Offshore",
"Off Shore",
"Off Coast",
"At Sea",
"Open Sea",
"Ocean",
"Near Shore",
"Unknown"
]
df["Location"] = df["Location"].replace(
generic_locations,
np.nan
)
[ ]:
# finally we check & control.
[88]:
# check the missing values.
df["Location"].isna().sum()
[86]:
np.int64(573)
[198]:
df["Location"]
[196]:
0 Kennedy Shoal +1 Assateague State Park +2 Horseshoe Reef Rottnest Island +3 Kooddoo +4 Middleton Beach Fleurieu Peninsula Adelaide + ... +7085 Roebuck Bay +7086 NaN +7087 Ocracoke Inlet +7088 Panama Bay 8ºn, 79ºw +7089 Below The English Fort, Trincomalee +Name: Location, Length: 7090, dtype: str
[89]:
# check unique localisation values.
df["Location"].nunique()
[87]:
4549
[91]:
# list the top 20 of the localisations.
df["Location"].value_counts().head(20)
[89]:
Location +New Smyrna Beach, Volusia County 198 +Daytona Beach, Volusia County 35 +Cocoa Beach, Brevard County 33 +Ponce Inlet, Volusia County 28 +Myrtle Beach, Horry County 22 +Melbourne Beach, Brevard County 21 +Isle Of Palms, Charleston County 18 +Durban 18 +Piedade 14 +Boa Viagem, Recife 14 +Jensen Beach, Martin County 13 +Jacksonville Beach, Duval County 13 +Ponce Inlet, New Smyrna Beach, Volusia County 13 +Juno Beach, Palm Beach County 12 +Ormond Beach, Volusia County 11 +Palm Beach, Palm Beach County 11 +Mossel Bay 11 +Ahvaz, On The Karun River 11 +New Smyrna Beach 10 +Florida Keys, Monroe County 10 +Name: count, dtype: int64
[ ]:
[ ]:
# 4. Time Column cleansing
# This column is very challenging, the dataset is very dirty but as we were coached by the best of the best we decided to give it a try.
# we need to ==> 1.clean the format, 2.Extract an hour when it exists, 3. Create a business‑oriented category(Morning, Afternoon, Evening, Night)
[179]:
# First we look at the unique values.
df['Time'].sample(20)
df['Time'].isna().sum()
df['Time'].value_counts().head(20)
[177]:
Time +Afternoon 263 +Morning 141 +11:00 140 +15:00 128 +16:00 126 +12:00 122 +14:00 117 +16:30 88 +13:00 87 +17:30 83 +17:00 81 +14:30 76 +11:30 75 +18:00 75 +10:00 74 +15:30 73 +13:30 70 +Night 64 +10:30 61 +09:00 61 +Name: count, dtype: int64
[180]:
# Now we manage the real missing values.
df["Time"] = df["Time"].replace(
["None", "", " "],
np.nan
)
[181]:
# We then apply a generic cleanup.
import numpy as np
df["Time"] = (
df["Time"]
.astype(str) # convert all the values into string/text to uniformize the dataset.
.str.strip() # suppress the unecessary spaces at the begining and the end.
.str.replace(r"\s+", " ", regex=True) # suppress the double spaces by just one.
)
[182]:
# We confirm that the type is a string :
df["Time"].dtype
[180]:
<StringDtype(storage='python', na_value=nan)>
[183]:
# We check the suppression of the spaces.
df["Time"].sample(20)
[181]:
4677 NaN +4985 NaN +632 14:00 +6780 NaN +2068 17:30 +1444 12:00 +2706 NaN +2467 03:00 +1161 19:30 +3491 09:45 +2166 Afternoon +2577 12:00 +3978 13:30 +2110 Afternoon +5517 14:00 +6466 NaN +5956 NaN +445 11:00 +1342 20:30 +5609 17:30 +Name: Time, dtype: str
[184]:
# Now we can standardize the time formats
# 1.We replace the format 17h30 => 17:30
# 2.We replace "h" by ":" , 17h30 => 17:30
# 3.We replace all the dots. ex: 13.30 ==> 13:30
df["Time"] = df["Time"].str.replace(r"(\d{1,2})h(\d{0,2})",
lambda m: f"{m.group(1)}:{m.group(2) if m.group(2) else '00'}",
regex=True)
df["Time"] = df["Time"].str.replace(r"^(\d{2})(\d{2})\b",
r"\1:\2",
regex=True)
df["Time"] = df["Time"].str.replace(r"(\d{1,2})\.(\d{2})",
r"\1:\2",
regex=True)
[185]:
# We control that the format are now harmonized :
df["Time"].value_counts().head(30)
[183]:
Time +Afternoon 263 +Morning 141 +11:00 140 +15:00 128 +16:00 126 +12:00 122 +14:00 117 +16:30 88 +13:00 87 +17:30 83 +17:00 81 +14:30 76 +11:30 75 +18:00 75 +10:00 74 +15:30 73 +13:30 70 +Night 64 +10:30 61 +09:00 61 +Evening 45 +08:00 43 +12:30 42 +09:30 39 +18:30 36 +07:30 35 +08:30 34 +? 30 +07:00 28 +19:00 26 +Name: count, dtype: int64
[186]:
# Now we control if some dirty formats remain:
df["Time"].str.contains(r"h", na=False).sum()
[184]:
np.int64(91)
[187]:
# We check if the HHMM is reduced.
df["Time"].str.contains(r"\d{4}", na=False).sum()
[185]:
np.int64(3)
[188]:
# Finally we check that format HH.MM have been removed.
df["Time"].str.contains(r"\.", na=False).sum()
[186]:
np.int64(29)
[190]:
# we need to harmonize the business values periods by using a custom/mapping dictionary.
# This will also allow to complete the missing hours data.
time_mapping = {
"Morning" : "Morning",
"Early Morning" : "Morning",
"Noon" : "Afternoon",
"Midday" : "Afternoon",
"Afternoon" : "Afternoon",
"Late afternoon" : "Afternoon",
"Evening" : "Evening",
"Night" : "Night",
"Late Night" : "Night",
"Dawn" : "Morning",
"Before Dawn" : "Morning",
"Sunset" : "Evening",
"Unknown" : np.nan
}
df["Time"] = df["Time"].replace(time_mapping)
[191]:
# Now we extract the hour as we want to create a business variable (morning, afternoon...)No need to keep the minutes or seconds.
df["Hour"] = pd.to_numeric(
df["Time"].str.extract(r"(\d{1,2})")[0],
errors="coerce"
)
[192]:
# Check the extracted hour.
# We try to see how many values have not been extracted
df["Hour"].describe()
df["Hour"].isna().sum()
[190]:
np.int64(4211)
[193]:
# Then we to create a business variable that will be usefull for the dashboard.
def time_period(hour):
if pd.isna(hour):
return np.nan
elif 5 <= hour < 12:
return "Morning"
elif 12 <= hour < 17:
return "Afternoon"
elif 17 <= hour < 21:
return "Evening"
else:
return "Night"
[194]:
# Application of the variable created.
df["Time_Period"] = df["Hour"].apply(time_period)
[195]:
print(df.columns) # We print all the columns to check the ones we have created.
Index(['Date', 'Year', 'Type', 'Country', 'State', 'Location', 'Activity', + 'Name', 'Sex', 'Age', 'Injury', 'Fatal Y/N', 'Time', 'Species ', + 'Source', 'pdf', 'href formula', 'href', 'Case Number', 'Case Number.1', + 'original order', 'Unnamed: 21', 'Unnamed: 22', 'Hour', 'Time_Period', + 'Time_Clean'], + dtype='str') +
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
# If we want to visualize the distribution of activity peaks.
df["Hour"].value_counts().sort_index()
[ ]:
# If we want to visualize through a clear graphic
df["Hour"].hist(bins=24)
[ ]:
[152]:
[153]:
[154]:
[155]:
[156]:
print(df.columns) # We print all the columns to check the ones we have created.
Index(['Date', 'Year', 'Type', 'Country', 'State', 'Location', 'Activity', + 'Name', 'Sex', 'Age', 'Injury', 'Fatal Y/N', 'Time', 'Species ', + 'Source', 'pdf', 'href formula', 'href', 'Case Number', 'Case Number.1', + 'original order', 'Unnamed: 21', 'Unnamed: 22', 'Hour', 'Time_Period', + 'Time_Clean'], + dtype='str') +
[ ]:
[ ]:
[ ]:
[ ]:
[159]:
# We check in the datframe what it contains.
df[['Time', 'Hour', 'Time_Period']].head(20)
[157]:
+
+
+
+
+
| + | Time | +Hour | +Time_Period | +
|---|---|---|---|
| 0 | +1100:00rs | +NaN | +1100:00rs | +
| 1 | +? | +NaN | +? | +
| 2 | +1000:00rs | +NaN | +1000:00rs | +
| 3 | +? | +NaN | +? | +
| 4 | +? | +NaN | +? | +
| 5 | +1730:00rs | +NaN | +1730:00rs | +
| 6 | +? | +NaN | +? | +
| 7 | +? | +NaN | +? | +
| 8 | +1715:00rs | +NaN | +1715:00rs | +
| 9 | +1015:00rs | +NaN | +1015:00rs | +
| 10 | +0800:00rs | +NaN | +0800:00rs | +
| 11 | +0815:00rs | +NaN | +0815:00rs | +
| 12 | +? | +NaN | +? | +
| 13 | +? | +NaN | +? | +
| 14 | +? | +NaN | +? | +
| 15 | +1100:00rs | +NaN | +1100:00rs | +
| 16 | +1815:00rs | +NaN | +1815:00rs | +
| 17 | +0830:00rs | +NaN | +0830:00rs | +
| 18 | +1145:00rs | +NaN | +1145:00rs | +
| 19 | +1820:00rs | +NaN | +1820:00rs | +
[ ]:
[166]:
df[['Time', 'Time_Clean']].sample(20)
[164]:
+
+
+
+
+
| + | Time | +Time_Clean | +
|---|---|---|
| 6117 | +NaN | +NaN | +
| 2843 | +09:30 | +NaN | +
| 3926 | +NaN | +NaN | +
| 2444 | +07:30 | +NaN | +
| 569 | +07:00 | +NaN | +
| 6882 | +NaN | +NaN | +
| 3933 | +NaN | +NaN | +
| 4680 | +NaN | +NaN | +
| 5612 | +NaN | +NaN | +
| 181 | +NaN | +NaN | +
| 2232 | +15:30 | +NaN | +
| 7066 | +NaN | +NaN | +
| 1171 | +17:51 | +NaN | +
| 4386 | +NaN | +NaN | +
| 5429 | +13:15 | +NaN | +
| 3797 | +NaN | +NaN | +
| 1411 | +NaN | +NaN | +
| 3443 | +15:30 | +NaN | +
| 3774 | +18:00 | +NaN | +
| 1616 | +NaN | +NaN | +
[167]:
# We check the total of non null values.
df['Time_Clean'].isna().sum()
[165]:
np.int64(7012)
[170]:
# We check the remained format that have HH:MM
df.loc[
(df['Time_Clean'].isna()) &
(df['Time'].str.contains(':', na=False)),
'Time'
].value_counts().head(30)
[168]:
Time +11:00 140 +15:00 128 +16:00 126 +12:00 122 +14:00 117 +16:30 88 +13:00 87 +17:30 83 +17:00 81 +14:30 76 +11:30 75 +18:00 75 +10:00 74 +15:30 73 +13:30 70 +10:30 61 +09:00 61 +08:00 43 +12:30 42 +09:30 39 +18:30 36 +07:30 35 +08:30 34 +07:00 28 +19:00 26 +13:20 20 +20:00 19 +11:45 19 +15:45 18 +11:15 18 +Name: count, dtype: int64
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[ ]:
[35]:
rows, cols = df_shark.shape
print(f"Number of observations: {rows}")
print(f"Number of variables: {cols}")
Number of observations: 7090 +Number of variables: 23 +
[36]:
df_shark.info()
<class 'pandas.DataFrame'> +RangeIndex: 7090 entries, 0 to 7089 +Data columns (total 23 columns): + # Column Non-Null Count Dtype +--- ------ -------------- ----- + 0 Date 7090 non-null object + 1 Year 7088 non-null float64 + 2 Type 7072 non-null str + 3 Country 7040 non-null str + 4 State 6603 non-null str + 5 Location 6523 non-null str + 6 Activity 6507 non-null str + 7 Name 6872 non-null str + 8 Sex 6512 non-null str + 9 Age 4096 non-null object + 10 Injury 7054 non-null str + 11 Fatal Y/N 6529 non-null object + 12 Time 3563 non-null object + 13 Species 3959 non-null str + 14 Source 7070 non-null object + 15 pdf 6799 non-null object + 16 href formula 6794 non-null str + 17 href 6796 non-null str + 18 Case Number 6798 non-null str + 19 Case Number.1 6797 non-null str + 20 original order 6799 non-null float64 + 21 Unnamed: 21 1 non-null str + 22 Unnamed: 22 2 non-null str +dtypes: float64(2), object(6), str(15) +memory usage: 692.5+ KB +
[37]:
df_shark.isnull().head()
[35]:
+
+
+
+
+
| + | Date | +Year | +Type | +Country | +State | +Location | +Activity | +Name | +Sex | +Age | +... | +Species | +Source | +href formula | +href | +Case Number | +Case Number.1 | +original order | +Unnamed: 21 | +Unnamed: 22 | +|
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | +False | +False | +False | +False | +False | +False | +False | +False | +False | +False | +... | +False | +False | +True | +True | +True | +True | +True | +True | +True | +True | +
| 1 | +False | +False | +False | +False | +False | +False | +False | +False | +False | +False | +... | +False | +False | +True | +True | +True | +True | +True | +True | +True | +True | +
| 2 | +False | +False | +False | +False | +False | +False | +False | +False | +False | +False | +... | +False | +False | +True | +True | +True | +True | +True | +True | +True | +True | +
| 3 | +False | +False | +False | +False | +False | +False | +False | +False | +False | +False | +... | +False | +False | +True | +True | +True | +True | +True | +True | +True | +True | +
| 4 | +False | +False | +False | +False | +False | +False | +False | +False | +False | +False | +... | +False | +False | +True | +True | +True | +True | +True | +True | +True | +True | +
5 rows × 23 columns
+[ ]:
[ ]:
# Col to keep: year _ type _ country _ state _ location _ activity _ age _ sex _ injury _ fatal.
[38]:
df_shark.isnull().sum()
[36]:
Date 0 +Year 2 +Type 18 +Country 50 +State 487 +Location 567 +Activity 583 +Name 218 +Sex 578 +Age 2994 +Injury 36 +Fatal Y/N 561 +Time 3527 +Species 3131 +Source 20 +pdf 291 +href formula 296 +href 294 +Case Number 292 +Case Number.1 293 +original order 291 +Unnamed: 21 7089 +Unnamed: 22 7088 +dtype: int64
[42]:
df_shark_geo = df_shark[["State", "Location", "Time"]] # We want to select the columns State, Location & Time
df_shark_geo.head()
[40]:
+
+
+
+
+
| + | State | +Location | +Time | +
|---|---|---|---|
| 0 | +Queensland | +Kennedy Shoal | +1100hrs | +
| 1 | +Maryland | +Assateague State Park | +? | +
| 2 | +Western Australia | +Horseshoe Reef Rottnest Island | +1000hrs | +
| 3 | +Gaafu Alif Atoll | +Kooddoo | +? | +
| 4 | +South Australia | +Middleton Beach Fleurieu Peninsula Adelaide | +? | +
[43]:
type(df_shark_geo) # to understand the type of df_shark = it is panda dataframe
[41]:
pandas.DataFrame
[44]:
df_shark_geo.isnull().sum() # we combine isnull() with sum() to have the total of nulls per column.
[42]:
State 487 +Location 567 +Time 3527 +dtype: int64
[45]:
# Find the most common value (mode) of State
most_common_state = df_shark_geo["State"].mode()[0]
print("Most common state:", most_common_state)
Most common state: Florida +
[50]:
# Fill the nulls in State with the most common state
# First step of our cleaning workflow, we save it as df_geo_clean
df_geo_clean = df_shark_geo.fillna({"State": most_common_state})
# then we verify that there is no more null in State
print("Nulls in State after fillna:", df_geo_clean["State"].isnull().sum())
print("Total rows :", df_geo_clean.shape[0])
Nulls in State after fillna: 0 +Total rows : 7090 +
[51]:
# Find the most common value (mode) of Location
most_common_location = df_shark_geo["Location"].mode()[0]
print("Most common location:", most_common_location)
Most common location: New Smyrna Beach, Volusia County +
[52]:
# We do the same exercise with the column Location now:
df_loca_clean = df_shark_geo.fillna({"Location": most_common_location})
print("Nulls in Location after fillna:", df_loca_clean["Location"].isnull().sum())
print("Total rows :", df_loca_clean.shape[0])
Nulls in Location after fillna: 0 +Total rows : 7090 +
[53]:
# Find the most common value (mode) of Time
most_common_time = df_shark_geo["Time"].mode()[0]
print("Most common time:", most_common_time)
Most common time: Afternoon +
[55]:
# We do the same exercise with the column Time now:
df_time_clean = df_shark_geo.fillna({"Time": most_common_time})
print("Nulls in Time after fillna:", df_time_clean["Time"].isnull().sum())
print("Total rows :", df_time_clean.shape[0])
Nulls in Time after fillna: 0 +Total rows : 7090 +
[ ]:
#Clean the
df_shark_geo["State"] = df_shark_geo["State"].str.strip()
[57]:
# Check the current data types of the shark dataset
df_shark.dtypes
[55]:
Date object +Year float64 +Type str +Country str +State str +Location str +Activity str +Name str +Sex str +Age object +Injury str +Fatal Y/N object +Time object +Species str +Source object +pdf object +href formula str +href str +Case Number str +Case Number.1 str +original order float64 +Unnamed: 21 str +Unnamed: 22 str +dtype: object
[ ]:
# "Time" is stored as "object"
# Pclass: 1, 2, or 3 --> First, Second, Third class
# Survived: 0 = No, 1 = Yes --> not real numbers
# Convert them from int64 to object (treat them as categories, not numbers)
# This returns a NEW DataFrame; we assign it back to keep the change
titanic_clean_df[["Pclass","Survived"]] = titanic_clean_df[["Pclass","Survived"]].astype("object")
# Verify the new dtypes (plural, because we are looking at multiple columns)
print("Dtypes of Pclass and Survived after astype:")
print(titanic_clean_df[["Pclass","Survived"]].dtypes)
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 
<h1>Table of Contents<span class="tocSkip"></span></h1>
<div class="toc"><ul class="toc-item"><li><span><a href="#APIS" data-toc-modified-id="APIS-1"><span class="toc-item-num">1 </span>APIS</a></span><ul class="toc-item"><li><span><a href="#Web-Communication-Fundamentals" data-toc-modified-id="Web-Communication-Fundamentals-1.1"><span class="toc-item-num">1.1 </span>Web Communication Fundamentals</a></span><ul class="toc-item"><li><span><a href="#DNS,-IP" data-toc-modified-id="DNS,-IP-1.1.1"><span class="toc-item-num">1.1.1 </span>DNS, IP</a></span></li><li><span><a href="#HTTP" data-toc-modified-id="HTTP-1.1.2"><span class="toc-item-num">1.1.2 </span>HTTP</a></span></li><li><span><a href="#URL" data-toc-modified-id="URL-1.1.3"><span class="toc-item-num">1.1.3 </span>URL</a></span></li></ul></li><li><span><a href="#HTTP-Requests-and-Responses" data-toc-modified-id="HTTP-Requests-and-Responses-1.2"><span class="toc-item-num">1.2 </span>HTTP Requests and Responses</a></span><ul class="toc-item"><li><span><a href="#Requests" data-toc-modified-id="Requests-1.2.1"><span class="toc-item-num">1.2.1 </span>Requests</a></span></li><li><span><a href="#Response" data-toc-modified-id="Response-1.2.2"><span class="toc-item-num">1.2.2 </span>Response</a></span></li></ul></li><li><span><a href="#APIs" data-toc-modified-id="APIs-1.3"><span class="toc-item-num">1.3 </span>APIs</a></span><ul class="toc-item"><li><span><a href="#RESTful-APIs" data-toc-modified-id="RESTful-APIs-1.3.1"><span class="toc-item-num">1.3.1 </span>RESTful APIs</a></span></li></ul></li><li><span><a href="#Requests-in-Python" data-toc-modified-id="Requests-in-Python-1.4"><span class="toc-item-num">1.4 </span>Requests in Python</a></span><ul class="toc-item"><li><span><a href="#GET" data-toc-modified-id="GET-1.4.1"><span class="toc-item-num">1.4.1 </span>GET</a></span><ul class="toc-item"><li><span><a href="#Accessing-an-endpoint" data-toc-modified-id="Accessing-an-endpoint-1.4.1.1"><span class="toc-item-num">1.4.1.1 </span>Accessing an endpoint</a></span></li><li><span><a href="#Response-headers" data-toc-modified-id="Response-headers-1.4.1.2"><span class="toc-item-num">1.4.1.2 </span>Response headers</a></span></li><li><span><a href="#Parameters" data-toc-modified-id="Parameters-1.4.1.3"><span class="toc-item-num">1.4.1.3 </span>Parameters</a></span></li><li><span><a href="#Request-Headers" data-toc-modified-id="Request-Headers-1.4.1.4"><span class="toc-item-num">1.4.1.4 </span>Request Headers</a></span></li></ul></li></ul></li><li><span><a href="#APIs-examples" data-toc-modified-id="APIs-examples-1.5"><span class="toc-item-num">1.5 </span>APIs examples</a></span><ul class="toc-item"><li><span><a href="#NewsAPI" data-toc-modified-id="NewsAPI-1.5.1"><span class="toc-item-num">1.5.1 </span>NewsAPI</a></span><ul class="toc-item"><li><span><a href="#💡-Check-for-understanding" data-toc-modified-id="💡-Check-for-understanding-1.5.1.1"><span class="toc-item-num">1.5.1.1 </span>💡 Check for understanding</a></span></li></ul></li><li><span><a href="#Pokemon-API" data-toc-modified-id="Pokemon-API-1.5.2"><span class="toc-item-num">1.5.2 </span>Pokemon API</a></span><ul class="toc-item"><li><span><a href="#json_normalize()" data-toc-modified-id="json_normalize()-1.5.2.1"><span class="toc-item-num">1.5.2.1 </span>json_normalize()</a></span></li></ul></li><li><span><a href="#Coincap-API" data-toc-modified-id="Coincap-API-1.5.3"><span class="toc-item-num">1.5.3 </span>Coincap API</a></span></li><li><span><a href="#Api-Jokes" data-toc-modified-id="Api-Jokes-1.5.4"><span class="toc-item-num">1.5.4 </span>Api Jokes</a></span></li><li><span><a href="#💡-Check-for-understanding" data-toc-modified-id="💡-Check-for-understanding-1.5.5"><span class="toc-item-num">1.5.5 </span>💡 Check for understanding</a></span></li></ul></li><li><span><a href="#API-Wrappers" data-toc-modified-id="API-Wrappers-1.6"><span class="toc-item-num">1.6 </span>API Wrappers</a></span></li><li><span><a href="#Summary" data-toc-modified-id="Summary-1.7"><span class="toc-item-num">1.7 </span>Summary</a></span></li><li><span><a href="#Glossary" data-toc-modified-id="Glossary-1.8"><span class="toc-item-num">1.8 </span>Glossary</a></span></li><li><span><a href="#Further-materials" data-toc-modified-id="Further-materials-1.9"><span class="toc-item-num">1.9 </span>Further materials</a></span></li></ul></li></ul></div>
## Web Communication Fundamentals
1.1. Web Communication Fundamentals¶
+
### DNS, IP
1.1.1. DNS, IP¶
+[How do we connect to www.google.com?](https://www.youtube.com/watch?v=sUhEqT_HSBI&ab_channel=ProfeSang)
* **DNS (Domain Name System)**: it's essentially the phonebook of the internet. Humans access information online through domain names, like "google.com". Web browsers, however, interact through Internet Protocol (IP) addresses. In this example, the DNS maps the internet address www.google.com to the server's IP: 216.58.222.196
* **IP**: Server identification. A code that allows information to be sent and received by the correct parties
* **Domain Providers**: Sell and purchase Internet domains
How do we connect to www.google.com?
+-
+
- DNS (Domain Name System): it's essentially the phonebook of the internet. Humans access information online through domain names, like "google.com". Web browsers, however, interact through Internet Protocol (IP) addresses. In this example, the DNS maps the internet address www.google.com to the server's IP: 216.58.222.196 +
- IP: Server identification. A code that allows information to be sent and received by the correct parties +
- Domain Providers: Sell and purchase Internet domains +
*When making API calls to specific domains, the DNS translates the human-readable domain name into an IP address.*
When making API calls to specific domains, the DNS translates the human-readable domain name into an IP address.
+### HTTP
1.1.2. HTTP¶
+**H**yper **T**ext **T**ransfer **P**rotocol
- HTTP is a communications protocol that provides a structure for requests between the client and the server on a network.
- For example, the web browser on the user's computer (the client) uses the HTTP protocol to request information from a website on a server.
*We will use APIs that employ the **HTTP** protocol as transport.*
Hyper Text Transfer Protocol
+-
+
- HTTP is a communications protocol that provides a structure for requests between the client and the server on a network. +
- For example, the web browser on the user's computer (the client) uses the HTTP protocol to request information from a website on a server. +
We will use APIs that employ the HTTP protocol as transport.
+### URL
Contains information about the resource being requested from the **server**.
1.1.3. URL¶
+Contains information about the resource being requested from the server.
+
Examples:
- https://www.google.com/webhp?authuser=2
- https://www.towardsdatascience.com
- https://www.ironhack.com/
- Protocol: https (https == http is the same, but https is encrypted)
- Domain Name --> ironhack
- TLD (Top-Level Domain) --> .com
Examples:
+-
+
- https://www.google.com/webhp?authuser=2 +
- https://www.towardsdatascience.com +
- https://www.ironhack.com/
-
+
- Protocol: https (https == http is the same, but https is encrypted) +
- Domain Name --> ironhack +
- TLD (Top-Level Domain) --> .com +
+
*When calling an API, you specify the URL of the API endpoint, which tells the system where to send the request.*
When calling an API, you specify the URL of the API endpoint, which tells the system where to send the request.
+## HTTP Requests and Responses
1.2. HTTP Requests and Responses¶
+
**Requests** and **responses** are fundamental components of the client-server communication model.
Requests and responses are fundamental components of the client-server communication model.
+### Requests
1.2.1. Requests¶
+These are queries or calls **sent by the client** (such as a web browser or other software) **to the server** in order to receive information (a **response**).
A request typically consists of:
- A method (such as GET, POST, PUT, DELETE) that defines the action to be performed
* GET: read the information of the resource, without modifying it in any way. Accessing the website from the browser **gets** information.
- The URL or endpoint specifying the resource
- Optional additional information such as:
- Headers (metadata): User-Agent, Accept-Language
- Parameters
- Body content.
For example, a client might send a GET request to retrieve information from a web page or a POST request to submit form data.
These are queries or calls sent by the client (such as a web browser or other software) to the server in order to receive information (a response).
+A request typically consists of:
+-
+
- A method (such as GET, POST, PUT, DELETE) that defines the action to be performed
-
+
- GET: read the information of the resource, without modifying it in any way. Accessing the website from the browser gets information. +
+ - The URL or endpoint specifying the resource +
- Optional additional information such as:
-
+
- Headers (metadata): User-Agent, Accept-Language +
- Parameters +
- Body content. +
+
For example, a client might send a GET request to retrieve information from a web page or a POST request to submit form data.
+### Response
1.2.2. Response¶
+These are the answers or data sent **by the server back to the client in reply to a request**.
A response typically includes:
- A status code that indicates the success or failure of the request
- Headers with meta-information about the server's behavior
- The actual content or data (if applicable), such as HTML, JSON (similar to a Python dictionary), images, or other media types.
An important part of the **header** is the **status code**. This code is a numerical value that indicates the server's result. There are different status codes depending on whether the server has managed to carry out the request or has not managed to do anything. These are some groups of status codes:
- **2xx successful**: the request was successfully received, understood, and accepted
- **3xx redirection**: more actions are required to complete the request
These are the answers or data sent by the server back to the client in reply to a request.
+A response typically includes:
+-
+
- A status code that indicates the success or failure of the request +
- Headers with meta-information about the server's behavior +
- The actual content or data (if applicable), such as HTML, JSON (similar to a Python dictionary), images, or other media types. +
An important part of the header is the status code. This code is a numerical value that indicates the server's result. There are different status codes depending on whether the server has managed to carry out the request or has not managed to do anything. These are some groups of status codes:
+-
+
- 2xx successful: the request was successfully received, understood, and accepted +
- 3xx redirection: more actions are required to complete the request +
- 4xx client error: the request contains incorrect syntax or cannot be fulfilled +
- 5xx server error: the server has failed to complete an apparently valid request +
Complete list: +https://en.wikipedia.org/wiki/List_of_HTTP_status_codes
+Much more fun: +https://http.cat/
+## APIs
1.3. APIs¶
+**A**pplication **P**rogramming **I**nterface
- APIs define a set of rules and protocols that allow different software applications to communicate with each other.
- The client calls the server through an API, and the server responds.
- Within a project, the backend may want to share information with the frontend without access to the database.
Application Programming Interface
+-
+
- APIs define a set of rules and protocols that allow different software applications to communicate with each other. +
- The client calls the server through an API, and the server responds. +
- Within a project, the backend may want to share information with the frontend without access to the database. +
### RESTful APIs
1.3.1. RESTful APIs¶
+As a data analyst, you often need to access data from various sources. RESTful APIs provide a **standardized way** to retrieve, update, or delete data from other systems.
As a data analyst, you often need to access data from various sources. RESTful APIs provide a standardized way to retrieve, update, or delete data from other systems.
+- Usually, **we (the client) send a request, and they (the server) return a response, often in JSON - JavaScript Object Notation - format**. While JSON looks similar to the dictionaries we're used to in Python, it's worth noting that when represented as raw text (e.g., in an API response), the entire JSON structure is a *string*. However, within that structure, JSON can represent various data types like numbers, booleans, and arrays, not just strings.
- APIs always need to provide documentation for their various services: **endpoints**. Each endpoint is a different URL.
- Sometimes we have to pass **parameters** to an API endpoint, similar to when we pass parameters to a Python function.
At this point in the Bootcamp, **reading documentation becomes essential**, as each API will "work" differently, and to use it, we will need to know what it requires.
[Here](https://github.com/public-apis/public-apis) is a repository where you can find several free APIs if you feel like exploring a bit.
-
+
Usually, we (the client) send a request, and they (the server) return a response, often in JSON - JavaScript Object Notation - format. While JSON looks similar to the dictionaries we're used to in Python, it's worth noting that when represented as raw text (e.g., in an API response), the entire JSON structure is a string. However, within that structure, JSON can represent various data types like numbers, booleans, and arrays, not just strings.
+
+APIs always need to provide documentation for their various services: endpoints. Each endpoint is a different URL.
+
+Sometimes we have to pass parameters to an API endpoint, similar to when we pass parameters to a Python function.
+
+
At this point in the Bootcamp, reading documentation becomes essential, as each API will "work" differently, and to use it, we will need to know what it requires.
+Here is a repository where you can find several free APIs if you feel like exploring a bit.
+## Requests in Python
1.4. Requests in Python¶
+You can make requests to RESTful APIs using libraries like `requests`.
You can make requests to RESTful APIs using libraries like requests.
[ ]:
# If you don't have it installed you can do use using pip or pip3
#!pip install requests
[ ]:
import requests
### GET
1.4.1. GET¶
+Making a `GET request` to the API is simply a call to a `URL` that returns information when provided with the appropriate `parameters`. We will only perform **GET** requests.
Making a GET request to the API is simply a call to a URL that returns information when provided with the appropriate parameters. We will only perform GET requests.
Here is an example on how to make a request with Python:
```python
url = "https://api.example.com/products"
response = requests.get(url)
if response.status_code == 200:
products = response.json()
# Now you can analyze and work with the products data in Python
```
Here is an example on how to make a request with Python:
+url = "https://api.example.com/products"
+response = requests.get(url)
+
+if response.status_code == 200:
+ products = response.json()
+ # Now you can analyze and work with the products data in Python#### Accessing an endpoint
As we mentioned above, APIs always need to provide documentation for their various services: **endpoints**. Each endpoint is a different URL.
**Example: ISS API**
1.4.1.1. Accessing an endpoint¶
+As we mentioned above, APIs always need to provide documentation for their various services: endpoints. Each endpoint is a different URL.
+Example: ISS API
+Let's get information from ISS (International Space Station)! We'll start looking at the [ISS API documentation](https://wheretheiss.at/w/developer)
Let's get information from ISS (International Space Station)! We'll start looking at the ISS API documentation
+This API allows you to access various data related to the International Space Station (ISS), including its current, past, or future position, timezone information for specific coordinates, and more.
**Key Features**
- **Authentication**: No authentication is currently required, but future endpoints may include this.
- **Rate Limiting**: Limited to approximately 1 request per second.
- **Responses**: Default to JSON format, with optional parameters to modify response appearance.
- **Endpoints**: Several endpoints provide different types of information:
- **satellites**: Information about satellites, including the ISS.
- **satellites/[id]**: Position, velocity, and related information for a satellite.
- **satellites/[id]/positions**: Position data for specific timestamps.
- **satellites/[id]/tles**: TLE (Two-Line Element Set) data in either JSON or text format.
This API allows you to access various data related to the International Space Station (ISS), including its current, past, or future position, timezone information for specific coordinates, and more.
+Key Features
+-
+
- Authentication: No authentication is currently required, but future endpoints may include this. +
- Rate Limiting: Limited to approximately 1 request per second. +
- Responses: Default to JSON format, with optional parameters to modify response appearance. +
- Endpoints: Several endpoints provide different types of information:
-
+
- satellites: Information about satellites, including the ISS. +
- satellites/[id]: Position, velocity, and related information for a satellite. +
- satellites/[id]/positions: Position data for specific timestamps. +
- satellites/[id]/tles: TLE (Two-Line Element Set) data in either JSON or text format. +
- coordinates/[lat,lon]: Timezone information for specific coordinates. +
+
**Examples**
- Satellite details: `https://api.wheretheiss.at/v1/satellites`
- ISS position: `https://api.wheretheiss.at/v1/satellites/25544`
- Coordinates information: `https://api.wheretheiss.at/v1/coordinates/37.795517,-122.393693`
Examples
+-
+
- Satellite details:
https://api.wheretheiss.at/v1/satellites
+ - ISS position:
https://api.wheretheiss.at/v1/satellites/25544
+ - Coordinates information:
https://api.wheretheiss.at/v1/coordinates/37.795517,-122.393693
+
**Endpoint satellites**
Endpoint satellites
+[ ]:
# We'll use the endpoint satellites, we read it gives us information about satellites
url = "https://api.wheretheiss.at/v1/satellites"
[ ]:
response = requests.get(url) # We use get method to make the request and get information from the API
[ ]:
type(response) # Lets look at the type of the response
[ ]:
response.status_code # We can access status code. 200 is OK
We can access the response content, which returns a string
We can access the response content, which returns a string
+[ ]:
response.content
If response Content-Type is json, we can access it better with `.json()`
If response Content-Type is json, we can access it better with .json()
[ ]:
info = response.json()
[ ]:
info
[ ]:
type(info) # It returns a list of dictionaries
[ ]:
info[0]["id"] # So we can access the id like this
#### Response headers
Response headers are part of the HTTP response that a server sends back to the client after a request has been made. These headers provide meta-information about the response and can affect how the client handles the response.
Here are some common response headers and what they typically represent:
1. **`Date`**: Represents the date and time at which the response was sent.
2. **`Server`**: Provides information about the software used by the originating server.
3. **`X-Rate-Limit`**: Sometimes used in APIs to inform the client about rate limiting policies, such as the number of allowed requests in a given time frame.
4. **`Content-Type`**: Specifies the media type of the resource or data the server is sending back. For example, it could be `application/json` for a JSON object, `text/html` for an HTML page, or `image/png` for an image.
1.4.1.2. Response headers¶
+Response headers are part of the HTTP response that a server sends back to the client after a request has been made. These headers provide meta-information about the response and can affect how the client handles the response.
+Here are some common response headers and what they typically represent:
+-
+
+Date: Represents the date and time at which the response was sent.
+
+Server: Provides information about the software used by the originating server.
+
+X-Rate-Limit: Sometimes used in APIs to inform the client about rate limiting policies, such as the number of allowed requests in a given time frame.
+
+Content-Type: Specifies the media type of the resource or data the server is sending back. For example, it could beapplication/jsonfor a JSON object,text/htmlfor an HTML page, orimage/pngfor an image.
+
And more.
+[ ]:
response.headers # We can also access the response headers
#### Parameters
1.4.1.3. Parameters¶
+As we mentioned above, sometimes we can pass **parameters** to an API endpoint, similar to when we pass parameters to a Python function.
As we mentioned above, sometimes we can pass parameters to an API endpoint, similar to when we pass parameters to a Python function.
+In the example above, we didn't use any parameters in the endpoint `satellites` since the [documentation]("https://wheretheiss.at/w/developer") said *Parameters: None*.
In the example above, we didn't use any parameters in the endpoint satellites since the documentation said Parameters: None.
API parameters are specific values that you include in a request to an API endpoint to filter, sort, or detail the data that you want to retrieve. They allow you to customize the request to get exactly the information you need.
API parameters are specific values that you include in a request to an API endpoint to filter, sort, or detail the data that you want to retrieve. They allow you to customize the request to get exactly the information you need.
+There are several types of parameters that can be used in API requests, such as Path Parameters, Query Parameters, Header Parameters and Request Body Parameters. Let's look at them with an example using another endpoint.
There are several types of parameters that can be used in API requests, such as Path Parameters, Query Parameters, Header Parameters and Request Body Parameters. Let's look at them with an example using another endpoint.
+##### 1. **Path Parameters**
These are embedded in the URL path and are used to identify a specific resource. For example, in the URL above `https://api.wheretheiss.at/v1/satellites/25544`, the number `25544` is a path parameter that identifies a specific sattelite.
1. Path Parameters¶
+These are embedded in the URL path and are used to identify a specific resource. For example, in the URL above https://api.wheretheiss.at/v1/satellites/25544, the number 25544 is a path parameter that identifies a specific sattelite.
**Endpoint satellites/id**
Endpoint satellites/id
+Lets use endpoint satellites/[id] with the id we got from the previous endpoint.
Important, we need to provide the whole url for the endpoint as a `string`.
Lets use endpoint satellites/[id] with the id we got from the previous endpoint.
+Important, we need to provide the whole url for the endpoint as a string.
[ ]:
url_iss_position = "https://api.wheretheiss.at/v1/satellites/"+str(info[0]["id"])
[ ]:
response = requests.get(url_iss_position)
[ ]:
response.status_code
[ ]:
response.json()
Every time we call the endpoint, the information changes since it gives us the current position of the satellite. Let's check that with a loop to gather information at different times.
Every time we call the endpoint, the information changes since it gives us the current position of the satellite. Let's check that with a loop to gather information at different times.
+[ ]:
import time
positions = []
for i in range(10):
response = requests.get(url_iss_position)
data = response.json()
positions.append(data)
time.sleep(0.5)
[ ]:
type(positions) # We created a list with responses
[ ]:
type(positions[0]) # Is a list of dictionaries
[ ]:
positions[0] # So we can access each dictionary by its position in the list
[ ]:
positions[9]
Let's extract 10 latitudes from the 10 dictionaries in the list, with 2 decimals.
Let's extract 10 latitudes from the 10 dictionaries in the list, with 2 decimals.
+[ ]:
latitudes = [round(p["latitude"], 2) for p in positions]
[ ]:
latitudes
[ ]:
import pandas as pd
pd.DataFrame(positions) # We can create a dataframe from the API response to work with it better
##### 2. **Query Parameters**
These are added to the end of the URL after a question mark (`?`) and are often used to filter or sort the response.
For example, in the ISS documentation, it mentions that there is a parameter `units` that can take values `miles` or `kilometers`. So in the URL `https://api.wheretheiss.at/v1/satellites/25544?units=miles`, we added `units=miles` which is a query parameter that shows the data in "miles".
In general, we add the parameters like this `?param1=value1¶m2=value2...` at the end of the URL.
2. Query Parameters¶
+These are added to the end of the URL after a question mark (?) and are often used to filter or sort the response.
For example, in the ISS documentation, it mentions that there is a parameter units that can take values miles or kilometers. So in the URL https://api.wheretheiss.at/v1/satellites/25544?units=miles, we added units=miles which is a query parameter that shows the data in "miles".
In general, we add the parameters like this ?param1=value1¶m2=value2... at the end of the URL.
[ ]:
url_iss_position
[ ]:
# We saw in the documentation that we can add a parameter units to use miles or kilometers
url_iss_position2 = 'https://api.wheretheiss.at/v1/satellites/25544?units=miles'
[ ]:
response = requests.get(url_iss_position2)
[ ]:
response.status_code
[ ]:
response.json()
We can also do it by passing to the argument `params` a dictionary with the parameters in the `get` method.
We can also do it by passing to the argument params a dictionary with the parameters in the get method.
[ ]:
parameters = {"units": "miles"}
[ ]:
url_iss_position
[ ]:
response = requests.get(
url=url_iss_position,
params=parameters
)
[ ]:
response.json()
##### 3. **Header Parameters**
These are included in the request header and can be used for various purposes, such as authentication (e.g., sending an API key or token), content negotiation (e.g., defining the response format), or custom settings defined by the API. If the API requires authentication, you might include an `Authorization` header with your API key.
3. Header Parameters¶
+These are included in the request header and can be used for various purposes, such as authentication (e.g., sending an API key or token), content negotiation (e.g., defining the response format), or custom settings defined by the API. If the API requires authentication, you might include an Authorization header with your API key.
ISS API mentions that *currently there is no authentication required*. So we'll look at an example with a different API of `authentication` in `header parameters`.
ISS API mentions that currently there is no authentication required. So we'll look at an example with a different API of authentication in header parameters.
#### Request Headers
1.4.1.4. Request Headers¶
+Request headers are key-value pairs sent in an HTTP request to provide information about the request itself.
Request headers are key-value pairs sent in an HTTP request to provide information about the request itself.
+Some request headers are:
1. **Content-Type**: Specifies the media type of the resource or data. Common examples include "application/json" for JSON data, "text/html" for HTML content, and "application/xml" for XML data.
3. **Authorization**: Contains credentials for authenticating the client with the server, often used with tokens or other forms of authentication.
4. **User-Agent**: Provides information about the client (browser or other client), including its version and operating system.
In Python's `requests` library, you can include headers in a request by using the `headers` argument, like this:
```python
headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer YOUR_TOKEN'}
response = requests.get(url, headers=headers)
```
Some request headers are:
+-
+
Content-Type: Specifies the media type of the resource or data. Common examples include "application/json" for JSON data, "text/html" for HTML content, and "application/xml" for XML data.
+
+Authorization: Contains credentials for authenticating the client with the server, often used with tokens or other forms of authentication.
+
+User-Agent: Provides information about the client (browser or other client), including its version and operating system.
+
+
In Python's requests library, you can include headers in a request by using the headers argument, like this:
headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer YOUR_TOKEN'}
+response = requests.get(url, headers=headers)⚠️🚨 ¡Careful with the authentication token! ⚠️🚨 A token is a personal credential for accessing an API, through which your request quota to the API is managed. Therefore, the ideal procedure is to ensure security by storing the token as a variable in an .env file. This way, you can call the environment variable without having to publicly display the token.
⚠️🚨 ¡Careful with the authentication token! ⚠️🚨 A token is a personal credential for accessing an API, through which your request quota to the API is managed. Therefore, the ideal procedure is to ensure security by storing the token as a variable in an .env file. This way, you can call the environment variable without having to publicly display the token.
+**Example: News API**
Example: News API
+The *News API* lets you locate articles and breaking news headlines from news sources and blogs across the web.
The News API lets you locate articles and breaking news headlines from news sources and blogs across the web.
+- Let's look at the [NewsAPI Authentication documentation](https://newsapi.org/docs/authentication). It mentions:
```
You can attach your API key to a request in one of three ways:
- Via the apiKey querystring parameter.
- Via the X-Api-Key HTTP header.
- Via the Authorization HTTP header. Including Bearer is optional, and be sure not to base 64 encode it like you may have seen in other authentication tutorials.
We strongly recommend the either of last two so that your API key isn't visible to others in logs or via request sniffing.
If you don't append your API key correctly, or your API key is invalid, you will receive a 401 - Unauthorized HTTP error.
```
- Let's look at the documentation for the endpoints to see what **parameters** they accept: [NewsAPI endpoints](https://newsapi.org/docs/endpoints). We see we have the endpoint `/v2/top-headlines` – *returns breaking news headlines for countries, categories, and singular publishers. This is perfect for use with news tickers or anywhere you want to use live up-to-date news headlines.*
-
+
- Let's look at the NewsAPI Authentication documentation. It mentions: +
You can attach your API key to a request in one of three ways:
+
+- Via the apiKey querystring parameter.
+- Via the X-Api-Key HTTP header.
+- Via the Authorization HTTP header. Including Bearer is optional, and be sure not to base 64 encode it like you may have seen in other authentication tutorials.
+We strongly recommend the either of last two so that your API key isn't visible to others in logs or via request sniffing.
+
+If you don't append your API key correctly, or your API key is invalid, you will receive a 401 - Unauthorized HTTP error.
+-
+
- Let's look at the documentation for the endpoints to see what parameters they accept: NewsAPI endpoints. We see we have the endpoint
/v2/top-headlines– returns breaking news headlines for countries, categories, and singular publishers. This is perfect for use with news tickers or anywhere you want to use live up-to-date news headlines. +If we look at the documentation of that endpoint we see we have more parameters we can add such ascountryorcategory.
+
[ ]:
url = "http://newsapi.org/v2/top-headlines"
[ ]:
response = requests.get(url)
[ ]:
response.status_code
Let's try a made up key, using the apiKey querystring parameter. As we read in the documentation, this is not the recommended way.
Let's try a made up key, using the apiKey querystring parameter. As we read in the documentation, this is not the recommended way.
+[ ]:
url = f"http://newsapi.org/v2/top-headlines?country=us&apiKey=manolonodoesnthavekey"
[ ]:
url
[ ]:
response = requests.get(url)
[ ]:
response.status_code
**4xx**: client error. This means, it is our error.
4xx: client error. This means, it is our error.
+Lets do with a correct key. You should get your own API key through their website. Once we've saved our key, we'll send it via the X-Api-Key HTTP header.
Lets do with a correct key. You should get your own API key through their website. Once we've saved our key, we'll send it via the X-Api-Key HTTP header.
+##### Saving the API Key
Saving the API Key¶
+Storing an API key directly in your code can expose sensitive information, especially if your code is publicly available (e.g., on a public GitHub repository). The best practice for saving and loading an API key in your code involves the following steps:
1. **Storing the API Key**:
- **Use Environment Variables**: Store your API key in an environment variable on your system. This keeps the key out of your codebase and allows you to change it without altering your code.
- **Create a .env File**: If you prefer, you can create a `.env` file (just call it `.env` nothing else before the `.`) in your project directory to store the API key. Inside this file, you would have something like:
```
API_KEY=your-api-key-here
```
- **Add .env to .gitignore**: If you're using a version control system like Git, make sure to add the `.env` file to your `.gitignore` file. This prevents the `.env` file (and therefore your API key) from being uploaded to any public repositories. We'll talk about Git and .gitignore in more detail soon.
Storing an API key directly in your code can expose sensitive information, especially if your code is publicly available (e.g., on a public GitHub repository). The best practice for saving and loading an API key in your code involves the following steps:
+-
+
Storing the API Key:
+-
+
Use Environment Variables: Store your API key in an environment variable on your system. This keeps the key out of your codebase and allows you to change it without altering your code.
+
+Create a .env File: If you prefer, you can create a
+.envfile (just call it.envnothing else before the.) in your project directory to store the API key. Inside this file, you would have something like:
+API_KEY=your-api-key-here +-
+
- Add .env to .gitignore: If you're using a version control system like Git, make sure to add the
.envfile to your.gitignorefile. This prevents the.envfile (and therefore your API key) from being uploaded to any public repositories. We'll talk about Git and .gitignore in more detail soon.
+
+- Add .env to .gitignore: If you're using a version control system like Git, make sure to add the
+Load the Key in Your Code:
+You can use libraries like
+python-dotenvto load the key into your code. + You would need to installpython-dotenvfirst.!pip install python-dotenv + +from dotenv import load_dotenv +import os + +load_dotenv() +api_key = os.getenv("API_KEY")Now,
+api_keycontains the value of your API key, and you can use it to authenticate your requests to the API.
+
Lets do the second approach. Lets save it in an env file and save it in a variable.
Lets do the second approach. Lets save it in an env file and save it in a variable.
+`Make sure your file is named .env and not .env.txt! Is one of the most common errors. If you need, look at the document properties to make sure it doesn't have .txt at the end even if its not showing the .txt when looking at your folder.`
Make sure your file is named .env and not .env.txt! Is one of the most common errors. If you need, look at the document properties to make sure it doesn't have .txt at the end even if its not showing the .txt when looking at your folder.
[ ]:
#!pip install python-dotenv
[ ]:
import os
from dotenv import load_dotenv, find_dotenv
load_dotenv()
my_key = os.getenv("API_KEY")
If you get an error, make sure you saved the .env file in the same directory as your jupyter notebook. You can use `os.getcwd()` to do so.
If you get an error, make sure you saved the .env file in the same directory as your jupyter notebook. You can use os.getcwd() to do so.
[ ]:
os.getcwd()
##### Using the API Key in the Header Parameters
Using the API Key in the Header Parameters¶
+[ ]:
url_top_headlines = "http://newsapi.org/v2/top-headlines"
[ ]:
parameters = {"country": "us", "category": "science"}
[ ]:
headers = {"X-Api-Key": my_key}
[ ]:
response = requests.get(
url=url_top_headlines,
params=parameters,
headers=headers
)
[ ]:
response.status_code
[ ]:
data = response.json()
[ ]:
data
## APIs examples
APIs examples¶
+### NewsAPI
1.5.1. NewsAPI¶
+[ ]:
type(data) # response.json() returns a dictionary
[ ]:
data.keys() # we can get the keys with method keys()
[ ]:
data.get("status") # we can get a key's value by using get method
[ ]:
data["totalResults"] # or by using dictionary notation to access values
[ ]:
# this is the same as above
data.get("totalResults")
`dict[key]` raises error if the key does not exits
`dict.get(key)` does not raise an error if the key does not exist
dict[key] raises error if the key does not exitsdict.get(key) does not raise an error if the key does not exist
[ ]:
# lets look at the articles
articles = data.get("articles")
[ ]:
type(articles) # lets see what type it is to know how to access it
[ ]:
len(articles)
[ ]:
articles[0] # since articles is a list, we can access its elements by the index
[ ]:
type(articles[0]) # each element inside the list is a dictionary
[ ]:
[art["title"] for art in articles] # lets get all the titles using list comprehension
[ ]:
df = pd.DataFrame(articles) # lets create a dataframe with the articles
[ ]:
df.head()
#### 💡 Check for understanding
1.5.1.1. 💡 Check for understanding¶
+1. What data type is the column *source*?
2. Since the column *source* is not very useful as it is, create a column called *name* that contains only the name inside the column *source*.
3. How many times each unique name appears in the "name" column?
-
+
- What data type is the column source? +
- Since the column source is not very useful as it is, create a column called name that contains only the name inside the column source. +
- How many times each unique name appears in the "name" column? +
[ ]:
# Your answer goes here
### Pokemon API
1.5.2. Pokemon API¶
+[PokeAPI](https://pokeapi.co/) is a full RESTful API linked to an extensive database detailing everything about the Pokémon main game series.
In the documentation we see that we can get lot's of data. Let's look at the endpoint *pokemon*: ```https://pokeapi.co/api/v2/pokemon/{id or name}/```
PokeAPI is a full RESTful API linked to an extensive database detailing everything about the Pokémon main game series.
+In the documentation we see that we can get lot's of data. Let's look at the endpoint pokemon: https://pokeapi.co/api/v2/pokemon/{id or name}/
[ ]:
res=requests.get('https://pokeapi.co/api/v2/pokemon/25')
res.status_code
[ ]:
data=res.json()
data
[ ]:
type(data)
[ ]:
data.keys()
[ ]:
# Lets look at the following attributes
print(data["name"])
print(data["weight"])
print(data["sprites"])
[ ]:
# Lets get the pokemon image from sprites - front_default.
# Remember we can read the documentation to understand how to get all the information
data['sprites']['front_default']
We will define a function, `get_pokemon`, that takes an ID number as input. This function fetches details about a Pokémon from the PokeAPI using the given ID, extracts its name, weight, and front-facing image, and then returns this data in a dictionary format. If the ID is invalid or there's an issue with the request, a ValueError is raised.
We will define a function, get_pokemon, that takes an ID number as input. This function fetches details about a Pokémon from the PokeAPI using the given ID, extracts its name, weight, and front-facing image, and then returns this data in a dictionary format. If the ID is invalid or there's an issue with the request, a ValueError is raised.
[ ]:
def get_pokemon(id_number):
res=requests.get('https://pokeapi.co/api/v2/pokemon/{}'.format(id_number))
if(res.status_code==200):
data = res.json()
return {"id": id_number,'name':data['name'], "weight":data["weight"],'image':data['sprites']['front_default']}
else:
raise ValueError("Cannot get pokemon, id does not exist")
[ ]:
get_pokemon(25) # Lets use the function for pokemon id 25
[ ]:
[get_pokemon(i+1) for i in range(5)] # Lets use list comprehension to gather information for the first 5 pokemons
We will now define a function, `print_pokemon`, that takes in a Pokémon dictionary and displays its ID, name, weight, and its image.
We will now define a function, print_pokemon, that takes in a Pokémon dictionary and displays its ID, name, weight, and its image.
[ ]:
from IPython.display import Image
def print_pokemon(poke):
print(str(poke["id"])+" " +poke["name"]+ " "+str(poke["weight"]) + " kg")
display(Image(url=poke['image']))
After defining this function, we will retrieve and display details for the first 10 Pokémon by iterating through their IDs and using both the `get_pokemon` and `print_pokemon` functions.
After defining this function, we will retrieve and display details for the first 10 Pokémon by iterating through their IDs and using both the get_pokemon and print_pokemon functions.
[ ]:
[print_pokemon(get_pokemon(i+1)) for i in range(10)]
#### json_normalize()
1.5.2.1. json_normalize()¶
+Want the data as a DataFrame instead of a dictionary?
Want the data as a DataFrame instead of a dictionary?
+The following line creates a DataFrame from the items of the dictionary data. Each key-value pair will be treated as a row, with two columns: the first for the key and the second for the value.
The following line creates a DataFrame from the items of the dictionary data. Each key-value pair will be treated as a row, with two columns: the first for the key and the second for the value.
+[ ]:
df = pd.DataFrame(data.items())
df
That format is still very hard to work with. Lets use `json_normalize` instead. `json_normalize` is used to normalize semi-structured JSON data into a flat table (DataFrame). It's particularly useful when dealing with nested dictionaries or lists inside JSON.
That format is still very hard to work with. Lets use json_normalize instead. json_normalize is used to normalize semi-structured JSON data into a flat table (DataFrame). It's particularly useful when dealing with nested dictionaries or lists inside JSON.
[ ]:
df = pd.json_normalize(data)
df
A lot better, even though we still have nested dictionaries or lists inside the dataframe.
Let's see again the difference between creating a dataframe directly or using `json_normalize` by using `data["abilities"]`.
A lot better, even though we still have nested dictionaries or lists inside the dataframe.
+Let's see again the difference between creating a dataframe directly or using json_normalize by using data["abilities"].
[ ]:
pd.DataFrame(data["abilities"])
[ ]:
pd.json_normalize(data["abilities"])
### Coincap API
1.5.3. Coincap API¶
+CoinCap is a useful tool for real-time pricing and market activity for over 1,000 cryptocurrencies. Check the documentation at https://docs.coincap.io/
CoinCap is a useful tool for real-time pricing and market activity for over 1,000 cryptocurrencies. Check the documentation at https://docs.coincap.io/
+[ ]:
url = "http://api.coincap.io/v2/assets"
[ ]:
response = requests.get(url)
[ ]:
print(response)
[ ]:
res = response.json()
[ ]:
res # it returns a dictionary with all the data
[ ]:
# Again, let's try to have it as a dataframe
data = pd.DataFrame(res["data"])
data.head()
[ ]:
# Lets see if the result would differ in this case using json_normalize
pd.json_normalize(res["data"])
### Api Jokes
JokeAPI is a REST API that serves uniformly and well formatted jokes.
It can be used without any API token, membership, registration or payment.
It supports a variety of filters that can be applied to get just the right jokes you need.
Check the documentation here: https://jokeapi.dev/
1.5.4. Api Jokes¶
+JokeAPI is a REST API that serves uniformly and well formatted jokes. +It can be used without any API token, membership, registration or payment. +It supports a variety of filters that can be applied to get just the right jokes you need. +Check the documentation here: https://jokeapi.dev/
+[ ]:
url_random_joke = "https://v2.jokeapi.dev/joke/dark?amount=3"
request = requests.get(url_random_joke).json()
request
### 💡 Check for understanding
1.5.5. 💡 Check for understanding¶
+Choose one API from the [Public APIs list](https://github.com/public-apis/public-apis). Attempt to call your selected API, either with or without parameters, and retrieve some valuable information. Document your findings.
Choose one API from the Public APIs list. Attempt to call your selected API, either with or without parameters, and retrieve some valuable information. Document your findings.
+[ ]:
# Your answer goes here
## API Wrappers
A Python wrapper is a Python library or module that provides a more convenient or more "Pythonic" interface to another software component, such as a library in another language, a system tool, or a web API. It "wraps" the functionality of that component in a way that abstracts away its complexities and makes it easier to use in a Python context. 🙌🏻
One example is the `tweepy` library, which makes obtaining data from Twitter's API relatively straightforward.
API Wrappers¶
+A Python wrapper is a Python library or module that provides a more convenient or more "Pythonic" interface to another software component, such as a library in another language, a system tool, or a web API. It "wraps" the functionality of that component in a way that abstracts away its complexities and makes it easier to use in a Python context. 🙌🏻
+One example is the tweepy library, which makes obtaining data from Twitter's API relatively straightforward.
## Summary
- Import the `requests` library.
- Store the necessary credentials, such as API key or OAuth tokens, if the API requires them.
- Execute a `request.get` to the desired API endpoint (the API's documentation will provide details on available endpoints).
- The API returns a JSON response.
- This JSON can be converted into a dataframe, or you can further explore its elements (typically a list of dictionaries).
Summary¶
+-
+
- Import the
requestslibrary.
+ - Store the necessary credentials, such as API key or OAuth tokens, if the API requires them. +
- Execute a
request.getto the desired API endpoint (the API's documentation will provide details on available endpoints).
+ - The API returns a JSON response. +
- This JSON can be converted into a dataframe, or you can further explore its elements (typically a list of dictionaries). +
## Glossary
1.8. Glossary¶
+* DNS: domain name system.
* HTTP: is the protocol used to transfer data over the web.
* API: application programming interface.
* REST: series of rules, architectural style.
-
+
- DNS: domain name system. +
- HTTP: is the protocol used to transfer data over the web. +
- API: application programming interface. +
- REST: series of rules, architectural style. +
* `requests`: Python library for interacting with APIs.
* URL: server name you want to ask information for.
* endpoint: server service you want to ask information for.
* parameters: extra parameters to your query.
* headers: metadata, invisible information.
-
+
requests: Python library for interacting with APIs.
+- URL: server name you want to ask information for. +
- endpoint: server service you want to ask information for. +
- parameters: extra parameters to your query. +
- headers: metadata, invisible information. +
## Further materials
1.9. Further materials¶
+[5 Simple-To-Use APIs for Beginners](https://dev.to/alanconstantino/5-simple-to-use-apis-for-beginners-2e0n)
[RapidAPI](https://rapidapi.com/category/Sports): access thousands of APIs.
RapidAPI: access thousands of APIs.
+- lab-python-data-structures.ipynb
- Untitled16.ipynb
- Untitled17.ipynb
- Untitled20.ipynb
- Untitled21.ipynb
- Project_shark_DF team.ipynb
- Untitled30.ipynb
- Untitled34.ipynb
- Untitled35.ipynb
- Challenge Katas.ipynb
- lab_error_handling_exercices.ipynb
- Untitled38.ipynb
- 2.5_apis.ipynb
- .env.txt
- Untitled41.ipynb