Fix empty values (#79)

Closes #78.

Adds helper methods to convert string values to `int` and `float`s to handle cases where Basketball Reference accidentally has an empty or blank value for a numerical field.

Author: jaebradley

basketball_reference_web_scraper/parsers/box_scores/players.py

@@ -1,6 +1,7 @@
 from lxml import html
 
 from basketball_reference_web_scraper.data import Location, Outcome, TEAM_ABBREVIATIONS_TO_TEAM
+from basketball_reference_web_scraper.utilities import str_to_int, str_to_float
 
 
 def parse_location(symbol):
@@ -39,25 +40,25 @@ def parse_player_box_score(row):
     return {
         "slug": str(row[1].get("data-append-csv")),
         "name": str(row[1].text_content()),
-        "team": TEAM_ABBREVIATIONS_TO_TEAM[row[2].text_content()],
+        "team": TEAM_ABBREVIATIONS_TO_TEAM.get(row[2].text_content()),
         "location": parse_location(row[3].text_content()),
-        "opponent": TEAM_ABBREVIATIONS_TO_TEAM[row[4].text_content()],
+        "opponent": TEAM_ABBREVIATIONS_TO_TEAM.get(row[4].text_content()),
         "outcome": parse_outcome(row[5].text_content()),
-        "seconds_played": int(parse_seconds_played(row[6].text_content())),
-        "made_field_goals": int(row[7].text_content()),
-        "attempted_field_goals": int(row[8].text_content()),
-        "made_three_point_field_goals": int(row[10].text_content()),
-        "attempted_three_point_field_goals": int(row[11].text_content()),
-        "made_free_throws": int(row[13].text_content()),
-        "attempted_free_throws": int(row[14].text_content()),
-        "offensive_rebounds": int(row[16].text_content()),
-        "defensive_rebounds": int(row[17].text_content()),
-        "assists": int(row[19].text_content()),
-        "steals": int(row[20].text_content()),
-        "blocks": int(row[21].text_content()),
-        "turnovers": int(row[22].text_content()),
-        "personal_fouls": int(row[23].text_content()),
-        "game_score": float(row[25].text_content()),
+        "seconds_played": parse_seconds_played(row[6].text_content()),
+        "made_field_goals": str_to_int(row[7].text_content()),
+        "attempted_field_goals": str_to_int(row[8].text_content()),
+        "made_three_point_field_goals": str_to_int(row[10].text_content()),
+        "attempted_three_point_field_goals": str_to_int(row[11].text_content()),
+        "made_free_throws": str_to_int(row[13].text_content()),
+        "attempted_free_throws": str_to_int(row[14].text_content()),
+        "offensive_rebounds": str_to_int(row[16].text_content()),
+        "defensive_rebounds": str_to_int(row[17].text_content()),
+        "assists": str_to_int(row[19].text_content()),
+        "steals": str_to_int(row[20].text_content()),
+        "blocks": str_to_int(row[21].text_content()),
+        "turnovers": str_to_int(row[22].text_content()),
+        "personal_fouls": str_to_int(row[23].text_content()),
+        "game_score": str_to_float(row[25].text_content()),
     }
 
 

basketball_reference_web_scraper/parsers/box_scores/teams.py

@@ -1,26 +1,27 @@
 from lxml import html
 
 from basketball_reference_web_scraper.data import TEAM_NAME_TO_TEAM
+from basketball_reference_web_scraper.utilities import str_to_int
 
 
 def parse_team_total(footer, team):
     cells = footer.xpath('tr/td')
     return {
         "team": team,
-        "minutes_played": int(cells[0].text_content()),
-        "made_field_goals": int(cells[1].text_content()),
-        "attempted_field_goals": int(cells[2].text_content()),
-        "made_three_point_field_goals": int(cells[4].text_content()),
-        "attempted_three_point_field_goals": int(cells[5].text_content()),
-        "made_free_throws": int(cells[7].text_content()),
-        "attempted_free_throws": int(cells[8].text_content()),
-        "offensive_rebounds": int(cells[10].text_content()),
-        "defensive_rebounds": int(cells[11].text_content()),
-        "assists": int(cells[13].text_content()),
-        "steals": int(cells[14].text_content()),
-        "blocks": int(cells[15].text_content()),
-        "turnovers": int(cells[16].text_content()),
-        "personal_fouls": int(cells[17].text_content()),
+        "minutes_played": str_to_int(cells[0].text_content()),
+        "made_field_goals": str_to_int(cells[1].text_content()),
+        "attempted_field_goals": str_to_int(cells[2].text_content()),
+        "made_three_point_field_goals": str_to_int(cells[4].text_content()),
+        "attempted_three_point_field_goals": str_to_int(cells[5].text_content()),
+        "made_free_throws": str_to_int(cells[7].text_content()),
+        "attempted_free_throws": str_to_int(cells[8].text_content()),
+        "offensive_rebounds": str_to_int(cells[10].text_content()),
+        "defensive_rebounds": str_to_int(cells[11].text_content()),
+        "assists": str_to_int(cells[13].text_content()),
+        "steals": str_to_int(cells[14].text_content()),
+        "blocks": str_to_int(cells[15].text_content()),
+        "turnovers": str_to_int(cells[16].text_content()),
+        "personal_fouls": str_to_int(cells[17].text_content()),
     }
 
 

basketball_reference_web_scraper/parsers/players_season_totals.py

@@ -1,31 +1,32 @@
 from lxml import html
 
 from basketball_reference_web_scraper.data import TEAM_ABBREVIATIONS_TO_TEAM, POSITION_ABBREVIATIONS_TO_POSITION
+from basketball_reference_web_scraper.utilities import str_to_int
 
 
 def parse_player_season_totals(row):
     return {
         "slug": str(row[1].get("data-append-csv")),
         "name": str(row[1].text_content()),
         "positions": parse_positions(row[2].text_content()),
-        "age": int(row[3].text_content()) if row[3].text_content() else None,
-        "team": TEAM_ABBREVIATIONS_TO_TEAM[row[4].text_content()],
-        "games_played": int(row[5].text_content()),
-        "games_started": int(row[6].text_content()),
-        "minutes_played": int(row[7].text_content()),
-        "made_field_goals": int(row[8].text_content()),
-        "attempted_field_goals": int(row[9].text_content()),
-        "made_three_point_field_goals": int(row[11].text_content()),
-        "attempted_three_point_field_goals": int(row[12].text_content()),
-        "made_free_throws": int(row[18].text_content()),
-        "attempted_free_throws": int(row[19].text_content()),
-        "offensive_rebounds": int(row[21].text_content()),
-        "defensive_rebounds": int(row[22].text_content()),
-        "assists": int(row[24].text_content()),
-        "steals": int(row[25].text_content()),
-        "blocks": int(row[26].text_content()),
-        "turnovers": int(row[27].text_content()),
-        "personal_fouls": int(row[28].text_content()),
+        "age": str_to_int(row[3].text_content(), default=None),
+        "team": TEAM_ABBREVIATIONS_TO_TEAM.get(row[4].text_content()),
+        "games_played": str_to_int(row[5].text_content()),
+        "games_started": str_to_int(row[6].text_content()),
+        "minutes_played": str_to_int(row[7].text_content()),
+        "made_field_goals": str_to_int(row[8].text_content()),
+        "attempted_field_goals": str_to_int(row[9].text_content()),
+        "made_three_point_field_goals": str_to_int(row[11].text_content()),
+        "attempted_three_point_field_goals": str_to_int(row[12].text_content()),
+        "made_free_throws": str_to_int(row[18].text_content()),
+        "attempted_free_throws": str_to_int(row[19].text_content()),
+        "offensive_rebounds": str_to_int(row[21].text_content()),
+        "defensive_rebounds": str_to_int(row[22].text_content()),
+        "assists": str_to_int(row[24].text_content()),
+        "steals": str_to_int(row[25].text_content()),
+        "blocks": str_to_int(row[26].text_content()),
+        "turnovers": str_to_int(row[27].text_content()),
+        "personal_fouls": str_to_int(row[28].text_content()),
     }
 
 
@@ -46,5 +47,10 @@ def parse_players_season_totals(page):
 
 
 def parse_positions(positions_content):
-    return list(map(lambda position_abbreviation: POSITION_ABBREVIATIONS_TO_POSITION[position_abbreviation],
-                    positions_content.split("-")))
+    parsed_positions = list(
+        map(
+            lambda position_abbreviation: POSITION_ABBREVIATIONS_TO_POSITION.get(position_abbreviation),
+            positions_content.split("-")
+        )
+    )
+    return [position for position in parsed_positions if position is not None]

basketball_reference_web_scraper/parsers/schedule.py

@@ -3,6 +3,7 @@
 import pytz
 
 from basketball_reference_web_scraper.data import Team
+from basketball_reference_web_scraper.utilities import str_to_int
 
 TEAM_NAME_TO_TEAM = {
     member.value: member
@@ -45,10 +46,10 @@ def parse_game(row):
     start_time = parse_start_time(formatted_date=row[0].text_content(), formatted_time_of_day=row[1].text_content())
     return {
         "start_time": start_time,
-        "away_team": TEAM_NAME_TO_TEAM[row[2].text_content().upper()],
-        "home_team": TEAM_NAME_TO_TEAM[row[4].text_content().upper()],
-        "away_team_score": int(row[3].text_content()) if row[3].text_content() else None,
-        "home_team_score": int(row[5].text_content()) if row[5].text_content() else None,
+        "away_team": TEAM_NAME_TO_TEAM.get(row[2].text_content().upper()),
+        "home_team": TEAM_NAME_TO_TEAM.get(row[4].text_content().upper()),
+        "away_team_score": str_to_int(row[3].text_content(), default=None),
+        "home_team_score": str_to_int(row[5].text_content(), default=None),
     }
 
 

basketball_reference_web_scraper/utilities.py

@@ -0,0 +1,14 @@
+def str_to_int(value, default=int(0)):
+    stripped_value = value.strip()
+    try:
+        return int(stripped_value)
+    except ValueError:
+        return default
+
+
+def str_to_float(value, default=float(0)):
+    stripped_value = value.strip()
+    try:
+        return float(stripped_value)
+    except ValueError:
+        return default

tests/12_12_2017_daily_leaders.html

@@ -7,6 +7,7 @@
 november_03_2003_daily_leaders_html = os.path.join(os.path.dirname(__file__), './11_03_2003_daily_leaders.html')
 november_01_2006_daily_leaders_html = os.path.join(os.path.dirname(__file__), './11_01_2006_daily_leaders.html')
 december_18_2015_daily_leaders_html = os.path.join(os.path.dirname(__file__), './12_18_2015_daily_leaders.html')
+december_12_2017_daily_leaders_html = os.path.join(os.path.dirname(__file__), './12_12_2017_daily_leaders.html')
 january_01_2017_daily_leaders_html = os.path.join(os.path.dirname(__file__), './01_29_2017_daily_leaders.html')
 
 
@@ -15,12 +16,23 @@ def setUp(self):
         self.november_01_2006_daily_leaders = open(november_01_2006_daily_leaders_html).read()
         self.december_18_2015_daily_leaders = open(december_18_2015_daily_leaders_html).read()
         self.november_03_2003_daily_leaders = open(november_03_2003_daily_leaders_html).read()
+        self.december_12_2017_daily_leaders = open(december_12_2017_daily_leaders_html).read()
         self.january_01_2017_daily_leaders = open(january_01_2017_daily_leaders_html).read()
 
     def test_box_scores_for_12_18_2015(self):
         parsed_box_score = players.parse_player_box_scores(self.december_18_2015_daily_leaders)
         self.assertEqual(len(parsed_box_score), 250)
 
+    def test_box_scores_for_12_12_2017(self):
+        parsed_box_score = players.parse_player_box_scores(self.december_12_2017_daily_leaders)
+        self.assertEqual(len(parsed_box_score), 149)
+
+    def test_parses_blank_value_for_andrew_bogut_on_12_12_2017(self):
+        parsed_box_score = players.parse_player_box_scores(self.december_12_2017_daily_leaders)
+        andrew_bogut = parsed_box_score[128]
+        self.assertEqual(andrew_bogut["made_three_point_field_goals"], 0)
+        self.assertEqual(andrew_bogut["attempted_three_point_field_goals"], 0)
+
     # Test for minutes played greater than or equal to 60 minutes
     def test_box_scores_for_01_01_2017(self):
         parsed_box_score = players.parse_player_box_scores(self.january_01_2017_daily_leaders)

tests/test_integration_parse_player_box_scores.py

@@ -0,0 +1,43 @@
+from unittest import TestCase
+
+from basketball_reference_web_scraper.utilities import str_to_int, str_to_float
+
+
+class TestStrToInt(TestCase):
+    def test_empty_string_is_zero(self):
+        self.assertEqual(str_to_int(""), 0)
+
+    def test_whitespace_is_zero(self):
+        self.assertEqual(str_to_int("    "), 0)
+
+    def test_stringified_number_is_converted(self):
+        self.assertEqual(str_to_int("10"), 10)
+
+    def test_stringified_number_with_leading_whitespace_is_converted(self):
+        self.assertEqual(str_to_int("  10"), 10)
+
+    def test_stringified_number_with_trailing_whitespace_is_converted(self):
+        self.assertEqual(str_to_int("10    "), 10)
+
+    def test_with_default(self):
+        self.assertIsNone(str_to_int("", default=None))
+
+
+class TestStrToFloat(TestCase):
+    def test_empty_string_is_zero(self):
+        self.assertEqual(str_to_float(""), 0.0)
+
+    def test_whitespace_is_zero(self):
+        self.assertEqual(str_to_float("    "), 0.0)
+
+    def test_stringified_number_is_converted(self):
+        self.assertEqual(str_to_float("1.234"), 1.234)
+
+    def test_stringified_number_with_leading_whitespace_is_converted(self):
+        self.assertEqual(str_to_float("  1.234"), 1.234)
+
+    def test_stringified_number_with_trailing_whitespace_is_converted(self):
+        self.assertEqual(str_to_float("1.234    "), 1.234)
+
+    def test_with_default(self):
+        self.assertIsNone(str_to_float("", default=None))