import pandas as pd
import numpy as np
from bs4 import BeautifulSoup
from urllib import request
from urllib.error import HTTPError, URLError
import time

team_url = request.urlopen('http://www.espn.com/nba/teams').read()
team_soup = BeautifulSoup(team_url,'lxml')
team_containers = team_soup.find_all('div',{'class':'pl3'})

team_names = [team.a.text for team in team_containers]
team_abbrs = [team.find_all('a',href=True)[1]['href'].split('/')[6] for team in team_containers]
teams = dict(zip(team_abbrs,team_names))

print(len(teams))
teams

# For the 2018-2019 regular season
year = '2019'

team_schedules = dict()
for team_abbr in list(teams.keys()):
    # Open the Team Schedule page and parse it.
    url = 'https://www.espn.com/nba/team/schedule/_/name/'+team_abbr+'/season/'+year+'/seasontype/2'
    schedule_url = request.urlopen(url).read()
    time.sleep(1)
    schedule_soup = BeautifulSoup(schedule_url,'lxml')

    # The class for each table row of a team's schedule contains the string "Table__TR".
    # So, search for that and exclude the first row which contains column names.
    game_rows = schedule_soup.select('tr[class*="Table__TR"]')[1:]

    # Grab the dates of each game.
    game_dates = [game.td.span.text for game in game_rows]
    # Append the team_abbr to each game ID. These will be used as dictionary keys later on.
    # For each row, get the all the <a> tag hyperlinks, select the third hyperlink,
    # split on the equals sign, and take the second element of the split to get the gameID.
    game_ids = [team_abbr+'_'+game.find_all('a',href=True)[2]['href'].split('=')[1] for game in game_rows]

    # The game locations are stored in a section element of class "flex items-center opponent-logo".
    game_logos = [game.find_all('div', attrs={'class':'flex items-center opponent-logo'}) for game in game_rows]
    # Game locations are indicated by "@" or "vs" for away and home, respectively.
    game_locs = [game[0].span.text for game in game_logos]

    # The result of a game is stored in a span element with a class containing the string "fw-bold clr-".
    game_results = [game.text for game in schedule_soup.select('span[class*="fw-bold clr-"]')]

    # Put table rows into a dictionary with a key of format teamABBR_gameID
    team_rows = list(zip(game_dates, game_locs, game_results))
    team_schedules.update(zip(game_ids,team_rows))

def get_box_score(table_data):
    """ Get all the cell data in a table. """
    box_data = []
    for tr in table_data:
        td = tr.find_all('td')
        row = [tr.text.strip('\t\n') for tr in td]
        box_data.append(row)
    return box_data

final_data = []
for key in list(team_schedules.keys()):
    team_abbr = key.split('_')[0]
    game_id = key.split('_')[1]

    # Make sure the URL opens
    try:
        box_url = request.urlopen('https://www.espn.com/nba/matchup?gameId='+game_id).read()
        time.sleep(1)
    # If there is an HTTP error, wait 2 mins and try again.
    except HTTPError as e:
        print(team_abbr, game_id, e.reason)
        time.sleep(120)
        box_url = request.urlopen('https://www.espn.com/nba/matchup?gameId='+game_id).read()
        pass

    # Make sure the URL has the same table format.
    try:
        box_soup = BeautifulSoup(box_url,'lxml')

        # Table containing the box score data
        box_table = box_soup.find_all('table',attrs={'class':'mod-data'})[0]

        # Get abbreviation and game points of the home team.
        home_team_container = box_soup.find_all('div', attrs={'class':'team home'})
        home_team_abbr = home_team_container[0].find_all('span', attrs={'class':'abbrev'})[0].text.lower()
        home_team_pts = box_soup.find_all('div', attrs={'class': 'score icon-font-before'})[0].text

        # Get abbreviation and game points of the away team.
        away_team_container = box_soup.find_all('div', attrs={'class':'team away'})
        away_team_abbr = away_team_container[0].find_all('span', attrs={'class':'abbrev'})[0].text.lower()
        away_team_pts = box_soup.find_all('div', attrs={'class': 'score icon-font-after'})[0].text

        # Put the game points and a column name in a list.
        points = ['PTS', away_team_pts, home_team_pts]

        # Table rows with an indent class
        indents = box_table.find_all('tr', attrs={'class':'indent'})
        # Table rows with a highlight class
        highlights = box_table.find_all('tr', attrs={'class':'highlight'})

        # Get the box scores
        highlights_data = get_box_score(highlights)
        indents_data = get_box_score(indents)

        # Concatenate points, highlights, and indents lists. Then transpose.
        box_data = np.concatenate(([points],highlights_data,indents_data)).T

        # List the team's data first then the opponent's data.
        # Also include the gameID, game date, game location, and game result.
        if team_abbr == away_team_abbr:
            row = np.concatenate(([team_abbr,teams[team_abbr],game_id],team_schedules[key],box_data[1],
                                  [home_team_abbr,teams[home_team_abbr]],box_data[2]))
        else:
            row = np.concatenate(([team_abbr,teams[team_abbr],game_id],team_schedules[key],box_data[2],
                                  [away_team_abbr,teams[away_team_abbr]],box_data[1]))
        final_data.append(row)

    # If the URL does not have the box score, skip this game ID and continue to the next one.
    except IndexError as e:
        print(team_abbr, game_id, e)
        continue

box_data[0]

col_names = ['teamABBR', 'teamName','gameID', 'gameDate', 'gameLoc', 'teamResult', 'teamPTS', 'teamFG', 'teamFG%',
             'team3PT', 'team3PT%', 'teamFT', 'teamFT%', 'teamTREB', 'teamASST', 'teamSTL', 'teamBLK',
             'teamTO', 'teamFB_PTS', 'teamPNT_PTS', 'teamFOUL', 'teamLG_LEAD', 'teamOREB', 'teamDREB',
             'teamTO_PTS', 'teamFOUL_T', 'teamFOUL_F',
             'opptABBR', 'opptName', 'opptPTS', 'opptFG', 'opptFG%',
             'oppt3PT', 'oppt3PT%', 'opptFT', 'opptFT%', 'opptTREB', 'opptASST', 'opptSTL', 'opptBLK',
             'opptTO', 'opptFB_PTS', 'opptPNT_PTS', 'opptFOUL', 'opptLG_LEAD', 'opptOREB', 'opptDREB',
             'opptTO_PTS', 'opptFOUL_T', 'opptFOUL_F']

# Create the data frame
df = pd.DataFrame(final_data, columns=col_names)

df[df.gameID=='401071855']

df.to_csv('../data/nba_team_box_scores_'+str(year)+'.csv')