
import math
from collections import Counter
from datetime import timedelta
from typing import Any, Dict, Optional

import matplotlib.patches as mpatches
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import shap
from IPython.display import Video
from sklearn.calibration import CalibratedClassifierCV, calibration_curve
from sklearn.metrics import brier_score_loss, log_loss, roc_auc_score
from sklearn.model_selection import GridSearchCV, TimeSeriesSplit, train_test_split
from xgboost import XGBClassifier

pd.set_option("display.max_columns", 2000)
pd.set_option("display.max_rows", 1000)


matches_df = pd.read_csv(
    "./epl_2015_16_data/epl_matches_15.csv"
)
events_df = pd.read_csv(
    "./epl_2015_16_data/epl_event_data_15.csv"
)

print("Matches shape:", matches_df.shape)
print("Events shape:", events_df.shape)

Matches shape: (380, 28)
Events shape: (1313783, 49)


matches_df.head(1)


print("number of matches", matches_df.match_id.nunique())
print("number of match weeks", matches_df.match_week.nunique())
print("number of competitions", matches_df["competition.competition_id"].nunique())
print("number of home teams", matches_df["home_team.home_team_id"].nunique())
print("number of away teams", matches_df["away_team.away_team_id"].nunique())
print(sorted(matches_df["home_team.home_team_name"].unique()))

number of matches 380
number of match weeks 38
number of competitions 1
number of home teams 20
number of away teams 20
['AFC Bournemouth', 'Arsenal', 'Aston Villa', 'Chelsea', 'Crystal Palace', 'Everton', 'Leicester City', 'Liverpool', 'Manchester City', 'Manchester United', 'Newcastle United', 'Norwich City', 'Southampton', 'Stoke City', 'Sunderland', 'Swansea City', 'Tottenham Hotspur', 'Watford', 'West Bromwich Albion', 'West Ham United']


events_df.head(1)


events_df["type.name"].value_counts()

type.name
Pass                 368619
Ball Receipt*        340324
Carry                276949
Pressure             115402
Ball Recovery         40943
Duel                  32290
Clearance             21645
Block                 14839
Dribble               13721
Goal Keeper           11777
Miscontrol            10786
Dispossessed          10520
Shot                   9908
Foul Committed         9512
Foul Won               9112
Interception           8920
Dribbled Past          8771
Substitution           2109
Half Start             1520
Half End               1520
Injury Stoppage        1032
Starting XI             760
Tactical Shift          625
50/50                   559
Shield                  503
Referee Ball-Drop       272
Bad Behaviour           193
Error                   178
Player Off              145
Player On               144
Offside                 109
Own Goal For             38
Own Goal Against         38
Name: count, dtype: int64


def plot_pitch():
    """Draw a detailed soccer pitch with 6-yard boxes, penalty boxes, and center circle"""
    fig, ax = plt.subplots(figsize=(10, 6))

    # Pitch outline & center line
    ax.plot([0, 0], [0, 80], color="black")
    ax.plot([0, 120], [80, 80], color="black")
    ax.plot([120, 120], [80, 0], color="black")
    ax.plot([120, 0], [0, 0], color="black")
    ax.plot([60, 60], [0, 80], color="black")

    # Left penalty area (18-yard box)
    ax.plot([0, 18], [18, 18], color="black")
    ax.plot([18, 18], [18, 62], color="black")
    ax.plot([0, 18], [62, 62], color="black")

    # Right penalty area (18-yard box)
    ax.plot([120, 102], [18, 18], color="black")
    ax.plot([102, 102], [18, 62], color="black")
    ax.plot([102, 120], [62, 62], color="black")

    # Left 6-yard box
    ax.plot([0, 6], [30, 30], color="black")
    ax.plot([6, 6], [30, 50], color="black")
    ax.plot([0, 6], [50, 50], color="black")

    # Right 6-yard box
    ax.plot([120, 114], [30, 30], color="black")
    ax.plot([114, 114], [30, 50], color="black")
    ax.plot([120, 114], [50, 50], color="black")

    # Center circle
    center_circle = plt.Circle((60, 40), 10, color="black", fill=False)
    ax.add_patch(center_circle)

    # Setup axes
    ax.set_xlim(0, 120)
    ax.set_ylim(0, 80)
    ax.set_axis_off()
    ax.invert_yaxis()  # match (0,0) top-left convention

    return fig, ax


def plot_actions(possess_df):
    match_id = possess_df["match_id"].iloc[0]
    possess_id = possess_df["possession"].iloc[0]

    event_color = {
        "Pass": "lightblue",
        "Ball Receipt*": "blue",
        "Ball Recovery": "darkgreen",
        "Interception": "green",
        "Carry": "purple",
        "Pressure": "red",
        "Block": "darkred",
        "Duel": "orange",
    }

    seen_coords = {}
    fig, ax = plot_pitch()

    for step, (_, row) in enumerate(possess_df.iterrows(), start=1):
        x, y = row["mirror_fix_x"], row["mirror_fix_y"]
        event = row["type.name"]
        player = row["player.name"]
        color = event_color.get(event, "gray")

        if pd.notnull(x) and pd.notnull(y):
            coord = (round(x, 1), round(y, 1))
            offset_count = seen_coords.get(coord, 0)
            jitter_x = x + 0.5 * offset_count
            jitter_y = y + 0.5 * offset_count
            seen_coords[coord] = offset_count + 1

            ax.text(
                jitter_x,
                jitter_y,
                str(step),
                fontsize=10,
                fontweight="bold",
                color=color,
                ha="center",
                va="center",
            )
            ax.text(
                jitter_x + 1,
                jitter_y + 1,
                player.split(" ")[-1],
                fontsize=8,
                color=color,
            )

    # Create custom legend
    legend_patches = [
        mpatches.Patch(color=color, label=etype) for etype, color in event_color.items()
    ]

    # Add legend to the right of the pitch
    ax.legend(
        handles=legend_patches,
        loc="center left",  # Align legend vertically center, left of anchor point
        bbox_to_anchor=(1.02, 0.5),  # Place legend just outside right of the plot
        borderaxespad=0,  # Remove internal padding
        fontsize=8,
    )


plot_pitch()

(<Figure size 1000x600 with 1 Axes>, <Axes: >)


test_match_id = 3754065


# get some info on the match
matches_df[matches_df["match_id"] == test_match_id]


events_mtest_df = events_df[events_df["match_id"] == test_match_id].reset_index(
    drop=True
)
print(events_mtest_df.shape)

(3106, 49)


# need to extract x,y coordinates for plotting
# Convert 'location' strings to x and y numeric columns
events_mtest_df["x"] = events_mtest_df["location"].apply(
    lambda s: float(s.split(",")[0].strip()) if pd.notnull(s) else None
)
events_mtest_df["y"] = events_mtest_df["location"].apply(
    lambda s: float(s.split(",")[1].strip()) if pd.notnull(s) else None
)


# Step 1: Create the possession flag
events_mtest_df["possess_team_event"] = (
    events_mtest_df["possession_team.name"] == events_mtest_df["team.name"]
)

# Step 2: Conditionally assign mirrored or original coordinates
events_mtest_df["mirror_fix_x"] = np.where(
    events_mtest_df["possess_team_event"],
    events_mtest_df["x"],
    120 - events_mtest_df["x"],
)
events_mtest_df["mirror_fix_y"] = np.where(
    events_mtest_df["possess_team_event"],
    events_mtest_df["y"],
    80 - events_mtest_df["y"],
)


def get_possess_df(event_m_df, possess_id):
    poss_df = event_m_df[event_m_df["possession"] == possess_id]
    return poss_df


possess_main_cols = [
    "timestamp",
    "duration",
    "type.name",
    "possession_team.name",
    "play_pattern.name",
    "team.name",
    "player.name",
    "player_possession",
    "mirror_fix_x",
    "mirror_fix_y",
    "ball_receipt.outcome.name",
    "pass.end_location",
]


test_possess3_df = get_possess_df(events_mtest_df, possess_id=3)
print(test_possess3_df.shape)
test_possess3_df[possess_main_cols]

(18, 54)


plot_actions(test_possess3_df)


Video("tott_vs_leic_poss_3.mp4", width=640, height=480)


test_possess48_df = get_possess_df(events_mtest_df, possess_id=48)
print(test_possess48_df.shape)
test_possess48_df[possess_main_cols]

(8, 54)


plot_actions(test_possess48_df)


Video("tott_vs_leic_poss_47_48_foul.mp4", width=640, height=480)


test_possess58_df = get_possess_df(events_mtest_df, possess_id=58)
print(test_possess58_df.shape)
test_possess58_df[possess_main_cols]

(15, 54)


plot_actions(test_possess58_df)


Video("tott_vs_leic_poss_58.mp4", width=640, height=480)


def parse_timestamp(ts):
    """Convert StatsBomb timestamp string into timedelta object."""
    minutes, seconds = ts.split(":")[1:]  # skip the initial 00 for hours
    sec, millisec = seconds.split(".")
    return timedelta(minutes=int(minutes), seconds=int(sec), milliseconds=int(millisec))


def extract_valid_possession_starts(df, time_diff_max_sec=0.5):
    """
    Extract valid possession start events from StatsBomb event data.
    A valid start is a successful Ball Receipt* or Ball Recovery, followed
    within time_diff_max_sec seconds by a Pass, Carry, Dribble, Shot, or Foul Won by the same player
    and player_possession.
    """
    df = df.copy().reset_index(drop=True)  # important to ensure index reset
    df["parsed_time"] = df["timestamp"].apply(parse_timestamp)
    # df['player_possession'] = df['player_possession'].fillna(method='ffill')

    valid_starts = []
    candidate_events = df[
        df["type.name"].isin(["Ball Receipt*", "Ball Recovery"])
    ].copy()

    # Filter successful Ball Receipt* only
    candidate_events = candidate_events[
        (candidate_events["type.name"] != "Ball Receipt*")
        | (candidate_events["ball_receipt.outcome.name"].isna())
    ]

    for idx, row in candidate_events.iterrows():
        if idx + 1 >= len(df):
            continue

        next_event = df.iloc[idx + 1]
        time_diff = (next_event["parsed_time"] - row["parsed_time"]).total_seconds()
        if (
            next_event["type.name"] in ["Pass", "Carry", "Dribble", "Shot", "Foul Won"]
            and next_event.get("player.id") == row.get("player.id")
            and next_event.get("player_possession") == row.get("player_possession")
            and 0 <= time_diff <= time_diff_max_sec
        ):
            valid_starts.append(row)

    return pd.DataFrame(valid_starts)


# Create test cases to validate different edge cases of valid possession extraction

# Sample mock data to simulate various edge cases
test_data = [
    # Case 1: Valid Ball Receipt* followed by Pass within 0.5s
    {
        "id": "c1",
        "timestamp": "00:01:00.000",
        "type.name": "Ball Receipt*",
        "ball_receipt.outcome.name": None,
        "player.id": "Player A",
        "player_possession": 1,
    },
    {
        "timestamp": "00:01:00.400",
        "type.name": "Pass",
        "ball_receipt.outcome.name": None,
        "player.id": "Player A",
        "player_possession": 1,
    },
    # Case 2: Ball Receipt* followed by Pass after 0.5s (Invalid)
    {
        "id": "c2",
        "timestamp": "00:01:10.000",
        "type.name": "Ball Receipt*",
        "ball_receipt.outcome.name": None,
        "player.id": "Player B",
        "player_possession": 2,
    },
    {
        "timestamp": "00:01:10.600",
        "type.name": "Pass",
        "ball_receipt.outcome.name": None,
        "player.id": "Player B",
        "player_possession": 2,
    },
    # Case 3: Ball Receipt* but outcome is Incomplete (Invalid)
    {
        "id": "c3",
        "timestamp": "00:01:20.000",
        "type.name": "Ball Receipt*",
        "ball_receipt.outcome.name": "Incomplete",
        "player.id": "Player C",
        "player_possession": 3,
    },
    {
        "timestamp": "00:01:20.300",
        "type.name": "Pass",
        "ball_receipt.outcome.name": None,
        "player.id": "Player C",
        "player_possession": 3,
    },
    # Case 4: Ball Recovery followed by Carry within 0.5s (Valid)
    {
        "id": "c4",
        "timestamp": "00:01:30.000",
        "type.name": "Ball Recovery",
        "ball_receipt.outcome.name": None,
        "player.id": "Player D",
        "player_possession": 4,
    },
    {
        "timestamp": "00:01:30.000",
        "type.name": "Carry",
        "ball_receipt.outcome.name": None,
        "player.id": "Player D",
        "player_possession": 4,
    },
    # Case 5: Ball Receipt* followed by Pass by different player (Invalid)
    {
        "id": "c5",
        "timestamp": "00:01:40.000",
        "type.name": "Ball Receipt*",
        "ball_receipt.outcome.name": None,
        "player.id": "Player E",
        "player_possession": 5,
    },
    {
        "timestamp": "00:01:40.200",
        "type.name": "Pass",
        "ball_receipt.outcome.name": None,
        "player.id": "Player F",
        "player_possession": 5,
    },
    # Case 6: Ball Receipt* followed by valid event but different player_possession (Invalid)
    {
        "id": "c6",
        "timestamp": "00:01:50.000",
        "type.name": "Ball Receipt*",
        "ball_receipt.outcome.name": None,
        "player.id": "Player G",
        "player_possession": 6,
    },
    {
        "timestamp": "00:01:50.300",
        "type.name": "Dribble",
        "ball_receipt.outcome.name": None,
        "player.id": "Player G",
        "player_possession": 7,
    },
]

# Convert to DataFrame
test_df = pd.DataFrame(test_data)

# Fill in required missing columns to match function expectations
test_df["duration"] = 0
test_df["possession_team.name"] = "TeamX"
test_df["play_pattern.name"] = "Regular Play"
test_df["team.name"] = "TeamX"
test_df["mirror_fix_x"] = 60
test_df["mirror_fix_y"] = 40
test_df["pass.end_location"] = None


# should be c1, c4
valid_test_results = extract_valid_possession_starts(test_df, time_diff_max_sec=0.5)
valid_test_results


# should be only c4
valid_test_results = extract_valid_possession_starts(test_df, time_diff_max_sec=0.001)
valid_test_results


match_id = 3754065


test_match_df = events_df[events_df["match_id"] == match_id].reset_index(drop=True)
print(test_match_df.shape)

(3106, 49)


# Lets first drill even further to a specific possession to see
poss_3_df = test_match_df[test_match_df["possession"] == 3]
print(poss_3_df.shape)

(18, 49)


poss_3_df["player.id"].iloc[0]

np.float64(3077.0)


display_cols = [
    # "id",
    "timestamp",
    "type.name",
    "player.id",
    "player.name",
    "player_possession",
    "possession_team.name",
    "team.name",
    "ball_receipt.outcome.name",
]


poss_3_df[display_cols]


extract_valid_possession_starts(poss_3_df, time_diff_max_sec=0.5)[display_cols]


poss_48_df = test_match_df[test_match_df["possession"] == 48].reset_index(drop=True)
print(poss_48_df.shape)

(8, 49)


poss_48_df[display_cols]


extract_valid_possession_starts(poss_48_df, time_diff_max_sec=0.5)[display_cols]


poss_58_df = test_match_df[test_match_df["possession"] == 58].reset_index(drop=True)
print(poss_58_df.shape)

(15, 49)


poss_58_df[display_cols]


extract_valid_possession_starts(poss_58_df, time_diff_max_sec=0.5)[display_cols]


events_df = events_df.sort_values(["match_id", "period", "timestamp"]).reset_index(
    drop=True
)


res_match_cols_save = [
    "match_id",
    "id",
    "period",
    "timestamp",
    "type.name",
    "location",
    "player.id",
    "possession",
    "player_possession",
]


%%time
# (probably could optimize this code to make faster but sufficient for this)
val_poss_match_df_list = []
for match_id in sorted(events_df.match_id.unique()):
    match_df = events_df[events_df.match_id == match_id].reset_index(drop=True)
    # get valid possesion start events for the match
    val_poss_match_df = extract_valid_possession_starts(
        match_df, time_diff_max_sec=0.5
    )[res_match_cols_save]
    val_poss_match_df_list.append(val_poss_match_df)
    # print(match_df.shape)
    # print(res_match_df.shape)
all_val_poss_df = pd.concat(val_poss_match_df_list).reset_index(drop=True)

CPU times: user 1min 20s, sys: 863 ms, total: 1min 21s
Wall time: 1min 24s


print(all_val_poss_df.shape)
print(all_val_poss_df["id"].nunique())
print(all_val_poss_df["match_id"].nunique())

(300891, 9)
300891
380


all_val_poss_df.head(2)


all_val_poss_df["type.name"].value_counts()

type.name
Ball Receipt*    265509
Ball Recovery     35382
Name: count, dtype: int64


events_df[events_df["type.name"].isin(["Ball Receipt*", "Ball Recovery"])][
    "type.name"
].value_counts()

type.name
Ball Receipt*    340324
Ball Recovery     40943
Name: count, dtype: int64


len(all_val_poss_df) / len(events_df)

0.2290264069484839


# all_val_poss_df.to_csv("valid_possess_start_events.csv", index=False)


# extract_valid_possession_starts(poss_3_df, time_diff_max_sec=0.5)[["match_id", "possession", "id"]]


# all_val_poss_df[(all_val_poss_df.match_id == 3754065) & (all_val_poss_df.possession == 3)]


# Define function to mark if a possession ends in 'Foul Won'
def mark_if_possession_ends_in_foul(start_events_df, all_events_df):
    """
    For each possession start event in `start_events_df`, determine if the corresponding player_possession
    includes a 'Foul Won' event by the same player.

    Returns:
        A copy of the input DataFrame with an additional boolean column 'ends_in_foul'.
    """
    results = start_events_df.copy()
    results["ends_in_foul"] = False  # default value

    # Ensure timestamp is parsed
    if "parsed_time" not in all_events_df.columns:
        all_events_df["parsed_time"] = all_events_df["timestamp"].apply(parse_timestamp)

    # Create a lookup of foul events by player.name and player_possession
    foul_events = all_events_df[
        (all_events_df["type.name"] == "Foul Won")
        & (all_events_df["player.id"].notna())
        & (all_events_df["player_possession"].notna())
    ][["player.id", "player_possession"]]

    foul_set = set(
        map(tuple, foul_events.values)
    )  # convert to set of tuples for fast lookup

    # Check each row
    for i, row in results.iterrows():
        key = (row["player.id"], row["player_possession"])
        if key in foul_set:
            results.at[i, "ends_in_foul"] = True

    return results


test_match_df = events_df[(events_df.match_id == 3754065)].reset_index(drop=True)
print(test_match_df.shape)

(3106, 49)


test_match_val_poss_df = all_val_poss_df[
    (all_val_poss_df.match_id == 3754065)
].reset_index(drop=True)
print(test_match_val_poss_df.shape)

(654, 9)


test_match_val_poss_df.head()


test_blah_res_df = mark_if_possession_ends_in_foul(
    test_match_val_poss_df, test_match_df
)


print(test_blah_res_df.shape)

(654, 10)


test_blah_res_df["ends_in_foul"].value_counts()

ends_in_foul
False    647
True       7
Name: count, dtype: int64


test_blah_res_df[test_blah_res_df.ends_in_foul == True]


test_match_df[test_match_df["type.name"] == "Foul Won"].shape

(24, 50)


test_match_df[test_match_df["type.name"] == "Foul Won"][display_cols].head()


test_match_df.iloc[775:782][display_cols]


test_match_df.iloc[890:897][display_cols]


test_match_df.iloc[1130:1135][display_cols]


%%time
foul_res_poss_df = mark_if_possession_ends_in_foul(all_val_poss_df, events_df)

CPU times: user 20 s, sys: 121 ms, total: 20.1 s
Wall time: 20.2 s


print(foul_res_poss_df.shape)
print(all_val_poss_df.shape)

(300891, 10)
(300891, 9)


foul_res_poss_df["ends_in_foul"].value_counts()

ends_in_foul
False    293304
True       7587
Name: count, dtype: int64


foul_res_poss_df["ends_in_foul"].value_counts(normalize=True)

ends_in_foul
False    0.974785
True     0.025215
Name: proportion, dtype: float64


# export final samples and labels
# foul_res_poss_df.to_csv("valid_poss_start_events_wlabel.csv", index=False)


# first want to join the matchweek and the date and team_id
# then I want to g
foul_res_poss_df.head(2)


tmp_matches_df = matches_df.copy()[["match_id", "match_date", "match_week"]]
print(tmp_matches_df.shape)
tmp_matches_df.head(2)

(380, 3)


# Ensure match_date is datetime
tmp_matches_df["match_date"] = pd.to_datetime(tmp_matches_df["match_date"])
# Sort by match_date
tmp_matches_df = tmp_matches_df.sort_values("match_date").reset_index(drop=True)
# Calculate 80/20 split index
split_index = int(len(tmp_matches_df) * 0.8)
# Create ml_split column
tmp_matches_df["ml_split"] = "test"
tmp_matches_df.loc[: split_index - 1, "ml_split"] = "train"


tmp_matches_df.ml_split.value_counts()

ml_split
train    304
test      76
Name: count, dtype: int64


print(tmp_matches_df[tmp_matches_df.ml_split == "train"].match_date.min())
print(tmp_matches_df[tmp_matches_df.ml_split == "train"].match_date.max())

print(tmp_matches_df[tmp_matches_df.ml_split == "test"].match_date.min())
print(tmp_matches_df[tmp_matches_df.ml_split == "test"].match_date.max())

2015-08-08 00:00:00
2016-04-02 00:00:00
2016-04-02 00:00:00
2016-05-17 00:00:00


train_match_ids = set(
    tmp_matches_df[tmp_matches_df.ml_split == "train"].match_id.unique()
)
test_match_ids = set(
    tmp_matches_df[tmp_matches_df.ml_split == "test"].match_id.unique()
)
print(len(train_match_ids))
print(len(test_match_ids))
# ensure no match_ids in both train and test
train_match_ids.isdisjoint(test_match_ids)

304
76

True


# now need to join train/test split info
foul_res_poss_df2 = pd.merge(
    foul_res_poss_df,
    tmp_matches_df[["match_id", "ml_split"]],
    how="left",
    on="match_id",
)
print(foul_res_poss_df2.shape)
foul_res_poss_df2.ml_split.value_counts(normalize=True)

(300891, 11)

ml_split
train    0.800054
test     0.199946
Name: proportion, dtype: float64


train_res_poss_df = foul_res_poss_df2[
    foul_res_poss_df2.ml_split == "train"
].reset_index(drop=True)

test_res_poss_df = foul_res_poss_df2[foul_res_poss_df2.ml_split == "test"].reset_index(
    drop=True
)

print(train_res_poss_df.shape, test_res_poss_df.shape)

(240729, 11) (60162, 11)


train_res_poss_df.ends_in_foul.value_counts(normalize=True)

ends_in_foul
False    0.974511
True     0.025489
Name: proportion, dtype: float64


test_res_poss_df.ends_in_foul.value_counts(normalize=True)

ends_in_foul
False    0.975882
True     0.024118
Name: proportion, dtype: float64


y_train = train_res_poss_df.ends_in_foul.values
y_test = test_res_poss_df.ends_in_foul.values

print(y_test.shape)
print(y_test.shape)

(60162,)
(60162,)


pd.Series(y_train).value_counts()

False    234593
True       6136
Name: count, dtype: int64


pd.Series(y_test).value_counts()

False    58711
True      1451
Name: count, dtype: int64


# export train and test id's and labels (ends_in_foul)
train_res_poss_df.to_csv("train_events.csv", index=False)
test_res_poss_df.to_csv("test_events.csv", index=False)


def compute_foul_win_rate_grid(df, bins=(5, 5), min_total_count=1):
    df = df.copy()
    df["x"] = df["location"].apply(
        lambda s: float(s.split(",")[0].strip()) if pd.notnull(s) else None
    )
    df["y"] = df["location"].apply(
        lambda s: float(s.split(",")[1].strip()) if pd.notnull(s) else None
    )

    x_bins = np.linspace(0, 120, bins[0] + 1)
    y_bins = np.linspace(0, 80, bins[1] + 1)

    df["x_bin"] = pd.cut(df["x"], bins=x_bins, labels=False, include_lowest=True)
    df["y_bin"] = pd.cut(df["y"], bins=y_bins, labels=False, include_lowest=True)
    df = df.dropna(subset=["x_bin", "y_bin"])

    grouped = df.groupby(["x_bin", "y_bin"])
    agg = (
        grouped["ends_in_foul"]
        .agg(total_poss_count="count", foul_win_count="sum")
        .reset_index()
    )

    agg["foul_win_rate"] = agg["foul_win_count"] / agg["total_poss_count"]

    # Compute standard deviation of a Bernoulli variable
    agg["foul_win_rate_std"] = agg.apply(
        lambda row: (
            np.sqrt(
                row["foul_win_rate"]
                * (1 - row["foul_win_rate"])
                / row["total_poss_count"]
            )
            if row["total_poss_count"] >= min_total_count
            else np.nan
        ),
        axis=1,
    )

    # Optionally suppress rate if too few samples
    agg["foul_win_rate"] = agg.apply(
        lambda row: (
            row["foul_win_rate"]
            if row["total_poss_count"] >= min_total_count
            else np.nan
        ),
        axis=1,
    )

    agg["x_bin"] = agg["x_bin"].astype(int)
    agg["y_bin"] = agg["y_bin"].astype(int)

    # Add unique grid_id
    num_y_bins = bins[1]
    agg["grid_id"] = agg["x_bin"] * num_y_bins + agg["y_bin"]
    agg["grid_id"].astype("int")

    return agg, x_bins, y_bins


def plot_foul_win_rate_grid_heatmap(
    grid_frate_df,
    x_bins,
    y_bins,
    vmax=0.05,
    annotate=True,
    show_grid_id=True,
    show_counts=True,
):
    fig, ax = plot_pitch()
    foul_rate_grid = np.full((len(x_bins) - 1, len(y_bins) - 1), np.nan)

    # Fill the grid
    for _, row in grid_frate_df.iterrows():
        x = int(row["x_bin"])
        y = int(row["y_bin"])
        if 0 <= x < foul_rate_grid.shape[0] and 0 <= y < foul_rate_grid.shape[1]:
            foul_rate_grid[x, y] = row["foul_win_rate"]

    # Plot heatmap
    pcm = ax.pcolormesh(
        x_bins, y_bins, foul_rate_grid.T, cmap="Reds", vmin=0, vmax=vmax
    )
    plt.colorbar(pcm, ax=ax, label="Foul Win Rate")
    plt.title("Foul Win Rate Heatmap with Annotations")

    # Annotate each bin
    if annotate:
        for _, row in grid_frate_df.iterrows():
            x_bin = int(row["x_bin"])
            y_bin = int(row["y_bin"])
            if np.isnan(row["foul_win_rate"]):
                continue

            x_center = (x_bins[x_bin] + x_bins[x_bin + 1]) / 2
            y_center = (y_bins[y_bin] + y_bins[y_bin + 1]) / 2

            text_lines = []
            if show_grid_id:
                text_lines.append(f"grid_id: : {int(row['grid_id'])}")
            text_lines.append(f"{row['foul_win_rate']:.2%}")
            if not pd.isna(row["foul_win_rate_std"]):
                text_lines.append(f"±{row['foul_win_rate_std']:.2%}")
            if show_counts:
                text_lines.append(f"n={int(row['total_poss_count'])}")
                text_lines.append(f"fouls={int(row['foul_win_count'])}")

            ax.text(
                x_center,
                y_center,
                "\n".join(text_lines),
                color="black",
                ha="center",
                va="center",
                fontsize=8,
            )

    plt.show()


grid_frate_df, x_bins, y_bins = compute_foul_win_rate_grid(
    train_res_poss_df, bins=(5, 5), min_total_count=5
)


plot_foul_win_rate_grid_heatmap(grid_frate_df, x_bins, y_bins)


cols_to_convert = [
    "position.id",
    "player.id",
]

for col in cols_to_convert:
    events_df[col] = events_df[col].astype("Int64")


events_df[["position.id", "player.id"]].iloc[3:8]


events_df[["timestamp", "index", "match_id"]].head()


del events_df["index"]  # remove old index column
events_df = events_df.reset_index()


%%time
events_dic = events_df.set_index("id").to_dict(orient="index")

CPU times: user 33.6 s, sys: 1.59 s, total: 35.1 s
Wall time: 35.4 s


%%time
matches_df = matches_df.reset_index()  # want index col explicitly so dont drop
matches_dic = matches_df.set_index("match_id").to_dict(orient="index")

CPU times: user 10.2 ms, sys: 2.79 ms, total: 12.9 ms
Wall time: 14.4 ms


test_id = "e7f3536d-3209-4ed2-a3a4-cd1519e44f7f"


def simple_single_fx(the_id, events_dic):
    """
    Extract and compute simple features from start possession event.

    Parameters:
    -----------
    the_id : str
        The unique identifier for the event.
    events_dic : dict
        Dictionary containing event data, where keys are event IDs and values are dictionaries of event attributes.

    Returns:
    --------
    fx_simple_dic : dict
        A dictionary containing simplified features derived from the event:
            - period: int (1 or 2, representing the half)
            - minute: int (match minute of the event)
            - x: float (x-coordinate on the pitch)
            - y: float (y-coordinate on the pitch)
            - is_recovery: int (1 if event is a 'Ball Recovery', else 0)
            - under_pressure: int (1 if event is under pressure, else 0)
    """
    event_dic = events_dic.get(the_id)
    if event_dic is None:
        raise ValueError(f"Event ID {the_id} not found in events_dic")

    # Determine if event is a 'Ball Recovery'
    type_name = event_dic.get("type.name", "")
    is_recovery = 1 if type_name == "Ball Recovery" else 0

    # Parse x and y coordinates from location string
    location = event_dic.get("location", "")
    x = float(location.split(",")[0].strip())
    y = float(location.split(",")[1].strip())

    # Convert 'under_pressure' to 0 (False) if missing or NaN, 1 (True) otherwise
    under_pressure = (
        0
        if event_dic.get("under_pressure") is np.nan
        else int(event_dic["under_pressure"])
    )

    # Build simplified feature dictionary
    fx_simple_dic = {
        "period": event_dic.get("period"),
        "minute": event_dic.get("minute"),
        "x": x,
        "y": y,
        "is_recovery": is_recovery,
        "under_pressure": under_pressure,
    }

    return fx_simple_dic


simple_single_fx(test_id, events_dic)

{'period': 1,
 'minute': 0,
 'x': 60.2,
 'y': 40.7,
 'is_recovery': 0,
 'under_pressure': 0}


def match_simple_fx(the_id, events_dic, matches_dic):
    """
    Extract match-level feature(s) for a given event, such as whether the player was on the home team.

    Parameters:
    -----------
    the_id : str
        Unique identifier for the event.
    events_dic : dict
        Dictionary mapping event IDs to event data dictionaries.
    matches_dic : dict
        Dictionary mapping match IDs to match data dictionaries.

    Returns:
    --------
    fx_match_simple_dic : dict
        A dictionary with match-level features:
            - is_home_team: int (1 if the player was on the home team, 0 otherwise)
    """
    event_dic = events_dic.get(the_id)
    if event_dic is None:
        raise ValueError(f"Event ID {the_id} not found in events_dic")

    # Extract match and team identifiers
    match_id = event_dic["match_id"]
    player_team_id = event_dic["team.id"]
    match_dic = matches_dic.get(match_id)
    if match_dic is None:
        raise ValueError(f"Match ID {match_id} not found in matches_dic")

    home_team_id = match_dic.get("home_team.home_team_id")

    # Determine whether the player's team is the home team
    is_home_team = 1 if player_team_id == home_team_id else 0

    # Build output dictionary with match-level feature(s)
    fx_match_simple_dic = {"is_home_team": is_home_team}

    return fx_match_simple_dic


match_simple_fx(test_id, events_dic, matches_dic)

{'is_home_team': 1}


def dist_angle_on_pitch(x1, y1, x2, y2):
    """
    Calculate the Euclidean distance and angle between two points on a soccer pitch.

    Parameters:
    - x1, y1: Coordinates of the first point (e.g., starting position)
    - x2, y2: Coordinates of the second point (e.g., ending position)

    Returns:
    - distance: Euclidean distance between the points
    - angle_deg: Angle in degrees from (x1, y1) to (x2, y2), 0° is right, 90° is up
    """
    dx = x2 - x1
    dy = y2 - y1
    distance = round(math.hypot(dx, dy), 1)
    angle_rad = math.atan2(dy, dx)
    angle_deg = round(math.degrees(angle_rad), 1)
    return distance, angle_deg


dist_angle_on_pitch(x1=15.7, y1=8.7, x2=120, y2=40)

(108.9, 16.7)


def distance_simple_fx(the_id, events_dic):
    event_dic = events_dic.get(the_id)
    if event_dic is None:
        raise ValueError(f"Event ID {the_id} not found in events_dic")

    # Parse x and y coordinates from location string
    location = event_dic.get("location", "")
    x = float(location.split(",")[0].strip())
    y = float(location.split(",")[1].strip())

    opp_goal_center_x = 120
    opp_goal_center_y = 40
    dist_opp_goal, angle_opp_goal = dist_angle_on_pitch(
        x, y, opp_goal_center_x, opp_goal_center_y
    )

    # Build simplified feature dictionary
    fx_dist_simple_dic = {
        "dist_opp_goal": dist_opp_goal,
        "angle_opp_goal": angle_opp_goal,
    }

    return fx_dist_simple_dic


distance_simple_fx(test_id, events_dic)

{'dist_opp_goal': 59.8, 'angle_opp_goal': -0.7}


train_df_match_ids = train_res_poss_df["match_id"].values
print(len(train_df_match_ids))

240729


train_match_events_df = events_df[events_df["match_id"].isin(train_df_match_ids)]
print(train_match_events_df.shape)

(1047566, 50)


def build_event_count_df(events_df, group_by_col, event_types):
    """
    Creates a DataFrame grouped by a specified column (e.g., 'team.id') with counts
    for each event type (e.g., 'Foul Won', 'Pass') from the 'type.name' column.

    Parameters:
    - events_df: DataFrame with event data
    - group_by_col: str, column name to group by (e.g., 'team.id', 'player.id')
    - event_types: list of str, event types to count (from 'type.name')

    Returns:
    - pd.DataFrame with group_by_col and one column per event_type
    """
    result_df = pd.DataFrame({group_by_col: events_df[group_by_col].unique()})

    for event in event_types:
        counts = (
            events_df[events_df["type.name"] == event][group_by_col]
            .value_counts()
            .rename(event)
            .reset_index()
            .rename(columns={"index": group_by_col})
        )
        result_df = result_df.merge(counts, on=group_by_col, how="left")

    result_df.fillna(0, inplace=True)
    result_df[event_types] = result_df[event_types].astype(int)

    return result_df


def get_foul_rate_df(events_df, group_by_col, dropna=True, min_possessions=100):
    """
    Builds a DataFrame grouped by the specified column (e.g., 'team.id', 'position.id'),
    counting key event types and calculating estimated foul rates.

    Parameters:
    - events_df: pd.DataFrame, the match events data
    - group_by_col: str, column to group by (e.g., 'team.id', 'position.id')
    - dropna: bool, whether to drop rows with NaN values

    Returns:
    - pd.DataFrame with counts for each event type and foul rate columns:
      'est_foul_won_rate' and 'est_foul_commit_rate'
    """
    # Define event types needed for foul rate calculation
    event_types = ["Foul Committed", "Foul Won", "Ball Receipt*", "Ball Recovery"]
    # Count specified event types per group
    frate_df = build_event_count_df(
        events_df, group_by_col=group_by_col, event_types=event_types
    )
    # Calculate total possessions (denominator for rate calculations)
    total_possessions = frate_df["Ball Receipt*"] + frate_df["Ball Recovery"]
    frate_df["total_possessions"] = total_possessions

    # Estimate foul rates
    frate_df["est_foul_win_rate"] = (frate_df["Foul Won"] / total_possessions).round(3)
    frate_df["est_foul_commit_rate"] = (
        frate_df["Foul Committed"] / total_possessions
    ).round(3)
    if dropna:
        frate_df = frate_df.dropna()

    # Compute averages across qualifying rows (those above threshold)
    valid_rows = frate_df["total_possessions"] >= min_possessions
    avg_est_foul_won_rate = frate_df.loc[valid_rows, "est_foul_win_rate"].mean()
    avg_est_foul_commit_rate = frate_df.loc[valid_rows, "est_foul_commit_rate"].mean()

    # Replace values with averages where possessions are below threshold
    frate_df.loc[~valid_rows, "est_foul_win_rate"] = avg_est_foul_won_rate
    frate_df.loc[~valid_rows, "est_foul_commit_rate"] = avg_est_foul_commit_rate

    frate_df[group_by_col] = frate_df[group_by_col].astype("int")

    return frate_df.sort_values(group_by_col).reset_index(drop=True)


team_frate_df = get_foul_rate_df(train_match_events_df, "team.id")
print(team_frate_df.shape)

# set a minimum possession to smooth things out, replace with global average if not enough samples
posi_frate_df = get_foul_rate_df(
    train_match_events_df, "position.id", min_possessions=500
)
print(posi_frate_df.shape)


player_frate_df = get_foul_rate_df(
    train_match_events_df, "player.id", min_possessions=100
)
print(player_frate_df.shape)

play_patt_frate_df = get_foul_rate_df(
    train_match_events_df, "play_pattern.id", min_possessions=100
)
print(play_patt_frate_df.shape)

(20, 8)
(23, 8)
(518, 8)
(9, 8)


team_frate_dic = team_frate_df.set_index("team.id").to_dict(orient="index")
position_frate_dic = posi_frate_df.set_index("position.id").to_dict(orient="index")
player_frate_dic = player_frate_df.set_index("player.id").to_dict(orient="index")
play_patt_frate_dic = play_patt_frate_df.set_index("play_pattern.id").to_dict(
    orient="index"
)


def frate_fx(
    the_id,
    events_dic,
    matches_dic,
    team_frate_dic,
    position_frate_dic,
    player_frate_dic,
    play_patt_frate_dic,
    default_win_rate=0.025,
):
    """
    Extract foul-rate related features for a given event with fallback to overall averages.

    Parameters:
    - the_id: str, ID of the event
    - *_frate_dic: dict, mapping IDs to rate dictionaries
    - default_win_rate: float, fallback to global avg value if key is missing from its respective dict

    Returns:
    - frate_fx_dic: dict with foul rate features
    """
    event_dic = events_dic.get(the_id)
    if event_dic is None:
        raise ValueError(f"Event ID {the_id} not found in events_dic")

    match_id = event_dic["match_id"]
    player_team_id = event_dic["team.id"]

    try:
        position_id = int(event_dic["position.id"])
        player_id = int(event_dic["player.id"])
        play_pattern_id = int(event_dic["play_pattern.id"])
    except (TypeError, ValueError) as e:
        raise ValueError(f"Invalid ID type in event {the_id}: {e}")

    match_dic = matches_dic.get(match_id)
    if match_dic is None:
        raise ValueError(f"Match ID {match_id} not found in matches_dic")

    home_team_id = match_dic.get("home_team.home_team_id")
    away_team_id = match_dic.get("away_team.away_team_id")

    # Determine opposing team
    if player_team_id == home_team_id:
        opp_team_id = away_team_id
    elif player_team_id == away_team_id:
        opp_team_id = home_team_id
    else:
        raise ValueError(f"Team ID {player_team_id} not found in match {match_id}")

    # Lookup with fallbacks
    team_foul_win_rate = team_frate_dic.get(player_team_id, {}).get(
        "est_foul_win_rate", default_win_rate
    )
    position_foul_win_rate = position_frate_dic.get(position_id, {}).get(
        "est_foul_win_rate", default_win_rate
    )
    player_foul_win_rate = player_frate_dic.get(player_id, {}).get(
        "est_foul_win_rate", default_win_rate
    )
    play_patt_foul_win_rate = play_patt_frate_dic.get(play_pattern_id, {}).get(
        "est_foul_win_rate", default_win_rate
    )
    opp_team_foul_commit_rate = team_frate_dic.get(opp_team_id, {}).get(
        "est_foul_commit_rate", default_win_rate
    )

    frate_fx_dic = {
        "team_foul_win_rate": team_foul_win_rate,
        "position_foul_win_rate": position_foul_win_rate,
        "player_foul_win_rate": player_foul_win_rate,
        "play_patt_foul_win_rate": play_patt_foul_win_rate,
        "opp_team_foul_commit_rate": opp_team_foul_commit_rate,
    }

    return frate_fx_dic


frate_fx(
    test_id,
    events_dic,
    matches_dic,
    team_frate_dic,
    position_frate_dic,
    player_frate_dic,
    play_patt_frate_dic,
)

{'team_foul_win_rate': 0.025,
 'position_foul_win_rate': 0.028,
 'player_foul_win_rate': 0.009,
 'play_patt_foul_win_rate': 0.015,
 'opp_team_foul_commit_rate': 0.021}


grid_frate_df, x_bins, y_bins = compute_foul_win_rate_grid(
    train_res_poss_df, bins=(5, 5)
)


plot_foul_win_rate_grid_heatmap(grid_frate_df, x_bins, y_bins)


grid_frate_df.head()


# convert dataframe into dictionary with grid_id as key
grid_frate_dic = grid_frate_df.set_index("grid_id").to_dict(orient="index")


grid_frate_dic[0]

{'x_bin': 0,
 'y_bin': 0,
 'total_poss_count': 2476,
 'foul_win_count': 67,
 'foul_win_rate': 0.02705977382875606,
 'foul_win_rate_std': 0.003260842635995429}


def get_grid_id_from_coordinate(x, y, x_bins, y_bins):
    """
    Given a coordinate (x, y), return the grid_id it falls into based on x_bins and y_bins.

    Returns:
        bin_id (int), or None if out of bounds.
    """
    x_bin = np.digitize([x], x_bins, right=False)[0] - 1
    y_bin = np.digitize([y], y_bins, right=False)[0] - 1

    if 0 <= x_bin < len(x_bins) - 1 and 0 <= y_bin < len(y_bins) - 1:
        num_y_bins = len(y_bins) - 1
        grid_id = x_bin * num_y_bins + y_bin
        return grid_id
    else:
        # should be very rare case if ever if coordinate outside football pitch dimensions
        # if so just put grid=12 center
        return 12


grid_id = get_grid_id_from_coordinate(45.0, 30.0, x_bins, y_bins)
print(f"Grid ID: {grid_id}")
grid_id = get_grid_id_from_coordinate(450, 30.0, x_bins, y_bins)  # test outside range
print(f"Grid ID: {grid_id}")

Grid ID: 6
Grid ID: 12


def get_grid_frate_metrics(grid_id, grid_frate_dic):
    grid_metrics_dic = grid_frate_dic[grid_id]

    grid_frate_metrics_dic = {
        "grid_id": grid_id,
        "grid_total_count": grid_metrics_dic["total_poss_count"],
        "grid_foul_count": grid_metrics_dic["foul_win_count"],
        "grid_foul_rate": grid_metrics_dic["foul_win_rate"],
        "grid_foul_rate_std": grid_metrics_dic["foul_win_rate_std"],
    }

    return grid_frate_metrics_dic


def grid_frate_fx(the_id, events_dic, x_bins, y_bins, grid_frate_dic):
    event_dic = events_dic.get(the_id)
    if event_dic is None:
        raise ValueError(f"Event ID {the_id} not found in events_dic")

    # Parse x and y coordinates from location string
    location = event_dic.get("location", "")
    x = float(location.split(",")[0].strip())
    y = float(location.split(",")[1].strip())

    # get which grid belongs too
    grid_id = get_grid_id_from_coordinate(x, y, x_bins, y_bins)

    # get grid metrics
    grid_frate_metrics_dic = get_grid_frate_metrics(grid_id, grid_frate_dic)

    grid_frate_fx_dic = grid_frate_metrics_dic

    return grid_frate_fx_dic


grid_frate_fx(test_id, events_dic, x_bins, y_bins, grid_frate_dic)

{'grid_id': np.int64(12),
 'grid_total_count': 14230,
 'grid_foul_count': 386,
 'grid_foul_rate': 0.027125790583274773,
 'grid_foul_rate_std': 0.0013618118142104132}


def time_to_seconds(row):
    return row["minute"] * 60 + row["second"] + (row["period"] - 1) * 2700


tmp_event_df = events_df.copy()
tmp_event_df = tmp_event_df.set_index("id")
# absolute seconds since start of match
tmp_event_df["abs_time_secs"] = tmp_event_df.apply(time_to_seconds, axis=1)
tmp_event_df["x"] = tmp_event_df["location"].apply(
    lambda s: float(s.split(",")[0].strip()) if pd.notnull(s) else None
)
tmp_event_df["y"] = tmp_event_df["location"].apply(
    lambda s: float(s.split(",")[1].strip()) if pd.notnull(s) else None
)
tmp_event_df["possess_team_event"] = (
    tmp_event_df["possession_team.name"] == tmp_event_df["team.name"]
)

# the following fixes the mirror issue to ensure non-possess team player locations align with poss team player locations
tmp_event_df["x"] = np.where(
    tmp_event_df["possess_team_event"], tmp_event_df["x"], 120 - tmp_event_df["x"]
)
tmp_event_df["y"] = np.where(
    tmp_event_df["possess_team_event"], tmp_event_df["y"], 80 - tmp_event_df["y"]
)


TARGET_EVENT_TYPES = [
    "Pressure",
    # "Duel",
    # "Clearance",
    # "Block",
    # "Miscontrol",
    # "Interception",
    # "Dribbled Past",
]


def get_preceding_rows_by_seconds(
    df: pd.DataFrame, the_id: str, seconds: float, lookback_limit: int = 30
) -> pd.DataFrame:
    """
    Return all preceding events within a given time window for the same match.

    Parameters:
    -----------
    df : pd.DataFrame
        The events DataFrame indexed by 'id'. Must include columns:
        - 'abs_time_secs': absolute time of each event in seconds
        - 'match_id': match identifier
    the_id : str
        The index value (event ID) for which we want to extract preceding events.
    seconds : float
        Time window in seconds to look back.

    Returns:
    --------
    pd.DataFrame
        A new DataFrame with all events that occurred within `seconds` before
        the target event, limited to the same match, and indexed by 'id'.
        Returned rows are in ascending time order.
    """

    if df.index.name != "id":
        raise ValueError("DataFrame must have 'id' as its index")

    if the_id not in df.index:
        return df.iloc[0:0]

    loc = df.index.get_loc(the_id)
    if isinstance(loc, slice) or isinstance(loc, list):
        raise ValueError("Row ID must be unique")

    target_row = df.iloc[loc]
    target_time = target_row["abs_time_secs"]
    target_match = target_row["match_id"]

    # Pull a fixed number of rows before current one
    start_idx = max(0, loc - lookback_limit)
    candidate_df = df.iloc[start_idx:loc]

    # Filter by match_id and time window
    mask = (candidate_df["match_id"] == target_match) & (
        candidate_df["abs_time_secs"] >= target_time - seconds
    )
    return candidate_df[mask]


tmp_event_df[tmp_event_df.index == "ec0b4585-41ad-4ada-817c-629d0866eeeb"][
    ["timestamp", "period", "abs_time_secs", "type.name"]
]


# test get preceding rows by timestamp works (last 3 seconds before)
get_preceding_rows_by_seconds(tmp_event_df, "ec0b4585-41ad-4ada-817c-629d0866eeeb", 3)[
    ["timestamp", "period", "abs_time_secs", "type.name"]
]


def count_event_types_in_time_window(
    df: pd.DataFrame, the_id: str, seconds: float
) -> Dict[str, int]:
    """
    Count the number of occurrences for each target event type within a time window
    preceding the specified event.

    Parameters:
    -----------
    df : pd.DataFrame
        Events DataFrame indexed by 'id', must include 'type.name' and 'abs_time_secs'.
    the_id : str
        The event ID to look back from.
    seconds : float
        Time window (in seconds) to consider before the target event.

    Returns:
    --------
    Dict[str, int]
        A dictionary mapping each event type in TARGET_EVENT_TYPES to its count
        within the specified time window.
    """
    # Get preceding events within the time window
    preceding = get_preceding_rows_by_seconds(df, the_id, seconds)
    # Count event type occurrences
    type_counts = Counter(preceding["type.name"])
    # Ensure all TARGET_EVENT_TYPES are included with default count of 0
    return {k: type_counts.get(k, 0) for k in TARGET_EVENT_TYPES}


def count_unique_possessions_in_time_window(
    df: pd.DataFrame, the_id: str, seconds: float
) -> int:
    """
    Count the number of unique possession IDs in a time window preceding the event.

    Parameters:
    -----------
    df : pd.DataFrame
        Events DataFrame indexed by 'id', must include 'possession' and 'abs_time_secs'.
    the_id : str
        The event ID to look back from.
    seconds : float
        Time window (in seconds) to consider before the target event.

    Returns:
    --------
    int
        The number of unique possessions in the time window (minimum 1).
    """
    # Get preceding events within the time window
    preceding = get_preceding_rows_by_seconds(df, the_id, seconds)
    # Clean and extract unique possession IDs
    possession_values = (
        pd.to_numeric(preceding["possession"], errors="coerce")  # ensure numeric
        .dropna()  # remove NaNs
        .astype(int)  # cast to int
        .unique()  # get unique values
    )
    # Return at least 1 to avoid divide-by-zero errors downstream
    return max(1, len(possession_values))


def count_counterpress_in_time_window(
    df: pd.DataFrame, the_id: str, seconds: float
) -> int:
    """
    Count how many of the preceding events are marked as 'counterpress' == True.

    Parameters:
    -----------
    df : pd.DataFrame
        Events DataFrame indexed by 'id', must include 'counterpress' and 'abs_time_secs'.
    the_id : str
        The event ID to look back from.
    seconds : float
        Time window (in seconds) to consider before the target event.

    Returns:
    --------
    int
        Number of preceding events with 'counterpress' == True.
    """
    # Get preceding events within the time window
    preceding = get_preceding_rows_by_seconds(df, the_id, seconds)
    # Count events where counterpress is explicitly True
    return (preceding["counterpress"] == True).sum()


def get_preceding_pass_features(
    df: pd.DataFrame, row_id: str
) -> Optional[Dict[str, Any]]:
    """
    Extract pass-related features from the event immediately before the given row,
    if that event is of type 'Pass'. Returns key pass details or None if not applicable.

    Parameters:
    -----------
    df : pd.DataFrame
        Events DataFrame indexed by 'id'. Should include:
        - 'type.name', 'pass.angle', 'pass.height.id',
          'pass.body_part.id', and 'pass.end_location'

    row_id : str
        ID of the current event for which to extract preceding pass data.

    Returns:
    --------
    Dict[str, Any] or None
        Dictionary of preceding pass features, or None if not a pass or unavailable.
    """
    try:
        loc = df.index.get_loc(row_id)
        if loc == 0:
            return None
        prev_row = df.iloc[loc - 1]
        if prev_row["type.name"] == "Pass":
            return {
                "pass.angle": prev_row.get("pass.angle"),
                "pass.height.id": prev_row.get("pass.height.id"),
                "pass.body_part.id": prev_row.get("pass.body_part.id"),
                # "pass.end_location": prev_row.get("pass.end_location"),
            }
    except Exception:
        return None
    return None


def count_event_types_in_time_distance(
    df: pd.DataFrame, the_id: str, seconds: float, max_dist: float
) -> Dict[str, int]:
    """
    Count how many preceding events of each type occurred within a specified time
    window and spatial distance from a target event.

    Parameters:
    -----------
    df : pd.DataFrame
        Events DataFrame indexed by 'id'. Must include columns:
        - 'abs_time_secs' for absolute event time in seconds,
        - 'x' and 'y' for event coordinates,
        - 'type.name' for event type names.
    the_id : str
        ID of the target event to look back from.
    seconds : float
        Time window (in seconds) to consider before the target event.
    max_dist : float
        Maximum spatial distance (in pitch units, e.g., meters) from the target event.

    Returns:
    --------
    Dict[str, int]
        A dictionary mapping each event type in TARGET_EVENT_TYPES to the number
        of times it occurred within both the time window and distance threshold.
    """
    # If the_id is missing from the DataFrame, return 0s
    if the_id not in df.index:
        return {k: 0 for k in TARGET_EVENT_TYPES}
    # Get the target event's location
    target = df.loc[the_id]
    x0, y0 = target["x"], target["y"]
    # If target event has no valid location, return 0s
    if pd.isna(x0) or pd.isna(y0):
        return {k: 0 for k in TARGET_EVENT_TYPES}

    nearby_types = []

    # Get all preceding events within the time window
    preceding = get_preceding_rows_by_seconds(df, the_id, seconds)
    # Loop through each preceding event and check spatial proximity
    for _, row in preceding.iterrows():
        x, y = row["x"], row["y"]
        if pd.notna(x) and pd.notna(y):
            # Compute Euclidean distance
            dist, _ = dist_angle_on_pitch(x0, y0, x, y)
            if dist <= max_dist:
                nearby_types.append(row["type.name"])

    # Count occurrences of event types within distance
    type_counts = Counter(nearby_types)
    # Ensure all TARGET_EVENT_TYPES are represented
    return {k: type_counts.get(k, 0) for k in TARGET_EVENT_TYPES}


def batch_prev_events_fx(
    events_df: pd.DataFrame, the_ids: list[str]
) -> Dict[str, Dict[str, Any]]:
    """
    Batch version of prev_events fx that processes all events grouped by match.
    Reduces DataFrame slicing overhead by working in bulk.

    Parameters:
        events_df: Full events dataframe, must be sorted by match_id and abs_time_secs.
        the_ids: List of event 'id' values to compute features for.

    Returns:
        A dictionary mapping each row_id to its features.
    """
    result = {}
    lookback_limit = 30
    time_windows = [1, 3, 5]
    proximity_thresholds = [10, 30]

    id_set = set(the_ids)

    for match_id, group in events_df.groupby("match_id"):
        group = group.sort_values("abs_time_secs")
        ids = group.index.tolist()
        times = group["abs_time_secs"].to_numpy()
        types = group["type.name"].tolist()
        xs = group["x"].to_numpy()
        ys = group["y"].to_numpy()
        possessions = group["possession"].to_numpy()
        counterpress = group["counterpress"].to_numpy()

        for i, row_id in enumerate(ids):
            if row_id not in id_set:
                continue

            feature_row = {}
            target_time = times[i]
            x0, y0 = xs[i], ys[i]

            for sec in time_windows:
                prefix = f"sec{str(sec).replace('.', '_')}"
                j_start = max(0, i - lookback_limit)

                # Filter time window
                mask = times[j_start:i] >= target_time - sec
                window_types = [
                    types[j] for j in range(j_start, i) if mask[j - j_start]
                ]
                window_possessions = [
                    possessions[j] for j in range(j_start, i) if mask[j - j_start]
                ]
                window_counterpress = [
                    counterpress[j] for j in range(j_start, i) if mask[j - j_start]
                ]
                type_counter = Counter(window_types)

                for k in TARGET_EVENT_TYPES:
                    feature_row[f"{prefix}_count_type_{k}"] = type_counter.get(k, 0)

                feature_row[f"{prefix}_unique_possessions"] = max(
                    1, len(set(p for p in window_possessions if pd.notna(p)))
                )
                feature_row[f"{prefix}_counterpress_true"] = sum(
                    1 for val in window_counterpress if val is True
                )

                for dist_thresh in proximity_thresholds:
                    nearby_types = []
                    for j in range(j_start, i):
                        if not mask[j - j_start]:
                            continue
                        x, y = xs[j], ys[j]
                        if (
                            pd.notna(x)
                            and pd.notna(y)
                            and pd.notna(x0)
                            and pd.notna(y0)
                        ):
                            dist = math.hypot(x - x0, y - y0)
                            if dist <= dist_thresh:
                                nearby_types.append(types[j])
                    dist_counter = Counter(nearby_types)
                    for k in TARGET_EVENT_TYPES:
                        feature_row[f"{prefix}_dist{dist_thresh}_type_{k}"] = (
                            dist_counter.get(k, 0)
                        )

            if i > 0 and types[i - 1] == "Pass":
                prev_row = group.iloc[i - 1]
                feature_row.update(
                    {
                        "pass.angle": prev_row.get("pass.angle"),
                        "pass.height.id": prev_row.get("pass.height.id"),
                        "pass.body_part.id": prev_row.get("pass.body_part.id"),
                    }
                )

            result[row_id] = feature_row

    return result


# test single event
batch_prev_events_fx(tmp_event_df, ["8ca81a7e-16f4-4c51-a708-bfcbbd63e1b2"])

{'8ca81a7e-16f4-4c51-a708-bfcbbd63e1b2': {'sec1_count_type_Pressure': 1,
  'sec1_unique_possessions': 1,
  'sec1_counterpress_true': 0,
  'sec1_dist10_type_Pressure': 1,
  'sec1_dist30_type_Pressure': 1,
  'sec3_count_type_Pressure': 1,
  'sec3_unique_possessions': 1,
  'sec3_counterpress_true': 0,
  'sec3_dist10_type_Pressure': 1,
  'sec3_dist30_type_Pressure': 1,
  'sec5_count_type_Pressure': 2,
  'sec5_unique_possessions': 1,
  'sec5_counterpress_true': 1,
  'sec5_dist10_type_Pressure': 2,
  'sec5_dist30_type_Pressure': 2,
  'pass.angle': np.float64(3.113431),
  'pass.height.id': np.float64(1.0),
  'pass.body_part.id': np.float64(40.0)}}


train_event_ids = train_res_poss_df["id"].values
print(len(train_event_ids))

240729


%%time
prev_train_fx_dic = batch_prev_events_fx(tmp_event_df, train_event_ids)
print(len(prev_train_fx_dic))

240729
CPU times: user 53.3 s, sys: 616 ms, total: 53.9 s
Wall time: 54.3 s


test_event_ids = test_res_poss_df["id"].values
print(len(test_event_ids))

60162


%%time
prev_test_fx_dic = batch_prev_events_fx(tmp_event_df, test_event_ids)
print(len(prev_test_fx_dic))

60162
CPU times: user 14.6 s, sys: 327 ms, total: 14.9 s
Wall time: 15 s


# prev_test_fx_dic.get(test_event_ids[400])


prev_events_fx_dic = prev_train_fx_dic | prev_test_fx_dic
print(len(prev_events_fx_dic))

300891


prev_events_fx_dic.get(test_event_ids[400])

{'sec1_count_type_Pressure': 0,
 'sec1_unique_possessions': 1,
 'sec1_counterpress_true': 0,
 'sec1_dist10_type_Pressure': 0,
 'sec1_dist30_type_Pressure': 0,
 'sec3_count_type_Pressure': 1,
 'sec3_unique_possessions': 2,
 'sec3_counterpress_true': 0,
 'sec3_dist10_type_Pressure': 0,
 'sec3_dist30_type_Pressure': 0,
 'sec5_count_type_Pressure': 2,
 'sec5_unique_possessions': 2,
 'sec5_counterpress_true': 0,
 'sec5_dist10_type_Pressure': 0,
 'sec5_dist30_type_Pressure': 0}


def fx(
    the_id,
    events_dic,
    matches_dic,
    team_frate_dic,
    position_frate_dic,
    player_frate_dic,
    play_patt_frate_dic,
    prev_events_fx_dic,
):
    """
    Generate a combined feature dictionary for a given event, including both event-level
    and match-level features.

    Parameters:
    -----------
    the_id : str
        Unique identifier for the event.
    events_dic : dict
        Dictionary mapping event IDs to event data dictionaries.
    matches_dic : dict
        Dictionary mapping match IDs to match data dictionaries.
    team_frate_dic : dict
        Dictionary mapping team IDs to foul rate stats.
    position_frate_dic : dict
        Dictionary mapping position IDs to foul rate stats.
    player_frate_dic : dict
        Dictionary mapping player IDs to foul rate stats.
    play_patt_frate_dic : dict
        Dictionary mapping play pattern IDs to foul rate stats.
    prev_events_fx_dic: dict
        Dictionary with id key which gives all 'previous events' features as returned dic

    Returns:
    --------
    final_fx_dic : dict
        A dictionary containing the merged features from:
            - simple_single_fx (single event-level features)
            - match_simple_fx (match-level features)
            - dist_simple_fx_dic (distance based features single event-level)
    """
    # Extract event-level features
    si_simple_fx_dic = simple_single_fx(the_id, events_dic)

    # Extract match-level features
    mat_simple_fx_dic = match_simple_fx(the_id, events_dic, matches_dic)

    # Extract simple-dist-based features
    dist_simple_fx_dic = distance_simple_fx(the_id, events_dic)

    # Extract foul-rate features
    frate_fx_dic = frate_fx(
        the_id,
        events_dic,
        matches_dic,
        team_frate_dic,
        position_frate_dic,
        player_frate_dic,
        play_patt_frate_dic,
    )

    # Extract grid foul rate based features
    grid_frate_fx_dic = grid_frate_fx(
        the_id, events_dic, x_bins, y_bins, grid_frate_dic
    )

    # Merge the many fx result dictionaries
    final_fx_dic = (
        si_simple_fx_dic
        | mat_simple_fx_dic
        | dist_simple_fx_dic
        | frate_fx_dic
        | grid_frate_fx_dic
        | prev_events_fx_dic[the_id]
    )

    return final_fx_dic


# testing single sample (event)
fx(
    test_id,
    events_dic,
    matches_dic,
    team_frate_dic,
    position_frate_dic,
    player_frate_dic,
    play_patt_frate_dic,
    prev_events_fx_dic,
)

{'period': 1,
 'minute': 0,
 'x': 60.2,
 'y': 40.7,
 'is_recovery': 0,
 'under_pressure': 0,
 'is_home_team': 1,
 'dist_opp_goal': 59.8,
 'angle_opp_goal': -0.7,
 'team_foul_win_rate': 0.025,
 'position_foul_win_rate': 0.028,
 'player_foul_win_rate': 0.009,
 'play_patt_foul_win_rate': 0.015,
 'opp_team_foul_commit_rate': 0.021,
 'grid_id': np.int64(12),
 'grid_total_count': 14230,
 'grid_foul_count': 386,
 'grid_foul_rate': 0.027125790583274773,
 'grid_foul_rate_std': 0.0013618118142104132,
 'sec1_count_type_Pressure': 0,
 'sec1_unique_possessions': 2,
 'sec1_counterpress_true': 0,
 'sec1_dist10_type_Pressure': 0,
 'sec1_dist30_type_Pressure': 0,
 'sec3_count_type_Pressure': 0,
 'sec3_unique_possessions': 2,
 'sec3_counterpress_true': 0,
 'sec3_dist10_type_Pressure': 0,
 'sec3_dist30_type_Pressure': 0,
 'sec5_count_type_Pressure': 0,
 'sec5_unique_possessions': 2,
 'sec5_counterpress_true': 0,
 'sec5_dist10_type_Pressure': 0,
 'sec5_dist30_type_Pressure': 0,
 'pass.angle': np.float64(2.4980915),
 'pass.height.id': np.float64(1.0),
 'pass.body_part.id': np.float64(40.0)}


print(train_res_poss_df.shape)
print(test_res_poss_df.shape)

(240729, 11)
(60162, 11)


train_res_poss_df.head(2)


def build_fx_df(
    source_df,
    events_dic,
    matches_dic,
    team_frate_dic,
    position_frate_dic,
    player_frate_dic,
    play_patt_frate_dic,
    prev_events_fx_dic,
):
    fx_res_list = []
    for i in range(len(source_df)):
        row = source_df.iloc[i]
        the_id = row["id"]
        fx_res_dic = fx(
            the_id,
            events_dic,
            matches_dic,
            team_frate_dic,
            position_frate_dic,
            player_frate_dic,
            play_patt_frate_dic,
            prev_events_fx_dic,
        )
        fx_res_list.append(fx_res_dic)

    fx_df = pd.DataFrame(fx_res_list)
    return fx_df


%%time
fx_train_df = build_fx_df(
    train_res_poss_df,
    events_dic,
    matches_dic,
    team_frate_dic,
    position_frate_dic,
    player_frate_dic,
    play_patt_frate_dic,
    prev_events_fx_dic,
)
print(fx_train_df.shape)

(240729, 37)
CPU times: user 24.9 s, sys: 999 ms, total: 25.9 s
Wall time: 25.6 s


%%time
fx_test_df = build_fx_df(
    test_res_poss_df,
    events_dic,
    matches_dic,
    team_frate_dic,
    position_frate_dic,
    player_frate_dic,
    play_patt_frate_dic,
    prev_events_fx_dic,
)
print(fx_test_df.shape)

(60162, 37)
CPU times: user 6.36 s, sys: 221 ms, total: 6.58 s
Wall time: 6.49 s


# fill NAs just in case for now
fx_train_df = fx_train_df.fillna(0)
fx_test_df = fx_test_df.fillna(0)


fx_train_df.head()


fx_train_df["sec5_counterpress_true"].value_counts()

sec5_counterpress_true
0    212338
1     19572
2      6506
3      1772
4       441
5        86
6        10
7         4
Name: count, dtype: int64


fx_train_df.under_pressure.value_counts()

under_pressure
0    214671
1     26058
Name: count, dtype: int64


fx_train_df["pass.height.id"].value_counts()

pass.height.id
0.0    133917
1.0     76608
2.0     15150
3.0     15054
Name: count, dtype: int64


fx_train_df.is_recovery.value_counts()

is_recovery
0    212219
1     28510
Name: count, dtype: int64


fx_test_df.head(2)


X_train = fx_train_df.copy()
X_test = fx_test_df.copy()

print(X_train.shape)
print(X_test.shape)
print(y_train.shape)
print(y_test.shape)

(240729, 37)
(60162, 37)
(240729,)
(60162,)


# Define XGBoost classifier
base_model = XGBClassifier(
    objective="binary:logistic",
    eval_metric="logloss",
    scale_pos_weight=(1 / 0.03),  # handle class imbalance
    random_state=42,
)


# setup param_grid for hyperparamter tuning, lets keep simple
param_grid = {
    "n_estimators": [100, 200],
    "max_depth": [3, 5],
    "learning_rate": [0.05, 0.1],
}


# Time-based CV for tuning
tscv = TimeSeriesSplit(n_splits=5)

grid_search = GridSearchCV(
    estimator=base_model,
    param_grid=param_grid,
    cv=tscv,
    scoring="neg_log_loss",
    n_jobs=-1,
    verbose=1,
)


%%time
# Step 3: Fit and select best model
grid_search.fit(X_train, y_train)
best_model = grid_search.best_estimator_

Fitting 5 folds for each of 8 candidates, totalling 40 fits
CPU times: user 12.2 s, sys: 878 ms, total: 13.1 s
Wall time: 24.9 s


print(f"Best hyperparameters: {grid_search.best_params_}")

Best hyperparameters: {'learning_rate': 0.1, 'max_depth': 5, 'n_estimators': 200}


%%time
# Wrap with calibration
calibrated_model = CalibratedClassifierCV(best_model, method="sigmoid", cv=3)
calibrated_model.fit(X_train, y_train)

CPU times: user 31.2 s, sys: 1.68 s, total: 32.9 s
Wall time: 4.42 s

CalibratedClassifierCV(cv=3,
                       estimator=XGBClassifier(base_score=None, booster=None,
                                               callbacks=None,
                                               colsample_bylevel=None,
                                               colsample_bynode=None,
                                               colsample_bytree=None,
                                               device=None,
                                               early_stopping_rounds=None,
                                               enable_categorical=False,
                                               eval_metric='logloss',
                                               feature_types=None,
                                               feature_weights=None, gamma=None,
                                               grow_policy=None,
                                               importance_type=None,
                                               interaction_constraints=None,
                                               learning_rate=0.1, max_bin=None,
                                               max_cat_threshold=None,
                                               max_cat_to_onehot=None,
                                               max_delta_step=None, max_depth=5,
                                               max_leaves=None,
                                               min_child_weight=None,
                                               missing=nan,
                                               monotone_constraints=None,
                                               multi_strategy=None,
                                               n_estimators=200, n_jobs=None,
                                               num_parallel_tree=None, ...))

CalibratedClassifierCV(cv=3,
                       estimator=XGBClassifier(base_score=None, booster=None,
                                               callbacks=None,
                                               colsample_bylevel=None,
                                               colsample_bynode=None,
                                               colsample_bytree=None,
                                               device=None,
                                               early_stopping_rounds=None,
                                               enable_categorical=False,
                                               eval_metric='logloss',
                                               feature_types=None,
                                               feature_weights=None, gamma=None,
                                               grow_policy=None,
                                               importance_type=None,
                                               interaction_constraints=None,
                                               learning_rate=0.1, max_bin=None,
                                               max_cat_threshold=None,
                                               max_cat_to_onehot=None,
                                               max_delta_step=None, max_depth=5,
                                               max_leaves=None,
                                               min_child_weight=None,
                                               missing=nan,
                                               monotone_constraints=None,
                                               multi_strategy=None,
                                               n_estimators=200, n_jobs=None,
                                               num_parallel_tree=None, ...))

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=None, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric='logloss',
              feature_types=None, feature_weights=None, gamma=None,
              grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=0.1, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=5, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=200, n_jobs=None,
              num_parallel_tree=None, ...)

XGBClassifier(base_score=None, booster=None, callbacks=None,
              colsample_bylevel=None, colsample_bynode=None,
              colsample_bytree=None, device=None, early_stopping_rounds=None,
              enable_categorical=False, eval_metric='logloss',
              feature_types=None, feature_weights=None, gamma=None,
              grow_policy=None, importance_type=None,
              interaction_constraints=None, learning_rate=0.1, max_bin=None,
              max_cat_threshold=None, max_cat_to_onehot=None,
              max_delta_step=None, max_depth=5, max_leaves=None,
              min_child_weight=None, missing=nan, monotone_constraints=None,
              multi_strategy=None, n_estimators=200, n_jobs=None,
              num_parallel_tree=None, ...)


# Predict on test set
y_pred_proba = calibrated_model.predict_proba(X_test)[:, 1]


%%time
# Fit base_model directly (needed for SHAP)
base_model.fit(X_train, y_train)
# Initialize TreeExplainer for XGBoost model
explainer = shap.TreeExplainer(base_model)
# Compute SHAP values on training set
shap_values = explainer.shap_values(X_train)

CPU times: user 7min 19s, sys: 1.51 s, total: 7min 21s
Wall time: 57.9 s


# Visualize
shap.summary_plot(
    shap_values, X_train, plot_type="bar", max_display=40
)  # Bar chart of mean absolute importance


print(
    y_pred_proba.min(), y_pred_proba.max()
)  # look at min/max predicted probabilties from model

0.004894436395792532 0.1697985709397434


fig, axes = plt.subplots(1, 2, figsize=(12, 5), sharey=True)

# Calibrated
pd.Series(y_pred_proba).hist(ax=axes[0], bins=30, color="skyblue", edgecolor="black")
axes[0].set_title("Calibrated Probability Predictions")
axes[0].set_xlabel("Probability")
axes[0].set_ylabel("Frequency")

# Non-Calibrated
base_xgb_pred_proba = base_model.predict_proba(X_test)[
    :, 1
]  # run non-calibrated xgb to generated predictions
pd.Series(base_xgb_pred_proba).hist(
    ax=axes[1], bins=30, color="salmon", edgecolor="black"
)
axes[1].set_title("Non-Calibrated Probability Predictions")
axes[1].set_xlabel("Probability")

plt.tight_layout()
plt.show()


# Calibration curve
prob_true, prob_pred = calibration_curve(
    y_test, y_pred_proba, n_bins=10, strategy="quantile"
)


# Build the table
calib_table = pd.DataFrame(
    {
        "Bin": [f"Bin {i+1}" for i in range(len(prob_pred))],
        "Avg Predicted Probability (prob_pred)": np.round(prob_pred, 5),
        "Actual Foul Rate (prob_true)": np.round(prob_true, 5),
    }
)
calib_table


# Plot calibration curve
plt.figure(figsize=(6, 5))
plt.plot(prob_pred, prob_true, marker="o", label="Model")
plt.plot([0, 1], [0, 1], "k--", label="Perfect Calibration")
plt.xlabel("Predicted Probability")
plt.ylabel("True Frequency")
plt.title("Calibration Curve")
plt.legend()
plt.grid()
plt.show()


pd.Series(y_train).value_counts(normalize=True)

False    0.974511
True     0.025489
Name: proportion, dtype: float64


p_foul_rate = pd.Series(y_train).value_counts(normalize=True)[True]
print(p_foul_rate)  # ~2.5%

0.025489243090778428


y_baseline_pred = np.full_like(y_test, 0.03, dtype=float)
print(y_baseline_pred.shape)

(60162,)


pd.Series(y_test).value_counts()

False    58711
True      1451
Name: count, dtype: int64


log_loss_baseline = log_loss(y_test, y_baseline_pred)
brier_baseline = brier_score_loss(y_test, y_baseline_pred)
# since model predicts same constant prob for every sample, we know no ranking, so random guessing
auc_baseline = 0.5

print("Baseline (foul win rate) Log Loss:", log_loss_baseline)
print("Baseline (foul win rate) Brier Score:", brier_baseline)

Baseline (foul win rate) Log Loss: 0.11429650011046871
Baseline (foul win rate) Brier Score: 0.02357112130580765


# from our xgboost model
log_loss_score = log_loss(y_test, y_pred_proba)
brier_score = brier_score_loss(y_test, y_pred_proba)
roc_auc = roc_auc_score(y_test, y_pred_proba)


metrics_df = pd.DataFrame(columns=["Model", "Log Loss", "Brier Score", "AUC-ROC"])
metrics_df.loc[len(metrics_df)] = [
    "baseline_foul_win_rate",
    log_loss_baseline,
    brier_baseline,
    auc_baseline,
]
metrics_df.loc[len(metrics_df)] = ["xgb_model", log_loss_score, brier_score, roc_auc]
metrics_df


print(
    "# of test set samples (valid player possession start): {}".format(
        len(y_pred_proba)
    )
)

# of test set samples (valid player possession start): 60162


# Calculated before (historical foul win rate from training dataset
base_frate = p_foul_rate
print("Base Rate (Foul Win Rate): {} %".format(round(base_frate * 100, 2)))

Base Rate (Foul Win Rate): 2.55 %


def threshold_model_predict_metrics(y_test, y_pred_proba, threshold, base_rate=None):
    """
    Prints metrics for predictions above a probability threshold:
    - Number of plays selected
    - % of all plays selected
    - Actual foul rate among selected plays
    - Optional lift over base foul rate

    Parameters:
    - y_test: np.array of true labels (0/1)
    - y_pred_proba: np.array of predicted probabilities
    - threshold: float, probability cutoff (e.g., 0.025)
    - base_rate: optional float, base foul rate for lift comparison
    """
    high_conf_mask = y_pred_proba >= threshold
    selected_count = np.sum(high_conf_mask)
    total_count = len(y_test)
    selected_pct = 100 * selected_count / total_count if total_count > 0 else 0

    precision_above = np.mean(y_test[high_conf_mask]) if selected_count > 0 else 0
    lift = precision_above / base_rate if base_rate and base_rate > 0 else None

    print(f"Threshold: {round(threshold * 100, 2)}% (prob ≥ {round(threshold, 5)})")
    print(
        f"Plays predicted above threshold: {selected_count} "
        f"({round(selected_pct, 2)}% of all plays)"
    )
    print(f"Foul rate among those plays: {round(precision_above * 100, 2)}%")
    if lift:
        print(f"Lift over base rate ({round(base_rate * 100, 2)}%): {round(lift, 2)}×")


# Assume you have y_test, y_pred_proba, and base_frate = 0.0252
threshold_model_predict_metrics(
    y_test, y_pred_proba, threshold=base_frate, base_rate=base_frate
)

Threshold: 2.55% (prob ≥ 0.02549)
Plays predicted above threshold: 19827 (32.96% of all plays)
Foul rate among those plays: 4.6%
Lift over base rate (2.55%): 1.81×


def threshold_performance_summary(
    y_test, y_pred_proba, base_rate=None, percentiles=None
):
    """
    Generate a table showing actual foul rate, lift, threshold probability,
    sample counts, and coverage at different top-K percentile cutoffs.

    Parameters:
    - y_test: np.array or pd.Series of true binary labels (0 or 1)
    - y_pred_proba: np.array of predicted probabilities
    - base_rate: (optional) base foul rate. If None, computed from y_test.
    - percentiles: list of percentiles (e.g., [95, 90] = top 5%, 10%)

    Returns:
    - pd.DataFrame with performance metrics per percentile threshold
    """
    if base_rate is None:
        base_rate = np.mean(y_test)

    if percentiles is None:
        percentiles = [99, 95, 90, 85, 80, 75, 70, 65, 60, 55, 50]

    total_count = len(y_test)
    results = []

    for p in percentiles:
        thresh = np.percentile(y_pred_proba, p)
        mask = y_pred_proba >= thresh
        selected = np.sum(mask)

        if selected > 0:
            actual_rate = np.mean(y_test[mask])
            lift = actual_rate / base_rate
        else:
            actual_rate, lift = 0.0, 0.0

        results.append(
            {
                "Top K Group": f"Top {100 - p}%",
                "Threshold (Prob ≥)": round(thresh, 5),
                "# of Samples (Poss. Start events) in Group": selected,
                "% of Samples (Poss. Start events Selected": round(
                    100 * selected / total_count, 2
                ),
                "Actual Foul Rate (%)": round(100 * actual_rate, 2),
                "Lift over Base": round(lift, 2),
            }
        )

    return pd.DataFrame(results)


res_table = threshold_performance_summary(y_test, y_pred_proba, base_rate=base_frate)
res_table


def plot_threshold_summary_metrics(table, y_test, base_rate_pct):
    """
    Plot Lift, Foul Rate, Sample Coverage, and Cumulative Recall from the lift table.

    Parameters:
    - table: DataFrame output from precision_lift_percentile_table
    - y_test: Original true labels (0/1) for recall calculation
    - base_rate_pct: Baseline foul rate (e.g., 0.0255)
    """
    top_k = table["Top K Group"]
    foul_rate = table["Actual Foul Rate (%)"]
    lift = table["Lift over Base"]
    sample_pct = table["% of Samples (Poss. Start events Selected"]
    sample_counts = table["# of Samples (Poss. Start events) in Group"]

    # Compute total fouls in each group
    thresholds = table["Threshold (Prob ≥)"]
    recall_counts = [np.sum((y_test >= 0) & (y_pred_proba >= t)) for t in thresholds]
    fouls_per_group = [np.sum(y_test[y_pred_proba >= t]) for t in thresholds]
    total_fouls = np.sum(y_test)
    recall_pct = [round(100 * f / total_fouls, 2) for f in fouls_per_group]

    fig, axes = plt.subplots(2, 2, figsize=(14, 10))

    # Plot 1: Lift
    axes[0, 0].bar(top_k, lift, color="skyblue")
    axes[0, 0].set_title("Lift over Base Rate")
    axes[0, 0].set_ylabel("Lift (×)")
    axes[0, 0].set_xticks(range(len(top_k)))
    axes[0, 0].set_xticklabels(top_k, rotation=45)

    # Plot 2: Actual Foul Rate
    axes[0, 1].bar(top_k, foul_rate, color="salmon")
    axes[0, 1].axhline(
        base_rate_pct * 100, color="gray", linestyle="--", label="Base Rate"
    )
    axes[0, 1].set_title("Actual Foul Rate by Top-K Group")
    axes[0, 1].set_ylabel("Foul Rate (%)")
    axes[0, 1].legend()
    axes[0, 1].set_xticks(range(len(top_k)))
    axes[0, 1].set_xticklabels(top_k, rotation=45)

    # Plot 3: Sample Coverage
    axes[1, 0].bar(top_k, sample_pct, color="lightgreen")
    axes[1, 0].set_title("Coverage (% of Possession Starts)")
    axes[1, 0].set_ylabel("% of Samples")
    axes[1, 0].set_xticks(range(len(top_k)))
    axes[1, 0].set_xticklabels(top_k, rotation=45)

    # Plot 4: Cumulative Recall (% of all fouls)
    axes[1, 1].bar(top_k, recall_pct, color="mediumpurple")
    axes[1, 1].set_title("Cumulative Recall (% of Fouls Captured)")
    axes[1, 1].set_ylabel("Recall (%)")
    axes[1, 1].set_xticks(range(len(top_k)))
    axes[1, 1].set_xticklabels(top_k, rotation=45)

    plt.tight_layout()
    plt.show()


plot_threshold_summary_metrics(res_table, y_test=y_test, base_rate_pct=base_frate)


def simulate_betting(y_test, y_pred_proba, threshold=0.05, odds=15.0, bet_amount=1.0):
    """
    Simulate a betting strategy where a bet is placed if predicted prob ≥ threshold.

    Parameters:
    - y_test: np.array of true labels (0/1)
    - y_pred_proba: np.array of predicted probabilities
    - threshold: float, prediction cutoff for placing a bet
    - odds: float, decimal odds (means: for every $1 bet, you get $15 total payout if you win which includes your $1 stake + $14 profit)
    - bet_amount: float, amount wagered per bet

    Returns:
    - Dictionary of results
    """
    mask = y_pred_proba >= threshold
    selected = np.sum(mask)

    if selected == 0:
        return {"error": "No bets placed at this threshold."}

    y_selected = y_test[mask]
    num_wins = np.sum(y_selected)
    num_losses = selected - num_wins

    total_bet = selected * bet_amount
    total_return = num_wins * odds * bet_amount
    net_profit = total_return - total_bet
    roi = net_profit / total_bet if total_bet > 0 else 0

    return {
        "Threshold (≥)": round(threshold, 5),
        "# Bets Placed": selected,
        "# Wins": int(num_wins),
        "# Losses": int(num_losses),
        "Win Rate": round(100 * num_wins / selected, 2),
        "Total Bet ($)": round(total_bet, 2),
        "Total Return ($)": round(total_return, 2),
        "Net Profit ($)": round(net_profit, 2),
        "ROI (%)": round(100 * roi, 2),
    }


# lets do 5% probability threshold and see results
simulate_betting(y_test, y_pred_proba, threshold=0.05, odds=15.0)

{'Threshold (≥)': 0.05,
 '# Bets Placed': np.int64(5625),
 '# Wins': 436,
 '# Losses': 5189,
 'Win Rate': np.float64(7.75),
 'Total Bet ($)': np.float64(5625.0),
 'Total Return ($)': np.float64(6540.0),
 'Net Profit ($)': np.float64(915.0),
 'ROI (%)': np.float64(16.27)}


def run_betting_simulations(
    y_test, y_pred_proba, thresholds, odds_list, bet_amount=1.0
):
    results = []

    for odds in odds_list:
        for threshold in thresholds:
            sim = simulate_betting(y_test, y_pred_proba, threshold, odds, bet_amount)
            if "error" not in sim:
                sim["Odds"] = odds
                results.append(sim)

    return pd.DataFrame(results)


def plot_betting_roi(results_df):
    plt.figure(figsize=(10, 6))

    for odds in sorted(results_df["Odds"].unique()):
        subset = results_df[results_df["Odds"] == odds]
        plt.plot(
            subset["Threshold (≥)"], subset["ROI (%)"], marker="o", label=f"Odds {odds}"
        )

    plt.axhline(0, color="gray", linestyle="--")
    plt.title("ROI vs. Threshold for Different Odds")
    plt.xlabel("Prediction Threshold")
    plt.ylabel("ROI (%)")
    plt.legend(title="Decimal Odds")
    plt.grid(True)
    plt.tight_layout()
    plt.show()


thresholds = [0.025, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.10]
odds_list = [10, 12, 15, 18, 20]

betting_results = run_betting_simulations(y_test, y_pred_proba, thresholds, odds_list)
plot_betting_roi(betting_results)

	position.id	player.id
3	<NA>	<NA>
4	2	3608
5	23	15996
6	19	3057
7	19	3057

	timestamp	index	match_id
0	00:00:00.000	1	3753972
1	00:00:00.000	2	3753972
2	00:00:00.000	3	3753972
3	00:00:00.000	4	3753972
4	00:00:00.178	1770	3753972

	y_bin	total_poss_count	foul_win_count	foul_win_rate	foul_win_rate_std	grid_id
0	0	2476	67	0.027060	0.003261	0
1	1	4415	85	0.019253	0.002068	1
2	2	6757	50	0.007400	0.001043	2
3	3	4243	59	0.013905	0.001798	3
4	4	2246	69	0.030721	0.003641	4

	timestamp	period	abs_time_secs	type.name
id
6bbafb9c-9e4f-45bf-80f2-6f93c8b4d5e2	00:00:00.000	1	0	Starting XI
07aa1c9f-aff7-42aa-b763-39a478b8e5ea	00:00:00.000	1	0	Starting XI
a893074f-957f-4272-8d64-92020f98337f	00:00:00.000	1	0	Half Start
0ee3b84e-9975-48e0-819a-daeee9dc8880	00:00:00.000	1	0	Half Start
d44ad288-509f-4957-9754-9aa361bc316b	00:00:00.178	1	0	Ball Receipt*
b1c7e9af-19b2-47d8-855f-14aa3209648c	00:00:00.548	1	0	Pass
e7f3536d-3209-4ed2-a3a4-cd1519e44f7f	00:00:00.898	1	0	Ball Receipt*
575d8b5a-1b10-4f34-98df-579cc0a52a2e	00:00:00.898	1	0	Pass
70fc4868-b11b-4f0d-adc7-1e819aa8abe9	00:00:01.731	1	1	Pressure

	Bin	Avg Predicted Probability (prob_pred)	Actual Foul Rate (prob_true)
0	Bin 1	0.00669	0.00482
1	Bin 2	0.00973	0.00947
2	Bin 3	0.01246	0.01213
3	Bin 4	0.01508	0.01430
4	Bin 5	0.01783	0.01596
5	Bin 6	0.02102	0.01912
6	Bin 7	0.02468	0.02178
7	Bin 8	0.02965	0.02809
8	Bin 9	0.03959	0.04006
9	Bin 10	0.07557	0.07545

	timestamp	duration	type.name	possession_team.name	play_pattern.name	team.name	player.name	player_possession	mirror_fix_x	mirror_fix_y	ball_receipt.outcome.name	pass.end_location
26	00:00:19.142	0.000000	Interception	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Jan Vertonghen	15	15.7	8.7	NaN	NaN
27	00:00:19.142	1.103334	Carry	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Jan Vertonghen	15	15.7	8.7	NaN	NaN
28	00:00:19.667	0.843299	Pressure	Tottenham Hotspur	Regular Play	Leicester City	Shinji Okazaki	15	18.0	14.3	NaN	NaN
29	00:00:20.246	1.568669	Pass	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Jan Vertonghen	15	15.7	7.6	NaN	38, 14.1
30	00:00:21.814	NaN	Ball Receipt*	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Bamidele Alli	16	38.0	14.1	NaN	NaN
31	00:00:21.814	0.612894	Pass	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Bamidele Alli	16	37.7	12.6	NaN	31.5, 13.2
32	00:00:22.427	NaN	Ball Receipt*	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Tom Carroll	17	31.5	13.2	NaN	NaN
33	00:00:22.443	0.832754	Pass	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Tom Carroll	17	32.6	13.4	NaN	44.5, 17.1
34	00:00:23.275	NaN	Ball Receipt*	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Christian Dannemann Eriksen	18	44.5	17.1	NaN	NaN
35	00:00:23.275	0.036602	Carry	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Christian Dannemann Eriksen	18	44.5	17.1	NaN	NaN
36	00:00:23.312	0.854946	Pass	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Christian Dannemann Eriksen	18	38.5	26.0	NaN	31.1, 32
37	00:00:24.167	NaN	Ball Receipt*	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Eric Dier	19	31.1	32.0	NaN	NaN
38	00:00:24.167	0.399729	Pass	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Eric Dier	19	31.1	32.0	NaN	38.1, 30.1
39	00:00:24.567	NaN	Ball Receipt*	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Christian Dannemann Eriksen	20	38.1	30.1	NaN	NaN
40	00:00:24.567	0.706365	Pass	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Christian Dannemann Eriksen	20	38.1	30.1	NaN	54.6, 23.9
41	00:00:25.273	NaN	Ball Receipt*	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Bamidele Alli	21	53.5	23.1	Incomplete	NaN
42	00:00:25.273	0.000000	Interception	Tottenham Hotspur	Regular Play	Leicester City	Danny Simpson	22	54.5	23.8	NaN	NaN
43	00:00:26.483	0.614010	Pressure	Tottenham Hotspur	Regular Play	Tottenham Hotspur	Ben Davies	22	41.8	15.1	NaN	NaN

	timestamp	duration	type.name	possession_team.name	play_pattern.name	team.name	player.name	player_possession	mirror_fix_x	mirror_fix_y	ball_receipt.outcome.name	pass.end_location
763	00:20:08.598	1.523599	Pass	Leicester City	Regular Play	Leicester City	Danny Simpson	346	63.6	64.7	NaN	66.6, 59.2
764	00:20:10.122	NaN	Ball Receipt*	Leicester City	Regular Play	Leicester City	N''Golo Kanté	347	66.6	59.2	NaN	NaN
765	00:20:10.122	1.041700	Pass	Leicester City	Regular Play	Leicester City	N''Golo Kanté	347	66.6	59.2	NaN	74.1, 68.2
766	00:20:10.901	0.726600	Pressure	Leicester City	Regular Play	Tottenham Hotspur	Ben Davies	347	80.0	66.8	NaN	NaN
767	00:20:11.163	NaN	Ball Receipt*	Leicester City	Regular Play	Leicester City	Riyad Mahrez	348	74.1	68.2	NaN	NaN
768	00:20:11.163	0.886997	Carry	Leicester City	Regular Play	Leicester City	Riyad Mahrez	348	74.1	68.2	NaN	NaN
769	00:20:12.050	0.000000	Foul Committed	Leicester City	Regular Play	Tottenham Hotspur	Ben Davies	348	77.2	67.6	NaN	NaN
770	00:20:12.050	0.000000	Foul Won	Leicester City	Regular Play	Leicester City	Riyad Mahrez	348	77.3	67.7	NaN	NaN

	timestamp	duration	type.name	possession_team.name	play_pattern.name	team.name	player.name	player_possession	mirror_fix_x	mirror_fix_y	ball_receipt.outcome.name	pass.end_location
881	00:24:27.319	0.000000	Duel	Leicester City	Regular Play	Leicester City	Shinji Okazaki	402	44.3	18.1	NaN	NaN
882	00:24:28.376	0.000000	Ball Recovery	Leicester City	Regular Play	Leicester City	Marc Albrighton	403	43.7	8.3	NaN	NaN
883	00:24:28.376	2.198197	Carry	Leicester City	Regular Play	Leicester City	Marc Albrighton	403	43.7	8.3	NaN	NaN
884	00:24:29.603	0.606193	Pressure	Leicester City	Regular Play	Tottenham Hotspur	Erik Lamela	403	38.9	15.0	NaN	NaN
885	00:24:30.574	0.000000	Dribbled Past	Leicester City	Regular Play	Tottenham Hotspur	Erik Lamela	403	36.1	12.7	NaN	NaN
886	00:24:30.574	0.000000	Dribble	Leicester City	Regular Play	Leicester City	Marc Albrighton	404	36.2	12.8	NaN	NaN
887	00:24:30.574	0.733596	Carry	Leicester City	Regular Play	Leicester City	Marc Albrighton	404	36.2	12.8	NaN	NaN
888	00:24:31.308	1.769800	Pass	Leicester City	Regular Play	Leicester City	Marc Albrighton	404	36.2	12.8	NaN	36.6, 3.4
889	00:24:33.077	NaN	Ball Receipt*	Leicester City	Regular Play	Leicester City	Christian Fuchs	405	36.6	3.4	NaN	NaN
890	00:24:33.077	0.070099	Carry	Leicester City	Regular Play	Leicester City	Christian Fuchs	405	36.6	3.4	NaN	NaN
891	00:24:33.148	2.855894	Pass	Leicester City	Regular Play	Leicester City	Christian Fuchs	405	36.6	3.4	NaN	64.7, 2.1
892	00:24:34.867	1.231761	Pressure	Leicester City	Regular Play	Tottenham Hotspur	Christian Dannemann Eriksen	405	53.0	5.8	NaN	NaN
893	00:24:36.003	NaN	Ball Receipt*	Leicester City	Regular Play	Leicester City	Danny Drinkwater	406	62.3	3.4	Incomplete	NaN
894	00:24:36.340	0.000000	Foul Committed	Leicester City	Regular Play	Tottenham Hotspur	Christian Dannemann Eriksen	406	64.8	2.4	NaN	NaN
895	00:24:36.340	0.000000	Foul Won	Leicester City	Regular Play	Leicester City	Danny Drinkwater	406	64.9	2.5	NaN	NaN

	id	timestamp	type.name	ball_receipt.outcome.name	player.id	player_possession	duration	possession_team.name	play_pattern.name	team.name	mirror_fix_x	mirror_fix_y	pass.end_location	parsed_time
0	c1	00:01:00.000	Ball Receipt*	None	Player A	1	0	TeamX	Regular Play	TeamX	60	40	None	0 days 00:01:00
6	c4	00:01:30.000	Ball Recovery	None	Player D	4	0	TeamX	Regular Play	TeamX	60	40	None	0 days 00:01:30

	timestamp	type.name	player.id	player.name	player_possession	possession_team.name	team.name	ball_receipt.outcome.name
26	00:00:19.142	Interception	3077.0	Jan Vertonghen	15	Tottenham Hotspur	Tottenham Hotspur	NaN
27	00:00:19.142	Carry	3077.0	Jan Vertonghen	15	Tottenham Hotspur	Tottenham Hotspur	NaN
28	00:00:19.667	Pressure	3810.0	Shinji Okazaki	15	Tottenham Hotspur	Leicester City	NaN
29	00:00:20.246	Pass	3077.0	Jan Vertonghen	15	Tottenham Hotspur	Tottenham Hotspur	NaN
30	00:00:21.814	Ball Receipt*	3094.0	Bamidele Alli	16	Tottenham Hotspur	Tottenham Hotspur	NaN
31	00:00:21.814	Pass	3094.0	Bamidele Alli	16	Tottenham Hotspur	Tottenham Hotspur	NaN
32	00:00:22.427	Ball Receipt*	3310.0	Tom Carroll	17	Tottenham Hotspur	Tottenham Hotspur	NaN
33	00:00:22.443	Pass	3310.0	Tom Carroll	17	Tottenham Hotspur	Tottenham Hotspur	NaN
34	00:00:23.275	Ball Receipt*	3043.0	Christian Dannemann Eriksen	18	Tottenham Hotspur	Tottenham Hotspur	NaN
35	00:00:23.275	Carry	3043.0	Christian Dannemann Eriksen	18	Tottenham Hotspur	Tottenham Hotspur	NaN
36	00:00:23.312	Pass	3043.0	Christian Dannemann Eriksen	18	Tottenham Hotspur	Tottenham Hotspur	NaN
37	00:00:24.167	Ball Receipt*	10956.0	Eric Dier	19	Tottenham Hotspur	Tottenham Hotspur	NaN
38	00:00:24.167	Pass	10956.0	Eric Dier	19	Tottenham Hotspur	Tottenham Hotspur	NaN
39	00:00:24.567	Ball Receipt*	3043.0	Christian Dannemann Eriksen	20	Tottenham Hotspur	Tottenham Hotspur	NaN
40	00:00:24.567	Pass	3043.0	Christian Dannemann Eriksen	20	Tottenham Hotspur	Tottenham Hotspur	NaN
41	00:00:25.273	Ball Receipt*	3094.0	Bamidele Alli	21	Tottenham Hotspur	Tottenham Hotspur	Incomplete
42	00:00:25.273	Interception	3270.0	Danny Simpson	22	Tottenham Hotspur	Leicester City	NaN
43	00:00:26.483	Pressure	3086.0	Ben Davies	22	Tottenham Hotspur	Tottenham Hotspur	NaN

	timestamp	type.name	player.id	player.name	player_possession	possession_team.name	team.name	ball_receipt.outcome.name
1	00:20:10.122	Ball Receipt*	3961.0	N''Golo Kanté	347	Leicester City	Leicester City	NaN
4	00:20:11.163	Ball Receipt*	3814.0	Riyad Mahrez	348	Leicester City	Leicester City	NaN

	timestamp	type.name	player.id	player.name	player_possession	possession_team.name	team.name	ball_receipt.outcome.name
1	00:24:28.376	Ball Recovery	3307.0	Marc Albrighton	403	Leicester City	Leicester City	NaN
8	00:24:33.077	Ball Receipt*	3812.0	Christian Fuchs	405	Leicester City	Leicester City	NaN

	match_id	id	period	timestamp	type.name	location	player.id	possession	player_possession
0	3754065	1132f7fa-8ca0-44f8-ae04-d5e72303458b	1	00:00:01.107	Ball Receipt*	59.5, 41	10960.0	2	6
1	3754065	02e97b03-47ab-4856-bdab-f3bb6409cd51	1	00:00:01.705	Ball Receipt*	52, 39.9	3662.0	2	7
2	3754065	4626ce49-1ea9-477d-a9c7-bd1ddd210dac	1	00:00:03.519	Ball Receipt*	44.3, 21.3	40123.0	2	8
3	3754065	b74061ae-79e0-4be3-9c3d-c3f70de143a1	1	00:00:07.842	Ball Receipt*	8, 38.9	3815.0	2	9
4	3754065	2999d133-b2e5-45fc-8abf-18653ea383fa	1	00:00:15.154	Ball Recovery	65.5, 57.9	3662.0	2	13

	match_id	id	period	timestamp	type.name	location	player.id	possession	player_possession	ends_in_foul
175	3754065	88813056-172c-4ab5-9a54-8f8300bf000c	1	00:20:11.163	Ball Receipt*	74.1, 68.2	3814.0	48	348	True
255	3754065	96e25b13-f217-446d-96da-a4dd17771c18	1	00:31:20.179	Ball Recovery	40.1, 9.6	3810.0	73	504	True
275	3754065	883bb648-dcc2-4119-a8b4-619979739eae	1	00:33:58.852	Ball Receipt*	70.4, 4.4	3094.0	77	546	True
356	3754065	408269ad-aa03-4104-af6f-6009ba830139	1	00:45:02.890	Ball Receipt*	34.3, 53.9	3585.0	100	715	True
458	3754065	1811e3b2-c6e4-4962-afd3-8aeaf50ae730	2	00:19:15.515	Ball Recovery	3.3, 48.4	3662.0	144	986	True
473	3754065	b2ea1821-b6c1-433c-9d4c-df0dc94b8306	2	00:21:47.843	Ball Receipt*	58.2, 24.4	10955.0	148	1030	True
627	3754065	c8462a95-7fbc-42bc-943e-4f5ce3557009	2	00:44:13.297	Ball Recovery	55.4, 65.3	3205.0	198	1345	True

	timestamp	type.name	player.id	player.name	player_possession	possession_team.name	team.name	ball_receipt.outcome.name
775	00:21:17.335	Ball Receipt*	10955.0	Harry Kane	351	Tottenham Hotspur	Tottenham Hotspur	Incomplete
776	00:21:17.335	Clearance	3812.0	Christian Fuchs	352	Tottenham Hotspur	Leicester City	NaN
777	00:21:18.442	Pressure	3662.0	Danny Drinkwater	352	Tottenham Hotspur	Leicester City	NaN
778	00:21:18.799	Foul Committed	3662.0	Danny Drinkwater	352	Tottenham Hotspur	Leicester City	NaN
779	00:21:18.799	Foul Won	3310.0	Tom Carroll	353	Tottenham Hotspur	Tottenham Hotspur	NaN
780	00:21:47.166	Pass	10956.0	Eric Dier	354	Tottenham Hotspur	Tottenham Hotspur	NaN
781	00:21:49.147	Ball Receipt*	3077.0	Jan Vertonghen	355	Tottenham Hotspur	Tottenham Hotspur	NaN

	match_id	match_date	match_week
0	3754058	2016-01-02	20
1	3754245	2015-10-17	9

	period	x	y	under_pressure	is_home_team	dist_opp_goal	angle_opp_goal	team_foul_win_rate	position_foul_win_rate	player_foul_win_rate	play_patt_foul_win_rate	opp_team_foul_commit_rate	grid_id	grid_total_count	grid_foul_count	grid_foul_rate	grid_foul_rate_std	sec1_unique_possessions	sec3_count_type_Pressure	sec3_unique_possessions	sec5_count_type_Pressure	sec5_unique_possessions	sec5_dist30_type_Pressure	pass.angle	pass.height.id	pass.body_part.id
0	1	60.2	40.7	0	1	59.8	-0.7	0.025	0.028	0.009	0.015	0.021	12	14230	386	0.027126	0.001362	2	0	2	0	2	0	2.498092	1.0	40.0
1	1	52.4	41.6	1	1	67.6	-1.4	0.025	0.020	0.024	0.015	0.021	12	14230	386	0.027126	0.001362	3	0	3	0	3	0	0.000000	0.0	0.0
2	1	33.2	60.9	0	1	89.3	-13.5	0.025	0.016	0.043	0.015	0.021	8	11820	238	0.020135	0.001292	1	1	1	1	3	0	2.389948	1.0	40.0
3	1	28.5	31.1	0	1	91.9	5.6	0.025	0.017	0.011	0.015	0.021	6	12595	268	0.021278	0.001286	1	0	1	1	1	1	-1.740478	1.0	40.0
4	1	26.8	60.2	0	1	95.4	-12.2	0.025	0.016	0.043	0.015	0.021	8	11820	238	0.020135	0.001292	1	0	1	0	1	0	1.635345	1.0	40.0

	period	minute	x	y	is_recovery	under_pressure	is_home_team	dist_opp_goal	angle_opp_goal	team_foul_win_rate	position_foul_win_rate	player_foul_win_rate	play_patt_foul_win_rate	opp_team_foul_commit_rate	grid_id	grid_total_count	grid_foul_count	grid_foul_rate	grid_foul_rate_std	sec1_count_type_Pressure	sec1_unique_possessions	sec1_counterpress_true	sec1_dist10_type_Pressure	sec1_dist30_type_Pressure	sec3_count_type_Pressure	sec3_unique_possessions	sec3_counterpress_true	sec3_dist10_type_Pressure	sec3_dist30_type_Pressure	sec5_count_type_Pressure	sec5_unique_possessions	sec5_counterpress_true	sec5_dist10_type_Pressure	sec5_dist30_type_Pressure	pass.angle	pass.height.id	pass.body_part.id
0	1	0	60.7	42.1	0	0	1	59.3	-2.0	0.026	0.024	0.022	0.015	0.027	12	14230	386	0.027126	0.001362	0	3	0	0	0	0	3	0	0	0	0	3	0	0	0	0.0	0.0	0.0
1	1	0	54.9	37.8	0	1	1	65.1	1.9	0.026	0.020	0.015	0.015	0.027	12	14230	386	0.027126	0.001362	1	1	0	1	1	1	3	0	1	1	1	3	0	1	1	0.0	0.0	0.0

	Model	Log Loss	Brier Score	AUC-ROC
0	baseline_foul_win_rate	0.114297	0.023571	0.500000
1	xgb_model	0.106650	0.023106	0.707539

	timestamp	type.name	player.id	player.name	player_possession	possession_team.name	team.name	ball_receipt.outcome.name
1130	00:31:30.777	Pass	3086.0	Ben Davies	509	Tottenham Hotspur	Tottenham Hotspur	NaN
1131	00:31:32.014	Pressure	3813.0	Wes Morgan	509	Tottenham Hotspur	Leicester City	NaN
1132	00:31:32.288	Foul Committed	3813.0	Wes Morgan	510	Tottenham Hotspur	Leicester City	NaN
1133	00:31:32.288	Foul Won	3094.0	Bamidele Alli	511	Tottenham Hotspur	Tottenham Hotspur	NaN
1134	00:32:03.971	Pass	3310.0	Tom Carroll	512	Tottenham Hotspur	Tottenham Hotspur	NaN

	timestamp	period	abs_time_secs	type.name
id
ec0b4585-41ad-4ada-817c-629d0866eeeb	00:00:01.808	1	1	Ball Receipt*

	Top K Group	Threshold (Prob ≥)	# of Samples (Poss. Start events) in Group	% of Samples (Poss. Start events Selected	Actual Foul Rate (%)	Lift over Base
0	Top 1%	0.11257	602	1.0	12.13	4.76
1	Top 5%	0.06769	3009	5.0	9.04	3.55
2	Top 10%	0.04845	6017	10.0	7.55	2.96
3	Top 15%	0.03896	9025	15.0	6.54	2.56
4	Top 20%	0.03313	12033	20.0	5.78	2.27
5	Top 25%	0.02951	15041	25.0	5.27	2.07
6	Top 30%	0.02679	18049	30.0	4.79	1.88
7	Top 35%	0.02465	21057	35.0	4.46	1.75
8	Top 40%	0.02272	24065	40.0	4.13	1.62
9	Top 45%	0.02100	27073	45.0	3.93	1.54
10	Top 50%	0.01935	30081	50.0	3.69	1.45

EPL 2015/16 Foul-Win Probability Modeling¶

Introduction¶

Notebook Outline (Table of Contents)¶

Read in Data¶

Initial Data Investigation and Analysis¶

Finding data specifications doc - StatsBomb¶

Visualizing the events on the pitch - function defs¶

Deep Dive Investigate - (tottenham vs leicester) match_id: 3754065¶

Fix (mirror) x,y coordinates for non-possession team events¶

posssession 3 investigate - no foul¶

posssession 48 investigate - 'foul won' example¶

posssession 58 investigate - 'foul won' but not valid possession¶

Model Framing and Assumptions¶

Model Framing Outline¶

Possession Event Extraction and Labeling¶

Step 1: get valid possession start events¶

Unit tests - possession event sequence extraction¶

Run on single match and spot check¶

Possession 3 test¶

Possesssion 48 test¶

Possesssion 58 test¶

Posssession Extract - Run on all matches¶

Step 2 - determine 'Foul Won' outcome¶

Testing on single match¶

Determine Foul Won - run on all matches¶

Time-based Train/Test Split¶

More EDA - Visualizing Spatial Grid-Based Foul Win Rates¶

Feature Engineering - FX¶

FX helper setup - convert event and match dfs to dics¶

Simple features (start possess event)¶

Simple features from match data¶

Basic distance-based features¶

Foul rate features¶

Calculate historical foul rate metrics on training dataset¶

Generate foul rate feature vectors¶

Pitch grid spatial foul win rate features¶

Previous events features - time-based and proximity based¶

Setup input data¶

Define helper functions¶

Previous events - fx batch run¶

Full FX - generate all feature vectors¶

ML Model - Xgb Training and Hyperparameter Tuning with Cross-Validation¶

SHAP - seeing most important features¶

ML Model Evaluation¶

Calibration Plots and understanding¶

Baseline (foul win rate) to compare against¶

Xgboost model compare vs baseline¶

Final Results Interpretation and Analysis¶

Threshold Performance Summary¶

Top 5% Group – Results Interpretation¶

Threshold Performance Summary Plots¶

Simulate Betting Strategy¶

Next Steps and Improvement Ideas¶