Add general pitch classification

convert.py

@@ -43,7 +43,7 @@ ball_kind = {
 
 ball_kind_code = {
     -1: '-',
-    31: 'FF',
+    31: '4S',
     32: 'SL',
     33: 'VS',
     34: 'SV',
@@ -59,14 +59,66 @@ ball_kind_code = {
     44: 'PB',
     45: 'KN',
     46: 'SH',
-    47: 'FT',
-    48: 'FW',
+    47: '2S',
+    48: '1S',
     49: 'FC',
     50: 'EP', # technically "super" eephus but I haven't encountered a normal one yet
     51: 'HS',
     52: 'HL'
 }
 
+general_ball_kind = {
+    -1: '-',
+    31: 'Fastball (4-seam)',
+    32: 'Slider',
+    33: 'Vertical Slider',
+    34: 'Slurve',
+    35: 'Curve',
+    36: 'Curve',
+    37: 'Curve',
+    38: 'Curve',
+    39: 'Splitter',
+    40: 'Splitter',
+    41: 'Changeup',
+    42: 'Sinker',
+    43: 'Screwball',
+    44: 'Palmball',
+    45: 'Knuckleball',
+    46: 'Sinker',
+    47: 'Sinker',
+    48: 'Sinker',
+    49: 'Cutter',
+    50: 'Eephus', # technically "super" eephus but I haven't encountered a normal one yet
+    51: 'Sinker',
+    52: 'Vertical Slider',
+}
+
+general_ball_kind_code = {
+    -1: '-',
+    31: '4S',
+    32: 'SL',
+    33: 'VS',
+    34: 'SV',
+    35: 'CU',
+    36: 'CU',
+    37: 'CU',
+    38: 'CU',
+    39: 'FS',
+    40: 'FS',
+    41: 'CH',
+    42: 'SI',
+    43: 'SB',
+    44: 'PB',
+    45: 'KN',
+    46: 'SI',
+    47: 'SI',
+    48: 'SI',
+    49: 'FC',
+    50: 'EP', # technically "super" eephus but I haven't encountered a normal one yet
+    51: 'SI',
+    52: 'VS'
+}
+
 lr = {1: 'l', 2: 'r'}
 
 presult = {
@@ -260,29 +312,32 @@ game_kind = {
 
 ball_kind_code_to_color = {
     '-': '',
-    'FF': 'crimson',
+    '4S': 'crimson',
     'SL': 'gold',
-    'VS': '',
-    'SV': '',
+    'VS': 'khaki',
+    'SV': 'lightsteelblue',
     'CU': 'paleturquoise',
     'SC': 'royalblue',
-    'PC': '',
+    'PC': 'turquoise',
     'KC': 'rebeccapurple',
     'FO': 'darkturquoise',
     'FS': 'cadetblue',
     'CH': 'mediumseagreen',
-    'SI': '',
-    'SB': '',
-    'PB': '',
+    'SI': 'orange',
+    'SB': 'lightgreen',
+    'PB': 'yellowgreen',
     'SH': 'tomato',
-    'FT': '',
-    'FW': '',
+    '2S': 'orangered',
+    '1S': 'lightsalmon',
     'FC': 'sienna',
-    'EP': '', # technically "super" eephus but I haven't encountered a normal one yet
-    'HS': '',
-    'HL': ''
+    'EP': 'darkgray', # technically "super" eephus but I haven't encountered a normal one yet
+    'HS': 'darkorange',
+    'HL': 'darkkhaki'
 }
+
+
 ball_kind_code_to_color = {k: v if v else 'C0' for k, v in ball_kind_code_to_color.items()}
+
 def get_text_color_from_color(color):
   if color in ['gold', 'paleturquoise']:
     return 'black'

data.py

@@ -4,15 +4,21 @@ from tqdm.auto import tqdm
 import pykakasi
 from huggingface_hub import snapshot_download
 
-from convert import aux_global_id_to_code, presult, ball_kind, ball_kind_code, lr, game_kind
+from convert import (
+  aux_global_id_to_code, presult, 
+  ball_kind, ball_kind_code, general_ball_kind, general_ball_kind_code, lr, 
+  game_kind
+)
 
 
 DATA_PATH = snapshot_download(
   repo_id='Ramos-Ramos/npb_data_app',
   repo_type='dataset',
   local_dir='./files',
-  cache_dir='./.cache'
+  cache_dir='./.cache',
+  allow_patterns=['*/pbp_data.parquet', '*/pbp_text.parquet', '*/pbp_aux.parquet', '*/schedule.parquet', '*/aux_schedule.parquet', 'players.parquet', 'players_kana.parquet']
 )
+
 SEASONS = [2021, 2022, 2023, 2024, 2025]
 
 data_df = pl.DataFrame()
@@ -212,6 +218,8 @@ data_df = (
         pl.col('presult').alias('presult_id'),
         pl.col('ballKind').replace_strict(ball_kind),
         pl.col('ballKind').replace_strict(ball_kind_code).alias('ballKind_code'),
+        pl.col('ballKind').replace_strict(general_ball_kind).alias('general_ballKind'),
+        pl.col('ballKind').replace_strict(general_ball_kind_code).alias('general_ballKind_code'),
         pl.col('batLR').replace_strict(lr),
         pl.col('date').str.to_date('%Y%m%d'),
         

pitcher_overview.py

@@ -31,7 +31,7 @@ def gr_create_pitcher_overview_card(name, season):
   
 def create_pitcher_overview(data_df):
   with gr.Blocks() as app:
-    gr.Markdown('Pitcher overview')
+    gr.Markdown('# Pitcher overview')
 
     with gr.Row():
       with gr.Column():

plotting.py

@@ -12,62 +12,23 @@ from datetime import date
 
 from data import data_df
 from convert import ball_kind_code_to_color, get_text_color_from_color
+from stats import filter_data_by_date_and_game_kind, compute_team_games, compute_pitch_stats
 
-mpl.use('Agg')
-
-def compute_team_games(data):
-  data = (
-      data
-      .with_columns(
-          pl.col('gameId').unique().len().over('HomeTeamNameES').alias('home_games'),
-          pl.col('gameId').unique().len().over('VisitorTeamNameES').alias('visitor_games')
-      )
-  )
-  game_data = (
-      data
-      .group_by('HomeTeamNameES')
-      .first()
-      [['HomeTeamNameES', 'home_games']]
-      .rename({'HomeTeamNameES': 'team'})
-      .join(
-          (
-              data
-              .group_by('VisitorTeamNameES')
-              .first()
-              [['VisitorTeamNameES', 'visitor_games']]
-              .rename({'VisitorTeamNameES': 'team'})
-          ),
-          on='team',
-      )
-      .with_columns((pl.col('home_games')+pl.col('visitor_games')).alias('games'))
-  )
 
-  return (
-      data
-      .drop('home_games', 'visitor_games')
-      .join(
-          game_data[['team', 'games']].rename({'games': 'home_games'}),
-          left_on='HomeTeamNameES',
-          right_on='team'
-      )
-      .join(
-          game_data[['team', 'games']].rename({'games': 'visitor_games'}),
-          left_on='VisitorTeamNameES',
-          right_on='team'
-      )
-  )
+mpl.use('Agg')
 
 
-def get_pitcher_stats(id, lr=None, game_kind=None, start_date=None, end_date=None, min_ip=1, min_pitches=1):
+def get_pitcher_stats(id, lr=None, game_kind=None, start_date=None, end_date=None, min_ip=1, min_pitches=1, pitch_class_type='specific'):
   source_data = data_df.filter(pl.col('ballKind_code') != '-')
 
-  if start_date is not None:
-    source_data = source_data.filter(pl.col('date') >= start_date)
-  if end_date is not None:
-    source_data = source_data.filter(pl.col('date') <= end_date)
-
-  if game_kind is not None:
-    source_data = source_data.filter(pl.col('coarse_game_kind') == game_kind)
+  # if start_date is not None:
+    # source_data = source_data.filter(pl.col('date') >= start_date)
+  # if end_date is not None:
+    # source_data = source_data.filter(pl.col('date') <= end_date)
+# 
+  # if game_kind is not None:
+    # source_data = source_data.filter(pl.col('coarse_game_kind') == game_kind)
+  source_data = filter_data_by_date_and_game_kind(source_data, start_date=start_date, end_date=end_date, game_kind=game_kind)
 
   source_data = (
       compute_team_games(source_data)
@@ -82,49 +43,10 @@ def get_pitcher_stats(id, lr=None, game_kind=None, start_date=None, end_date=Non
   else:
     source_data = source_data.with_columns((pl.col('IP') >= min_ip).alias('qualified'))
 
-
   if lr is not None:
     source_data = source_data.filter(pl.col('batLR') == lr)
 
-  pitch_stats = (
-      source_data
-      # .with_columns(
-      #     pl.col('presult').is_in(['None', 'Balk', 'Batter interference', 'Catcher interference', 'Pitcher delay', 'Intentional walk', 'Unknown']).not_().alias('pitch'),
-      #     pl.col('presult').is_in(['Swinging strike', 'Swinging strikeout']).alias('whiff'),
-      # )
-      # .with_columns(
-      #     (pl.col('pitch') & pl.col('presult').is_in(['Hit by pitch', 'Sacrifice bunt', 'Sacrifice fly', 'Looking strike', 'Ball', 'Walk', 'Looking strikeout', 'Sacrifice hit error', 'Sacrifice fly error', "Sacrifice fielder's choice", 'Bunt strikeout']).not_()).alias('swing'),
-      #     (pl.col('whiff') | pl.col('presult').is_in(['Looking strike', 'Uncaught third strike', 'Looking strikeout'])).alias('csw')
-      # )
-      .group_by('pitId', 'ballKind_code')
-      .agg(
-          pl.len().alias('count'),
-          pl.col('aux_bresult').struct.field('batType').drop_nulls().value_counts(normalize=True),
-          (pl.col('whiff').sum() / pl.col('pitch').sum()).alias('SwStr%'),
-          (pl.col('whiff').sum() / pl.col('swing').sum()).alias('Whiff%'),
-          (pl.col('csw').sum() / pl.col('pitch').sum()).alias('CSW%')
-      )
-      .with_columns(
-          (pl.col('count')/pl.sum('count').over('pitId')).alias('usage'),
-          (pl.col('count') >= min_pitches).alias('qualified')
-      )
-      .explode('batType')
-      .unnest('batType')
-      .pivot(on='batType', values='proportion')
-      .fill_null(0)
-      .with_columns(
-          (pl.col('G') + pl.col('B')).alias('GB%'),
-          (pl.col('F') + pl.col('P')).alias('FB%'),
-          pl.col('L').alias('LD%').round(2),
-      )
-      .drop('G', 'F', 'B', 'P', 'L')
-      .with_columns(
-          (pl.when(pl.col('qualified')).then(pl.col(stat)).rank()/pl.when(pl.col('qualified')).then(pl.col(stat)).count()).alias(f'{stat}_pctl')
-          for stat in ['SwStr%', 'Whiff%', 'CSW%', 'GB%']
-      )
-      .sort('pitId', 'count', descending=[False, True])
-      .filter(pl.col('pitId') == id)
-  )
+  pitch_stats = compute_pitch_stats(source_data, player_type='pitcher', pitch_class_type=pitch_class_type, min_pitches=min_pitches).filter(pl.col('pitId') == id)
 
   pitch_shapes = (
       source_data
@@ -134,7 +56,7 @@ def get_pitcher_stats(id, lr=None, game_kind=None, start_date=None, end_date=Non
           pl.col('y').is_not_null() &
           (pl.col('ballSpeed') > 0)
       )
-      [['pitId', 'ballKind_code', 'ballSpeed', 'x', 'y']]
+      [['pitId', 'general_ballKind_code', 'ballKind_code', 'ballSpeed', 'x', 'y']]
   )
 
   pitcher_stats = (
@@ -224,7 +146,7 @@ def plot_loc(ax, locs):
   ax.add_patch(plt.Rectangle((-40, 75), width=80, height=100, facecolor='ivory', edgecolor='darkgray'))
   ax.add_patch(plt.Polygon([(0, -10), (45, -30), (51, -50), (-51, -50), (-45, -30), (0, -10)], facecolor='snow', edgecolor='darkgray'))
 
-  for (pitch,), _locs in locs.sort(pl.len().over('ballKind_code'), descending=True).group_by('ballKind_code', maintain_order=True):
+  for (pitch,), _locs in locs.sort(pl.len().over('general_ballKind_code'), descending=True).group_by('general_ballKind_code', maintain_order=True):
     if len(_locs) <= 2:
       continue
     
@@ -243,7 +165,7 @@ def plot_loc(ax, locs):
 
 def plot_velo(ax, velos):
   trans = transforms.blended_transform_factory(ax.transData, ax.transAxes)
-  for (pitch,), _velos in velos.group_by('ballKind_code'):
+  for (pitch,), _velos in velos.group_by('general_ballKind_code'):
     if len(_velos) <= 1:
       continue
 
@@ -317,7 +239,7 @@ font = load_google_font('Saira Extra Condensed', weight='medium')
 
 
 def create_pitcher_overview_card(id, season, dpi=300):
-  data = get_card_data(id, start_date=date(season, 1, 1), end_date=date(season, 12, 31), game_kind='Regular Season', min_pitches=100)
+  data = get_card_data(id, start_date=date(season, 1, 1), end_date=date(season, 12, 31), game_kind='Regular Season', min_pitches=100, pitch_class_type='general')
 
   fig = plt.figure(figsize=(1080/300, 1350/300), dpi=dpi)
   gs = fig.add_gridspec(8, 6, height_ratios=[1, 1, 1.5, 6, 1, 3, 1, 0.5])

stats.py

@@ -0,0 +1,95 @@
+import polars as pl
+
+def filter_data_by_date_and_game_kind(data, start_date=None, end_date=None, game_kind=None):
+  if start_date is not None:
+    data = data.filter(pl.col('date') >= start_date)
+  if end_date is not None:
+    data = data.filter(pl.col('date') <= end_date)
+  if game_kind is not None:
+    data = data.filter(pl.col('coarse_game_kind') == game_kind)
+  return data
+
+def compute_team_games(data):
+  data = (
+      data
+      .with_columns(
+          pl.col('gameId').unique().len().over('HomeTeamNameES').alias('home_games'),
+          pl.col('gameId').unique().len().over('VisitorTeamNameES').alias('visitor_games')
+      )
+  )
+  game_data = (
+      data
+      .group_by('HomeTeamNameES')
+      .first()
+      [['HomeTeamNameES', 'home_games']]
+      .rename({'HomeTeamNameES': 'team'})
+      .join(
+          (
+              data
+              .group_by('VisitorTeamNameES')
+              .first()
+              [['VisitorTeamNameES', 'visitor_games']]
+              .rename({'VisitorTeamNameES': 'team'})
+          ),
+          on='team',
+      )
+      .with_columns((pl.col('home_games')+pl.col('visitor_games')).alias('games'))
+  )
+
+  return (
+      data
+      .drop('home_games', 'visitor_games')
+      .join(
+          game_data[['team', 'games']].rename({'games': 'home_games'}),
+          left_on='HomeTeamNameES',
+          right_on='team'
+      )
+      .join(
+          game_data[['team', 'games']].rename({'games': 'visitor_games'}),
+          left_on='VisitorTeamNameES',
+          right_on='team'
+      )
+  )
+
+
+def compute_pitch_stats(data, player_type, pitch_class_type, min_pitches=1):
+  assert player_type in ('pitcher', 'batter')
+  assert pitch_class_type in ('general', 'specific')
+  id_col = 'pitId' if player_type == 'pitcher' else 'batId'
+  pitch_col = 'ballKind_code' if pitch_class_type == 'specific' else 'general_ballKind_code'
+  pitch_name_col = 'ballKind' if pitch_class_type == 'specific' else 'general_ballKind'
+  pitch_stats = (
+      data
+      .group_by(id_col, pitch_col)
+      .agg(
+          pl.first('pitcher_name'),
+          *([pl.first('general_ballKind')] if pitch_class_type == 'specific' else []),
+          pl.first(pitch_name_col),
+          pl.len().alias('count'),
+          pl.col('aux_bresult').struct.field('batType').drop_nulls().value_counts(normalize=True),
+          (pl.col('whiff').sum() / pl.col('pitch').sum()).alias('SwStr%'),
+          (pl.col('whiff').sum() / pl.col('swing').sum()).alias('Whiff%'),
+          (pl.col('csw').sum() / pl.col('pitch').sum()).alias('CSW%')
+      )
+      .with_columns(
+          (pl.col('count')/pl.sum('count').over('pitId')).alias('usage'),
+          (pl.col('count') >= min_pitches).alias('qualified')
+      )
+      .explode('batType')
+      .unnest('batType')
+      .pivot(on='batType', values='proportion')
+      .fill_null(0)
+      .with_columns(
+          (pl.col('G') + pl.col('B')).alias('GB%'),
+          (pl.col('F') + pl.col('P')).alias('FB%'),
+          pl.col('L').alias('LD%').round(2),
+      )
+      .drop('G', 'F', 'B', 'P', 'L', 'null')
+      .with_columns(
+          (pl.when(pl.col('qualified')).then(pl.col(stat)).rank()/pl.when(pl.col('qualified')).then(pl.col(stat)).count()).alias(f'{stat}_pctl')
+          for stat in ['SwStr%', 'Whiff%', 'CSW%', 'GB%', 'FB%', 'LD%']
+      )
+      .rename({pitch_col: 'ballKind_code', pitch_name_col: 'ballKind'} if pitch_class_type == 'general' else {})
+      .sort(id_col, 'count', descending=[False, True])
+  )
+  return pitch_stats