Fix player name translation

data.py

@@ -47,109 +47,6 @@ for season in tqdm(SEASONS):
   _aux_sched_df = pl.read_parquet(os.path.join(DATA_PATH, str(season), 'aux_schedule.parquet'))
   aux_sched_df = pl.concat((aux_sched_df, _aux_sched_df))
 
-
-def select_name(names):
-  '''
-  When given mutiple names, 
-  prioritizes the name with ASCII characters (ex. R. マルティネス > マルティネス), 
-  followed by the shorter name (ex. 大勢 > 翁田 大勢)
-  Names with ASCII characters help differentiate between foreign players,
-  whlie shorter names are more accurate for players going by shorter names
-  '''
-  lens = []
-  for name in names:
-    if any([char in ascii_letters for char in name]):
-      return name
-    else:
-      lens.append(len(name))
-  return names[np.argmin(lens).item()]
-
-# load player dfs
-players_df = (
-    pl.read_parquet('files/players.parquet')
-    .with_columns(pl.col('playerName').str.normalize('NFKC').str.replace_all('・', ' '))
-    .group_by('playerId').agg(pl.col('playerName').map_elements(select_name, return_dtype=pl.String))
-)
-translated_df = (
-    pl.read_parquet('files/players_translated.parquet')
-    .with_columns(pl.col('name_jp').str.normalize('NFKC').str.replace_all('・', ' '))
-    ['name_jp', 'name_kana', 'name_en']
-    .unique()
-)
-manual_translated_df = pl.read_parquet('files/players_translated_manual.parquet')
-
-# names with no romanization are approximated with kana translation
-kks = pykakasi.kakasi()
-
-# take names in parenthesis when they contain an ascii character
-translated_df = (
-    translated_df
-    .with_columns(
-        pl.when(pl.col('name_jp').str.contains(r'\('))
-        .then(pl.col('name_jp').str.extract(r'.*\(', 0).str.strip_chars_end(' ('))
-        .otherwise(pl.col('name_jp'))
-        .str.replace_all('・', ' ')
-        .alias('name_jp')
-    )
-    .with_columns(pl.col('name_kana').str.normalize('NFKC').str.replace_all('・', ' '))
-    .with_columns(pl.col('name_kana').str.extract(r'\(.*\)', 0).str.strip_chars('()').alias('in_parentheses'))
-    .with_columns(pl.col('name_kana').str.extract(r'.*\(', 0).str.strip_chars_end('(').alias('before_parentheses'))
-    .with_columns(
-        pl.when(pl.col('name_en').is_null())
-        .then
-         (
-            pl.when(pl.col('in_parentheses').is_not_null() | pl.col('before_parentheses').is_not_null())
-            .then(
-                pl.when(pl.col('in_parentheses').map_elements(lambda name: any([char in ascii_letters for char in name]), pl.Boolean))
-                .then(pl.col('in_parentheses'))
-                .otherwise(pl.col('before_parentheses'))
-            )
-            .otherwise(pl.col('name_kana').map_elements(lambda name: ''.join([word['hepburn'].capitalize() for word in kks.convert(name)]), return_dtype=pl.String))
-        )
-        .otherwise(pl.col('name_en'))
-        .alias('name_en')
-    )
-    .with_columns(pl.col('name_en').str.replace_all(',', '').str.to_titlecase())
-)
-
-# handle inconsistent kanji between sources
-for old_char, new_char in [
-    ('崎', '﨑'),
-    ('高', '髙'),
-    ('徳', '德'),
-    ('濱', '濵'),
-    ('瀬', '瀨')
-]:
-  players_df = (
-      players_df.with_columns(
-          pl.when(~pl.col('playerName').is_in(translated_df['name_jp']))
-          .then(pl.col('playerName').str.replace(old_char, new_char))
-          .otherwise('playerName')
-      )
-  )
-
-# merge player dfs
-players_df = (
-    players_df
-    .join(manual_translated_df.rename({'name_en': 'name_en_manual'}), on='playerId', how='left')
-    .join(
-        (
-            translated_df
-            .with_columns(
-                pl.when(pl.col('name_jp').str.contains(r'\.') & ~pl.col('name_jp').is_in(players_df.filter(pl.len().over('playerName') == 1)['playerName']))
-                .then(pl.col('name_jp').str.strip_chars(ascii_letters+'.'))
-                .otherwise('name_jp')
-            )
-            [['name_jp', 'name_en']]
-        ),
-        left_on='playerName', right_on='name_jp', how='left'
-    )
-    .with_columns(pl.coalesce('name_en_manual', 'name_en').alias('name_en'))
-    .unique() # remove duplicates from names with multiple matches in other dataframes
-    .drop('name_en_manual', 'name_jp')
-    # .filter(pl.col('name_en').is_null())
-)
-
 aux_df = (
     aux_df
     .filter(pl.col('type') != 'RUNNER')
@@ -257,9 +154,9 @@ data_df = (
         on='universal_code',
         how='left'
     )
-    .join(
-        players_df.rename({'name_en': 'pitcher_name'}), left_on='pitId', right_on='playerId', how='left'
-    )
+    # .join(
+        # players_df.rename({'name_en': 'pitcher_name'}), left_on='pitId', right_on='playerId', how='left'
+    # )
     .join(
         text_df[['GameID', 'GameKindID']].with_columns(
             pl.col('GameID').cast(pl.Int32),
@@ -294,7 +191,10 @@ data_df = (
         .alias('coarse_game_kind'),
 
         pl.when(pl.col('half_inning').str.ends_with(1)).then('HomeTeamNameES').otherwise('VisitorTeamNameES').alias('pitcher_team'),
-        pl.when(pl.col('half_inning').str.ends_with(1)).then('home_team_name_short').otherwise('visitor_team_name_short').alias('pitcher_team_name_short')
+        pl.when(pl.col('half_inning').str.ends_with(1)).then('home_team_name_short').otherwise('visitor_team_name_short').alias('pitcher_team_name_short'),
+
+        pl.when(pl.col('half_inning').str.ends_with(2)).then('HomeTeamNameES').otherwise('VisitorTeamNameES').alias('batter_team'),
+        pl.when(pl.col('half_inning').str.ends_with(2)).then('home_team_name_short').otherwise('visitor_team_name_short').alias('batter_team_name_short')
     )
     .with_columns(
         pl.col('presult_id').replace_strict(presult).alias('presult')
@@ -314,5 +214,123 @@ data_df = (
     .filter(pl.col('ballKind_code') != '-')
 )
 
+
+def select_name(names):
+  '''
+  When given mutiple names, 
+  prioritizes the name with ASCII characters (ex. R. マルティネス > マルティネス), 
+  followed by the shorter name (ex. 大勢 > 翁田 大勢)
+  Names with ASCII characters help differentiate between foreign players,
+  whlie shorter names are more accurate for players going by shorter names
+  '''
+  lens = []
+  for name in names:
+    if any([char in ascii_letters for char in name]):
+      return name
+    else:
+      lens.append(len(name))
+  return names[np.argmin(lens).item()]
+
+
+# load player dfs
+players_df = (
+    pl.read_parquet('files/players.parquet')
+    .with_columns(pl.col('playerName').str.normalize('NFKC').str.replace_all('・', ' '))
+    .group_by('playerId').agg(pl.col('playerName').map_elements(select_name, return_dtype=pl.String))
+)
+translated_df = (
+    pl.read_parquet('files/players_translated.parquet')
+    .with_columns(pl.col('name_jp').str.normalize('NFKC').str.replace_all('・', ' '))
+    # ['name_jp', 'name_kana', 'name_en']
+)
+manual_translated_df = pl.read_parquet('files/players_translated_manual.parquet')
+
+# get seasons and teams per player id
+batter_df = (
+    data_df
+    .with_columns(pl.col('date').dt.year().alias('season'))
+    .unique(['batId', 'batter_team', 'season'])
+    ['batId', 'batter_team', 'season']
+    .rename({'batId': 'playerId', 'batter_team': 'team'})
+)
+pitcher_df = (
+    data_df
+    .with_columns(pl.col('date').dt.year().alias('season'))
+    .unique(['pitId', 'pitcher_team', 'season'])
+    ['pitId', 'pitcher_team', 'season']
+    .rename({'pitId': 'playerId', 'pitcher_team': 'team'})
+)
+players_df = players_df.join(pl.concat((pitcher_df, batter_df)).unique(), on='playerId')
+
+# names with no romanization are approximated with kana translation
+kks = pykakasi.kakasi()
+
+# take names in parenthesis when they contain an ascii character
+translated_df = (
+    translated_df
+    .with_columns(
+        pl.when(pl.col('name_jp').str.contains(r'\('))
+        .then(pl.col('name_jp').str.extract(r'.*\(', 0).str.strip_chars_end(' ('))
+        .otherwise(pl.col('name_jp'))
+        .str.replace_all('・', ' ')
+        .alias('name_jp')
+    )
+    .with_columns(pl.col('name_kana').str.normalize('NFKC').str.replace_all('・', ' '))
+    .with_columns(pl.col('name_kana').str.extract(r'\(.*\)', 0).str.strip_chars('()').alias('in_parentheses'))
+    .with_columns(pl.col('name_kana').str.extract(r'.*\(', 0).str.strip_chars_end('(').alias('before_parentheses'))
+    .with_columns(
+        pl.when(pl.col('name_en').is_null())
+        .then
+         (
+            pl.when(pl.col('in_parentheses').is_not_null() | pl.col('before_parentheses').is_not_null())
+            .then(
+                pl.when(pl.col('in_parentheses').map_elements(lambda name: any([char in ascii_letters for char in name]), pl.Boolean))
+                .then(pl.col('in_parentheses'))
+                .otherwise(pl.col('before_parentheses'))
+            )
+            .otherwise(pl.col('name_kana').map_elements(lambda name: ''.join([word['hepburn'].capitalize() for word in kks.convert(name)]), return_dtype=pl.String))
+        )
+        .otherwise(pl.col('name_en'))
+        .alias('name_en')
+    )
+    .with_columns(pl.col('name_en').str.replace_all(',', '').str.to_titlecase())
+)
+
+# handle inconsistent kanji between sources
+for old_char, new_char in [
+    ('崎', '﨑'),
+    ('高', '髙'),
+    ('徳', '德'),
+    ('濱', '濵'),
+    ('瀬', '瀨')
+]:
+  players_df = (
+      players_df.with_columns(
+          pl.when(~pl.col('playerName').is_in(translated_df['name_jp']))
+          .then(pl.col('playerName').str.replace(old_char, new_char))
+          .otherwise('playerName')
+      )
+  )
+
+# merge player dfs
+players_df = (
+    players_df
+    .join(
+        translated_df
+        .with_columns(
+            pl.when(pl.col('name_jp').str.contains(r'\.') & ~pl.col('name_jp').is_in(players_df['playerName'].implode()))
+            .then(pl.col('name_jp').str.strip_chars(ascii_letters+'.'))
+            .otherwise('name_jp')
+        )
+        [['name_jp', 'name_en', 'team', 'season']], 
+        left_on=['playerName', 'season', 'team'], 
+        right_on=['name_jp', 'season', 'team']
+    )
+)
+print(players_df.filter(pl.len().over('playerId', 'team', 'season') > 1))
+players_df = pl.concat((players_df.group_by('playerId').agg(pl.first('name_en')), manual_translated_df[['playerId', 'name_en']]))
+
+# join players to data
+data_df = data_df.join(players_df.rename({'name_en': 'pitcher_name'}), left_on='pitId', right_on='playerId', how='left')
 if __name__ == '__main__':
   breakpoint()