Initial for supporting smolagent translation

agent/handler.py

@@ -13,12 +13,14 @@ from agent.workflow import (
 )
 from pr_generator.searcher import find_reference_pr_simple_stream
 from translator.content import get_full_prompt, get_content, preprocess_content
+from translator.project_config import get_available_projects, get_project_config
 
 
 # State management
 class ChatState:
     def __init__(self):
-        self.step = "welcome"  # welcome -> find_files -> translate -> create_github_pr
+        self.step = "welcome"  # welcome -> select_project -> find_files -> translate -> create_github_pr
+        self.selected_project = "transformers"  # Default project
         self.target_language = "ko"
         self.k_files = 10
         self.files_to_translate = []
@@ -53,25 +55,26 @@ def _extract_content_for_display(content: str) -> str:
 
 
 def get_welcome_message():
-    """Initial welcome message with file finding controls"""
+    """Initial welcome message with project selection"""
     return """**👋 Welcome to 🌐 Hugging Face i18n Translation Agent!**
 
 I'll help you find files that need translation and translate them in a streamlined workflow.
 
-**🔎 Let's start by finding files that need translation.**
+**🎯 First, select which project you want to translate:**
 
-Use the **`Quick Controls`** on the right or **ask me `what`, `how`, or `help`** to get started.
+Use the **`Quick Controls`** on the right to select a project, or **ask me `what`, `how`, or `help`** to get started.
 """
 
 
-def process_file_search_handler(lang: str, k: int, history: list) -> tuple:
+def process_file_search_handler(project: str, lang: str, k: int, history: list) -> tuple:
     """Process file search request and update Gradio UI components."""
     global state
+    state.selected_project = project
     state.target_language = lang
     state.k_files = k
     state.step = "find_files"
 
-    status_report, files_list = report_translation_target_files(lang, k)
+    status_report, files_list = report_translation_target_files(project, lang, k)
     state.files_to_translate = (
         [file[0] for file in files_list]
         if files_list
@@ -87,8 +90,10 @@ def process_file_search_handler(lang: str, k: int, history: list) -> tuple:
 """
 
     if state.files_to_translate:
+        config = get_project_config(state.selected_project)
         for i, file in enumerate(state.files_to_translate, 1):
-            response += f"\n{i}. `{file}`"
+            file_link = f"{config.repo_url}/blob/main/{file}"
+            response += f"\n{i}. [`{file}`]({file_link})"
 
         # if len(state.files_to_translate) > 5:
         #     response += f"\n... and {len(state.files_to_translate) - 5} more files"
@@ -138,9 +143,8 @@ def start_translation_process():
         p.parent.mkdir(parents=True, exist_ok=True)
         p.write_text(translated, encoding="utf-8")
 
-        original_file_link = (
-            "https://github.com/huggingface/transformers/blob/main/" + current_file
-        )
+        config = get_project_config(state.selected_project)
+        original_file_link = f"{config.repo_url}/blob/main/{current_file}"
         print("Compeleted translation:\n")
         print(translated)
         print("----------------------------")
@@ -226,12 +230,12 @@ def handle_user_message(message, history):
 
 def update_status():
     if state.step == "welcome":
-        return """
+        return f"""
         <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px;padding: 10px; background: rgba(0, 0, 0, 0.25); border-radius: 8px;">
             <div><strong>🔄 Step:</strong> Welcome</div>
+            <div><strong>🎯 Project:</strong> {state.selected_project}</div>
             <div><strong>📁 Files:</strong> 0</div>
-            <div><strong>🌍 Language:</strong> ko</div>
-            <div><strong>⏳ Progress:</strong> Ready</div>
+            <div><strong>🌍 Language:</strong> {state.target_language}</div>
         </div>
         """
 
@@ -267,6 +271,7 @@ def update_status():
     status_html = f"""
     <div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px; padding: 10px; background: rgba(0, 0, 0, 0.25); border-radius: 8px;">
         <div><strong>🔄 Step:</strong> {step_map.get(state.step, state.step)}</div>
+        <div><strong>🎯 Project:</strong> {state.selected_project}</div>
         <div><strong>📁 Files:</strong> {len(state.files_to_translate)}</div>
         <div><strong>🌍 Language:</strong> {state.target_language}</div>
         <div><strong>⏳ Progress:</strong> {progress_map.get(state.step, 'In progress')}</div>
@@ -292,14 +297,18 @@ def update_github_config(token, owner, repo, reference_pr_url):
     if token:
         os.environ["GITHUB_TOKEN"] = token
 
+    # Get default reference PR URL from project config if not provided
+    if not reference_pr_url:
+        config = get_project_config(state.selected_project)
+        reference_pr_url = config.reference_pr_url
+
     # Save GitHub configuration to state
     state.github_config.update(
         {
             "token": token,
             "owner": owner,
             "repo_name": repo,
-            "reference_pr_url": reference_pr_url
-            or state.github_config["reference_pr_url"],
+            "reference_pr_url": reference_pr_url,
         }
     )
 

agent/workflow.py

@@ -26,19 +26,20 @@ from logger.github_logger import GitHubLogger
 
 
 def report_translation_target_files(
-    translate_lang: str, top_k: int = 1
+    project: str, translate_lang: str, top_k: int = 1
 ) -> tuple[str, list[list[str]]]:
     """Return the top-k files that need translation, excluding files already in progress.
 
     Args:
+        project: Project to translate (e.g., "transformers", "smolagents")
         translate_lang: Target language to translate
         top_k: Number of top-first files to return for translation. (Default 1)
     """
     # Get files in progress
-    docs_in_progress, pr_info_list = get_github_issue_open_pr(translate_lang)
+    docs_in_progress, pr_info_list = get_github_issue_open_pr(project, translate_lang)
 
     # Get all available files for translation
-    all_status_report, all_filepath_list = report(translate_lang, top_k * 2)  # Get more to account for filtering
+    all_status_report, all_filepath_list = report(project, translate_lang, top_k * 2)  # Get more to account for filtering
 
     # Filter out files that are already in progress
     available_files = [f for f in all_filepath_list if f not in docs_in_progress]
@@ -52,7 +53,7 @@ def report_translation_target_files(
     if docs_in_progress:
         status_report += f"\n\n🤖 Found {len(docs_in_progress)} files in progress for translation:"
         for i, file in enumerate(docs_in_progress):
-            status_report += f"\n{i+1}. `{file}`: {pr_info_list[i]}"
+            status_report += f"\n{i+1}. [`{file}`]({pr_info_list[i]})"
         status_report += f"\n\n📋 Showing {len(filepath_list)} available files (excluding in-progress):"
 
     return status_report, [[file] for file in filepath_list]

app.py

@@ -19,6 +19,7 @@ from agent.handler import (
     update_github_config,
 )
 from translator.model import Languages
+from translator.project_config import get_available_projects
 
 load_dotenv()
 
@@ -125,6 +126,11 @@ with gr.Blocks(
                 with gr.Tabs(elem_classes="simple-tabs") as control_tabs:
                     with gr.TabItem("1. Find Files", id=0):
                         with gr.Group():
+                            project_dropdown = gr.Radio(
+                                choices=get_available_projects(),
+                                label="🎯 Select Project",
+                                value="transformers",
+                            )
                             lang_dropdown = gr.Radio(
                                 choices=[language.value for language in Languages],
                                 label="🌍 Translate To",
@@ -226,7 +232,7 @@ with gr.Blocks(
 
     find_btn.click(
         fn=process_file_search_handler,
-        inputs=[lang_dropdown, k_input, chatbot],
+        inputs=[project_dropdown, lang_dropdown, k_input, chatbot],
         outputs=[chatbot, msg_input, status_display, control_tabs, files_to_translate],
     )
 

example.env

@@ -13,6 +13,5 @@ HF_SPACE_NAME=
 
 # Secrets for logging to Github
 LOG_REPO=
-LOG_GITHUB_TOKEN=
 LOG_BRANCH=
 LOG_FILE_PATH=

logger/github_logger.py

@@ -12,7 +12,7 @@ class GitHubLogger:
     """Dedicated logger that appends JSONL entries to a GitHub repo/branch/file.
 
     Env vars:
-      - LOG_GITHUB_TOKEN (fallback: GITHUB_TOKEN)
+      - GITHUB_TOKEN
       - LOG_REPO (format: owner/repo)
       - LOG_BRANCH (default: 'log_event')
       - LOG_FILE_PATH (default: 'pr_success.log')
@@ -21,9 +21,9 @@ class GitHubLogger:
     def __init__(self):
         if not LIBS_OK:
             raise ImportError("PyGithub not installed. Please install PyGithub.")
-        token = os.environ.get("LOG_GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN")
+        token = os.environ.get("GITHUB_TOKEN")
         if not token:
-            raise ValueError("Missing LOG_GITHUB_TOKEN or GITHUB_TOKEN for logging")
+            raise ValueError("Missing GITHUB_TOKEN for logging")
         self._client = Github(token)
 
         repo_spec = os.environ.get("LOG_REPO")

translator/project_config.py

@@ -0,0 +1,48 @@
+"""Project configuration for different HuggingFace repositories."""
+
+from dataclasses import dataclass
+from typing import Dict
+
+
+@dataclass
+class ProjectConfig:
+    """Configuration for a specific HuggingFace project."""
+    name: str
+    repo_url: str
+    api_url: str
+    docs_path: str
+    github_issues: Dict[str, str]  # language -> issue_id
+    reference_pr_url: str
+
+
+# Project configurations
+PROJECTS = {
+    "transformers": ProjectConfig(
+        name="Transformers",
+        repo_url="https://github.com/huggingface/transformers",
+        api_url="https://api.github.com/repos/huggingface/transformers/git/trees/main?recursive=1",
+        docs_path="docs/source",
+        github_issues={"ko": "20179"},
+        reference_pr_url="https://github.com/huggingface/transformers/pull/24968"
+    ),
+    "smolagents": ProjectConfig(
+        name="SmolAgents",
+        repo_url="https://github.com/huggingface/smolagents",
+        api_url="https://api.github.com/repos/huggingface/smolagents/git/trees/main?recursive=1",
+        docs_path="docs/source", 
+        github_issues={"ko": "20179"},  # To be filled when issue is created
+        reference_pr_url="https://github.com/huggingface/smolagents/pull/1581"  # To be filled with actual PR URL
+    )
+}
+
+
+def get_project_config(project_key: str) -> ProjectConfig:
+    """Get project configuration by key."""
+    if project_key not in PROJECTS:
+        raise ValueError(f"Unknown project: {project_key}. Available: {list(PROJECTS.keys())}")
+    return PROJECTS[project_key]
+
+
+def get_available_projects() -> list[str]:
+    """Get list of available project keys."""
+    return list(PROJECTS.keys())

translator/retriever.py

@@ -5,15 +5,22 @@ from pathlib import Path
 import requests
 
 from .model import Languages, Summary, TranslationDoc
+from .project_config import get_project_config
 
-URL = "https://api.github.com/repos/huggingface/transformers/git/trees/main?recursive=1"
 
-
-def get_github_repo_files():
+def get_github_repo_files(project: str = "transformers"):
     """
     Get github repo files
     """
-    response = requests.get(URL)
+    config = get_project_config(project)
+    
+    # Add GitHub token if available to avoid rate limiting
+    headers = {}
+    github_token = os.environ.get("GITHUB_TOKEN")
+    if github_token:
+        headers["Authorization"] = f"token {github_token}"
+    
+    response = requests.get(config.api_url, headers=headers)
 
     data = response.json()
     all_items = data.get("tree", [])
@@ -26,27 +33,33 @@ def get_github_repo_files():
     return file_paths
 
 
-def get_github_issue_open_pr(lang: str = "ko"):
+def get_github_issue_open_pr(project: str = "transformers", lang: str = "ko"):
     """
-    Get open PR in the github issue, filtered by title starting with '🌐 [i18n-KO]'.
+    Get open PR in the github issue, filtered by title containing '[i18n-KO]'.
     """
-    if lang == "ko":
-        issue_id = "20179"
-    else:
-        raise ValueError(
-            "No Github issue has been registered to the server. (Only 'ko' is supported - please contact us to support this.)"
-        )
+    config = get_project_config(project)
+    issue_id = config.github_issues.get(lang)
+    
+    # For projects without GitHub issue tracking, still search for PRs
+    if not issue_id:
+        raise ValueError(f"⚠️ No GitHub issue registered for {project}.")
 
     headers = {
         "Accept": "application/vnd.github+json",
     }
     
+    # Add GitHub token if available to avoid rate limiting
+    github_token = os.environ.get("GITHUB_TOKEN")
+    if github_token:
+        headers["Authorization"] = f"token {github_token}"
+    
     all_open_prs = []
     page = 1
     per_page = 100  # Maximum allowed by GitHub API
     
     while True:
-        url = f"https://api.github.com/repos/huggingface/transformers/pulls?state=open&page={page}&per_page={per_page}"
+        repo_path = config.repo_url.replace("https://github.com/", "")
+        url = f"https://api.github.com/repos/{repo_path}/pulls?state=open&page={page}&per_page={per_page}"
         response = requests.get(url, headers=headers)
         
         if response.status_code != 200:
@@ -63,17 +76,20 @@ def get_github_issue_open_pr(lang: str = "ko"):
         if len(page_prs) < per_page:
             break
 
-    filtered_prs = [pr for pr in all_open_prs if pr["title"].startswith("🌐 [i18n-KO]")]
+    filtered_prs = [pr for pr in all_open_prs if "[i18n-KO]" in pr["title"]]
 
-    pattern = re.compile(r"`([^`]+\.md)`")
+    # Pattern to match both `filename.md` and filename.md formats
+    pattern = re.compile(r"(?:`([^`]+\.md)`|(\w+\.md))")
 
-    filenames = [
-        "docs/source/en/" + match.group(1)
-        for pr in filtered_prs
-        if (match := pattern.search(pr["title"]))
-    ]
+    filenames = []
+    for pr in filtered_prs:
+        match = pattern.search(pr["title"])
+        if match:
+            # Use group 1 (with backticks) or group 2 (without backticks)
+            filename = match.group(1) or match.group(2)
+            filenames.append("docs/source/en/" + filename)
     pr_info_list = [
-        f"https://github.com/huggingface/transformers/pull/{pr['url'].rstrip('/').split('/')[-1]}"
+        f"{config.repo_url}/pull/{pr['url'].rstrip('/').split('/')[-1]}"
         for pr in filtered_prs
     ]
     return filenames, pr_info_list
@@ -99,11 +115,11 @@ def retrieve(summary: Summary, table_size: int = 10) -> tuple[str, list[str]]:
     return report, first_missing_docs
 
 
-def report(target_lang: str, top_k: int = 1) -> tuple[str, list[str]]:
+def report(project: str, target_lang: str, top_k: int = 1) -> tuple[str, list[str]]:
     """
     Generate a report for the translated docs
     """
-    docs_file = get_github_repo_files()
+    docs_file = get_github_repo_files(project)
 
     base_docs_path = Path("docs/source")
     en_docs_path = Path("docs/source/en")