init commit

.gitignore

@@ -1,2 +1,2 @@
 __pycache__/
-
+.env

.gradio/certificate.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw
+TzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh
+cmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4
+WhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu
+ZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY
+MTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc
+h77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+
+0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U
+A5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW
+T8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH
+B5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC
+B5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv
+KBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn
+OlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn
+jh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw
+qHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI
+rU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV
+HRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq
+hkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL
+ubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ
+3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK
+NFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5
+ORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur
+TkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC
+jNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc
+oyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq
+4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA
+mRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d
+emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=
+-----END CERTIFICATE-----

README.md

@@ -1,18 +1,70 @@
 ---
-title: PDF to Page Images Dataset
-emoji: 📂🤗
-colorFrom: pink
-colorTo: red
+title: Research Paper Pitfall Checker
+emoji: 🔍📄
+colorFrom: blue
+colorTo: purple
 sdk: gradio
 sdk_version: 4.44.0
 app_file: app.py
 pinned: false
-hf_oauth: true
-hf_oauth_scopes:
-- read-repos
-- write-repos
-- manage-repos
 license: agpl-3.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Research Paper Pitfall Checker
+
+A Gradio application that analyzes research papers to identify potential evaluation pitfalls using AI-powered analysis.
+
+## Features
+
+- **PDF Text Extraction**: Extracts text content from uploaded research papers
+- **AI-Powered Analysis**: Uses OpenRouter API with Qwen model for intelligent pitfall detection
+- **Comprehensive Pitfall Detection**: Identifies common evaluation pitfalls including:
+  - 🔒 The Lock-In Effect
+  - 🍎🍊 Apples-to-Oranges Comparisons  
+  - 💧 Contamination Leak
+  - 🤖❓ Unvalidated Automation
+  - 🧐 Vague Scales
+
+## How to Use
+
+1. **Get an API Key**: Sign up at [OpenRouter.ai](https://openrouter.ai) to get your API key
+2. **Set Environment Variable**: Set your API key as an environment variable
+3. **Upload a PDF**: Upload your research paper PDF file
+4. **Analyze**: Click "Analyze Paper for Pitfalls" to get a detailed analysis
+5. **Review Results**: Review the analysis report for potential issues and improvement suggestions
+
+## Setup
+
+1. Install dependencies:
+   ```bash
+   pip install -r requirements.in
+   ```
+
+2. Set your OpenRouter API key as an environment variable:
+   ```bash
+   export OPENROUTER_API_KEY="your-api-key-here"
+   ```
+   
+   Or create a `.env` file (copy from `env.example`):
+   ```
+   OPENROUTER_API_KEY=your-api-key-here
+   ```
+
+3. Run the application:
+   ```bash
+   python app.py
+   ```
+
+## Configuration
+
+The pitfalls are defined in `pitfalls.json`. You can modify this file to add or remove specific pitfalls to check for.
+
+## API Requirements
+
+- OpenRouter API key (get one from [OpenRouter.ai](https://openrouter.ai))
+- Uses the `qwen/qwen2.5-next-80b-a3b-instruct` model via OpenRouter
+- API key must be set as the `OPENROUTER_API_KEY` environment variable
+
+## License
+
+AGPL-3.0

app.py

@@ -1,303 +1,342 @@
-import multiprocessing
+import json
 import os
-import random
-import shutil
 import tempfile
-import zipfile
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime
+from typing import List, Dict, Any
 
 import fitz  # PyMuPDF
 import gradio as gr
-from huggingface_hub import DatasetCard, DatasetCardData, HfApi
-from PIL import Image
+from openai import OpenAI
 
+# Load API key from environment variable
+OPENROUTER_API_KEY = os.getenv("OPENROUTER_API_KEY")
 
-from dataset_card_template import DATASET_CARD_TEMPLATE
 
-os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
+def load_pitfalls() -> List[Dict[str, Any]]:
+    """Load pitfalls from the JSON file."""
+    try:
+        with open("pitfalls.json", "r") as f:
+            data = json.load(f)
+            return data.get("pitfalls", [])
+    except FileNotFoundError:
+        gr.Warning("pitfalls.json file not found!")
+        return []
+    except json.JSONDecodeError:
+        gr.Warning("Invalid JSON in pitfalls.json file!")
+        return []
+
+
+def extract_text_from_pdf(pdf_file) -> str:
+    """Extract text content from a PDF file."""
+    try:
+        pdf_path = pdf_file.name
+        doc = fitz.open(pdf_path)
+        text_content = ""
 
+        for page_num in range(len(doc)):
+            page = doc[page_num]
+            text_content += f"\n--- Page {page_num + 1} ---\n"
+            text_content += page.get_text()
 
-CPU_COUNT = multiprocessing.cpu_count()
-MAX_WORKERS = min(32, CPU_COUNT)  # Use CPU count directly for processes
+        doc.close()
+        return text_content
+    except Exception as e:
+        raise gr.Error(f"Error extracting text from {pdf_file.name}: {str(e)}")
 
 
-def process_pdf(pdf_file, sample_percentage, temp_dir):
-    try:
-        pdf_path = pdf_file.name
-        doc = fitz.open(pdf_path)
-        total_pages = len(doc)
+def format_paper_text(paper_text: str) -> Dict[str, Any]:
+    """First stage: Format the paper text to make it more readable and suitable for analysis."""
 
-        pages_to_convert = int(total_pages * (sample_percentage / 100))
-        pages_to_convert = max(
-            1, min(pages_to_convert, total_pages)
-        )  # Ensure at least one page and not more than total pages
+    # Check if API key is available
+    if not OPENROUTER_API_KEY:
+        return {
+            "formatted_text": None,
+            "success": False,
+            "error": "OpenRouter API key not found. Please set the OPENROUTER_API_KEY environment variable.",
+        }
 
-        selected_pages = (
-            sorted(random.sample(range(total_pages), pages_to_convert))
-            if 0 < sample_percentage < 100
-            else range(total_pages)
-        )
+    # Initialize OpenAI client with OpenRouter
+    client = OpenAI(
+        base_url="https://openrouter.ai/api/v1",
+        api_key=OPENROUTER_API_KEY,
+    )
 
-        images = []
-        for page_num in selected_pages:
-            page = doc[page_num]
-            pix = page.get_pixmap()  # Remove the Matrix scaling
-            image = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
-            image_path = os.path.join(
-                temp_dir, f"{os.path.basename(pdf_path)}_page_{page_num+1}.jpg"
-            )
-            image.save(image_path, "JPEG", quality=85, optimize=True)
-            images.append(image_path)
+    format_prompt = f"""You are an expert academic text processor. Your task is to clean and format the following research paper text to make it more readable and suitable for detailed analysis.
 
-        doc.close()
-        return images, None, len(images)
+Please:
+1. Remove excessive whitespace and formatting artifacts
+2. Organize the text into clear sections (Abstract, Introduction, Methods, Results, Discussion, Conclusion, References, Appendix)
+3. Preserve all important content including figures, tables, and equations
+4. Make the text flow better while maintaining academic integrity
+5. Ensure all evaluation-related content is clearly identifiable
+6. Keep the text under 8000 characters while preserving key information
+
+Original paper text:
+{paper_text}
+
+Please provide the cleaned and formatted text:"""
+
+    try:
+        completion = client.chat.completions.create(
+            extra_headers={
+                "HTTP-Referer": "https://github.com/paper-eval-checker",
+                "X-Title": "Paper Evaluation Pitfall Checker",
+            },
+            model="x-ai/grok-4-fast:free",
+            messages=[{"role": "user", "content": format_prompt}],
+            temperature=0.1,  # Very low temperature for consistent formatting
+            max_tokens=3000,
+        )
+
+        return {
+            "formatted_text": completion.choices[0].message.content,
+            "success": True,
+            "error": None,
+        }
     except Exception as e:
-        return [], f"Error processing {pdf_file.name}: {str(e)}", 0
+        return {"formatted_text": None, "success": False, "error": str(e)}
+
+
+def analyze_paper_for_pitfalls(
+    formatted_text: str, pitfalls: List[Dict[str, Any]]
+) -> Dict[str, Any]:
+    """Second stage: Use OpenRouter API with Grok model to analyze the formatted paper for potential pitfalls."""
+
+    # Check if API key is available
+    if not OPENROUTER_API_KEY:
+        return {
+            "analysis": None,
+            "success": False,
+            "error": "OpenRouter API key not found. Please set the OPENROUTER_API_KEY environment variable.",
+        }
+
+    # Initialize OpenAI client with OpenRouter
+    client = OpenAI(
+        base_url="https://openrouter.ai/api/v1",
+        api_key=OPENROUTER_API_KEY,
+    )
+
+    # Create the prompt for pitfall analysis
+    pitfalls_description = "\n\n".join(
+        [
+            f"**{pitfall['name']}** {pitfall['emoji']}\n"
+            f"Category: {pitfall['category']}\n"
+            f"Description: {pitfall['description']}\n"
+            f"Subjective/Objective: {pitfall['subjective_objective']}\n"
+            f"Actors Most Affected: {', '.join(pitfall['actors_most_affected'])}\n"
+            f"Evaluation Use: {pitfall['evaluation_use']}\n"
+            f"Modalities: {', '.join(pitfall['modalities'])}"
+            for pitfall in pitfalls
+        ]
+    )
+
+    analysis_prompt = f"""You are an expert research paper reviewer specializing in identifying evaluation pitfalls in academic papers. 
+
+Your task is to analyze the provided formatted research paper text and identify any potential pitfalls from the following list:
+
+{pitfalls_description}
+
+Please analyze the paper carefully and provide:
+1. A list of potential pitfalls found (if any)
+2. For each pitfall found, provide:
+   - The pitfall name
+   - Specific evidence from the paper that suggests this pitfall
+   - The section/page where this evidence appears
+   - A confidence level (High/Medium/Low) for your assessment
+   - Suggestions for improvement
+3. Be concise, and use markdown formatting.
+4. If you find evidence of a pitfall, make sure to look at ALL of the paper to see if it is mitigated elsewhere -- make sure to check the appendix of the paper as well.
 
+The output format:
 
-def pdf_to_images(pdf_files, sample_percentage, temp_dir, progress=gr.Progress()):
-    if not os.path.exists(temp_dir):
-        os.makedirs(temp_dir)
+# Overall
+<img src="https://img.shields.io/severity/high-red" alt="Severity: High"> (for low use green, for medium use yellow, for high use red)
+<img src="https://img.shields.io/evaluation/objective-blue" alt="Objective evaluation"> (either write 'subjective', 'objective', or include two images in case both are present in the paper)
+[One sentence summary of evaluation use]
 
-    progress(0, desc="Starting conversion")
-    all_images = []
-    skipped_pdfs = []
+# Pitfall 
 
-    total_pages = sum(len(fitz.open(pdf.name)) for pdf in pdf_files)
-    processed_pages = 0
+## Evidence
+"specific evidence from the paper"
 
-    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
-        future_to_pdf = {
-            executor.submit(process_pdf, pdf, sample_percentage, temp_dir): pdf
-            for pdf in pdf_files
+If no pitfalls are found, please state that clearly.
+
+Formatted paper text to analyze:
+{formatted_text}
+
+Please provide your analysis in a structured format."""
+
+    try:
+        completion = client.chat.completions.create(
+            extra_headers={
+                "HTTP-Referer": "https://github.com/paper-eval-checker",
+                "X-Title": "Paper Evaluation Pitfall Checker",
+            },
+            model="x-ai/grok-4-fast:free",
+            messages=[{"role": "user", "content": analysis_prompt}],
+            temperature=0.3,  # Lower temperature for more consistent analysis
+            max_tokens=2000,
+        )
+
+        return {
+            "analysis": completion.choices[0].message.content,
+            "success": True,
+            "error": None,
         }
+    except Exception as e:
+        return {"analysis": None, "success": False, "error": str(e)}
+
+
+def process_paper(pdf_file, progress=gr.Progress()):
+    """Main function to process a research paper for pitfall detection using two-stage approach."""
 
-        for future in as_completed(future_to_pdf):
-            pdf = future_to_pdf[future]
-            images, error, pages_processed = future.result()
-            if error:
-                skipped_pdfs.append(error)
-                gr.Info(error)
-            else:
-                all_images.extend(images)
-
-            processed_pages += pages_processed
-            progress((processed_pages / total_pages), desc=f"Processing {pdf.name}")
-
-    message = f"Saved {len(all_images)} images to temporary directory"
-    if skipped_pdfs:
-        message += f"\nSkipped {len(skipped_pdfs)} PDFs due to errors: {', '.join(skipped_pdfs)}"
-    return all_images, message
-
-
-def get_size_category(num_images):
-    if num_images < 1000:
-        return "n<1K"
-    elif num_images < 10000:
-        return "1K<n<10K"
-    elif num_images < 100000:
-        return "10K<n<100K"
-    elif num_images < 1000000:
-        return "100K<n<1M"
-    else:
-        return "n>1M"
-
-
-def process_pdfs(
-    pdf_files,
-    sample_percentage,
-    hf_repo,
-    create_zip,
-    private_repo,
-    oauth_token: gr.OAuthToken | None,
-    progress=gr.Progress(),
-):
-    if not pdf_files:
-        return (
-            None,
-            None,
-            gr.Markdown(
-                "⚠️ No PDF files uploaded. Please upload at least one PDF file."
-            ),
+    if not pdf_file:
+        return gr.Markdown(
+            "⚠️ No PDF file uploaded. Please upload a research paper PDF."
         )
 
-    # if oauth_token is None:
-    #     return (
-    #         None,
-    #         None,
-    #         gr.Markdown(
-    #             "⚠️ Not logged in to Hugging Face. Please log in to upload to a Hugging Face dataset."
-    #         ),
-    #     )
+    if not OPENROUTER_API_KEY:
+        return gr.Markdown(
+            "⚠️ OpenRouter API key not found. Please set the OPENROUTER_API_KEY environment variable."
+        )
 
     try:
-        temp_dir = tempfile.mkdtemp()
-        images_dir = os.path.join(temp_dir, "images")
-        os.makedirs(images_dir)
-
-        progress(0, desc="Starting PDF processing")
-        images, message = pdf_to_images(pdf_files, sample_percentage, images_dir)
-
-        # Create a new directory for sampled images
-        sampled_images_dir = os.path.join(temp_dir, "sampled_images")
-        os.makedirs(sampled_images_dir)
-
-        # Move sampled images to the new directory and update paths
-        updated_images = []
-        for image in images:
-            new_path = os.path.join(sampled_images_dir, os.path.basename(image))
-            shutil.move(image, new_path)
-            updated_images.append(new_path)
-
-        # Update the images list with new paths
-        images = updated_images
-
-        zip_path = None
-        if create_zip:
-            # Create a zip file of the sampled images
-            zip_path = os.path.join(temp_dir, "converted_images.zip")
-            with zipfile.ZipFile(zip_path, "w") as zipf:
-                progress(0, desc="Zipping images")
-                for image in progress.tqdm(images, desc="Zipping images"):
-                    zipf.write(
-                        os.path.join(sampled_images_dir, os.path.basename(image)),
-                        os.path.basename(image),
-                    )
-            message += f"\nCreated zip file with {len(images)} images"
-
-        if hf_repo:
-            if oauth_token is None:
-                raise gr.Error(
-                    "Not logged in to Hugging Face. Please log in to upload to a Hugging Face dataset."
-                )
-            try:
-                hf_api = HfApi(token=oauth_token.token)
-                hf_api.create_repo(
-                    hf_repo,
-                    repo_type="dataset",
-                    private=private_repo,
-                )
-                # Upload only the sampled images directory
-                hf_api.upload_folder(
-                    folder_path=sampled_images_dir,
-                    repo_id=hf_repo,
-                    repo_type="dataset",
-                    path_in_repo="images",
-                )
-
-                # Determine size category
-                size_category = get_size_category(len(images))
-
-                # Create DatasetCardData instance
-                card_data = DatasetCardData(
-                    tags=["created-with-pdfs-to-page-images-converter", "pdf-to-image"],
-                    size_categories=[size_category],
-                )
-
-                # Create and populate the dataset card
-                card = DatasetCard.from_template(
-                    card_data,
-                    template_path=None,  # Use default template
-                    hf_repo=hf_repo,
-                    num_images=len(images),
-                    num_pdfs=len(pdf_files),
-                    sample_size=sample_percentage
-                    if sample_percentage > 0
-                    else "All pages",
-                    creation_date=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-                )
-
-                # Add our custom content to the card
-                card.text = DATASET_CARD_TEMPLATE.format(
-                    hf_repo=hf_repo,
-                    num_images=len(images),
-                    num_pdfs=len(pdf_files),
-                    sample_size=sample_percentage
-                    if sample_percentage > 0
-                    else "All pages",
-                    creation_date=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-                    size_category=size_category,
-                )
-
-                repo_url = f"https://huggingface.co/datasets/{hf_repo}"
-                message += f"\nUploaded dataset card to Hugging Face repo: [{hf_repo}]({repo_url})"
-
-                card.push_to_hub(hf_repo, token=oauth_token.token)
-            except Exception as e:
-                message += f"\nFailed to upload to Hugging Face: {str(e)}"
-
-        return images, zip_path, message
+        # Step 1: Load pitfalls
+        progress(0.1, desc="Loading pitfalls definitions...")
+        pitfalls = load_pitfalls()
+
+        if not pitfalls:
+            return gr.Markdown(
+                "⚠️ No pitfalls definitions found. Please check pitfalls.json file."
+            )
+
+        # Step 2: Extract text from PDF
+        progress(0.2, desc="Extracting text from PDF...")
+        paper_text = extract_text_from_pdf(pdf_file)
+
+        if not paper_text.strip():
+            return gr.Markdown(
+                "⚠️ No text content found in the PDF. Please check if the PDF contains readable text."
+            )
+
+        # Step 3: Format paper text (First AI call)
+        progress(0.3, desc="Formatting paper text for analysis...")
+        format_result = format_paper_text(paper_text)
+
+        if not format_result["success"]:
+            return gr.Markdown(
+                f"❌ Error during text formatting: {format_result['error']}"
+            )
+
+        # Step 4: Analyze for pitfalls (Second AI call)
+        progress(0.7, desc="Analyzing paper for potential pitfalls...")
+        analysis_result = analyze_paper_for_pitfalls(
+            format_result["formatted_text"], pitfalls
+        )
+
+        if not analysis_result["success"]:
+            return gr.Markdown(f"❌ Error during analysis: {analysis_result['error']}")
+
+        # Step 5: Format final results
+        progress(0.9, desc="Preparing final report...")
+        analysis_text = analysis_result["analysis"]
+
+        # Create a formatted markdown report
+        report = f"""# Research Paper Pitfall Analysis Report
+
+## Analysis Results
+
+{analysis_text}
+
+---
+*Analysis completed using OpenRouter API with Grok model (two-stage processing)*
+"""
+
+        progress(1.0, desc="Analysis complete!")
+        return gr.Markdown(report)
+
     except Exception as e:
-        if "temp_dir" in locals():
-            shutil.rmtree(temp_dir)
-        return None, None, f"An error occurred: {str(e)}"
+        return gr.Markdown(f"❌ An error occurred: {str(e)}")
 
 
 # Define the Gradio interface
-with gr.Blocks() as demo:
+with gr.Blocks(title="Research Paper Pitfall Checker") as demo:
     gr.HTML(
-        """<h1 style='text-align: center;'> PDFs to Page Images Converter</h1>
-        <center><i> &#128193; Convert PDFs to an image dataset, splitting pages into individual images &#128193; </i></center>"""
+        """<h1 style='text-align: center;'>🔍 Research Paper Pitfall Checker</h1>
+        <center><i>Identify potential evaluation pitfalls in academic research papers</i></center>"""
     )
+
     gr.HTML(
         """
-        <div style="display: flex; justify-content: center; align-items: center; max-width: 1000px; margin: 0 auto;">
-            <div style="flex: 1; padding-right: 20px;">
-                <p>This app allows you to:</p>
-                <ol>
-                    <li>Upload one or more PDF files</li>
-                    <li>Convert each page of the PDFs into separate image files</li>
-                    <li>(Optionally) sample a specific number of pages from each PDF</li>
-                    <li>(Optionally) Create a downloadable ZIP file of the converted images</li>
-                    <li>(Optionally) Upload the images to a Hugging Face dataset repository</li>
+        <div style="max-width: 800px; margin: 0 auto; padding: 20px;">
+            <h3>How it works:</h3>
+            <ol>
+                <li><strong>Upload a PDF</strong> of your research paper</li>
+                <li><strong>Click "Analyze Paper"</strong> to scan for potential pitfalls</li>
+                <li><strong>Review the analysis</strong> to identify areas for improvement</li>
                 </ol>
-            </div>
-            <div style="flex: 1;">
-                <img src="https://huggingface.co/spaces/Dataset-Creation-Tools/pdf-to-page-images-dataset/resolve/main/assets/PDF%20page%20split%20illustration.png" 
-                     alt="PDF page split illustration" 
-                     style="max-width: 50%; height: auto;">
-            </div>
+            
+            <h3>Supported Pitfalls:</h3>
+            <ul>
+                <li>🔒 The Lock-In Effect</li>
+                <li>🍎🍊 Apples-to-Oranges Comparisons</li>
+                <li>💧 Contamination Leak</li>
+                <li>🤖❓ Unvalidated Automation</li>
+                <li>🧐 Vague Scales</li>
+            </ul>
         </div>
         """
     )
 
     with gr.Row():
-        pdf_files = gr.File(
-            file_count="multiple", label="Upload PDF(s)", file_types=["*.pdf"]
-        )
+        with gr.Column(scale=3):
+            pdf_file = gr.File(label="Upload Research Paper (PDF)", file_types=[".pdf"])
+
+        with gr.Column(scale=1):
+            analyze_button = gr.Button(
+                "🔍 Analyze Paper for Pitfalls",
+                variant="primary",
+                size="lg",
+                elem_id="analyze-btn",
+            )
 
     with gr.Row():
-        sample_percentage = gr.Slider(
-            minimum=0,
-            maximum=100,
-            value=100,
-            step=1,
-            label="Percentage of pages to sample per PDF",
-            info="0% for no sampling (all pages), 100% for all pages",
+        results = gr.Markdown(
+            value="Upload a PDF to get started with pitfall analysis.",
+            elem_id="results",
         )
-        create_zip = gr.Checkbox(label="Create ZIP file of images?", value=False)
-
-    with gr.Accordion("Hugging Face Upload Options", open=True):
-        gr.LoginButton(size="sm")
-        with gr.Row():
-            hf_repo = gr.Textbox(
-                label="Hugging Face Repo",
-                placeholder="username/repo-name",
-                info="Enter the Hugging Face repository name in the format 'username/repo-name'",
-            )
-            private_repo = gr.Checkbox(label="Make repository private?", value=False)
-
-    with gr.Accordion("View converted images", open=False):
-        output_gallery = gr.Gallery(label="Converted Images")
 
-    status_text = gr.Markdown(label="Status")
-    download_button = gr.File(label="Download Converted Images")
-
-    submit_button = gr.Button("Convert PDFs to page images")
-    submit_button.click(
-        process_pdfs,
-        inputs=[pdf_files, sample_percentage, hf_repo, create_zip, private_repo],
-        outputs=[output_gallery, download_button, status_text],
+    # Add loading animation CSS
+    demo.css = """
+    #analyze-btn {
+        background: linear-gradient(45deg, #ff6b6b, #4ecdc4, #45b7d1, #96ceb4, #feca57);
+        background-size: 400% 400%;
+        animation: gradient 3s ease infinite;
+        border: none;
+        color: white;
+        font-weight: bold;
+    }
+    
+    @keyframes gradient {
+        0% { background-position: 0% 50%; }
+        50% { background-position: 100% 50%; }
+        100% { background-position: 0% 50%; }
+    }
+    
+    #results {
+        min-height: 200px;
+        padding: 20px;
+        border: 1px solid #e0e0e0;
+        border-radius: 8px;
+        background-color: #fafafa;
+    }
+    """
+
+    # Connect the button to the processing function
+    analyze_button.click(
+        fn=process_paper,
+        inputs=[pdf_file],
+        outputs=[results],
     )
-demo.launch()
+
+if __name__ == "__main__":
+    demo.launch(share=True, server_name="localhost", server_port=9090)

dataset_card_template.py

@@ -1,40 +0,0 @@
-DATASET_CARD_TEMPLATE = """
-# Dataset Card for {hf_repo}
-
-## Dataset Description
-
-This dataset contains images converted from PDFs using the PDFs to Page Images Converter Space.
-
-- **Number of images:** {num_images}
-- **Number of PDFs processed:** {num_pdfs}
-- **Sample size per PDF:** {sample_size}
-- **Created on:** {creation_date}
-
-## Dataset Creation
-
-### Source Data
-
-The images in this dataset were generated from user-uploaded PDF files.
-
-### Processing Steps
-
-1. PDF files were uploaded to the PDFs to Page Images Converter.
-2. Each PDF was processed, converting selected pages to images.
-3. The resulting images were saved and uploaded to this dataset.
-
-## Dataset Structure
-
-The dataset consists of JPEG images, each representing a single page from the source PDFs.
-
-### Data Fields
-
-- `images/`: A folder containing all the converted images.
-
-### Data Splits
-
-This dataset does not have specific splits.
-
-## Additional Information
-
-- **Contributions:** Thanks to the PDFs to Page Images Converter for creating this dataset.
-"""

env.example

@@ -0,0 +1,3 @@
+# OpenRouter API Key
+# Get your API key from https://openrouter.ai
+OPENROUTER_API_KEY=your-api-key-here

packages.txt

@@ -1 +0,0 @@
-poppler-utils 

pitfalls.json

@@ -0,0 +1,92 @@
+{
+    "pitfalls": [
+        {
+            "name": "The Lock-In Effect",
+            "emoji": "🔒",
+            "category": "General",
+            "description": "Practices known to be problematic remain widespread simply because they are already widespread, making it difficult for new, better methods to be adopted.",
+            "subjective_objective": "Both",
+            "actors_most_affected": [
+                "Academic researcher",
+                "Model creator"
+            ],
+            "evaluation_use": "Compare models",
+            "modalities": [
+                "General"
+            ],
+            "sources": []
+        },
+        {
+            "name": "Apples-to-Oranges",
+            "emoji": "🍎🍊",
+            "category": "General",
+            "description": "Models or data are compared on an unequal footing, such as evaluating models using a different number of examples or under different conditions.",
+            "subjective_objective": "Both",
+            "actors_most_affected": [
+                "Academic researcher",
+                "Model deployer"
+            ],
+            "evaluation_use": "Compare models",
+            "modalities": [
+                "General",
+                "NLP",
+                "Speech"
+            ],
+            "sources": []
+        },
+        {
+            "name": "Contamination Leak",
+            "emoji": "💧",
+            "category": "General",
+            "description": "The model has already been exposed to the evaluation data during its training phase, which invalidates the results. This is a widespread and subtle problem.",
+            "subjective_objective": "Both",
+            "actors_most_affected": [
+                "Academic researcher",
+                "Model creator"
+            ],
+            "evaluation_use": "Compare models, assess system reliability",
+            "modalities": [
+                "General"
+            ],
+            "sources": []
+        },
+        {
+            "name": "Unvalidated Automation",
+            "emoji": "🤖❓",
+            "category": "NLP",
+            "description": "Using an LLM-as-a-judge to evaluate outputs without first validating the judge LLM's performance against human experts or established criteria. While LLMs can scale evaluation, they are not yet reliable enough to be the sole evaluators.",
+            "subjective_objective": "Both",
+            "actors_most_affected": [
+                "Academic researcher",
+                "Model creator",
+                "Model deployer"
+            ],
+            "evaluation_use": "Assess system reliability",
+            "modalities": [
+                "Text",
+                "General"
+            ],
+            "sources": [
+                "The LLM Evaluation guidebook"
+            ]
+        },
+        {
+            "name": "Vague Scales",
+            "emoji": "🧐",
+            "category": "TTS",
+            "description": "Papers on synthetic speech fail to report crucial details, such as whether they are evaluating 'quality' or 'naturalness,' or do not disclose the labels used in their Mean Opinion Score (MOS) scale.",
+            "subjective_objective": "Subjective",
+            "actors_most_affected": [
+                "Academic researcher"
+            ],
+            "evaluation_use": "Compare models, assess system reliability",
+            "modalities": [
+                "Speech"
+            ],
+            "sources": [
+                "Good practices for evaluation of synthesized speech",
+                "Hot topics in speech synthesis evaluation"
+            ]
+        }
+    ]
+}

requirements.in

@@ -1,6 +1,4 @@
 
-Pillow
-gradio[oauth]==4.44.0
-huggingface_hub[hf_transfer]
-pdf2image
+gradio==4.44.0
+openai
 PyMuPDF