backup

app.py

@@ -110,12 +110,13 @@ scheduler = CommitScheduler(
 )
 
 
-def save_evaluation(post_id, model_a, model_b, verdict, username, start_time, end_time):
-    """Save evaluation results to CSV including timing and username information."""
+def save_evaluation(
+    post_id, model_a, model_b, verdict, username, start_time, end_time, dataset_idx
+):
+    """Save evaluation results to CSV including timing, username and dataset index information."""
     timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    duration = end_time - start_time  # Calculate duration in seconds
+    duration = end_time - start_time
 
-    # Create data directory if it doesn't exist
     os.makedirs("data", exist_ok=True)
     filename = "data/evaluation_results_exp.csv"
 
@@ -134,6 +135,7 @@ def save_evaluation(post_id, model_a, model_b, verdict, username, start_time, en
                     "start_time",
                     "end_time",
                     "duration_seconds",
+                    "dataset_idx",
                 ]
             )
 
@@ -151,6 +153,7 @@ def save_evaluation(post_id, model_a, model_b, verdict, username, start_time, en
                 start_time,
                 end_time,
                 duration,
+                dataset_idx,
             ]
         )
 
@@ -159,57 +162,52 @@ def save_evaluation(post_id, model_a, model_b, verdict, username, start_time, en
     )
 
 
-def get_previously_annotated_posts(username):
-    """Get list of post_ids already annotated by this username."""
+def get_annotated_indices(username):
+    """Get list of dataset indices already annotated by this user"""
     filename = "data/evaluation_results_exp.csv"
     if not os.path.exists(filename):
         return set()
 
-    df = pd.read_csv(filename)
-    user_annotations = df[df["username"] == username]["post_id"].unique()
-    return set(user_annotations)
+    try:
+        df = pd.read_csv(filename)
+        if "dataset_idx" not in df.columns or "username" not in df.columns:
+            return set()
+        user_annotations = df[df["username"] == username]["dataset_idx"].tolist()
+        return set(user_annotations)
+    except:
+        return set()
 
 
 def get_random_sample(username):
-    """Get a random sample from the dataset, avoiding previously annotated items."""
-    # Get previously annotated posts for this user
-    annotated_posts = get_previously_annotated_posts(username)
-
-    # Get available post_ids (not yet annotated by this user)
-    available_posts = set(dataset_post_ids) - annotated_posts
-
-    if not available_posts:
-        # If user has annotated all posts, allow repeats
-        available_posts = set(dataset_post_ids)
-        print(f"User {username} has annotated all posts, allowing repeats")
-
-    # Select random post_id from available ones
-    selected_post_id = random.choice(list(available_posts))
-
-    # Find the corresponding dataset index
-    for idx, item in enumerate(dataset):
-        if item["post_id"] == selected_post_id:
-            sample = item
-            break
-    else:
-        raise ValueError(f"Could not find post_id {selected_post_id} in dataset")
+    """Get a random sample excluding previously annotated items"""
+    # Get indices already annotated by this user
+    annotated_indices = get_annotated_indices(username)
+
+    # Get all valid indices that haven't been annotated
+    all_indices = set(range(len(dataset)))
+    available_indices = list(all_indices - annotated_indices)
+
+    if not available_indices:
+        # If user has annotated all items, allow repeats
+        available_indices = list(all_indices)
+
+    # Randomly select from available indices
+    idx = random.choice(available_indices)
+    sample = dataset[idx]
 
     # Randomly decide which image goes to position A and B
     if random.choice([True, False]):
         # AI edit is A, human edit is B
         image_a = sample["ai_edited_image"]
         image_b = sample["human_edited_image"]
-        model_a = sample["model"]  # AI model name
-        model_b = "HUMAN"  # Human edit identifier
+        model_a = sample["model"]
+        model_b = "HUMAN"
     else:
         # Human edit is A, AI edit is B
         image_a = sample["human_edited_image"]
         image_b = sample["ai_edited_image"]
-        model_a = "HUMAN"  # Human edit identifier
-        model_b = sample["model"]  # AI model name
-
-    print(f"Selected post_id: {sample['post_id']}")
-    print(f"AI model: {sample['model']}")
+        model_a = "HUMAN"
+        model_b = sample["model"]
 
     return {
         "post_id": sample["post_id"],
@@ -222,6 +220,7 @@ def get_random_sample(username):
         "image_b": image_b,
         "model_a": model_a,
         "model_b": model_b,
+        "dataset_idx": idx,  # Include the dataset index in the return
     }
 
 
@@ -260,11 +259,11 @@ def evaluate(verdict, state):
         state["username"],
         state["start_time"],
         end_time,
+        state["dataset_idx"],
     )
 
-    # Get next sample, passing username to avoid repeats
+    # Get next sample using username to avoid repeats
     next_sample = get_random_sample(state["username"])
-
     # Preserve username in state and set new start time
     next_state = next_sample.copy()
     next_state["username"] = state["username"]
@@ -339,7 +338,7 @@ def handle_username_submit(email, current_page):
 
 def initialize(username):
     """Initialize the interface with first sample"""
-    sample = get_random_sample(username)  # Pass username to avoid repeats
+    sample = get_random_sample(username)
     # Create state with username and start time included
     state = sample.copy()
     state["username"] = username