correct JSON and filtering

input_output/wjec-gce-as-a-economics-specification-from-2015.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eef6fa3102f82c9a3e0eb99a8c7a08f86df01c2ba7636ff4bef8cbd7f780e7b6
+size 3543551

pearson_json/_subtopics.json

@@ -0,0 +1,202 @@
+[
+  {
+    "title": "Content",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_1.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_3.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_4.jpg_r1_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_5.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_6.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_9.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_15.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_16.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_18.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_19.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_20.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_22.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_23.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_27.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_28.jpg_r0_c0.png"
+      },
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_29.jpg_r0_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "Factors influencing demand and supply in product markets",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_2.jpg_r1_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "Why and how governments intervene in markets",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_7.jpg_r1_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "The circular flow of income model",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_8.jpg_r2_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "Government policy objectives",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_10.jpg_r1_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "Fiscal policy",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_11.jpg_r1_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "Monetary policy",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_12.jpg_r1_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "Exchange rates and exchange rate policy",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_13.jpg_r1_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "Free trade and protectionism",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_14.jpg_r1_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "Monopoly",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_17.jpg_r2_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "Economic growth",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_21.jpg_r1_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "Inflation and deflation",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_24.jpg_r1_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "The balance of payments",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_25.jpg_r2_c0.png"
+      }
+    ],
+    "children": []
+  },
+  {
+    "title": "Control of the national (public sector) debt",
+    "contents": [
+      {
+        "type": "image",
+        "key": "/topic-extraction/cells/img_26.jpg_r1_c0.png"
+      }
+    ],
+    "children": []
+  }
+]

topic_extr.py

@@ -207,6 +207,13 @@ class s3Writer:
             logger.error(f"Failed to upload to S3: {str(e)}")
             raise
 
+    def delete(self, path: str) -> None:
+        try:
+            self.client.delete_object(Bucket=self.bucket, Key=path)
+        except Exception as e:
+            logger.error(f"Failed to delete from S3: {str(e)}")
+            raise
+
 def preprocess_image(image_data: bytes, max_dim: int = 600, quality: int = 60) -> bytes:
     arr = np.frombuffer(image_data, np.uint8)
     img = cv2.imdecode(arr, cv2.IMREAD_COLOR)
@@ -238,11 +245,6 @@ The two-column 'table' image includes such key features:
     - Two columns
     - Headers like 'Subject content', 'Additional information'
     - Possibly sections (e.g. 2.1, 3.4, G2, G3, )
-The empty image include such key features:
-    - Does not include anything at all (like a blank white or black image)
-    - Truncated image with words like 'Subject content', 'What students need to learn' with blue background.
-    - Truncated image with words like 'Topics', 'What students need to learn', 'Content' with grey background ((166, 166, 166) or (180,180,180) RGB color code).
-If the image is an empty image, respond with 'EMPTY_IMAGE'.
 If the image is a relevant table with 2 columns, respond with 'TWO_COLUMN'.
 If the image is a relevant table with 3 columns, respond with 'THREE_COLUMN'.
 If the image is non-empty but does not show a table, respond with 'NO_TABLE'.
@@ -309,7 +311,7 @@ Your task is to extract:
 
 Follow these rules:
 
-(1) **If the cell shows a main topic in the format "<number> <Topic Name>",** for example "2 Algebra and functions continued", then:
+(1) **If the cell shows a main topic in the format "<number> <Topic Name>",** for example "2 Algebra and functions continued", (remove the word "continued") then:
     - Put that text (without the word "continued") in "title". (e.g. "2 Algebra and functions")
     - "subtopics" should be an empty array, unless you also see smaller subtopic numbers.
 
@@ -318,11 +320,11 @@ Follow these rules:
     - "title" in this case should be an empty string if you only detect subtopics. 
       (Example: If text is "2.5 Solve linear inequalities...", then "title" = "", "subtopics" = ["2.5"]).
 
-(3) **If neither a main topic nor a subtopic is detected,** return empty values:
-    {
+(3) If no main topic or subtopic is detected but the text appears to be a heading (e.g. "Scarcity, choice and opportunity cost"), return:
+    {{
       "title": "",
       "subtopics": []
-    }
+    }}
 
 (4) **If there is no numeric value in the left column** (e.g. "2.1" or "2 <Topic name>" not found) but the left column text appears to be a heading (for instance "Scarcity, choice and opportunity cost"), then:
     - Use the **left column text** as "title".
@@ -344,6 +346,15 @@ Follow these rules:
       "subtopics": [...]
     }
 
+(7) If the image is blank or truncated, defined as:
+    - Contains no words at all (e.g. a blank white or black image)
+    - Contains only a truncated snippet of words such as "Topics", "What students need to learn" with blue background
+    - Contains a truncated snippet with words like "Topics", "What students need to learn", "Content" with gray background (RGB (166,166,166) or (180,180,180)) then return:
+    {{
+        "title": "EMPTY_IMAGE",
+        "subtopics": []
+    }}
+
 **Examples**:
 
 - If the image text is `"2 Algebra and functions continued"`, return:
@@ -411,6 +422,8 @@ Follow these rules:
 
             title = data.get("title", "")
             subtopics = data.get("subtopics", [])
+            if title.upper() == "EMPTY_IMAGE":
+                return {"title": "EMPTY_IMAGE", "subtopics": []}
             if not isinstance(subtopics, list):
                 subtopics = []
             return {"title": title, "subtopics": subtopics}
@@ -454,21 +467,27 @@ class S3ImageWriter(DataWriter):
             for p, info in self.descriptions.items()
         }
         results = await asyncio.gather(*tasks.values(), return_exceptions=True)
-        for p, result in zip(tasks.keys(), results):
+        for p, result in zip(list(self.descriptions.keys()), results):
             if isinstance(result, Exception):
                 logger.error(f"Table classification error for {p}: {result}")
                 self.descriptions[p]['table_classification'] = "NO_TABLE"
             else:
                 self.descriptions[p]['table_classification'] = result
 
-        for p, info in self.descriptions.items():
+        # Process each image description.
+        for p, info in list(self.descriptions.items()):
             cls = info['table_classification']
             if cls == "TWO_COLUMN":
                 info['final_alt'] = "HAS TO BE PROCESSED - two column table"
             elif cls == "THREE_COLUMN":
                 info['final_alt'] = "HAS TO BE PROCESSED - three column table"
             elif cls == "EMPTY_IMAGE":
+                # Remove markdown reference, delete from descriptions and S3.
                 md_content = md_content.replace(f"![]({key}{p})", "")
+                try:
+                    self.s3_writer.delete(info['s3_path'])
+                except Exception as e:
+                    logger.error(f"Error deleting S3 object {info['s3_path']}: {e}")
                 del self.descriptions[p]
                 continue
             else:
@@ -477,7 +496,7 @@ class S3ImageWriter(DataWriter):
 
         md_content = await self._process_table_images_in_markdown(key, md_content)
 
-        # Filter final lines to keep only lines with images
+        # Filter final lines to keep only lines with images.
         final_lines = [
             line.strip() for line in md_content.split("\n")
             if re.match(r"^\!\[.*\]\(.*\)", line.strip())
@@ -558,12 +577,20 @@ class S3ImageWriter(DataWriter):
                         with open(cell_path, "rb") as cf:
                             cell_image_data = cf.read()
 
-                        # Save cell image to S3.
                         cell_key = f"{self.base_path}cells/{os.path.basename(s3_key)}_r{i}_c{j}.png"
                         self.s3_writer.write(cell_key, cell_image_data)
-                        
+
+                        #extract subtopic info from the cell image.
                         info = call_gemini_for_subtopic_identification_image(cell_image_data, self.gemini_api_key)
-                        # logger.info(f"Gemini subtopic extraction result for cell {cell_path}: {info}")
+                        
+                        # Check if the image is empty.
+                        if info.get("title", "").upper() == "EMPTY_IMAGE":
+                            try:
+                                self.s3_writer.delete(cell_key)
+                                logger.info(f"Deleted empty cell image from S3: {cell_key}")
+                            except Exception as e:
+                                logger.error(f"Error deleting empty cell image {cell_key}: {e}")
+                            continue  # Skip processing this cell further
 
                         if info["title"] and not recognized_main_topic:
                             recognized_main_topic = info["title"]
@@ -706,6 +733,15 @@ In that scenario, your output might look like:
     "2.3 A2 Unit 3": [24, 30],
     "2.4 A2 Unit 4": [31, 35]
 }}
+or
+2.1 AS units 6
+2.2 AS units 23
+In that scenario, your output might look like:
+{{
+    "2.1 AS Unit 1": [6, 2],
+    "2.2 AS Unit 2": [23, 43]
+}}
+
 4. Another example might list subtopics:
 3.1 Overarching themes 11
 3.2 A: Proof 12
@@ -912,7 +948,7 @@ class MineruNoTextProcessor:
             subtopic_list = list(writer.extracted_subtopics.values())
             subtopic_list = merge_topics(subtopic_list)
 
-            out_path = os.path.join(self.output_folder, "subtopics.json")
+            out_path = os.path.join(self.output_folder, "_subtopics.json")
             with open(out_path, "w", encoding="utf-8") as f:
                 json.dump(subtopic_list, f, indent=2)
             logger.info(f"Final subtopics JSON saved locally at {out_path}")
@@ -925,7 +961,7 @@ class MineruNoTextProcessor:
             self.cleanup_gpu()
 
 if __name__ == "__main__":
-    input_pdf = "/home/user/app/input_output/a-level-pearson-mathematics-specification.pdf"
+    input_pdf = "/home/user/app/input_output/wjec-gce-as-a-economics-specification-from-2015.pdf"
     output_dir = "/home/user/app/pearson_json"
     gemini_key = os.getenv("GEMINI_API_KEY", "AIzaSyDtoakpXa2pjJwcQB6TJ5QaXHNSA5JxcrU")
     try: