Update update_docx_with_pdf.py

update_docx_with_pdf.py

@@ -15,6 +15,28 @@ from collections import OrderedDict  # <-- add this
 def _nz(x):
     return x if isinstance(x, str) and x.strip() else ""
 
+def _fix_ocr_date_noise(date_str: str) -> str:
+    """Clean up OCR date noise and standardize date format."""
+    if not date_str:
+        return ""
+    
+    # Remove common OCR artifacts
+    cleaned = re.sub(r'\s+', ' ', date_str.strip())
+    cleaned = re.sub(r'[^\w\s/\-]', '', cleaned)
+    
+    # Try to extract month/year patterns
+    month_year_match = re.search(r'([A-Za-z]+)\s+(\d{4})', cleaned)
+    if month_year_match:
+        return f"{month_year_match.group(1)} {month_year_match.group(2)}"
+    
+    # Try to extract date patterns like "21st October 2022"
+    date_match = re.search(r'(\d{1,2})(?:st|nd|rd|th)?\s+([A-Za-z]+)\s+(\d{4})', cleaned)
+    if date_match:
+        return f"{date_match.group(1)} {date_match.group(2)} {date_match.group(3)}"
+    
+    # Return cleaned version if no specific pattern found
+    return cleaned
+
 SUMMARY_SECTIONS = {
     "MAINTENANCE MANAGEMENT": "Maintenance Management Summary",
     "MASS MANAGEMENT": "Mass Management Summary",
@@ -770,6 +792,34 @@ class NHVASMerger:
             page_num = table.get("page", 0)
             self.log_debug(f"Processing table on page {page_num} with headers: {headers[:3]}...")
 
+            # NEW: Check for single-column Nature of Business table
+            if (len(headers) == 1 and 
+                "nature of the operators business" in str(headers[0]).lower() and
+                len(data_rows) > 0 and len(data_rows[0]) > 0):
+                
+                text = str(data_rows[0][0])
+                self.log_debug(f"Found Nature of Business table with text: {text[:100]}...")
+                
+                # Extract inline expiry date and accreditation number
+                m_exp = re.search(r"\b(?:Mass and Maintenance\s+)?Expiry\s*Date[:\s-]*([0-9\.\/\-]+)", text, flags=re.I)
+                m_acc = re.search(r"\bAccreditation\s*Number[:\s-]*([A-Za-z0-9\s\-\/]+)", text, flags=re.I)
+                
+                if m_exp:
+                    exp_date = m_exp.group(1).strip()
+                    extracted.setdefault("business_summary_extras", {})["expiry_date"] = exp_date
+                    self.log_debug(f"Extracted expiry date: {exp_date}")
+                    
+                if m_acc:
+                    acc_num = m_acc.group(1).strip()
+                    extracted.setdefault("business_summary_extras", {})["accreditation_number"] = acc_num
+                    self.log_debug(f"Extracted accreditation number: {acc_num}")
+                    
+                # Store the clean text (without the inline date/number)
+                clean_text = re.sub(r"\s*(?:Mass and Maintenance\s+)?Expiry\s*Date[:\s-]*[0-9\.\/\-]+", "", text, flags=re.I)
+                clean_text = re.sub(r"\s*Accreditation\s*Number[:\s-]*[A-Za-z0-9\s\-\/]+", "", clean_text, flags=re.I)
+                extracted["business_summary"] = clean_text.strip()
+                continue
+
             # 🔧 NEW: collapse possible multi-line headers once up front
             collapsed_headers, collapsed_rows = self._collapse_multiline_headers(headers, data_rows)
 
@@ -1052,78 +1102,122 @@ class NHVASMerger:
         return added
 
     def _extract_driver_table(self, headers: List[str], data_rows: List[List], extracted: Dict):
-        """Header-driven extraction for Driver / Scheduler Records."""
+        """Enhanced header-driven extraction for Driver / Scheduler Records."""
         drivers = []
-        ch = [_canon_header(h) for h in headers or []]
-
-        # helpers
-        def find_col(needles: list[str]) -> Optional[int]:
-            for i, h in enumerate(ch):
-                if any(n in h for n in needles):
-                    return i
-            return None
-
-        def find_col_rx(patterns: list[str]) -> Optional[int]:
-            for i, h in enumerate(ch):
-                if any(re.search(p, h) for p in patterns):
-                    return i
-            return None
-
-        name_idx   = find_col_rx([r"\bdriver\s*/\s*scheduler\s*name\b",
-                              r"\bdriver\s+name\b", r"\bscheduler\s+name\b", r"\bname\b"])
-        tlif_d_idx = find_col(["driver tlif"])
-        tlif_s_idx = find_col(["scheduler tlif"])
-        medical_idx= find_col(["medical", "expiry"])
-        roster_idx = find_col_rx([r"\broster\b", r"\bsafe\s+driving\s+plan\b", r"\bschedule\b(?!r\b)"])
-        fit_idx    = find_col(["fit for duty"])
-        diary_idx  = find_col(["work diary", "electronic work diary", "page numbers"])
-
-        for row in data_rows:
+        
+        self.log_debug(f"Driver table has {len(data_rows)} rows")
+        
+        # Skip header continuation rows - look for the first row that starts with a number
+        actual_data_start = 0
+        for i, row in enumerate(data_rows):
+            if row and str(row[0]).strip().startswith(('1.', '1')):
+                actual_data_start = i
+                self.log_debug(f"Found actual data starting at row {i}")
+                break
+        
+        if actual_data_start == 0:
+            self.log_debug("Warning: Could not find numbered data rows")
+        
+        # Process only the actual data rows (skip header continuation rows)
+        for row_idx, row in enumerate(data_rows[actual_data_start:], start=actual_data_start):
             if not row:
                 continue
+                
+            self.log_debug(f"Processing data row {row_idx}: {row}")
+            
+            # Check if this is a numbered row (1., 2., etc.)
+            first_cell = str(row[0]).strip()
+            if not (first_cell.endswith('.') and first_cell[:-1].isdigit()):
+                self.log_debug(f"Skipping row {row_idx} - not a numbered data row")
+                continue
 
+            # Based on the raw data structure, extract from fixed positions
             name = None
-            if name_idx is not None and name_idx < len(row):
-                name = _smart_space(str(row[name_idx]).strip())
+            driver_tlif = ""
+            scheduler_tlif = ""
+            medical = ""
+            roster = ""
+            fit_duty = ""
+            work_diary = ""
+            
+            # Look for name in columns around index 3-4
+            for i in range(2, min(6, len(row))):
+                candidate = _smart_space(str(row[i]).strip())
+                if (candidate and 
+                    len(candidate) > 3 and 
+                    any(c.isalpha() for c in candidate) and
+                    candidate.lower() not in ['entry', 'n/a', 'yes', 'no', 'name'] and
+                    not candidate.isdigit() and
+                    not candidate.endswith('.')):
+                    name = candidate
+                    self.log_debug(f"Found name at column {i}: {name}")
+                    break
+            
             if not name:
+                self.log_debug(f"Skipping row {row_idx} - no valid name found")
                 continue
 
-            d = {"name": name}
-
-            if tlif_d_idx is not None and tlif_d_idx < len(row):
-                d["driver_tlif"] = _smart_space(str(row[tlif_d_idx]).strip())
-            if tlif_s_idx is not None and tlif_s_idx < len(row):
-                d["scheduler_tlif"] = _smart_space(str(row[tlif_s_idx]).strip())
-            if medical_idx is not None and medical_idx < len(row):
-                d["medical_expiry"] = _smart_space(str(row[medical_idx]).strip())
-
-            # Roster/Schedule/SDP: prefer the detected column; accept only date/range-like, not the name
-            if roster_idx is not None and roster_idx < len(row):
-                raw_roster = _smart_space(str(row[roster_idx]).strip())
-                if raw_roster and re.search(r"[0-9/–-]", raw_roster) and raw_roster.lower() != name.lower():
-                    d["roster_schedule"] = raw_roster
-
-            # Fallback: scan the row for the first date/range-like cell that's not the name cell
-            if "roster_schedule" not in d:
-                for j, cell in enumerate(row):
-                    if j == name_idx:
-                        continue
-                    s = _smart_space(str(cell).strip())
-                if s and re.search(r"[0-9/–-]", s) and s.lower() != name.lower():
-                    d["roster_schedule"] = s
+            # Extract other fields from approximate positions based on raw data
+            # Driver TLIF around column 6
+            for i in range(5, min(8, len(row))):
+                val = str(row[i]).strip()
+                if val and val.lower() in ['yes', 'no']:
+                    driver_tlif = val.title()
+                    break
+                    
+            # Scheduler TLIF around column 9  
+            for i in range(8, min(12, len(row))):
+                val = str(row[i]).strip()
+                if val and val.lower() in ['yes', 'no']:
+                    scheduler_tlif = val.title()
+                    break
+                    
+            # Medical around column 12
+            for i in range(11, min(15, len(row))):
+                val = _smart_space(str(row[i]).strip())
+                if val and val.lower() not in ['', 'entry']:
+                    medical = val
+                    break
+                    
+            # Roster around column 15
+            for i in range(14, min(18, len(row))):
+                val = _smart_space(str(row[i]).strip())
+                if val:
+                    roster = val
+                    break
+                    
+            # Fit for Duty around column 18
+            for i in range(17, min(21, len(row))):
+                val = str(row[i]).strip()
+                if val and val.lower() in ['yes', 'no']:
+                    fit_duty = val.title()
+                    break
+                    
+            # Work Diary around column 21
+            for i in range(20, min(len(row), 24)):
+                val = _smart_space(str(row[i]).strip())
+                if val:
+                    work_diary = val
                     break
 
-            if fit_idx is not None and fit_idx < len(row):
-                d["fit_for_duty"] = _smart_space(str(row[fit_idx]).strip())
-            if diary_idx is not None and diary_idx < len(row):
-                d["work_diary"] = _smart_space(str(row[diary_idx]).strip())
-
+            d = {
+                "name": name,
+                "driver_tlif": driver_tlif,
+                "scheduler_tlif": scheduler_tlif,
+                "medical_expiry": medical,
+                "roster_schedule": roster,
+                "fit_for_duty": fit_duty,
+                "work_diary": work_diary
+            }
+            
             drivers.append(d)
+            self.log_debug(f"Added driver: {d}")
 
         if drivers:
             extracted["drivers_detailed"] = drivers
-            self.log_debug(f"Driver rows extracted (header-based): {len(drivers)}")
-
+            self.log_debug(f"Driver rows extracted: {len(drivers)}")
+        else:
+            self.log_debug("No drivers extracted")
 
     def _extract_management_table(self, data_rows: List[List], extracted: Dict, headers: List[str]):
         txt = " ".join(str(h) for h in headers).lower()
@@ -1177,7 +1271,7 @@ class NHVASMerger:
 
             # Defensive trimming of trailing uppercase boilerplate or table header noise
             candidate = re.sub(
-                r"(ACCREDITATION VEHICLE SUMMARY|AUDIT OBSERVATIONS|NHVAS AUDIT SUMMARY REPORT|STD\s+\d+\.).*$",
+                r"(Mass and Maintenance Expiry Date:|ACCREDITATION DRIVER SUMMARY|ACCREDITATION VEHICLE SUMMARY|AUDIT OBSERVATIONS|NHVAS AUDIT SUMMARY REPORT|STD\s+\d+\.).*$",
                 "",
                 candidate,
                 flags=re.I | re.DOTALL,
@@ -1189,7 +1283,7 @@ class NHVASMerger:
 
                 # Extract Accreditation Number / Expiry only if they appear inline in this small block
                 m_acc = re.search(r"\bAccreditation\s*Number[:\s-]*([A-Za-z0-9\s\-\/]+)", candidate, flags=re.I)
-                m_exp = re.search(r"\bExpiry\s*Date[:\s-]*([A-Za-z0-9\s,\/\-]+)", candidate, flags=re.I)
+                m_exp = re.search(r"\b(?:Mass and Maintenance\s+)?Expiry\s*Date[:\s-]*([A-Za-z0-9\s,\/\-\.]+)", candidate, flags=re.I)
                 if m_acc:
                     acc = re.sub(r"\s+", " ", m_acc.group(1)).strip()
                     acc = re.sub(r"[^\d]", "", acc) or acc
@@ -1483,7 +1577,17 @@ class NHVASMerger:
     
         # Business summary
         if "business_summary" in pdf_extracted and "Nature of the Operators Business (Summary)" in merged:
-            merged["Nature of the Operators Business (Summary)"]["Nature of the Operators Business (Summary):"] = [_smart_space(pdf_extracted["business_summary"])]
+            # Clean the main text by removing either inline date pattern
+            business_text = pdf_extracted["business_summary"]
+            clean_text = re.sub(r"\s*(?:Mass and Maintenance\s+)?Expiry Date[:\s]*[A-Za-z0-9\s,\/\-\.]+", "", business_text, flags=re.I)
+            merged["Nature of the Operators Business (Summary)"]["Nature of the Operators Business (Summary):"] = [_smart_space(clean_text)]
+            
+            # Override with extracted inline values
+            extras = pdf_extracted.get("business_summary_extras", {})
+            if extras.get("expiry_date"):
+                merged["Nature of the Operators Business (Summary)"]["Expiry Date"] = [extras["expiry_date"]]
+            if extras.get("accreditation_number"):  
+                merged["Nature of the Operators Business (Summary)"]["Accreditation Number"] = [extras["accreditation_number"]]
 
         # Vehicle summary
         if "vehicle_summary" in pdf_extracted:
@@ -1526,18 +1630,19 @@ class NHVASMerger:
             )
 
 
-        # replace the whole Drivers/Scheduler block with:
+       # Complete driver mapping - add these lines:
         if "drivers_detailed" in pdf_extracted and "Driver / Scheduler Records Examined" in merged:
             drivers = pdf_extracted["drivers_detailed"]
-
-            def _looks_like_range(s):
-                return bool(re.search(r"[0-9]{1,2}[/-]", s or ""))
-
+            
+            # Map ALL the driver fields
+            merged["Driver / Scheduler Records Examined"]["Driver / Scheduler Name"] = [d.get("name","") for d in drivers]
+            merged["Driver / Scheduler Records Examined"]["Driver TLIF Course # Completed"] = [d.get("driver_tlif","") for d in drivers]
+            merged["Driver / Scheduler Records Examined"]["Scheduler TLIF Course # Completed"] = [d.get("scheduler_tlif","") for d in drivers]
+            merged["Driver / Scheduler Records Examined"]["Medical Certificates (Current Yes/No) Date of expiry"] = [d.get("medical_expiry","") for d in drivers]
             merged["Driver / Scheduler Records Examined"]["Roster / Schedule / Safe Driving Plan (Date Range)"] = [d.get("roster_schedule","") for d in drivers]
-            merged["Driver / Scheduler Records Examined"]["Fit for Duty Statement Completed (Yes/No)"]          = [d.get("fit_for_duty","") for d in drivers]
+            merged["Driver / Scheduler Records Examined"]["Fit for Duty Statement Completed (Yes/No)"] = [d.get("fit_for_duty","") for d in drivers]
             merged["Driver / Scheduler Records Examined"]["Work Diary Pages (Page Numbers) Electronic Work Diary Records (Date Range)"] = [d.get("work_diary","") for d in drivers]
 
-
         # --- Print accreditation name (robust, no UnboundLocalError) ---
         if "Print accreditation name" in merged:
             acc_name = ""  # init