Update master_key.py

master_key.py

@@ -1,6 +1,7 @@
 # master_key.py
 """
 Comprehensive Master Key for NHVAS Audit extraction (updated & hardened)
+
 Usage:
  - Import TABLE_SCHEMAS, HEADING_PATTERNS, PARAGRAPH_PATTERNS, GLOBAL_SETTINGS
  - The pipeline should:
@@ -45,83 +46,6 @@ GLOBAL_SETTINGS = {
     ]
 }
 
-# ---------------------------
-# Extra header synonyms for common OCR variants and noisy long phrases
-# ---------------------------
-# NOTE: keys in this mapping should be applied to a *normalized* header key
-# (lowercased, punctuation removed, whitespace collapsed) before lookup.
-# Example normalized keys: "registrationnumber", "subcontractoryesno", "rfssuspensioncertificationn/a"
-EXTRA_HEADER_SYNONYMS = {
-    # Registration / common short variants
-    "registrationnumber": "Registration Number",
-    "registration number": "Registration Number",
-    "registrationno": "Registration Number",
-    "reg no": "Registration Number",
-    "regno": "Registration Number",
-    "registration": "Registration Number",
-    "no": "No.",
-
-    # Roadworthiness / maintenance
-    "roadworthinesscertificates": "Roadworthiness Certificates",
-    "roadworthiness certificate": "Roadworthiness Certificates",
-    "roadworthiness certificates": "Roadworthiness Certificates",
-
-    "maintenancerecords": "Maintenance Records",
-    "maintenance records": "Maintenance Records",
-    "triprecords": "Maintenance Records",
-    "trip records": "Maintenance Records",
-
-    "dailychecks": "Daily Checks",
-    "daily check": "Daily Checks",
-    "daily checks": "Daily Checks",
-
-    # Faults
-    "faultrecordingreporting": "Fault Recording/ Reporting",
-    "fault recording reporting": "Fault Recording/ Reporting",
-    "fault recording/reporting": "Fault Recording/ Reporting",
-    "faultrecording/reportingonsuspensionsystem": "Fault Recording/ Reporting",
-
-    "faultrepair": "Fault Repair",
-    "fault repair": "Fault Repair",
-
-    # Sub-contractor / compliance long forms
-    "subcontractoryesno": "Sub-contractor (Yes/No)",
-    "sub-contractor(yesno)": "Sub-contractor (Yes/No)",
-    "sub-contractedvehiclesstatementofcompliance": "Sub-contracted Vehicles Statement of Compliance (Yes/No)",
-    "sub contracted vehicles statement of compliance": "Sub-contracted Vehicles Statement of Compliance (Yes/No)",
-    "sub contracted": "Sub-contractor (Yes/No)",
-    "sub-contracted": "Sub-contractor (Yes/No)",
-    "sub contractor": "Sub-contractor (Yes/No)",
-
-    # RFS / suspension / weight verification
-    "rfs suspension certification": "RFS Suspension Certification # (N/A if not applicable)",
-    "rfs suspension certification # (n/a if not applicable)": "RFS Suspension Certification # (N/A if not applicable)",
-    "weightverificationrecords": "Weight Verification Records (Date Range)",
-    "weight verification records": "Weight Verification Records (Date Range)",
-    "suspensionsystemmaintenance": "Suspension System Maintenance (Date Range)",
-    "suspension system maintenance": "Suspension System Maintenance (Date Range)",
-
-    # NHVR / auditor registration
-    "nhvrorexemplarglobalauditorregistrationnumber": "NHVR or Exemplar Global Auditor Registration Number",
-    "nhvr auditor registration number": "NHVR or Exemplar Global Auditor Registration Number",
-    "nhvr auditor reg no": "NHVR or Exemplar Global Auditor Registration Number",
-
-    # Print/Accreditation
-    "printname": "Print Name",
-    "print accreditation name": "(print accreditation name)",
-
-    # Operator Declaration/Position short forms
-    "positiontitle": "Position Title",
-    "position": "Position Title",
-
-    # Misc helpful fallbacks
-    "details": "DETAILS",
-    "management": "Management",
-    "maintenance management": "MAINTENANCE MANAGEMENT",
-    "mass management": "MASS MANAGEMENT",
-    "fatigue management": "FATIGUE MANAGEMENT"
-}
-
 # ---------------------------
 # Table schemas
 # ---------------------------
@@ -576,4 +500,31 @@ PARAGRAPH_PATTERNS = {
     "date_line": r"^\s*\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]+\s+\d{4}\s*$|^Date$",
     # extra patterns to help skip/identify date-like strings in name fields
     "date_like": GLOBAL_SETTINGS["date_like_pattern"]
-}
+}
+
+# ---------------------------
+# Helpful hints for pipeline implementer (non-executable comments)
+# ---------------------------
+# 1) Matching order recommendation (pipeline):
+#    - Try exact qualified keys in schema['preferred_json_keys']
+#    - Try exact label match against schema['labels'] and schema['label_synonyms']
+#    - Apply normalization rules from GLOBAL_SETTINGS then try clean match
+#    - Fuzzy match using GLOBAL_SETTINGS['fuzzy_thresholds'] keyed by schema priority
+#    - If schema has 'fallback_keys', try them last
+#
+# 2) Date-safety: for any schema field listed under 'skip_if_date_like', verify JSON value
+#    or cell content is not date-like (use PARAGRAPH_PATTERNS['date_like']) before replacing.
+#
+# 3) Multi-value cells: If schema contains 'split_on', join JSON list items using '\n'
+#    or split a single JSON string on these delimiters to create separate lines.
+#
+# 4) OCR repairs: run GLOBAL_SETTINGS['ocr_repair_rules'] on extracted header text
+#    before attempting matching (fix common OCR confusion).
+#
+# 5) Processed flags: keep per-table attribute (e.g. table._processed_operator_declaration = True)
+#    to avoid running two different handlers on the same Operator Declaration table.
+#
+# 6) Tuning: If a schema is consistently mis-matched, add the exact observed header string
+#    into schema['label_synonyms'] for deterministic matching.
+#
+# End of master key