Update master_key.py

master_key.py

@@ -1,7 +1,6 @@
 # master_key.py
 """
 Comprehensive Master Key for NHVAS Audit extraction (updated & hardened)
-
 Usage:
  - Import TABLE_SCHEMAS, HEADING_PATTERNS, PARAGRAPH_PATTERNS, GLOBAL_SETTINGS
  - The pipeline should:
@@ -46,6 +45,83 @@ GLOBAL_SETTINGS = {
     ]
 }
 
+# ---------------------------
+# Extra header synonyms for common OCR variants and noisy long phrases
+# ---------------------------
+# NOTE: keys in this mapping should be applied to a *normalized* header key
+# (lowercased, punctuation removed, whitespace collapsed) before lookup.
+# Example normalized keys: "registrationnumber", "subcontractoryesno", "rfssuspensioncertificationn/a"
+EXTRA_HEADER_SYNONYMS = {
+    # Registration / common short variants
+    "registrationnumber": "Registration Number",
+    "registration number": "Registration Number",
+    "registrationno": "Registration Number",
+    "reg no": "Registration Number",
+    "regno": "Registration Number",
+    "registration": "Registration Number",
+    "no": "No.",
+
+    # Roadworthiness / maintenance
+    "roadworthinesscertificates": "Roadworthiness Certificates",
+    "roadworthiness certificate": "Roadworthiness Certificates",
+    "roadworthiness certificates": "Roadworthiness Certificates",
+
+    "maintenancerecords": "Maintenance Records",
+    "maintenance records": "Maintenance Records",
+    "triprecords": "Maintenance Records",
+    "trip records": "Maintenance Records",
+
+    "dailychecks": "Daily Checks",
+    "daily check": "Daily Checks",
+    "daily checks": "Daily Checks",
+
+    # Faults
+    "faultrecordingreporting": "Fault Recording/ Reporting",
+    "fault recording reporting": "Fault Recording/ Reporting",
+    "fault recording/reporting": "Fault Recording/ Reporting",
+    "faultrecording/reportingonsuspensionsystem": "Fault Recording/ Reporting",
+
+    "faultrepair": "Fault Repair",
+    "fault repair": "Fault Repair",
+
+    # Sub-contractor / compliance long forms
+    "subcontractoryesno": "Sub-contractor (Yes/No)",
+    "sub-contractor(yesno)": "Sub-contractor (Yes/No)",
+    "sub-contractedvehiclesstatementofcompliance": "Sub-contracted Vehicles Statement of Compliance (Yes/No)",
+    "sub contracted vehicles statement of compliance": "Sub-contracted Vehicles Statement of Compliance (Yes/No)",
+    "sub contracted": "Sub-contractor (Yes/No)",
+    "sub-contracted": "Sub-contractor (Yes/No)",
+    "sub contractor": "Sub-contractor (Yes/No)",
+
+    # RFS / suspension / weight verification
+    "rfs suspension certification": "RFS Suspension Certification # (N/A if not applicable)",
+    "rfs suspension certification # (n/a if not applicable)": "RFS Suspension Certification # (N/A if not applicable)",
+    "weightverificationrecords": "Weight Verification Records (Date Range)",
+    "weight verification records": "Weight Verification Records (Date Range)",
+    "suspensionsystemmaintenance": "Suspension System Maintenance (Date Range)",
+    "suspension system maintenance": "Suspension System Maintenance (Date Range)",
+
+    # NHVR / auditor registration
+    "nhvrorexemplarglobalauditorregistrationnumber": "NHVR or Exemplar Global Auditor Registration Number",
+    "nhvr auditor registration number": "NHVR or Exemplar Global Auditor Registration Number",
+    "nhvr auditor reg no": "NHVR or Exemplar Global Auditor Registration Number",
+
+    # Print/Accreditation
+    "printname": "Print Name",
+    "print accreditation name": "(print accreditation name)",
+
+    # Operator Declaration/Position short forms
+    "positiontitle": "Position Title",
+    "position": "Position Title",
+
+    # Misc helpful fallbacks
+    "details": "DETAILS",
+    "management": "Management",
+    "maintenance management": "MAINTENANCE MANAGEMENT",
+    "mass management": "MASS MANAGEMENT",
+    "fatigue management": "FATIGUE MANAGEMENT"
+}
+
 # ---------------------------
 # Table schemas
 # ---------------------------
@@ -500,31 +576,4 @@ PARAGRAPH_PATTERNS = {
     "date_line": r"^\s*\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]+\s+\d{4}\s*$|^Date$",
     # extra patterns to help skip/identify date-like strings in name fields
     "date_like": GLOBAL_SETTINGS["date_like_pattern"]
-}
-
-# ---------------------------
-# Helpful hints for pipeline implementer (non-executable comments)
-# ---------------------------
-# 1) Matching order recommendation (pipeline):
-#    - Try exact qualified keys in schema['preferred_json_keys']
-#    - Try exact label match against schema['labels'] and schema['label_synonyms']
-#    - Apply normalization rules from GLOBAL_SETTINGS then try clean match
-#    - Fuzzy match using GLOBAL_SETTINGS['fuzzy_thresholds'] keyed by schema priority
-#    - If schema has 'fallback_keys', try them last
-#
-# 2) Date-safety: for any schema field listed under 'skip_if_date_like', verify JSON value
-#    or cell content is not date-like (use PARAGRAPH_PATTERNS['date_like']) before replacing.
-#
-# 3) Multi-value cells: If schema contains 'split_on', join JSON list items using '\n'
-#    or split a single JSON string on these delimiters to create separate lines.
-#
-# 4) OCR repairs: run GLOBAL_SETTINGS['ocr_repair_rules'] on extracted header text
-#    before attempting matching (fix common OCR confusion).
-#
-# 5) Processed flags: keep per-table attribute (e.g. table._processed_operator_declaration = True)
-#    to avoid running two different handlers on the same Operator Declaration table.
-#
-# 6) Tuning: If a schema is consistently mis-matched, add the exact observed header string
-#    into schema['label_synonyms'] for deterministic matching.
-#
-# End of master key
+}