master_key.py

# master_key.py
"""
Comprehensive Master Key for NHVAS Audit extraction (updated & hardened)

Usage:
 - Import TABLE_SCHEMAS, HEADING_PATTERNS, PARAGRAPH_PATTERNS, GLOBAL_SETTINGS
 - The pipeline should:
    1) normalize candidate header/key using GLOBAL_SETTINGS['normalize']
    2) attempt exact matches using schema['preferred_json_keys'] (if provided)
    3) try direct label matches (schema['labels'] + schema['label_synonyms'])
    4) fallback to fuzzy matching using GLOBAL_SETTINGS thresholds
    5) respect schema['skip_if_date_like'] for sensitive fields (e.g. Print Name)
    6) for multi-value cells, use schema['split_on'] to split JSON values into lines
"""

import re

# ---------------------------
# Global matching / normalization settings
# ---------------------------
GLOBAL_SETTINGS = {
    "normalize": {
        # Normalization pipeline (applied in this order)
        "lower": True,
        "strip_punctuation": True,   # remove punctuation like ()'.,–
        "collapse_whitespace": True,
        "replace_smart_dashes": True # normalize – — to -
    },
    "fuzzy_thresholds": {
        # Minimum score thresholds (0-100) for different priority tiers
        "high_priority": 70,
        "medium_priority": 60,
        "low_priority": 45
    },
    "fuzzy_algorithm": "token_set_ratio",  # recommendation
    "split_on": [" – ", " - ", ";", "\n", " / ", " & "],  # common attendance / multi-name delimiters
    "date_like_pattern": r"^\s*(\d{1,2}(st|nd|rd|th)?\s+[A-Za-z]+|\d{1,2}\/\d{1,2}\/\d{2,4}|\d{1,2}\.\d{1,2}\.\d{2,4}|\d{1,2}\s+[A-Za-z]{3,})", 
    # used for skip_if_date_like checks
    "ocr_repair_rules": [
        # small fixes for common OCR noise in headers
        (r"\s*\(\s*Yes\s*/\s*No\s*\)", " (Yes/No)"),
        (r"R[e3]gistrat[i1]on", "Registration"),
        (r"Prin?t", "Print"),
        (r"Accredi[ta]tion", "Accreditation"),
        (r"[^\w\s\-\&\(\)\/:]", " "),  # remove stray symbols but keep parentheses/slash
    ]
}

# ---------------------------
# Table schemas
# ---------------------------
TABLE_SCHEMAS = {
    # Basic top-of-form tick boxes
    "Tick as appropriate": {
        "headings": [
            {"level": 1, "text": r"NHVAS\s+Audit\s+Summary\s+Report"},
        ],
        "orientation": "left",
        "labels": ["Tick as appropriate"],
        "label_synonyms": ["Tick as appropriate:", "Tick where appropriate", "Tick as required"],
        "priority": 90,
        "preferred_json_keys": ["Tick as appropriate", "Tick as appropriate.Tick as appropriate"],
        "context_keywords": ["tick", "appropriate"]
    },

    "Audit Information": {
        "orientation": "left",
        "labels": [
            "Date of Audit",
            "Location of audit",
            "Auditor name",
            "Audit Matrix Identifier (Name or Number)",
            "Auditor Exemplar Global Reg No.",
            "Expiry Date",
            "NHVR Auditor Registration Number"
        ],
        "label_synonyms": [
            "Audit Date", "Date of audit", "Location", "Auditor", "Exemplar Global Reg No.",
            "Auditor Reg No", "Auditor Registration Number"
        ],
        "priority": 80,
        "preferred_json_keys": [
            "Audit Information.Date of Audit",
            "Audit Information.Auditor name",
            "Audit Information.Location of audit"
        ],
        "context_keywords": ["audit", "auditor", "matrix", "registration"]
    },

    "Operator Information": {
        "headings": [
            {"level": 1, "text": r"Operator\s+Information"}
        ],
        "orientation": "left",
        "labels": [
            "Operator name (Legal entity)",
            "Operator’s Name (legal entity)",
            "NHVAS Accreditation No. (If applicable)",
            "Registered trading name/s",
            "Australian Company Number",
            "NHVAS Manual (Policies and Procedures) developed by"
        ],
        "label_synonyms": [
            "Operator name", "Operator name (Legal entity)", "Operator’s name", "Operator's Name (legal entity)"
        ],
        "priority": 85,
        "preferred_json_keys": [
            "Operator Information.Operator name (Legal entity)",
            "Operator name (Legal entity)"
        ],
        "context_keywords": ["operator", "legal entity", "accreditation", "company number"]
    },

    "Operator contact details": {
        "orientation": "left",
        "labels": [
            "Operator business address",
            "Operator Postal address",
            "Email address",
            "Operator Telephone Number"
        ],
        "label_synonyms": ["Business address", "Postal address", "Email", "Telephone", "Phone"],
        "priority": 75,
        "context_keywords": ["contact", "address", "email", "telephone", "phone"]
    },

    "Attendance List (Names and Position Titles)": {
        "headings": [
            {"level": 1, "text": r"Attendance\s+List\s*\(?Names\s+and\s+Position\s+Titles\)?", "re": True}
        ],
        "orientation": "row1",
        "labels": ["Attendance List (Names and Position Titles)"],
        "label_synonyms": [
            "Attendance List", "Attendees", "Attendance", "Names and Position Titles",
            "Names & Position Titles", "Attendance list (names & positions)"
        ],
        "priority": 90,
        "preferred_json_keys": [
            "Attendance List (Names and Position Titles).Attendance List (Names and Position Titles)",
            "Attendance List (Names and Position Titles)"
        ],
        "split_on": [" – ", " - ", ";", "\n", " / "],  # ensure one person per line
        "context_keywords": ["attendance", "attendees", "names", "position"],
        "skip_if_date_like": False
    },

    "Nature of the Operators Business (Summary)": {
        "orientation": "row1",
        "labels": ["Nature of the Operators Business (Summary):", "Nature of the Operator's Business (Summary):"],
        "label_synonyms": ["Nature of the Operators Business", "Nature of Business", "Nature of the Operator Business"],
        "split_labels": ["Accreditation Number:", "Expiry Date:"],
        "priority": 85,
        "context_keywords": ["nature", "business", "operators business", "summary"]
    },

    "Accreditation Vehicle Summary": {
        "orientation": "left",
        "labels": ["Number of powered vehicles", "Number of trailing vehicles"],
        "label_synonyms": ["powered vehicles", "trailing vehicles", "Number of vehicles"],
        "priority": 80,
        "context_keywords": ["vehicle", "powered vehicles", "trailing"]
    },

    "Accreditation Driver Summary": {
        "orientation": "left",
        "labels": ["Number of drivers in BFM", "Number of drivers in AFM"],
        "priority": 80,
        "context_keywords": ["driver", "bfm", "afm"]
    },

    "Compliance Codes": {
        "orientation": "left",
        "labels": ["V", "NC", "TNC", "SFI", "NAP", "NA"],
        "priority": 70,
        "context_exclusions": ["MASS MANAGEMENT", "MAINTENANCE MANAGEMENT", "FATIGUE MANAGEMENT"]
    },

    "Corrective Action Request Identification": {
        "orientation": "row1",
        "labels": ["Title", "Abbreviation", "Description"],
        "label_synonyms": ["Corrective Action Request", "CAR Identification", "CAR"],
        "priority": 80,
        "context_keywords": ["corrective", "action", "CAR"]
    },

    # Basic Management schemas (lower priority than summary versions)
    "Maintenance Management": {
        "headings": [
            {"level": 1, "text": r"NHVAS\s+AUDIT\s+SUMMARY\s+REPORT", "re": True}
        ],
        "orientation": "left",
        "labels": [
            "Std 1. Daily Check",
            "Std 2. Fault Recording and Reporting",
            "Std 3. Fault Repair",
            "Std 4. Maintenance Schedules and Methods",
            "Std 5. Records and Documentation",
            "Std 6. Responsibilities",
            "Std 7. Internal Review",
            "Std 8. Training and Education"
        ],
        "label_synonyms": [
            "Daily Check", "Fault Recording/ Reporting", "Fault Repair",
            "Maintenance Schedules", "Records and Documentation", "Internal Review"
        ],
        "priority": 60,
        "context_keywords": ["maintenance"],
        "context_exclusions": ["summary", "details", "audit findings"]
    },

    "Mass Management": {
        "headings": [
            {"level": 1, "text": r"NHVAS\s+AUDIT\s+SUMMARY\s+REPORT", "re": True}
        ],
        "orientation": "left",
        "labels": [
            "Std 1. Responsibilities",
            "Std 2. Vehicle Control",
            "Std 3. Vehicle Use",
            "Std 4. Records and Documentation",
            "Std 5. Verification",
            "Std 6. Internal Review",
            "Std 7. Training and Education",
            "Std 8. Maintenance of Suspension"
        ],
        "priority": 60,
        "context_keywords": ["mass"],
        "context_exclusions": ["summary", "details", "audit findings"]
    },

    "Fatigue Management": {
        "headings": [
            {"level": 1, "text": r"NHVAS\s+AUDIT\s+SUMMARY\s+REPORT", "re": True}
        ],
        "orientation": "left",
        "labels": [
            "Std 1. Scheduling and Rostering",
            "Std 2. Health and wellbeing for performed duty",
            "Std 3. Training and Education",
            "Std 4. Responsibilities and management practices",
            "Std 5. Internal Review",
            "Std 6. Records and Documentation",
            "Std 7. Workplace conditions"
        ],
        "priority": 60,
        "context_keywords": ["fatigue"],
        "context_exclusions": ["summary", "details", "audit findings"]
    },

    # Summary management schemas (higher priority)
    "Maintenance Management Summary": {
        "headings": [
            {"level": 1, "text": r"Audit\s+Observations\s+and\s+Comments", "re": True},
            {"level": 2, "text": r"Maintenance\s+Management\s+Summary\s+of\s+Audit\s+findings", "re": True}
        ],
        "orientation": "left",
        "columns": ["MAINTENANCE MANAGEMENT", "DETAILS"],
        "labels": [
            "Std 1. Daily Check",
            "Std 2. Fault Recording and Reporting",
            "Std 3. Fault Repair",
            "Std 4. Maintenance Schedules and Methods",
            "Std 5. Records and Documentation",
            "Std 6. Responsibilities",
            "Std 7. Internal Review",
            "Std 8. Training and Education"
        ],
        "label_synonyms": ["Details", "Management", "Maintenance Management Summary"],
        "priority": 85,
        "context_keywords": ["maintenance", "summary", "details", "audit findings"]
    },

    "Mass Management Summary": {
        "headings": [
            {"level": 1, "text": r"Mass\s+Management\s+Summary\s+of\s+Audit\s+findings", "re": True}
        ],
        "orientation": "left",
        "columns": ["MASS MANAGEMENT", "DETAILS"],
        "labels": [
            "Std 1. Responsibilities",
            "Std 2. Vehicle Control",
            "Std 3. Vehicle Use",
            "Std 4. Records and Documentation",
            "Std 5. Verification",
            "Std 6. Internal Review",
            "Std 7. Training and Education",
            "Std 8. Maintenance of Suspension"
        ],
        "priority": 85,
        "context_keywords": ["mass", "summary", "details", "audit findings"]
    },

    "Fatigue Management Summary": {
        "headings": [
            {"level": 1, "text": r"Fatigue\s+Management\s+Summary\s+of\s+Audit\s+findings", "re": True}
        ],
        "orientation": "left",
        "columns": ["FATIGUE MANAGEMENT", "DETAILS"],
        "labels": [
            "Std 1. Scheduling and Rostering",
            "Std 2. Health and wellbeing for performed duty",
            "Std 3. Training and Education",
            "Std 4. Responsibilities and management practices",
            "Std 5. Internal Review",
            "Std 6. Records and Documentation",
            "Std 7. Workplace conditions"
        ],
        "priority": 85,
        "context_keywords": ["fatigue", "summary", "details", "audit findings"]
    },

    # Vehicle registration tables (mass / maintenance variants)
    "Vehicle Registration Numbers Mass": {
        "headings": [
            {"level": 1, "text": r"Vehicle\s+Registration\s+Numbers\s+of\s+Records\s+Examined", "re": True},
            {"level": 2, "text": r"MASS\s+MANAGEMENT", "re": True}
        ],
        "orientation": "row1",
        "labels": [
            "No.", "Registration Number", "Sub contractor", "Sub-contractor (Yes/No)",
            "Sub-contracted Vehicles Statement of Compliance",
            "Weight Verification Records", "RFS Suspension Certification #",
            "Suspension System Maintenance", "Trip Records",
            "Fault Recording/ Reporting on Suspension System"
        ],
        "label_synonyms": [
            "Reg No", "RegistrationNo", "RegistrationNumber",
            "Sub-contractor", "Sub contracted", "Sub contractor"
        ],
        "priority": 90,
        "context_keywords": ["mass", "vehicle registration", "rfs suspension", "weight verification"],
        "context_exclusions": ["maintenance", "daily checks"]
    },

    "Vehicle Registration Numbers Maintenance": {
        "headings": [
            {"level": 1, "text": r"Vehicle\s+Registration\s+Numbers\s+of\s+Records\s+Examined", "re": True},
            {"level": 2, "text": r"Maintenance\s+Management", "re": True}
        ],
        "orientation": "row1",
        "labels": [
            "No.", "Registration Number", "Roadworthiness Certificates",
            "Maintenance Records", "Daily Checks", "Fault Recording/ Reporting", "Fault Repair"
        ],
        "label_synonyms": ["Roadworthiness", "Daily Checks", "Maintenance Records"],
        "priority": 85,
        "context_keywords": ["maintenance", "vehicle registration", "roadworthiness", "daily checks"]
    },

    "Driver / Scheduler Records Examined": {
        "headings": [
            {"level": 1, "text": r"Driver\s*/\s*Scheduler\s+Records\s+Examined", "re": True},
            {"level": 2, "text": r"FATIGUE\s+MANAGEMENT", "re": True}
        ],
        "orientation": "row1",
        "labels": [
            "No.",
            "Driver / Scheduler Name",
            "Driver TLIF Course # Completed",
            "Scheduler TLIF Course # Completed",
            "Medical Certificates (Current Yes/No) Date of expiry",
            "Roster / Schedule / Safe Driving Plan (Date Range)",
            "Fit for Duty Statement Completed (Yes/No)",
            "Work Diary Pages (Page Numbers) Electronic Work Diary Records (Date Range)"
        ],
        "label_synonyms": ["Driver Name", "Scheduler Name", "Work Diary", "Roster / Schedule"],
        "priority": 80,
        "context_keywords": ["driver", "scheduler", "fatigue"]
    },

    # Other tables
    "Operator's Name (legal entity)": {
        "headings": [
            {"level": 1, "text": r"CORRECTIVE\s+ACTION\s+REQUEST\s+\(CAR\)", "re": True}
        ],
        "orientation": "left",
        "labels": ["Operator's Name (legal entity)", "Operator’s Name (legal entity)"],
        "label_synonyms": ["Operator's Name", "Operator Name (legal)"],
        "priority": 85
    },

    "Non-conformance and CAR details": {
        "orientation": "left",
        "labels": [
            "Non-conformance agreed close out date",
            "Module and Standard",
            "Corrective Action Request (CAR) Number",
            "Observed Non-conformance:",
            "Corrective Action taken or to be taken by operator:",
            "Operator or Representative Signature",
            "Position",
            "Date",
            "Comments:",
            "Auditor signature"
        ],
        "label_synonyms": ["Non-conformance", "CAR Number", "Observed Non-conformance"],
        "priority": 75,
        "context_keywords": ["non-conformance", "corrective action", "CAR"]
    },

    "NHVAS Approved Auditor Declaration": {
        "headings": [
            {"level": 1, "text": r"NHVAS\s+APPROVED\s+AUDITOR\s+DECLARATION", "re": True}
        ],
        "orientation": "row1",
        "labels": ["Print Name", "NHVR or Exemplar Global Auditor Registration Number", "Auditor name"],
        "label_synonyms": ["Print Name (Auditor)", "Auditor Print Name", "NHVR Auditor Reg No"],
        "priority": 90,
        "context_exclusions": ["manager", "operator declaration"],
        # Defend against date-like values accidentally being used as a name
        "skip_if_date_like": ["Print Name", "Auditor name"]
    },

    "Audit Declaration dates": {
        "headings": [
            {"level": 1, "text": r"Audit Declaration dates", "re": True}
        ],
        "orientation": "left",
        "labels": [
            "Audit was conducted on",
            "Unconditional CARs closed out on:",
            "Conditional CARs to be closed out by:"
        ],
        "label_synonyms": ["Audit was conducted", "Audit date", "CARs closed out"],
        "priority": 80
    },

    "Print accreditation name": {
        "headings": [
            {"level": 1, "text": r"\(?print accreditation name\)?", "re": True}
        ],
        "orientation": "left",
        "labels": ["(print accreditation name)", "Print accreditation name", "Print Accreditation Name"],
        "label_synonyms": ["Print name accreditation", "Print accreditation"],
        "priority": 85,
        "preferred_json_keys": [
            "Print accreditation name.(print accreditation name)",
            "(print accreditation name)"
        ]
    },

    "Operator Declaration": {
        "headings": [
            {"level": 1, "text": r"Operator\s+Declaration", "re": True}
        ],
        "orientation": "row1",
        "labels": ["Print Name", "Position Title"],
        "label_synonyms": [
            "Print Name", "Printed Name", "Name (Print)", "Position", "Position Title", "Title"
        ],
        "priority": 90,
        "preferred_json_keys": [
            "Operator Declaration.Print Name",
            "Operator Declaration.Position Title",
            "Operator Declaration.Print Name.Print Name"
        ],
        "fallback_keys": ["Print Name", "Position Title", "Print accreditation name.(print accreditation name)"],
        "context_keywords": ["operator declaration", "manager", "operator"],
        "context_exclusions": ["auditor", "nhvas approved"],
        "skip_if_date_like": ["Print Name"],  # do not replace name field with date-like values
        "split_on": [" – ", " - ", "\n"],      # in case JSON contains multiple names
        # NB: pipeline should attempt exact-qualified keys first, then fallbacks
    }
}

# ---------------------------
# Heading / paragraph patterns (expanded)
# ---------------------------
HEADING_PATTERNS = {
    "main": [
        r"NHVAS\s+Audit\s+Summary\s+Report",
        r"NATIONAL\s+HEAVY\s+VEHICLE\s+ACCREDITATION\s+AUDIT\s+SUMMARY\s+REPORT",
        r"NHVAS\s+AUDIT\s+SUMMARY\s+REPORT"
    ],
    "sub": [
        r"AUDIT\s+OBSERVATIONS\s+AND\s+COMMENTS",
        r"MAINTENANCE\s+MANAGEMENT",
        r"MASS\s+MANAGEMENT",
        r"FATIGUE\s+MANAGEMENT",
        r"Fatigue\s+Management\s+Summary\s+of\s+Audit\s+findings",
        r"MAINTENANCE\s+MANAGEMENT\s+SUMMARY\s+OF\s+AUDIT\s+FINDINGS",
        r"MASS\s+MANAGEMENT\s+SUMMARY\s+OF\s+AUDIT\s+FINDINGS",
        r"Vehicle\s+Registration\s+Numbers\s+of\s+Records\s+Examined",
        r"CORRECTIVE\s+ACTION\s+REQUEST\s+\(CAR\)",
        r"NHVAS\s+APPROVED\s+AUDITOR\s+DECLARATION",
        r"Operator\s+Declaration",
        r"Operator\s+Information",
        # catch small variants / OCR noise
        r"OPERATOR'?S\s+NAME",
    ]
}

# ---------------------------
# Paragraph patterns (improved)
# ---------------------------
PARAGRAPH_PATTERNS = {
    "findings_summary": r"Provide a summary of findings based on the evidence gathered during the audit\.?",
    "declaration_text": r"I hereby acknowledge and agree with the findings.*",
    "introductory_note": r"This audit assesses the.*",
    "date_line": r"^\s*\d{1,2}(?:st|nd|rd|th)?\s+[A-Za-z]+\s+\d{4}\s*$|^Date$",
    # extra patterns to help skip/identify date-like strings in name fields
    "date_like": GLOBAL_SETTINGS["date_like_pattern"]
}

# ---------------------------
# Helpful hints for pipeline implementer (non-executable comments)
# ---------------------------
# 1) Matching order recommendation (pipeline):
#    - Try exact qualified keys in schema['preferred_json_keys']
#    - Try exact label match against schema['labels'] and schema['label_synonyms']
#    - Apply normalization rules from GLOBAL_SETTINGS then try clean match
#    - Fuzzy match using GLOBAL_SETTINGS['fuzzy_thresholds'] keyed by schema priority
#    - If schema has 'fallback_keys', try them last
#
# 2) Date-safety: for any schema field listed under 'skip_if_date_like', verify JSON value
#    or cell content is not date-like (use PARAGRAPH_PATTERNS['date_like']) before replacing.
#
# 3) Multi-value cells: If schema contains 'split_on', join JSON list items using '\n'
#    or split a single JSON string on these delimiters to create separate lines.
#
# 4) OCR repairs: run GLOBAL_SETTINGS['ocr_repair_rules'] on extracted header text
#    before attempting matching (fix common OCR confusion).
#
# 5) Processed flags: keep per-table attribute (e.g. table._processed_operator_declaration = True)
#    to avoid running two different handlers on the same Operator Declaration table.
#
# 6) Tuning: If a schema is consistently mis-matched, add the exact observed header string
#    into schema['label_synonyms'] for deterministic matching.
#
# End of master key