box/bin/lookup_engine.py

#!/usr/bin/env python3
"""lookup_engine.py — Shared Zero-Downtime Hot-Reloading Pattern & Schema Engine.

Provides authoritative runtime access to lookup_internal/ databases for
daemons (response-harvester, self_main_loop, job-dispatch), CLI commands,
and agents.

Features:
- Dynamic mtime-based zero-downtime hot reloading of compiled regex patterns.
- Pre-flight soft validation of outbound agent sentences and work orders.
- Direct helper functions for core protocol regexes (RESULT, VERB, TOOL, etc.).
"""

import json
import os
import re
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple

NETVM_ROOT = Path(__file__).resolve().parent.parent
LOOKUP_INTERNAL = NETVM_ROOT / "lookup_internal"
if not LOOKUP_INTERNAL.exists() and (NETVM_ROOT / "docs_internal").exists():
    LOOKUP_INTERNAL = NETVM_ROOT / "docs_internal"

# In-memory cache structures with modification timestamps
_CACHE_MTIMES: Dict[str, float] = {}
_RAW_CACHE: Dict[str, Any] = {}
_COMPILED_PATTERNS: Dict[str, re.Pattern] = {}

# Fallback hardcoded regexes in case files are missing or unreadable
_FALLBACK_RESULT_RE = re.compile(r"\[RESULT\s+([A-Za-z0-9_/-]+)\]\s*(.*?)(?=\[RESULT\s|\Z)", re.S)
_FALLBACK_VERB_RE = re.compile(r"\[(ACK|CLAIM|RESULT|DECLINE|NO-ACTION)\s+([A-Za-z0-9_/-]+)\]")
_FALLBACK_TOOL_RE = re.compile(r"\[(TOOL|EXEC)\s+([a-zA-Z0-9_.-]+)\s+(\{.*?\})\]", re.S)
_FALLBACK_CONTRACT_FOOTER = (
    "Reply: [ACK id] seen | [CLAIM id] mine | "
    "[RESULT id] done | [DECLINE id] | [NO-ACTION id]."
)


def load_lookup_json(filename: str) -> Dict[str, Any]:
    """Load JSON from lookup_internal/ with mtime-based caching."""
    target_path = LOOKUP_INTERNAL / filename
    if not target_path.is_file():
        return {}

    try:
        current_mtime = os.path.getmtime(target_path)
    except OSError:
        return _RAW_CACHE.get(filename, {})

    if filename in _RAW_CACHE and _CACHE_MTIMES.get(filename) == current_mtime:
        return _RAW_CACHE[filename]

    try:
        with open(target_path, "r", encoding="utf-8") as f:
            data = json.load(f)
        _RAW_CACHE[filename] = data
        _CACHE_MTIMES[filename] = current_mtime
        return data
    except Exception as e:
        print(f"[lookup_engine] Warning: Error reading {target_path}: {e}", file=sys.stderr)
        return _RAW_CACHE.get(filename, {})


def _refresh_compiled_patterns_if_needed():
    """Checks regex_patterns.json mtime and recompiles if changed."""
    global _COMPILED_PATTERNS
    data = load_lookup_json("regex_patterns.json")
    patterns_data = data.get("patterns", {})

    target_path = LOOKUP_INTERNAL / "regex_patterns.json"
    current_mtime = _CACHE_MTIMES.get("regex_patterns.json", 0.0)
    compiled_mtime = _CACHE_MTIMES.get("_compiled_patterns_mtime", 0.0)

    if current_mtime == compiled_mtime and _COMPILED_PATTERNS:
        return

    new_compiled = {}
    for key, entry in patterns_data.items():
        raw_pat = entry.get("pattern", "")
        flag_names = entry.get("flags", [])
        flags = 0
        for fn in flag_names:
            if hasattr(re, fn):
                flags |= getattr(re, fn)
        try:
            new_compiled[key] = re.compile(raw_pat, flags)
        except Exception as e:
            print(f"[lookup_engine] Warning: Failed to compile pattern '{key}': {e}", file=sys.stderr)

    _COMPILED_PATTERNS = new_compiled
    _CACHE_MTIMES["_compiled_patterns_mtime"] = current_mtime


def get_compiled_pattern(name: str) -> Optional[re.Pattern]:
    """Retrieve a compiled pattern by name, hot-reloading if the database was modified."""
    _refresh_compiled_patterns_if_needed()
    return _COMPILED_PATTERNS.get(name)


def get_all_compiled_patterns() -> Dict[str, re.Pattern]:
    """Retrieve all compiled patterns with automatic hot-reloading."""
    _refresh_compiled_patterns_if_needed()
    return dict(_COMPILED_PATTERNS)


def get_result_regex() -> re.Pattern:
    """Return the canonical [RESULT ...] regex."""
    p = get_compiled_pattern("result")
    return p if p is not None else _FALLBACK_RESULT_RE


def get_verb_regex() -> re.Pattern:
    """Return the canonical [VERB ...] regex (ACK|CLAIM|RESULT|DECLINE|NO-ACTION)."""
    p = get_compiled_pattern("verb")
    return p if p is not None else _FALLBACK_VERB_RE


def get_tool_regex() -> re.Pattern:
    """Return the canonical [TOOL ...] regex."""
    p = get_compiled_pattern("tool_call")
    return p if p is not None else _FALLBACK_TOOL_RE


def get_contract_footer() -> str:
    """Return standard contract footer string."""
    struct_data = load_lookup_json("sentence_structure.json")
    cf = struct_data.get("structures", {}).get("contract_footer", {})
    return cf.get("example") or _FALLBACK_CONTRACT_FOOTER


def parse_agent_utterance(text: str) -> List[Dict[str, Any]]:
    """Pass text through all registered patterns and extract matched tokens."""
    _refresh_compiled_patterns_if_needed()
    patterns_data = load_lookup_json("regex_patterns.json").get("patterns", {})
    matches = []

    for key, compiled in _COMPILED_PATTERNS.items():
        m = compiled.search(text)
        if m:
            entry = patterns_data.get(key, {})
            matches.append({
                "pattern_key": key,
                "pattern_name": entry.get("name", key),
                "matched_text": m.group(0),
                "named_groups": m.groupdict(),
                "span": m.span()
            })
    return matches


def validate_outbound_sentence(text: str) -> Tuple[bool, Optional[str], Optional[str]]:
    """Pre-flight check for outbound messages sent via CLI.

    Returns:
        (is_valid, matched_kind, warning_or_hint)
    """
    cleaned = text.strip()
    # If message starts with bracketed protocol marker
    if cleaned.startswith("["):
        marker = cleaned.split("]")[0] + "]"
        upper_marker = marker.upper()

        if upper_marker.startswith("[WO:") or upper_marker.startswith("[WORKORDER:"):
            wo_pat = get_compiled_pattern("work_order")
            if wo_pat and not wo_pat.search(cleaned):
                hint = (
                    "Notice: Message starts with a Work Order marker but does not match canonical structure.\n"
                    "  Expected format: [WO:<id>] [from <sender>] <title> — <body>\n"
                    "  Example: [WO:7fce46e0] [from super] Audit endpoints — Check GET /api/stats\n"
                    "  Hint: Query 'box lookup sentence work_order' for full spec."
                )
                return False, "work_order", hint
            return True, "work_order", None

        if upper_marker.startswith("[ACK:") or upper_marker.startswith("[ACK "):
            ack_pat = get_compiled_pattern("ack")
            verb_pat = get_compiled_pattern("verb")
            if (ack_pat and not ack_pat.search(cleaned)) and (verb_pat and not verb_pat.search(cleaned)):
                hint = (
                    "Notice: Message looks like an ACK but deviates from standard syntax.\n"
                    "  Expected format: [ACK:<id>] [from <sender>] or [ACK <id>]\n"
                    "  Example: [ACK:7fce46e0] [from 646]"
                )
                return False, "ack", hint
            return True, "ack", None

        if upper_marker.startswith("[RESULT"):
            res_pat = get_result_regex()
            if not res_pat.search(cleaned):
                hint = (
                    "Notice: Message starts with [RESULT] but deviates from standard syntax.\n"
                    "  Expected format: [RESULT <job_id>] <status> <summary>\n"
                    "  Example: [RESULT 7fce46e0] OK Task completed successfully"
                )
                return False, "result", hint
            return True, "result", None

    return True, "plain_message", None