feat(sidechats): headless gateway new channel auto-provisioning and dedicated heartbeat channel
This commit is contained in:
operator
@@ -598,6 +598,60 @@ def dm_send(agent, target, message, verify=True, raw=False,
|
||||
except Exception as _e:
|
||||
log_event({"type": "gateway_fallback", "id": msg_id, "error": str(_e)[:100]})
|
||||
|
||||
# Fast gateway auto-provisioning for named sidechat targets:
|
||||
if not is_uuid and recipient in VALID_AGENTS and target != "main":
|
||||
try:
|
||||
import muse_hybrid
|
||||
matched_uuid = None
|
||||
gw_threads, gw_t_err = muse_hybrid.get_threads(recipient)
|
||||
if not gw_t_err and gw_threads:
|
||||
for t in gw_threads:
|
||||
t_title = (t.get("title") or "").strip().lower()
|
||||
if t_title == target.strip().lower():
|
||||
matched_uuid = t.get("session_id")
|
||||
break
|
||||
|
||||
if not matched_uuid:
|
||||
# Spawn a brand new sidechat channel via fast headless gateway
|
||||
res, err = muse_hybrid.start_session(recipient, title=target)
|
||||
if res and not err and res.get("session_id"):
|
||||
matched_uuid = res.get("session_id")
|
||||
log_event({"type": "sidechat_autoprovisioned", "id": msg_id, "target": target,
|
||||
"thread_uuid": matched_uuid, "agent": recipient, "transport": "gateway"})
|
||||
|
||||
if matched_uuid:
|
||||
# Save mapping to job-sidechats.json
|
||||
sc_data = load_sidechat_map()
|
||||
save_key = f"{target}@{recipient}" if f"{target}@{recipient}" in sc_data else target
|
||||
sc_data[save_key] = {
|
||||
"thread_uuid": matched_uuid,
|
||||
"agent": recipient,
|
||||
"title": target,
|
||||
"created_at": datetime.now(timezone.utc).isoformat()
|
||||
}
|
||||
tmp_sc = f"{SIDCHAT_MAP_FILE}.tmp.{os.getpid()}"
|
||||
with open(tmp_sc, "w", encoding="utf-8") as f:
|
||||
json.dump(sc_data, f, indent=2)
|
||||
os.replace(tmp_sc, SIDCHAT_MAP_FILE)
|
||||
|
||||
# Send immediately via gateway
|
||||
gw_res, gw_err = muse_hybrid.send_message(recipient, tagged, thread_id=matched_uuid, wait=0)
|
||||
if gw_res and not gw_err:
|
||||
thread_uuid = matched_uuid
|
||||
tags["thread"] = thread_uuid
|
||||
log_event({"type": "verified", "id": msg_id, "agent": agent, "to": recipient,
|
||||
"target": target, "thread_uuid": thread_uuid, "transport": "gateway",
|
||||
"placement": "confirmed"})
|
||||
log_event({"type": "send_done", "id": msg_id, "agent": agent, "to": recipient, "target": target})
|
||||
log_event({"type": "sent", "id": msg_id, "agent": agent, "to": recipient, "target": target,
|
||||
"verified": True, "transport": "gateway", "tags": tags})
|
||||
print(f"DM {msg_id} from {agent} to {recipient}/{target}: SENT and VERIFIED thread={thread_uuid} (gateway, auto-provisioned)")
|
||||
if tags.get("reply:expected"):
|
||||
_register_followup(msg_id, agent, recipient, target, tags)
|
||||
return msg_id
|
||||
except Exception as _ge:
|
||||
log_event({"type": "gateway_autoprovision_fallback", "id": msg_id, "error": str(_ge)[:100]})
|
||||
|
||||
if target == "main":
|
||||
_rc, _out, _err = run_full(f"{NETVM_EXEC} {recipient} -- python3 {API} --account {recipient} sidechat main")
|
||||
else:
|
||||
|
||||
+47
-4
@@ -455,14 +455,57 @@ def main():
|
||||
target = p_entry.get("target") or f"pipe-{pipeline_run_id.split('-')[-1]}"
|
||||
else:
|
||||
target = f"pipe-{pipeline_run_id.split('-')[-1]}"
|
||||
elif job.get("sidechat", {}).get("create"):
|
||||
sc_cfg = job.get("sidechat", {})
|
||||
sc_name = render_prompt(sc_cfg.get("name_template", "job-{job_name}-{date}"), variables)
|
||||
reuse_key = sc_cfg.get("reuse_key")
|
||||
|
||||
# Check if reuse_key exists in job-sidechats.json and thread is still alive
|
||||
sc_state = load_sidechat_state()
|
||||
reused_uuid = None
|
||||
if reuse_key and reuse_key in sc_state:
|
||||
val = sc_state[reuse_key]
|
||||
cand_uuid = val.get("thread_uuid") if isinstance(val, dict) else val
|
||||
if cand_uuid:
|
||||
try:
|
||||
import muse_hybrid
|
||||
threads, err = muse_hybrid.get_threads(agent)
|
||||
if not err and threads:
|
||||
thread_ids = [t.get("session_id") for t in threads]
|
||||
if cand_uuid in thread_ids:
|
||||
reused_uuid = cand_uuid
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if reused_uuid:
|
||||
target = reused_uuid
|
||||
else:
|
||||
# Spawn a brand new sidechat/channel via fast headless gateway!
|
||||
channel_title = sc_name
|
||||
try:
|
||||
import muse_hybrid
|
||||
res, err = muse_hybrid.start_session(agent, title=channel_title)
|
||||
if res and not err and res.get("session_id"):
|
||||
new_uuid = res.get("session_id")
|
||||
key_to_save = reuse_key or sc_name
|
||||
sc_state[key_to_save] = {
|
||||
"thread_uuid": new_uuid,
|
||||
"agent": agent,
|
||||
"title": channel_title,
|
||||
"created_at": datetime.now(timezone.utc).isoformat()
|
||||
}
|
||||
save_sidechat_state(sc_state)
|
||||
target = new_uuid
|
||||
print(f"Spawned new sidechat channel '{channel_title}' ({new_uuid}) for {agent}")
|
||||
else:
|
||||
target = sc_name
|
||||
except Exception as e:
|
||||
sys.stderr.write(f"warning: fast gateway session-start exception ({e}), falling back to name {sc_name}\n")
|
||||
target = sc_name
|
||||
elif job.get("dm_target"):
|
||||
target = job.get("dm_target").strip()
|
||||
elif job.get("target"):
|
||||
target = job.get("target").strip()
|
||||
elif job.get("sidechat", {}).get("create"):
|
||||
sc_cfg = job.get("sidechat", {})
|
||||
sc_name = render_prompt(sc_cfg.get("name_template", "job-{job_name}-{date}"), variables)
|
||||
target = sc_name
|
||||
|
||||
# Sidechat-first policy (2026-10-04): refuse to dispatch to main chat
|
||||
# unless the job explicitly opts in. Never fall back to main silently.
|
||||
|
||||
+57
-4
@@ -92,14 +92,16 @@
|
||||
"purpose": "Main loop coordination: health reports, digests, alerts"
|
||||
},
|
||||
"heartbeat": {
|
||||
"thread_uuid": "5f18476d-8994-49e7-a9e0-4838732363fe",
|
||||
"thread_uuid": "51695451-ffba-49e2-8c6d-ffa4f511ce5a",
|
||||
"agent": "opm",
|
||||
"description": "Alias for main-loop brain heartbeat"
|
||||
"description": "Dedicated heartbeat thread (provisioned 2026-10-05 via headless gateway)",
|
||||
"updated_at": "2026-10-05T02:22:19+00:00"
|
||||
},
|
||||
"heartbeat-opm": {
|
||||
"thread_uuid": "5f18476d-8994-49e7-a9e0-4838732363fe",
|
||||
"thread_uuid": "51695451-ffba-49e2-8c6d-ffa4f511ce5a",
|
||||
"agent": "opm",
|
||||
"description": "Reuse key for OPM heartbeat"
|
||||
"description": "Reuse key for OPM heartbeat -> dedicated heartbeat thread",
|
||||
"updated_at": "2026-10-05T02:22:19+00:00"
|
||||
},
|
||||
"646-pip@646": {
|
||||
"thread_uuid": "e3442b78-0300-4034-8cbe-7a0ab6c5764a",
|
||||
@@ -156,5 +158,56 @@
|
||||
"agent": "dev",
|
||||
"created_at": "2026-10-04T23:37:01+00:00",
|
||||
"note": "onboarding-alpha-probe"
|
||||
},
|
||||
"onboarding-def": {
|
||||
"thread_uuid": "466005b1-7065-4798-bf31-bbe785d65f3a",
|
||||
"agent": "def",
|
||||
"created_at": "2026-10-04T23:45:37+00:00",
|
||||
"note": "onboarding-def-probe"
|
||||
},
|
||||
"main-loop-brain": {
|
||||
"thread_uuid": "74d05d6b-af95-423f-8999-f733ab647785",
|
||||
"agent": "646",
|
||||
"created_at": "2026-10-05T00:53:28.473884+00:00"
|
||||
},
|
||||
"dev-audit-channel": {
|
||||
"thread_uuid": "729d31e0-5154-44ad-97c8-748d36daa448",
|
||||
"agent": "dev",
|
||||
"title": "dev-audit-channel",
|
||||
"created_at": "2026-10-05T02:23:45.941926+00:00"
|
||||
},
|
||||
"canary-test": {
|
||||
"thread_uuid": "bcda3950-9966-4edd-a014-f931d8b17701",
|
||||
"agent": "opm",
|
||||
"title": "canary-test",
|
||||
"created_at": "2026-10-05T02:24:25.475505+00:00"
|
||||
},
|
||||
"heartbeat-thread": {
|
||||
"thread_uuid": "51695451-ffba-49e2-8c6d-ffa4f511ce5a",
|
||||
"agent": "opm",
|
||||
"description": "Alias for heartbeat (dedicated heartbeat channel provisioned 2026-10-05)"
|
||||
},
|
||||
"main-loop": {
|
||||
"thread_uuid": "5f18476d-8994-49e7-a9e0-4838732363fe",
|
||||
"agent": "opm",
|
||||
"description": "Alias for main-loop brain; added 2026-10-05 to stop autoprovision junk"
|
||||
},
|
||||
"box-http-health": {
|
||||
"thread_uuid": "d9c596c4-5ace-44dc-9619-9862cf56968d",
|
||||
"agent": "646",
|
||||
"title": "box-http-health",
|
||||
"created_at": "2026-10-05T02:25:41.573556+00:00"
|
||||
},
|
||||
"box-service-health": {
|
||||
"thread_uuid": "94ab9f9f-012c-41e5-ad35-dc44f2b24281",
|
||||
"agent": "646",
|
||||
"title": "box-service-health",
|
||||
"created_at": "2026-10-05T02:26:11.075842+00:00"
|
||||
},
|
||||
"box-deep-health": {
|
||||
"thread_uuid": "6628c035-4413-4d9f-863c-56ee861c8c83",
|
||||
"agent": "646",
|
||||
"title": "box-deep-health",
|
||||
"created_at": "2026-10-05T02:26:37.043639+00:00"
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@
|
||||
"agent": "646",
|
||||
"chain_next": null,
|
||||
"description": "Box deep health \u2014 daily comprehensive check",
|
||||
"dm_target": "646 tasks",
|
||||
"followup": {
|
||||
"escalate": "opm",
|
||||
"expect_reply": true,
|
||||
@@ -15,7 +14,9 @@
|
||||
"prompt_template": "Box deep health check (daily).\nJob ID: {job_id}\nTime: {datetime}\n\nRun the FULL Box health suite on the VM:\n ssh dev-operator-646@34.139.37.135 \"/srv/box/bin/box-health-check.sh all\"\n\nThen verify these endpoints and services:\n1. https://box.muse-dev.online/ loads (Fleet Console UI, assets box.js and box.css 200 OK)\n2. https://box.muse-dev.online/api/box/fleet returns live fleet status\n3. https://box.muse-dev.online/api/box/dm/log returns live DM log entries\n4. https://box.muse-dev.online/followups loads (operator view, follow-up dashboard)\n5. https://box.muse-dev.online/api/box/followups/summary returns valid JSON\n6. Check /srv/box/dm-log.jsonl tail for errors in the last 24h\n7. Confirm the request-sweeper processed follow-ups in the last hour\n (grep dm_followup /srv/box/box_requests.jsonl | tail -5)\n\nReport any degradation, even if checks pass (slow responses, log warnings).\n\nReply with OK or FAIL <summary>, plus any observations worth tracking.",
|
||||
"schedule": "0 9 * * *",
|
||||
"sidechat": {
|
||||
"create": false
|
||||
"create": true,
|
||||
"name_template": "box-deep-health",
|
||||
"reuse_key": "box-deep-health"
|
||||
},
|
||||
"timeout": 1800
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
"agent": "646",
|
||||
"chain_next": null,
|
||||
"description": "Box HTTP endpoint health \u2014 every 15 minutes",
|
||||
"dm_target": "646 tasks",
|
||||
"followup": {
|
||||
"escalate": "opm",
|
||||
"expect_reply": true,
|
||||
@@ -15,7 +14,9 @@
|
||||
"prompt_template": "Box HTTP health check.\nJob ID: {job_id}\nTime: {datetime}\n\nRun the Box HTTP checks on the VM:\n ssh dev-operator-646@34.139.37.135 \"/srv/box/bin/box-health-check.sh http\"\n(or run /srv/box/bin/box-health-check.sh http via any VM access you have)\n\nAlso verify the new live Box Console endpoints:\n1. https://box.muse-dev.online/ (Front-Door Console loads, assets box.js and box.css return 200)\n2. https://box.muse-dev.online/api/box/fleet (Returns 200 with live fleet array)\n3. https://box.muse-dev.online/api/box/dm/log (Returns 200 with recent DM log entries)\n\nExpected: all OK. If any FAIL, investigate immediately (check board.service,\nCaddy, recent deploys) and include the failure lines in your reply.\n\nReply with OK or FAIL <one-line summary>.",
|
||||
"schedule": "*/15 * * * *",
|
||||
"sidechat": {
|
||||
"create": false
|
||||
"create": true,
|
||||
"name_template": "box-http-health",
|
||||
"reuse_key": "box-http-health"
|
||||
},
|
||||
"timeout": 600
|
||||
}
|
||||
|
||||
@@ -1 +1,22 @@
|
||||
{"agent":"646","chain_next":null,"description":"Box service + data health — every 15 minutes (offset 7m)","dm_target":"646 tasks","followup":{"escalate":"opm","expect_reply":true,"nudges":2,"route":"box-health","timeout":"30m"},"name":"box-service-health","on_failure":"alert","prompt_template":"Box service health check.\nJob ID: {job_id}\nTime: {datetime}\n\nRun the Box service and data checks on the VM:\n ssh dev-operator-646@34.139.37.135 \"/srv/box/bin/box-health-check.sh services\"\n ssh dev-operator-646@34.139.37.135 \"/srv/box/bin/box-health-check.sh data\"\n(or run /srv/box/bin/box-health-check.sh via any VM access you have)\n\nExpected: board.service active, caddy active, sweeper timer firing,\n/srv/box writable. If any FAIL, investigate and include failure lines.\n\nReply with [RESULT {job_id}] OK or [RESULT {job_id}] FAIL <one-line summary>.","schedule":"7,22,37,52 * * * *","sidechat":{"create":false},"timeout":600}
|
||||
{
|
||||
"agent": "646",
|
||||
"chain_next": null,
|
||||
"description": "Box service + data health — every 15 minutes (offset 7m)",
|
||||
"followup": {
|
||||
"escalate": "opm",
|
||||
"expect_reply": true,
|
||||
"nudges": 2,
|
||||
"route": "box-health",
|
||||
"timeout": "30m"
|
||||
},
|
||||
"name": "box-service-health",
|
||||
"on_failure": "alert",
|
||||
"prompt_template": "Box service health check.\nJob ID: {job_id}\nTime: {datetime}\n\nRun the Box service and data checks on the VM:\n ssh dev-operator-646@34.139.37.135 \"/srv/box/bin/box-health-check.sh services\"\n ssh dev-operator-646@34.139.37.135 \"/srv/box/bin/box-health-check.sh data\"\n(or run /srv/box/bin/box-health-check.sh via any VM access you have)\n\nExpected: board.service active, caddy active, sweeper timer firing,\n/srv/box writable. If any FAIL, investigate and include failure lines.\n\nReply with [RESULT {job_id}] OK or [RESULT {job_id}] FAIL <one-line summary>.",
|
||||
"schedule": "7,22,37,52 * * * *",
|
||||
"sidechat": {
|
||||
"create": true,
|
||||
"name_template": "box-service-health",
|
||||
"reuse_key": "box-service-health"
|
||||
},
|
||||
"timeout": 600
|
||||
}
|
||||
+14
-1
@@ -1 +1,14 @@
|
||||
{"agent":"opm","chain_next":null,"description":"Canary job to test the dispatcher - sends a simple DM","dm_target":"heartbeat","name":"canary-test","on_failure":"alert","prompt_template":"This is a canary test from the job dispatcher.\nJob ID: {job_id}\nDate: {date}\n\nPlease reply to confirm you received this.","schedule":"manual","sidechat":{"create":false},"timeout":300}
|
||||
{
|
||||
"agent": "opm",
|
||||
"chain_next": null,
|
||||
"description": "Canary job to test the dispatcher - sends a simple DM",
|
||||
"name": "canary-test",
|
||||
"on_failure": "alert",
|
||||
"prompt_template": "This is a canary test from the job dispatcher.\nJob ID: {job_id}\nDate: {date}\n\nPlease reply to confirm you received this.",
|
||||
"schedule": "manual",
|
||||
"sidechat": {
|
||||
"create": true,
|
||||
"name_template": "canary-test"
|
||||
},
|
||||
"timeout": 300
|
||||
}
|
||||
Reference in New Issue
Block a user