From 1e86ed8a447da7693da4c2258385b9c6e2459aac Mon Sep 17 00:00:00 2001
From: operator <operator@netvm.local>
Date: Mon, 5 Oct 2026 16:37:14 +0000
Subject: [PATCH] retention piece 1: immediate rotations for chat-history /
 job-log / followups (b67084660385)

- chat-history.jsonl rotates at 10MB or 7d -> logs/archive/*.jsonl.gz + .sha256
- job-log.jsonl rotates at 2MB or 14d -> same archive layout
- followups.json archives resolved>7d / escalated>30d -> followups.archive.jsonl (append-only)
- verify wrapper: sha256 -c, gzip -t, clean listing, spot-extract JSON + ts bounds
- driver + hourly systemd user timer (retention-rotations.timer)
- archive-only: nothing is ever deleted; thread backfill excluded (needs human-reviewed preview)
First run 2026-10-05 16:35Z: chat-history 29.7MB + job-log 4.7MB rotated and verified; followups 0 eligible of 163.
---
 bin/retention-archive-followups.py   | 93 ++++++++++++++++++++++++++++
 bin/retention-rotate-chat-history.sh | 45 ++++++++++++++
 bin/retention-rotate-job-log.sh      | 45 ++++++++++++++
 bin/retention-run-rotations.sh       | 46 ++++++++++++++
 bin/retention-verify-archive.sh      | 67 ++++++++++++++++++++
 docs/RETENTION-ROTATIONS-P1.md       | 81 ++++++++++++++++++++++++
 systemd/retention-rotations.service  | 10 +++
 systemd/retention-rotations.timer    |  9 +++
 8 files changed, 396 insertions(+)
 create mode 100755 bin/retention-archive-followups.py
 create mode 100755 bin/retention-rotate-chat-history.sh
 create mode 100755 bin/retention-rotate-job-log.sh
 create mode 100755 bin/retention-run-rotations.sh
 create mode 100755 bin/retention-verify-archive.sh
 create mode 100644 docs/RETENTION-ROTATIONS-P1.md
 create mode 100644 systemd/retention-rotations.service
 create mode 100644 systemd/retention-rotations.timer

diff --git a/bin/retention-archive-followups.py b/bin/retention-archive-followups.py
new file mode 100755
index 0000000..4ae2d52
--- /dev/null
+++ b/bin/retention-archive-followups.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""retention-archive-followups.py — retention piece 1 (board bug b67084660385).
+
+Archives resolved follow-ups older than 7 days and escalated follow-ups older
+than 30 days from followups.json into followups.archive.jsonl (append-only).
+
+ARCHIVE-ONLY: records are moved from the live file to the archive file; the
+archive itself is never pruned or hard-deleted by this script.
+
+Concurrency: the live file is written atomically (tmp + os.replace) by both
+this script and bin/followup-sweeper.py. This script snapshots the file's
+(mtime_ns, size) before deciding the archival set and aborts (rc=2) if the
+file changed in the meantime — the next run picks it up.
+"""
+import json
+import os
+import sys
+from datetime import datetime, timezone, timedelta
+
+ROOT = os.environ.get("NETVM_ROOT", "/home/super/Projects/NetVM")
+LIVE = os.path.join(ROOT, "followups.json")
+ARCHIVE = os.path.join(ROOT, "followups.archive.jsonl")
+RESOLVED_AFTER_DAYS = 7
+ESCALATED_AFTER_DAYS = 30
+
+
+def parse_ts(ts):
+    if not ts:
+        return None
+    try:
+        dt = datetime.fromisoformat(str(ts).replace("Z", "+00:00"))
+        return dt if dt.tzinfo is not None else dt.replace(tzinfo=timezone.utc)
+    except Exception:
+        return None
+
+
+def main():
+    if not os.path.exists(LIVE):
+        print("SKIP followups: no live file at %s" % LIVE)
+        return 0
+
+    with open(LIVE, encoding="utf-8") as f:
+        raw = f.read()
+    st_before = os.stat(LIVE)
+    data = json.loads(raw)
+    now = datetime.now(timezone.utc)
+
+    archive, keep = [], {}
+    for key, rec in data.items():
+        status = rec.get("status")
+        eligible = False
+        reason = ""
+        if status == "resolved":
+            dt = parse_ts(rec.get("resolved_at")) or parse_ts(rec.get("sent_at"))
+            if dt and (now - dt) > timedelta(days=RESOLVED_AFTER_DAYS):
+                eligible, reason = True, "resolved>7d"
+        elif status == "escalated":
+            dt = parse_ts(rec.get("escalated_at")) or parse_ts(rec.get("sent_at"))
+            if dt and (now - dt) > timedelta(days=ESCALATED_AFTER_DAYS):
+                eligible, reason = True, "escalated>30d"
+        if eligible:
+            out = dict(rec)
+            out["_archived_at"] = now.isoformat()
+            out["_archive_reason"] = reason
+            archive.append(out)
+        else:
+            keep[key] = rec
+
+    if not archive:
+        print("SKIP followups: 0 eligible of %d records (resolved>7d / escalated>30d)" % len(data))
+        return 0
+
+    st_after = os.stat(LIVE)
+    if (st_after.st_mtime_ns, st_after.st_size) != (st_before.st_mtime_ns, st_before.st_size):
+        print("ABORT followups: live file changed during archival decision; retry next run",
+              file=sys.stderr)
+        return 2
+
+    with open(ARCHIVE, "a", encoding="utf-8") as f:
+        for rec in archive:
+            f.write(json.dumps(rec) + "\n")
+
+    tmp = "%s.tmp.retention.%d" % (LIVE, os.getpid())
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump(keep, f, indent=2)
+    os.replace(tmp, LIVE)
+
+    print("OK followups: archived %d records -> %s (live now %d)" % (len(archive), ARCHIVE, len(keep)))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/bin/retention-rotate-chat-history.sh b/bin/retention-rotate-chat-history.sh
new file mode 100755
index 0000000..c613c0c
--- /dev/null
+++ b/bin/retention-rotate-chat-history.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# retention-rotate-chat-history.sh — retention piece 1 (board bug b67084660385).
+# Rotates logs/chat-history.jsonl when it exceeds 10MB or is older than 7 days.
+#
+# Live-writer safety: the harvester (bin/response-harvester.py) opens the file
+# in append mode per write and closes it immediately, so an atomic `mv` of the
+# live file is safe — the next write recreates a fresh file. We also `touch`
+# the new live file for tidiness and to keep ownership/visibility stable.
+#
+# Archives land in logs/archive/chat-history-YYYYMMDD-HHMMSS.jsonl.gz with a
+# sha256 sidecar. Nothing is ever deleted by this script.
+set -euo pipefail
+
+ROOT="${NETVM_ROOT:-/home/super/Projects/NetVM}"
+LIVE="$ROOT/logs/chat-history.jsonl"
+ARCHIVE_DIR="$ROOT/logs/archive"
+SIZE_LIMIT=$((10 * 1024 * 1024))   # 10MB
+AGE_LIMIT_DAYS=7
+TS="$(date -u +%Y%m%d-%H%M%S)"
+
+mkdir -p "$ARCHIVE_DIR"
+
+if [ ! -f "$LIVE" ]; then
+  echo "SKIP chat-history: no live file at $LIVE"
+  exit 0
+fi
+
+size="$(stat -c %s "$LIVE")"
+age_days=$(( ( $(date +%s) - $(stat -c %Y "$LIVE") ) / 86400 ))
+
+if [ "$size" -lt "$SIZE_LIMIT" ] && [ "$age_days" -lt "$AGE_LIMIT_DAYS" ]; then
+  echo "SKIP chat-history: ${size}B, ${age_days}d old (limits ${SIZE_LIMIT}B / ${AGE_LIMIT_DAYS}d)"
+  exit 0
+fi
+
+lines="$(wc -l < "$LIVE")"
+base="chat-history-$TS"
+echo "ROTATE chat-history: ${size}B / ${lines} lines (${age_days}d old) -> $ARCHIVE_DIR/$base.jsonl.gz"
+
+mv "$LIVE" "$ARCHIVE_DIR/$base.jsonl"
+gzip -9 "$ARCHIVE_DIR/$base.jsonl"
+( cd "$ARCHIVE_DIR" && sha256sum "$base.jsonl.gz" > "$base.jsonl.gz.sha256" )
+touch "$LIVE"
+
+echo "OK chat-history: archived $base.jsonl.gz, live file recreated empty"
diff --git a/bin/retention-rotate-job-log.sh b/bin/retention-rotate-job-log.sh
new file mode 100755
index 0000000..8b546d5
--- /dev/null
+++ b/bin/retention-rotate-job-log.sh
@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+# retention-rotate-job-log.sh — retention piece 1 (board bug b67084660385).
+# Rotates job-log.jsonl when it exceeds 2MB or is older than 14 days.
+#
+# Live-writer safety: all writers (bin/job-dispatch.py, bin/followup-sweeper.py)
+# open the file in append mode per write, so an atomic `mv` of the live file
+# is safe — the next write recreates a fresh file. We also `touch` the new
+# live file for tidiness.
+#
+# Archives land in logs/archive/job-log-YYYYMMDD-HHMMSS.jsonl.gz with a
+# sha256 sidecar. Nothing is ever deleted by this script.
+set -euo pipefail
+
+ROOT="${NETVM_ROOT:-/home/super/Projects/NetVM}"
+LIVE="$ROOT/job-log.jsonl"
+ARCHIVE_DIR="$ROOT/logs/archive"
+SIZE_LIMIT=$((2 * 1024 * 1024))    # 2MB
+AGE_LIMIT_DAYS=14
+TS="$(date -u +%Y%m%d-%H%M%S)"
+
+mkdir -p "$ARCHIVE_DIR"
+
+if [ ! -f "$LIVE" ]; then
+  echo "SKIP job-log: no live file at $LIVE"
+  exit 0
+fi
+
+size="$(stat -c %s "$LIVE")"
+age_days=$(( ( $(date +%s) - $(stat -c %Y "$LIVE") ) / 86400 ))
+
+if [ "$size" -lt "$SIZE_LIMIT" ] && [ "$age_days" -lt "$AGE_LIMIT_DAYS" ]; then
+  echo "SKIP job-log: ${size}B, ${age_days}d old (limits ${SIZE_LIMIT}B / ${AGE_LIMIT_DAYS}d)"
+  exit 0
+fi
+
+lines="$(wc -l < "$LIVE")"
+base="job-log-$TS"
+echo "ROTATE job-log: ${size}B / ${lines} lines (${age_days}d old) -> $ARCHIVE_DIR/$base.jsonl.gz"
+
+mv "$LIVE" "$ARCHIVE_DIR/$base.jsonl"
+gzip -9 "$ARCHIVE_DIR/$base.jsonl"
+( cd "$ARCHIVE_DIR" && sha256sum "$base.jsonl.gz" > "$base.jsonl.gz.sha256" )
+touch "$LIVE"
+
+echo "OK job-log: archived $base.jsonl.gz, live file recreated empty"
diff --git a/bin/retention-run-rotations.sh b/bin/retention-run-rotations.sh
new file mode 100755
index 0000000..4b7ad02
--- /dev/null
+++ b/bin/retention-run-rotations.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# retention-run-rotations.sh — retention piece 1 driver (board bug b67084660385).
+# Runs all three piece-1 rotations (chat-history, job-log, followups archive)
+# then verifies every archive touched. Idempotent: no-op when under threshold.
+# A full run report is appended to logs/retention-runs/retention-run-TS.log.
+# Exits 2 if any rotation or verification fails (so the timer run is loud).
+set -uo pipefail
+
+ROOT="${NETVM_ROOT:-/home/super/Projects/NetVM}"
+BIN="$ROOT/bin"
+ARCHIVE_DIR="$ROOT/logs/archive"
+RUN_TS="$(date -u +%Y%m%d-%H%M%S)"
+LOGDIR="$ROOT/logs/retention-runs"
+mkdir -p "$LOGDIR"
+LOG="$LOGDIR/retention-run-$RUN_TS.log"
+
+{
+echo "=== retention-run $RUN_TS (UTC) ==="
+
+fail=0
+
+"$BIN/retention-rotate-chat-history.sh" || fail=1
+"$BIN/retention-rotate-job-log.sh"     || fail=1
+"$BIN/retention-archive-followups.py"  || fail=1
+
+echo "--- verification ---"
+
+# Verify the newest chat-history / job-log archives (skip if none exist)
+for pattern in "chat-history-*.jsonl.gz" "job-log-*.jsonl.gz"; do
+  newest="$(ls -t "$ARCHIVE_DIR"/$pattern 2>/dev/null | head -1 || true)"
+  if [ -n "$newest" ]; then
+    "$BIN/retention-verify-archive.sh" "$newest" || fail=1
+  else
+    echo "SKIP verify: no $pattern in $ARCHIVE_DIR"
+  fi
+done
+
+# Verify the followups archive (append-only, cumulative)
+"$BIN/retention-verify-archive.sh" "$ROOT/followups.archive.jsonl" || fail=1
+
+echo "--- live sizes after run ---"
+ls -la "$ROOT/logs/chat-history.jsonl" "$ROOT/job-log.jsonl" "$ROOT/followups.json" 2>/dev/null || true
+
+echo "=== retention-run done rc=$fail ==="
+exit $fail
+} 2>&1 | tee -a "$LOG"
diff --git a/bin/retention-verify-archive.sh b/bin/retention-verify-archive.sh
new file mode 100755
index 0000000..8189a5e
--- /dev/null
+++ b/bin/retention-verify-archive.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# retention-verify-archive.sh — retention piece 1 (board bug b67084660385).
+# Verifies one archive file: checksum, integrity, clean listing, spot-extract.
+# Usage: retention-verify-archive.sh <path-to-.jsonl.gz-or-.jsonl>
+# Exits non-zero on any verification failure. Missing file = SKIP (rc 0).
+set -uo pipefail
+
+fail() { echo "FAIL verify $1: $2"; exit 1; }
+
+P="${1:?usage: $0 <archive-path>}"
+
+if [ ! -f "$P" ]; then
+  echo "SKIP verify: $P does not exist"
+  exit 0
+fi
+
+echo "== verify $P =="
+
+# 1. checksum sidecar
+if [ -f "$P.sha256" ]; then
+  ( cd "$(dirname "$P")" && sha256sum -c "$(basename "$P").sha256" ) \
+    || fail "$P" "sha256 mismatch"
+  echo "  checksum: OK"
+else
+  echo "  checksum: no sidecar (archiver writes one for new .gz files)"
+fi
+
+# 2. integrity + clean listing for gzip archives
+if [[ "$P" == *.gz ]]; then
+  gzip -t "$P" || fail "$P" "gzip integrity test failed"
+  echo "  integrity: gzip -t OK"
+  echo "  listing:"; gzip -l "$P" | sed 's/^/    /'
+  reader="zcat"
+else
+  reader="cat"
+fi
+
+# 3. spot-extract: first 3 and last 3 lines must be valid JSON
+n=0
+while IFS= read -r line; do
+  n=$((n+1))
+  echo "$line" | python3 -c 'import json,sys; json.loads(sys.stdin.read())' \
+    || fail "$P" "line $n is not valid JSON"
+done < <( { $reader "$P" | head -3; $reader "$P" | tail -3; } 2>/dev/null )
+
+total="$( $reader "$P" | wc -l )"
+echo "  spot-extract: first/last 3 lines valid JSON ($total total lines)"
+
+# 4. spot-extract timestamps: report newest/oldest ts seen in the sample
+sample="$($reader "$P" | head -2000)"
+ts_line="$(echo "$sample" | python3 -c '
+import json,sys
+keys=("ts","timestamp","time","created_at","sent_at","resolved_at")
+seen=[]
+for line in sys.stdin:
+    line=line.strip()
+    if not line: continue
+    try: rec=json.loads(line)
+    except Exception: continue
+    for k in keys:
+        if rec.get(k): seen.append(str(rec[k])); break
+seen=sorted(set(seen))
+print(("oldest="+seen[0]+" newest="+seen[-1]) if seen else "no-ts-field")
+')"
+echo "  timestamps: $ts_line"
+
+echo "OK verify $P"
diff --git a/docs/RETENTION-ROTATIONS-P1.md b/docs/RETENTION-ROTATIONS-P1.md
new file mode 100644
index 0000000..00c6b53
--- /dev/null
+++ b/docs/RETENTION-ROTATIONS-P1.md
@@ -0,0 +1,81 @@
+# Retention piece 1 — immediate rotations (board bug b67084660385)
+
+Owner: operator-646 (per opm verdict GO WITH MODIFICATIONS, 2026-10-05).
+Branch: `dev/operator-646/retention-rotations-p1`.
+
+Scope is deliberately narrow: the three files whose thresholds were already
+exceeded on bl (opm-verified 2026-10-05 ~10:35 UTC). Everything else from the
+retention draft — the 344-thread backfill (needs human-reviewed preview),
+jobs/ archival, subagent-session state fix, swarm blob summarizer, dispatch
+reuse-key hardening — is a later piece.
+
+## Policy implemented
+
+| File | Threshold (opm-verified) | Action | Archive layout |
+|------|--------------------------|--------|----------------|
+| `logs/chat-history.jsonl` | 10MB or 7d | rotate | `logs/archive/chat-history-YYYYMMDD-HHMMSS.jsonl.gz` + `.sha256` |
+| `job-log.jsonl` | 2MB or 14d | rotate | `logs/archive/job-log-YYYYMMDD-HHMMSS.jsonl.gz` + `.sha256` |
+| `followups.json` | resolved >7d / escalated >30d | archive records | `followups.archive.jsonl` (append-only) |
+
+Archive-only: no script here deletes anything. The draft's 180d hard-delete
+of archive entries is explicitly NOT implemented in this piece.
+
+## Files
+
+- `bin/retention-rotate-chat-history.sh` — 10MB/7d rotate
+- `bin/retention-rotate-job-log.sh` — 2MB/14d rotate
+- `bin/retention-archive-followups.py` — resolved>7d / escalated>30d archival
+- `bin/retention-verify-archive.sh <path>` — checksum, `gzip -t`, clean
+  listing (`gzip -l`), spot-extract (first/last 3 lines valid JSON + ts
+  bounds), line counts
+- `bin/retention-run-rotations.sh` — driver: runs all three, verifies the
+  newest archives + the cumulative followups archive, appends a full report
+  to `logs/retention-runs/retention-run-TS.log`
+- `systemd/retention-rotations.service` + `systemd/retention-rotations.timer`
+  — hourly user timer (`OnCalendar=hourly`, Persistent), same shape as
+  `followup-sweeper.timer`
+
+## Live-writer safety
+
+- `chat-history.jsonl`: the harvester opens the file in append mode per write
+  (`bin/response-harvester.py`, `with open(path, "a")`), so the script's
+  atomic `mv` of the live file is safe — the next write recreates it. The
+  script also `touch`es the new live file.
+- `job-log.jsonl`: same pattern (`bin/job-dispatch.py`, `bin/followup-sweeper.py`).
+- `followups.json`: both this script and `bin/followup-sweeper.py` write
+  atomically (tmp + rename). This script snapshots `(mtime_ns, size)` before
+  deciding the archival set and aborts (rc=2) if the sweeper rewrote the
+  file mid-decision; the next hourly run picks it up.
+
+## Operations
+
+Run once now (or any time): `bin/retention-run-rotations.sh`
+Dry check without acting: each rotate script prints SKIP when under threshold.
+Verify one archive: `bin/retention-verify-archive.sh <path>`
+
+Install the timer (bl, user units):
+```
+cp systemd/retention-rotations.{service,timer} ~/.config/systemd/user/
+systemctl --user daemon-reload
+systemctl --user enable --now retention-rotations.timer
+systemctl --user list-timers | grep retention
+```
+Uninstall: `systemctl --user disable --now retention-rotations.timer`
+
+## First-run report (2026-10-05, run by operator-646)
+
+- chat-history.jsonl: 29,746,435 B (25,240 lines) -> rotated to
+  `logs/archive/chat-history-20261005-163558.jsonl.gz` (5,904,954 B compressed,
+  80.1% ratio). Verify: sha256 OK, gzip -t OK, clean listing, spot-extract
+  first/last 3 lines valid JSON (25,240 lines), live file 0 B after rotation
+  and growing again within a minute (writers healthy on the new file).
+- job-log.jsonl: 4,724,788 B (16,143 lines) -> rotated to
+  `logs/archive/job-log-20261005-163559.jsonl.gz` (594,762 B compressed,
+  87.4% ratio). Same verification, all OK. Live file already 287 B seconds
+  after rotation (job-dispatch writes flowing).
+- followups.json: 163 records (161 resolved, 1 escalated, 1 pending),
+  0 eligible under resolved>7d / escalated>30d -> no archival, machinery
+  verified by dry logic + schema check; `followups.archive.jsonl` not
+  created yet (created on first eligible archival run)
+- Timer installed and enabled; first hourly run after install is a no-op
+  unless thresholds are exceeded again.
diff --git a/systemd/retention-rotations.service b/systemd/retention-rotations.service
new file mode 100644
index 0000000..61ea1f3
--- /dev/null
+++ b/systemd/retention-rotations.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=NetVM retention rotations (piece 1: chat-history / job-log / followups)
+After=network.target
+
+[Service]
+Type=oneshot
+ExecStart=/home/super/Projects/NetVM/bin/retention-run-rotations.sh
+WorkingDirectory=/home/super/Projects/NetVM
+StandardOutput=journal
+StandardError=journal
diff --git a/systemd/retention-rotations.timer b/systemd/retention-rotations.timer
new file mode 100644
index 0000000..2e1b5b7
--- /dev/null
+++ b/systemd/retention-rotations.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Run NetVM retention rotations hourly
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+
+[Install]
+WantedBy=timers.target