retention piece 1: immediate rotations for chat-history / job-log / followups (b67084660385)
- chat-history.jsonl rotates at 10MB or 7d -> logs/archive/*.jsonl.gz + .sha256 - job-log.jsonl rotates at 2MB or 14d -> same archive layout - followups.json archives resolved>7d / escalated>30d -> followups.archive.jsonl (append-only) - verify wrapper: sha256 -c, gzip -t, clean listing, spot-extract JSON + ts bounds - driver + hourly systemd user timer (retention-rotations.timer) - archive-only: nothing is ever deleted; thread backfill excluded (needs human-reviewed preview) First run 2026-10-05 16:35Z: chat-history 29.7MB + job-log 4.7MB rotated and verified; followups 0 eligible of 163.
This commit is contained in:
Executable
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
# retention-verify-archive.sh — retention piece 1 (board bug b67084660385).
|
||||
# Verifies one archive file: checksum, integrity, clean listing, spot-extract.
|
||||
# Usage: retention-verify-archive.sh <path-to-.jsonl.gz-or-.jsonl>
|
||||
# Exits non-zero on any verification failure. Missing file = SKIP (rc 0).
|
||||
set -uo pipefail
|
||||
|
||||
fail() { echo "FAIL verify $1: $2"; exit 1; }
|
||||
|
||||
P="${1:?usage: $0 <archive-path>}"
|
||||
|
||||
if [ ! -f "$P" ]; then
|
||||
echo "SKIP verify: $P does not exist"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "== verify $P =="
|
||||
|
||||
# 1. checksum sidecar
|
||||
if [ -f "$P.sha256" ]; then
|
||||
( cd "$(dirname "$P")" && sha256sum -c "$(basename "$P").sha256" ) \
|
||||
|| fail "$P" "sha256 mismatch"
|
||||
echo " checksum: OK"
|
||||
else
|
||||
echo " checksum: no sidecar (archiver writes one for new .gz files)"
|
||||
fi
|
||||
|
||||
# 2. integrity + clean listing for gzip archives
|
||||
if [[ "$P" == *.gz ]]; then
|
||||
gzip -t "$P" || fail "$P" "gzip integrity test failed"
|
||||
echo " integrity: gzip -t OK"
|
||||
echo " listing:"; gzip -l "$P" | sed 's/^/ /'
|
||||
reader="zcat"
|
||||
else
|
||||
reader="cat"
|
||||
fi
|
||||
|
||||
# 3. spot-extract: first 3 and last 3 lines must be valid JSON
|
||||
n=0
|
||||
while IFS= read -r line; do
|
||||
n=$((n+1))
|
||||
echo "$line" | python3 -c 'import json,sys; json.loads(sys.stdin.read())' \
|
||||
|| fail "$P" "line $n is not valid JSON"
|
||||
done < <( { $reader "$P" | head -3; $reader "$P" | tail -3; } 2>/dev/null )
|
||||
|
||||
total="$( $reader "$P" | wc -l )"
|
||||
echo " spot-extract: first/last 3 lines valid JSON ($total total lines)"
|
||||
|
||||
# 4. spot-extract timestamps: report newest/oldest ts seen in the sample
|
||||
sample="$($reader "$P" | head -2000)"
|
||||
ts_line="$(echo "$sample" | python3 -c '
|
||||
import json,sys
|
||||
keys=("ts","timestamp","time","created_at","sent_at","resolved_at")
|
||||
seen=[]
|
||||
for line in sys.stdin:
|
||||
line=line.strip()
|
||||
if not line: continue
|
||||
try: rec=json.loads(line)
|
||||
except Exception: continue
|
||||
for k in keys:
|
||||
if rec.get(k): seen.append(str(rec[k])); break
|
||||
seen=sorted(set(seen))
|
||||
print(("oldest="+seen[0]+" newest="+seen[-1]) if seen else "no-ts-field")
|
||||
')"
|
||||
echo " timestamps: $ts_line"
|
||||
|
||||
echo "OK verify $P"
|
||||
Reference in New Issue
Block a user