operator
1e86ed8a44
- chat-history.jsonl rotates at 10MB or 7d -> logs/archive/*.jsonl.gz + .sha256 - job-log.jsonl rotates at 2MB or 14d -> same archive layout - followups.json archives resolved>7d / escalated>30d -> followups.archive.jsonl (append-only) - verify wrapper: sha256 -c, gzip -t, clean listing, spot-extract JSON + ts bounds - driver + hourly systemd user timer (retention-rotations.timer) - archive-only: nothing is ever deleted; thread backfill excluded (needs human-reviewed preview) First run 2026-10-05 16:35Z: chat-history 29.7MB + job-log 4.7MB rotated and verified; followups 0 eligible of 163.
68 lines
2.0 KiB
Bash
Executable File
68 lines
2.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# retention-verify-archive.sh — retention piece 1 (board bug b67084660385).
|
|
# Verifies one archive file: checksum, integrity, clean listing, spot-extract.
|
|
# Usage: retention-verify-archive.sh <path-to-.jsonl.gz-or-.jsonl>
|
|
# Exits non-zero on any verification failure. Missing file = SKIP (rc 0).
|
|
set -uo pipefail
|
|
|
|
fail() { echo "FAIL verify $1: $2"; exit 1; }
|
|
|
|
P="${1:?usage: $0 <archive-path>}"
|
|
|
|
if [ ! -f "$P" ]; then
|
|
echo "SKIP verify: $P does not exist"
|
|
exit 0
|
|
fi
|
|
|
|
echo "== verify $P =="
|
|
|
|
# 1. checksum sidecar
|
|
if [ -f "$P.sha256" ]; then
|
|
( cd "$(dirname "$P")" && sha256sum -c "$(basename "$P").sha256" ) \
|
|
|| fail "$P" "sha256 mismatch"
|
|
echo " checksum: OK"
|
|
else
|
|
echo " checksum: no sidecar (archiver writes one for new .gz files)"
|
|
fi
|
|
|
|
# 2. integrity + clean listing for gzip archives
|
|
if [[ "$P" == *.gz ]]; then
|
|
gzip -t "$P" || fail "$P" "gzip integrity test failed"
|
|
echo " integrity: gzip -t OK"
|
|
echo " listing:"; gzip -l "$P" | sed 's/^/ /'
|
|
reader="zcat"
|
|
else
|
|
reader="cat"
|
|
fi
|
|
|
|
# 3. spot-extract: first 3 and last 3 lines must be valid JSON
|
|
n=0
|
|
while IFS= read -r line; do
|
|
n=$((n+1))
|
|
echo "$line" | python3 -c 'import json,sys; json.loads(sys.stdin.read())' \
|
|
|| fail "$P" "line $n is not valid JSON"
|
|
done < <( { $reader "$P" | head -3; $reader "$P" | tail -3; } 2>/dev/null )
|
|
|
|
total="$( $reader "$P" | wc -l )"
|
|
echo " spot-extract: first/last 3 lines valid JSON ($total total lines)"
|
|
|
|
# 4. spot-extract timestamps: report newest/oldest ts seen in the sample
|
|
sample="$($reader "$P" | head -2000)"
|
|
ts_line="$(echo "$sample" | python3 -c '
|
|
import json,sys
|
|
keys=("ts","timestamp","time","created_at","sent_at","resolved_at")
|
|
seen=[]
|
|
for line in sys.stdin:
|
|
line=line.strip()
|
|
if not line: continue
|
|
try: rec=json.loads(line)
|
|
except Exception: continue
|
|
for k in keys:
|
|
if rec.get(k): seen.append(str(rec[k])); break
|
|
seen=sorted(set(seen))
|
|
print(("oldest="+seen[0]+" newest="+seen[-1]) if seen else "no-ts-field")
|
|
')"
|
|
echo " timestamps: $ts_line"
|
|
|
|
echo "OK verify $P"
|