Resume timer after pause

README.md

@@ -1,6 +1,6 @@
 # cleanup-smartcam-data 部署说明
 
-![cleanup-smartcam-data 自动清理策略](assets/cleanup-smartcam-data-flow.png)
+![cleanup-smartcam-data 自动清理策略](assets/cleanup-smartcam-data-flow.svg)
 
 ## 文件结构
 
@@ -8,11 +8,13 @@
 cleanup-smartcam-data/
 ├── README.md
 ├── assets/
-│   └── cleanup-smartcam-data-flow.png
+│   ├── cleanup-smartcam-data-flow.svg
+│   └── cleanup-smartcam-data-flow.png（旧版兼容文件）
 ├── bin/
 │   └── cleanup-smartcam-data
 ├── systemd/
 │   ├── cleanup-smartcam-data.service
+│   ├── cleanup-smartcam-data-resume.timer
 │   └── cleanup-smartcam-data.timer
 └── scripts/
     ├── install.sh
@@ -26,6 +28,7 @@ cleanup-smartcam-data/
 | `bin/cleanup-smartcam-data` | `/usr/local/sbin/cleanup-smartcam-data` |
 | `systemd/cleanup-smartcam-data.service` | `/etc/systemd/system/cleanup-smartcam-data.service` |
 | `systemd/cleanup-smartcam-data.timer` | `/etc/systemd/system/cleanup-smartcam-data.timer` |
+| `systemd/cleanup-smartcam-data-resume.timer` | `/etc/systemd/system/cleanup-smartcam-data-resume.timer` |
 
 运行状态目录由程序自动创建：
 
@@ -69,7 +72,7 @@ sudo ./scripts/install.sh
 
 - 开机2分钟后首次检查；
 - 之后每60分钟检查一次；
-- `/userdata` 低于15GiB时开始安全清理；
+- `/userdata` 空闲低于10GiB时开始安全清理；
 - 只清理 `processed_files.log` 中确认成功、且超过60分钟的数据；
 - 同时删除同一任务的：
 
@@ -80,9 +83,10 @@ sudo ./scripts/install.sh
 ```
 
 - 超过60分钟的0字节 raw，如果未被进程占用，也会按上述三件套删除；
-- 低于8GiB时暂停采集，Producer继续上传；
-- 恢复到15GiB后自动恢复采集；
-- 低于5GiB时输出 CRITICAL 告警；
+- 空闲低于5GiB时暂停采集，Producer继续上传；
+- 暂停期间每2分钟检查一次；
+- 空闲恢复到10GiB后自动恢复采集，并停止2分钟检查；
+- 空闲低于3GiB时输出 CRITICAL 告警；
 - 无法读取成功记录时不删除数据。
 
 ## 验证

assets/cleanup-smartcam-data-flow.svg

@@ -0,0 +1,56 @@
+<svg xmlns="http://www.w3.org/2000/svg" width="1800" height="1100" viewBox="0 0 1800 1100">
+  <defs>
+    <linearGradient id="bg" x1="0" y1="0" x2="0" y2="1"><stop stop-color="#f7faff"/><stop offset="1" stop-color="#edf4fb"/></linearGradient>
+    <linearGradient id="head" x1="0" y1="0" x2="1" y2="0"><stop stop-color="#063b73"/><stop offset="1" stop-color="#0862a6"/></linearGradient>
+    <filter id="shadow"><feDropShadow dx="0" dy="5" stdDeviation="8" flood-opacity=".12"/></filter>
+    <marker id="arrow" markerWidth="10" markerHeight="10" refX="8" refY="3" orient="auto"><path d="M0,0 L0,6 L9,3 z" fill="#184d80"/></marker>
+    <style>
+      text{font-family:"Noto Sans CJK SC","WenQuanYi Micro Hei",sans-serif;fill:#11345b}.title{font-size:55px;font-weight:700}.h{font-size:32px;font-weight:700;fill:#fff}.foot{font-size:24px;fill:#ddecff}.t{font-size:25px;font-weight:600}.s{font-size:21px}.box{fill:#fff;stroke:#1b4f82;stroke-width:3;rx:14}.green{fill:#edf9ef;stroke:#16843a}.orange{fill:#fff7e9;stroke:#e87512}.red{fill:#fff1f1;stroke:#c82532}.line{fill:none;stroke:#184d80;stroke-width:4;marker-end:url(#arrow)}
+    </style>
+  </defs>
+  <rect width="1800" height="1100" fill="#f3f7fc"/>
+  <text x="900" y="75" text-anchor="middle" class="title">cleanup-smartcam-data 自动清理与恢复策略</text>
+
+  <g transform="translate(45 115)">
+    <rect width="525" height="790" rx="18" fill="#fff" stroke="#b7cee5" stroke-width="2"/>
+    <path d="M18 0h489q18 0 18 18v62H0V18Q0 0 18 0" fill="#07508d"/><text x="262" y="52" text-anchor="middle" class="h">检查频率</text>
+    <rect class="box" x="95" y="120" width="335" height="78"/><text class="t" x="262" y="168" text-anchor="middle">开机 2 分钟后首次检查</text>
+    <path class="line" d="M262 198v58"/>
+    <rect class="box green" x="95" y="270" width="335" height="105"/><text class="t" x="262" y="312" text-anchor="middle">正常采集状态</text><text class="s" x="262" y="347" text-anchor="middle">每 60 分钟检查一次</text>
+    <path class="line" d="M262 375v58"/>
+    <rect class="box red" x="95" y="447" width="335" height="115"/><text class="t" x="262" y="490" text-anchor="middle">采集暂停状态</text><text class="s" x="262" y="526" text-anchor="middle">1 分钟后首次检查</text><text class="s" x="262" y="553" text-anchor="middle">之后每 2 分钟检查</text>
+    <path class="line" d="M262 562v58"/>
+    <rect class="box" x="72" y="635" width="380" height="105"/><text class="t" x="262" y="678" text-anchor="middle">恢复到 10 GiB</text><text class="s" x="262" y="714" text-anchor="middle">立即恢复采集，停止快速检查</text>
+  </g>
+
+  <g transform="translate(592 115)">
+    <rect width="555" height="790" rx="18" fill="#fff" stroke="#b7cee5" stroke-width="2"/>
+    <path d="M18 0h519q18 0 18 18v62H0V18Q0 0 18 0" fill="#07508d"/><text x="277" y="52" text-anchor="middle" class="h">安全清理</text>
+    <rect class="box" x="85" y="115" width="385" height="82"/><text class="t" x="277" y="165" text-anchor="middle">检查 /userdata 空闲空间</text>
+    <path class="line" d="M277 197v55"/>
+    <polygon points="277,245 450,335 277,425 104,335" fill="#fff" stroke="#1b4f82" stroke-width="3"/>
+    <text class="t" x="277" y="325" text-anchor="middle">空闲 &lt; 10 GiB？</text><text class="s" x="277" y="362" text-anchor="middle">是：读取成功上传名单</text>
+    <path class="line" d="M277 425v45"/>
+    <rect class="box orange" x="55" y="482" width="445" height="120"/><text class="t" x="277" y="523" text-anchor="middle">只删除已确认上传且超过 60 分钟</text><text class="s" x="277" y="558" text-anchor="middle">videos + videos_raw + annotations</text><text class="s" x="277" y="588" text-anchor="middle">三件套按时间从旧到新删除</text>
+    <path class="line" d="M277 602v43"/>
+    <rect class="box red" x="55" y="657" width="445" height="90"/><text class="t" x="277" y="696" text-anchor="middle">未上传 / 上传失败 / 正在上传</text><text class="s" x="277" y="729" text-anchor="middle">禁止删除；成功名单不可读时也不删除</text>
+  </g>
+
+  <g transform="translate(1170 115)">
+    <rect width="585" height="790" rx="18" fill="#fff" stroke="#b7cee5" stroke-width="2"/>
+    <path d="M18 0h549q18 0 18 18v62H0V18Q0 0 18 0" fill="#07508d"/><text x="292" y="52" text-anchor="middle" class="h">磁盘保护与自动恢复</text>
+    <rect class="box orange" x="75" y="112" width="435" height="86"/><text class="t" x="292" y="148" text-anchor="middle">空闲 &lt; 10 GiB</text><text class="s" x="292" y="180" text-anchor="middle">开始清理已确认成功的三件套</text>
+    <path class="line" d="M292 198v45"/>
+    <rect class="box red" x="75" y="256" width="435" height="98"/><text class="t" x="292" y="296" text-anchor="middle">空闲 &lt; 5 GiB</text><text class="s" x="292" y="331" text-anchor="middle">暂停 smart_camera；上传程序继续运行</text>
+    <path class="line" d="M292 354v45"/>
+    <rect class="box red" x="75" y="412" width="435" height="86"/><text class="t" x="292" y="449" text-anchor="middle">空闲 &lt; 3 GiB</text><text class="s" x="292" y="480" text-anchor="middle">输出 CRITICAL 严重告警</text>
+    <path class="line" d="M292 498v45"/>
+    <rect class="box" x="75" y="556" width="435" height="82"/><text class="t" x="292" y="606" text-anchor="middle">SFTP / Kafka 继续消化积压</text>
+    <path class="line" d="M292 638v42"/>
+    <rect class="box green" x="75" y="692" width="435" height="72"/><text class="t" x="292" y="737" text-anchor="middle">空闲 ≥ 10 GiB → 自动恢复采集</text>
+  </g>
+
+  <rect x="170" y="945" width="1460" height="105" rx="18" fill="#07508d"/>
+  <text x="900" y="990" text-anchor="middle" class="h">核心原则：只删除确认上传成功的数据；空间不足时暂停采集，不停止上传</text>
+  <text x="900" y="1028" text-anchor="middle" class="foot">空闲恢复到 10 GiB 后自动继续采集，并恢复每 60 分钟检查</text>
+</svg>

bin/cleanup-smartcam-data

@@ -9,11 +9,11 @@ STATE_DIR="/var/lib/cleanup-smartcam-data"
 PAUSED_STATE="${STATE_DIR}/capture-paused"
 LOCK_FILE="/run/cleanup-smartcam-data.lock"
 
-# Start deleting confirmed bundles below 15 GiB, pause capture below 8 GiB,
-# and resume only after the filesystem has recovered to 15 GiB.
-CLEAN_TARGET_BYTES="${CLEAN_TARGET_BYTES:-16106127360}"
-PAUSE_BELOW_BYTES="${PAUSE_BELOW_BYTES:-8589934592}"
-CRITICAL_BELOW_BYTES="${CRITICAL_BELOW_BYTES:-5368709120}"
+# Start deleting confirmed bundles below 10 GiB, pause capture below 5 GiB,
+# raise a critical alert below 3 GiB, and resume after recovery to 10 GiB.
+CLEAN_TARGET_BYTES="${CLEAN_TARGET_BYTES:-10737418240}"
+PAUSE_BELOW_BYTES="${PAUSE_BELOW_BYTES:-5368709120}"
+CRITICAL_BELOW_BYTES="${CRITICAL_BELOW_BYTES:-3221225472}"
 MIN_FILE_AGE_MINUTES="${MIN_FILE_AGE_MINUTES:-60}"
 SAFE_WINDOW_SECONDS="${SAFE_WINDOW_SECONDS:-10}"
 
@@ -70,6 +70,15 @@ camera_is_stopped() {
   ps -o stat= -p "$pid" 2>/dev/null | grep -q '^T'
 }
 
+start_fast_resume_checks() {
+  systemctl start cleanup-smartcam-data-resume.timer 2>/dev/null ||
+    log ERROR "failed to start fast resume checks"
+}
+
+stop_fast_resume_checks() {
+  systemctl --no-block stop cleanup-smartcam-data-resume.timer 2>/dev/null || true
+}
+
 wait_for_safe_window() {
   local newest size1 size2
   newest="$(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f -name '*.mp4' \
@@ -91,6 +100,7 @@ pause_capture() {
   fi
   if camera_is_stopped "$pid"; then
     printf 'pid=%s\npaused_at=%s\n' "$pid" "$(date --iso-8601=seconds)" > "$PAUSED_STATE"
+    start_fast_resume_checks
     return 0
   fi
 
@@ -99,6 +109,7 @@ pause_capture() {
   fi
   kill -STOP "$pid"
   printf 'pid=%s\npaused_at=%s\n' "$pid" "$(date --iso-8601=seconds)" > "$PAUSED_STATE"
+  start_fast_resume_checks
   alert CRITICAL "capture paused at free=$(human_bytes "$free_bytes"); SFTP/Kafka uploader remains running; automatic resume target=$(human_bytes "$CLEAN_TARGET_BYTES")"
 }
 
@@ -110,16 +121,19 @@ resume_capture_if_ready() {
   if ! valid_camera_pid "$pid"; then
     log WARNING "discarding stale capture pause state (pid=${pid:-unknown})"
     rm -f "$PAUSED_STATE"
+    stop_fast_resume_checks
     return 0
   fi
   if ! camera_is_stopped "$pid"; then
     log WARNING "capture is already running; clearing stale pause state"
     rm -f "$PAUSED_STATE"
+    stop_fast_resume_checks
     return 0
   fi
   if [[ "$free_bytes" -ge "$CLEAN_TARGET_BYTES" ]]; then
     kill -CONT "$pid"
     rm -f "$PAUSED_STATE"
+    stop_fast_resume_checks
     alert RECOVERED "free space reached $(human_bytes "$free_bytes"); capture resumed automatically"
   else
     log WARNING "capture remains paused: free=$(human_bytes "$free_bytes"), resume target=$(human_bytes "$CLEAN_TARGET_BYTES")"

scripts/install.sh

@@ -33,9 +33,21 @@ install -o root -g root -m 0644 \
   "$ROOT_DIR/systemd/cleanup-smartcam-data.timer" \
   /etc/systemd/system/cleanup-smartcam-data.timer
 
+install -o root -g root -m 0644 \
+  "$ROOT_DIR/systemd/cleanup-smartcam-data-resume.timer" \
+  /etc/systemd/system/cleanup-smartcam-data-resume.timer
+
 systemctl daemon-reload
 systemctl enable --now cleanup-smartcam-data.timer
 
+# The fast timer is started by the cleanup program only while capture is paused.
+systemctl disable cleanup-smartcam-data-resume.timer 2>/dev/null || true
+if [[ -f /var/lib/cleanup-smartcam-data/capture-paused ]]; then
+  systemctl start cleanup-smartcam-data-resume.timer
+else
+  systemctl stop cleanup-smartcam-data-resume.timer 2>/dev/null || true
+fi
+
 # Remove obsolete installed names only after the new timer is active.
 rm -f \
   /etc/systemd/system/userdata-video-cleanup.timer \
@@ -47,4 +59,5 @@ echo "安装完成。"
 echo "程序: /usr/local/sbin/cleanup-smartcam-data"
 echo "服务: /etc/systemd/system/cleanup-smartcam-data.service"
 echo "定时器: /etc/systemd/system/cleanup-smartcam-data.timer"
+echo "暂停恢复定时器: /etc/systemd/system/cleanup-smartcam-data-resume.timer"
 systemctl --no-pager status cleanup-smartcam-data.timer || true

scripts/uninstall.sh

@@ -7,8 +7,10 @@ if [[ "${EUID}" -ne 0 ]]; then
 fi
 
 systemctl disable --now cleanup-smartcam-data.timer 2>/dev/null || true
+systemctl disable --now cleanup-smartcam-data-resume.timer 2>/dev/null || true
 rm -f \
   /etc/systemd/system/cleanup-smartcam-data.timer \
+  /etc/systemd/system/cleanup-smartcam-data-resume.timer \
   /etc/systemd/system/cleanup-smartcam-data.service \
   /usr/local/sbin/cleanup-smartcam-data
 systemctl daemon-reload

systemd/cleanup-smartcam-data-resume.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Check smart-cam disk recovery every 2 minutes while capture is paused
+Documentation=man:systemd.timer(5)
+
+[Timer]
+OnActiveSec=1min
+OnUnitActiveSec=2min
+AccuracySec=15s
+Unit=cleanup-smartcam-data.service
+