diff --git a/README.md b/README.md index f3e23d2..74ea994 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # cleanup-smartcam-data 部署说明 -![cleanup-smartcam-data 自动清理策略](assets/cleanup-smartcam-data-flow.png) +![cleanup-smartcam-data 自动清理策略](assets/cleanup-smartcam-data-flow.svg) ## 文件结构 @@ -8,11 +8,13 @@ cleanup-smartcam-data/ ├── README.md ├── assets/ -│ └── cleanup-smartcam-data-flow.png +│ ├── cleanup-smartcam-data-flow.svg +│ └── cleanup-smartcam-data-flow.png（旧版兼容文件） ├── bin/ │ └── cleanup-smartcam-data ├── systemd/ │ ├── cleanup-smartcam-data.service +│ ├── cleanup-smartcam-data-resume.timer │ └── cleanup-smartcam-data.timer └── scripts/ ├── install.sh @@ -26,6 +28,7 @@ cleanup-smartcam-data/ | `bin/cleanup-smartcam-data` | `/usr/local/sbin/cleanup-smartcam-data` | | `systemd/cleanup-smartcam-data.service` | `/etc/systemd/system/cleanup-smartcam-data.service` | | `systemd/cleanup-smartcam-data.timer` | `/etc/systemd/system/cleanup-smartcam-data.timer` | +| `systemd/cleanup-smartcam-data-resume.timer` | `/etc/systemd/system/cleanup-smartcam-data-resume.timer` | 运行状态目录由程序自动创建： @@ -69,7 +72,7 @@ sudo ./scripts/install.sh - 开机2分钟后首次检查； - 之后每60分钟检查一次； -- `/userdata` 低于15GiB时开始安全清理； +- `/userdata` 空闲低于10GiB时开始安全清理； - 只清理 `processed_files.log` 中确认成功、且超过60分钟的数据； - 同时删除同一任务的： @@ -80,9 +83,10 @@ sudo ./scripts/install.sh ``` - 超过60分钟的0字节 raw，如果未被进程占用，也会按上述三件套删除； -- 低于8GiB时暂停采集，Producer继续上传； -- 恢复到15GiB后自动恢复采集； -- 低于5GiB时输出 CRITICAL 告警； +- 空闲低于5GiB时暂停采集，Producer继续上传； +- 暂停期间每2分钟检查一次； +- 空闲恢复到10GiB后自动恢复采集，并停止2分钟检查； +- 空闲低于3GiB时输出 CRITICAL 告警； - 无法读取成功记录时不删除数据。 ## 验证 diff --git a/assets/cleanup-smartcam-data-flow.png b/assets/cleanup-smartcam-data-flow.png index ec2e6d7..f5e6047 100644 Binary files a/assets/cleanup-smartcam-data-flow.png and b/assets/cleanup-smartcam-data-flow.png differ diff --git a/assets/cleanup-smartcam-data-flow.svg b/assets/cleanup-smartcam-data-flow.svg new file mode 100644 index 0000000..c97d970 --- /dev/null +++ b/assets/cleanup-smartcam-data-flow.svg @@ -0,0 +1,56 @@ + + + + + + + + + + cleanup-smartcam-data 自动清理与恢复策略 + + + + 检查频率 + 开机 2 分钟后首次检查 + + 正常采集状态每 60 分钟检查一次 + + 采集暂停状态1 分钟后首次检查之后每 2 分钟检查 + + 恢复到 10 GiB立即恢复采集，停止快速检查 + + + + + 安全清理 + 检查 /userdata 空闲空间 + + + 空闲 < 10 GiB？是：读取成功上传名单 + + 只删除已确认上传且超过 60 分钟videos + videos_raw + annotations三件套按时间从旧到新删除 + + 未上传 / 上传失败 / 正在上传禁止删除；成功名单不可读时也不删除 + + + + + 磁盘保护与自动恢复 + 空闲 < 10 GiB开始清理已确认成功的三件套 + + 空闲 < 5 GiB暂停 smart_camera；上传程序继续运行 + + 空闲 < 3 GiB输出 CRITICAL 严重告警 + + SFTP / Kafka 继续消化积压 + + 空闲 ≥ 10 GiB → 自动恢复采集 + + + + 核心原则：只删除确认上传成功的数据；空间不足时暂停采集，不停止上传 + 空闲恢复到 10 GiB 后自动继续采集，并恢复每 60 分钟检查 + diff --git a/bin/cleanup-smartcam-data b/bin/cleanup-smartcam-data index 29ebed3..426fe35 100755 --- a/bin/cleanup-smartcam-data +++ b/bin/cleanup-smartcam-data @@ -9,11 +9,11 @@ STATE_DIR="/var/lib/cleanup-smartcam-data" PAUSED_STATE="${STATE_DIR}/capture-paused" LOCK_FILE="/run/cleanup-smartcam-data.lock" -# Start deleting confirmed bundles below 15 GiB, pause capture below 8 GiB, -# and resume only after the filesystem has recovered to 15 GiB. -CLEAN_TARGET_BYTES="${CLEAN_TARGET_BYTES:-16106127360}" -PAUSE_BELOW_BYTES="${PAUSE_BELOW_BYTES:-8589934592}" -CRITICAL_BELOW_BYTES="${CRITICAL_BELOW_BYTES:-5368709120}" +# Start deleting confirmed bundles below 10 GiB, pause capture below 5 GiB, +# raise a critical alert below 3 GiB, and resume after recovery to 10 GiB. +CLEAN_TARGET_BYTES="${CLEAN_TARGET_BYTES:-10737418240}" +PAUSE_BELOW_BYTES="${PAUSE_BELOW_BYTES:-5368709120}" +CRITICAL_BELOW_BYTES="${CRITICAL_BELOW_BYTES:-3221225472}" MIN_FILE_AGE_MINUTES="${MIN_FILE_AGE_MINUTES:-60}" SAFE_WINDOW_SECONDS="${SAFE_WINDOW_SECONDS:-10}" @@ -70,6 +70,15 @@ camera_is_stopped() { ps -o stat= -p "$pid" 2>/dev/null | grep -q '^T' } +start_fast_resume_checks() { + systemctl start cleanup-smartcam-data-resume.timer 2>/dev/null || + log ERROR "failed to start fast resume checks" +} + +stop_fast_resume_checks() { + systemctl --no-block stop cleanup-smartcam-data-resume.timer 2>/dev/null || true +} + wait_for_safe_window() { local newest size1 size2 newest="$(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f -name '*.mp4' \ @@ -91,6 +100,7 @@ pause_capture() { fi if camera_is_stopped "$pid"; then printf 'pid=%s\npaused_at=%s\n' "$pid" "$(date --iso-8601=seconds)" > "$PAUSED_STATE" + start_fast_resume_checks return 0 fi @@ -99,6 +109,7 @@ pause_capture() { fi kill -STOP "$pid" printf 'pid=%s\npaused_at=%s\n' "$pid" "$(date --iso-8601=seconds)" > "$PAUSED_STATE" + start_fast_resume_checks alert CRITICAL "capture paused at free=$(human_bytes "$free_bytes"); SFTP/Kafka uploader remains running; automatic resume target=$(human_bytes "$CLEAN_TARGET_BYTES")" } @@ -110,16 +121,19 @@ resume_capture_if_ready() { if ! valid_camera_pid "$pid"; then log WARNING "discarding stale capture pause state (pid=${pid:-unknown})" rm -f "$PAUSED_STATE" + stop_fast_resume_checks return 0 fi if ! camera_is_stopped "$pid"; then log WARNING "capture is already running; clearing stale pause state" rm -f "$PAUSED_STATE" + stop_fast_resume_checks return 0 fi if [[ "$free_bytes" -ge "$CLEAN_TARGET_BYTES" ]]; then kill -CONT "$pid" rm -f "$PAUSED_STATE" + stop_fast_resume_checks alert RECOVERED "free space reached $(human_bytes "$free_bytes"); capture resumed automatically" else log WARNING "capture remains paused: free=$(human_bytes "$free_bytes"), resume target=$(human_bytes "$CLEAN_TARGET_BYTES")" diff --git a/scripts/install.sh b/scripts/install.sh index 62b9306..526637a 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -33,9 +33,21 @@ install -o root -g root -m 0644 \ "$ROOT_DIR/systemd/cleanup-smartcam-data.timer" \ /etc/systemd/system/cleanup-smartcam-data.timer +install -o root -g root -m 0644 \ + "$ROOT_DIR/systemd/cleanup-smartcam-data-resume.timer" \ + /etc/systemd/system/cleanup-smartcam-data-resume.timer + systemctl daemon-reload systemctl enable --now cleanup-smartcam-data.timer +# The fast timer is started by the cleanup program only while capture is paused. +systemctl disable cleanup-smartcam-data-resume.timer 2>/dev/null || true +if [[ -f /var/lib/cleanup-smartcam-data/capture-paused ]]; then + systemctl start cleanup-smartcam-data-resume.timer +else + systemctl stop cleanup-smartcam-data-resume.timer 2>/dev/null || true +fi + # Remove obsolete installed names only after the new timer is active. rm -f \ /etc/systemd/system/userdata-video-cleanup.timer \ @@ -47,4 +59,5 @@ echo "安装完成。" echo "程序: /usr/local/sbin/cleanup-smartcam-data" echo "服务: /etc/systemd/system/cleanup-smartcam-data.service" echo "定时器: /etc/systemd/system/cleanup-smartcam-data.timer" +echo "暂停恢复定时器: /etc/systemd/system/cleanup-smartcam-data-resume.timer" systemctl --no-pager status cleanup-smartcam-data.timer || true diff --git a/scripts/uninstall.sh b/scripts/uninstall.sh index a57a777..15ad381 100755 --- a/scripts/uninstall.sh +++ b/scripts/uninstall.sh @@ -7,8 +7,10 @@ if [[ "${EUID}" -ne 0 ]]; then fi systemctl disable --now cleanup-smartcam-data.timer 2>/dev/null || true +systemctl disable --now cleanup-smartcam-data-resume.timer 2>/dev/null || true rm -f \ /etc/systemd/system/cleanup-smartcam-data.timer \ + /etc/systemd/system/cleanup-smartcam-data-resume.timer \ /etc/systemd/system/cleanup-smartcam-data.service \ /usr/local/sbin/cleanup-smartcam-data systemctl daemon-reload diff --git a/systemd/cleanup-smartcam-data-resume.timer b/systemd/cleanup-smartcam-data-resume.timer new file mode 100644 index 0000000..c9d5293 --- /dev/null +++ b/systemd/cleanup-smartcam-data-resume.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Check smart-cam disk recovery every 2 minutes while capture is paused +Documentation=man:systemd.timer(5) + +[Timer] +OnActiveSec=1min +OnUnitActiveSec=2min +AccuracySec=15s +Unit=cleanup-smartcam-data.service +