rewrite time limit

This commit is contained in:
Ankle committed 2026-07-23 08:44:44 +08:00
1 parent cf920dc76f
commit 7ed62d66e9
3 files changed
+24 -24

No files matched your search

+6 -7
View File
@@ -1,6 +1,6 @@
# cleanup-smartcam-data 部署说明
![cleanup-smartcam-data 自动清理策略](assets/cleanup-smartcam-data-flow.svg)
![cleanup-smartcam-data 自动清理策略](assets/cleanup-smartcam-data-flow.png)
## 文件结构
@@ -8,8 +8,8 @@
cleanup-smartcam-data/
├── README.md
├── assets/
│ ├── cleanup-smartcam-data-flow.svg
│ └── cleanup-smartcam-data-flow.png(旧版兼容文件)
│ ├── cleanup-smartcam-data-flow.svg(旧版源文件)
│ └── cleanup-smartcam-data-flow.png(当前流程图)
├── bin/
│ └── cleanup-smartcam-data
├── systemd/
@@ -72,7 +72,7 @@ sudo ./scripts/install.sh
- 开机2分钟后首次检查;
- 之后每60分钟检查一次;
- `/userdata` 空闲低于10GiB时开始安全清理;
- 每次定时检查都会安全清理已确认上传成功的数据,不以空闲空间阈值作为清理条件;
- 只清理 `processed_files.log` 中确认成功、且超过60分钟的数据;
- 同时删除同一任务的:
@@ -83,10 +83,9 @@ sudo ./scripts/install.sh
```
- 超过60分钟的0字节 raw,如果未被进程占用,也会按上述三件套删除;
- 空闲低于5GiB时暂停采集,Producer继续上传;
- 空闲低于3GiB时暂停整个 `smart_camera` 进程(持续检测、录像和识别一起暂停),Producer继续上传,并输出 CRITICAL 告警;
- 暂停期间每2分钟检查一次;
- 空闲恢复到10GiB后自动恢复采集,并停止2分钟检查;
- 空闲低于3GiB时输出 CRITICAL 告警;
- 清理使空闲恢复到5GiB后自动恢复 `smart_camera`;程序重新持续检测,达到鸟类确认门控后再录像和识别,并停止2分钟快速检查;
- 无法读取成功记录时不删除数据。
## 验证
Binary file not shown.

Before

Width:  |  Height:  |  Size: 236 KiB

After

Width:  |  Height:  |  Size: 1.3 MiB

+18 -17
View File
@@ -9,10 +9,10 @@ STATE_DIR="/var/lib/cleanup-smartcam-data"
PAUSED_STATE="${STATE_DIR}/capture-paused"
LOCK_FILE="/run/cleanup-smartcam-data.lock"
# Start deleting confirmed bundles below 10 GiB, pause capture below 5 GiB,
# raise a critical alert below 3 GiB, and resume after recovery to 10 GiB.
CLEAN_TARGET_BYTES="${CLEAN_TARGET_BYTES:-10737418240}"
PAUSE_BELOW_BYTES="${PAUSE_BELOW_BYTES:-5368709120}"
# Delete confirmed bundles on every scheduled run. Pause capture and raise a
# critical alert below 3 GiB, and resume after recovery to 5 GiB.
RESUME_TARGET_BYTES="${RESUME_TARGET_BYTES:-5368709120}"
PAUSE_BELOW_BYTES="${PAUSE_BELOW_BYTES:-3221225472}"
CRITICAL_BELOW_BYTES="${CRITICAL_BELOW_BYTES:-3221225472}"
MIN_FILE_AGE_MINUTES="${MIN_FILE_AGE_MINUTES:-60}"
SAFE_WINDOW_SECONDS="${SAFE_WINDOW_SECONDS:-10}"
@@ -110,7 +110,7 @@ pause_capture() {
kill -STOP "$pid"
printf 'pid=%s\npaused_at=%s\n' "$pid" "$(date --iso-8601=seconds)" > "$PAUSED_STATE"
start_fast_resume_checks
alert CRITICAL "capture paused at free=$(human_bytes "$free_bytes"); SFTP/Kafka uploader remains running; automatic resume target=$(human_bytes "$CLEAN_TARGET_BYTES")"
alert CRITICAL "capture paused at free=$(human_bytes "$free_bytes"); SFTP/Kafka uploader remains running; automatic resume target=$(human_bytes "$RESUME_TARGET_BYTES")"
}
resume_capture_if_ready() {
@@ -130,13 +130,13 @@ resume_capture_if_ready() {
stop_fast_resume_checks
return 0
fi
if [[ "$free_bytes" -ge "$CLEAN_TARGET_BYTES" ]]; then
if [[ "$free_bytes" -ge "$RESUME_TARGET_BYTES" ]]; then
kill -CONT "$pid"
rm -f "$PAUSED_STATE"
stop_fast_resume_checks
alert RECOVERED "free space reached $(human_bytes "$free_bytes"); capture resumed automatically"
else
log WARNING "capture remains paused: free=$(human_bytes "$free_bytes"), resume target=$(human_bytes "$CLEAN_TARGET_BYTES")"
log WARNING "capture remains paused: free=$(human_bytes "$free_bytes"), resume target=$(human_bytes "$RESUME_TARGET_BYTES")"
fi
}
@@ -170,7 +170,7 @@ build_safe_bundles() {
}
delete_safe_bundles() {
local bundles="$1" free_bytes deleted_count=0 deleted_bytes=0 record base path size
local bundles="$1" deleted_count=0 deleted_bytes=0 record base path size
while IFS= read -r record; do
base="${record#* }"
for path in \
@@ -187,10 +187,6 @@ delete_safe_bundles() {
log ERROR "failed to delete confirmed file: ${path}"
fi
done
free_bytes="$(available_bytes)"
if [[ "$free_bytes" -ge "$CLEAN_TARGET_BYTES" ]]; then
break
fi
done < "$bundles"
log INFO "safe cleanup result: deleted=${deleted_count} files bytes=${deleted_bytes} free=$(human_bytes "$(available_bytes)")"
}
@@ -232,9 +228,17 @@ main() {
local free_bytes processed bundles
delete_stale_zero_bundles
free_bytes="$(available_bytes)"
# Protect the disk before cleanup work: once free space is below 3 GiB,
# stop continuous detection/recording/inference first while upload stays up.
if [[ "$free_bytes" -lt "$CRITICAL_BELOW_BYTES" ]]; then
alert CRITICAL "free space below critical threshold: $(human_bytes "$free_bytes")"
fi
if [[ "$free_bytes" -lt "$PAUSE_BELOW_BYTES" ]]; then
pause_capture "$free_bytes"
fi
resume_capture_if_ready "$free_bytes"
if [[ "$free_bytes" -lt "$CLEAN_TARGET_BYTES" ]]; then
processed="$(mktemp /tmp/userdata-processed.XXXXXX)"
bundles="$(mktemp /tmp/userdata-bundles.XXXXXX)"
trap 'rm -f "${processed:-}" "${bundles:-}"' EXIT
@@ -244,14 +248,11 @@ main() {
if [[ -s "$bundles" ]]; then
delete_safe_bundles "$bundles"
else
log WARNING "free space is low, but no confirmed old bundles are safe to delete"
log INFO "no confirmed old bundles are ready for safe cleanup"
fi
else
alert ERROR "cannot read confirmed-upload list; fail-safe mode prevents all deletion"
fi
else
log INFO "free space is sufficient: $(human_bytes "$free_bytes") available"
fi
free_bytes="$(available_bytes)"
if [[ "$free_bytes" -lt "$CRITICAL_BELOW_BYTES" ]]; then