first commit
This commit is contained in:
Ankle committed
commit
24f5b3fdfa
7 files changed
+494
No files matched your search
@@ -0,0 +1,152 @@
|
||||
# cleanup-smartcam-data 部署说明
|
||||
|
||||
|
||||
|
||||
## 文件结构
|
||||
|
||||
```text
|
||||
cleanup-smartcam-data/
|
||||
├── README.md
|
||||
├── assets/
|
||||
│ └── cleanup-smartcam-data-flow.png
|
||||
├── bin/
|
||||
│ └── cleanup-smartcam-data
|
||||
├── systemd/
|
||||
│ ├── cleanup-smartcam-data.service
|
||||
│ └── cleanup-smartcam-data.timer
|
||||
└── scripts/
|
||||
├── install.sh
|
||||
└── uninstall.sh
|
||||
```
|
||||
|
||||
## RK3588 安装位置
|
||||
|
||||
| 项目文件 | 设备安装位置 |
|
||||
|---|---|
|
||||
| `bin/cleanup-smartcam-data` | `/usr/local/sbin/cleanup-smartcam-data` |
|
||||
| `systemd/cleanup-smartcam-data.service` | `/etc/systemd/system/cleanup-smartcam-data.service` |
|
||||
| `systemd/cleanup-smartcam-data.timer` | `/etc/systemd/system/cleanup-smartcam-data.timer` |
|
||||
|
||||
运行状态目录由程序自动创建：
|
||||
|
||||
```text
|
||||
/var/lib/cleanup-smartcam-data/
|
||||
/run/cleanup-smartcam-data.lock
|
||||
```
|
||||
|
||||
## 环境要求
|
||||
|
||||
- smart-cam 容器名：`smart-cam`
|
||||
- 数据目录：`/userdata/data`
|
||||
- 容器内成功记录：`/smart-cam/kafka_sender/processed_files.log`
|
||||
- 采集进程命令包含：`build/bin/smart_camera`
|
||||
- 宿主机使用 Docker 和 systemd
|
||||
|
||||
## 安装
|
||||
|
||||
将整个目录复制到目标设备，例如：
|
||||
|
||||
```text
|
||||
/home/zhangw/cleanup-smartcam-data
|
||||
```
|
||||
|
||||
执行：
|
||||
|
||||
```bash
|
||||
cd /home/zhangw/cleanup-smartcam-data
|
||||
sudo ./scripts/install.sh
|
||||
```
|
||||
|
||||
安装脚本会：
|
||||
|
||||
1. 校验清理脚本语法；
|
||||
2. 安装程序和 systemd unit；
|
||||
3. 启用 `cleanup-smartcam-data.timer`；
|
||||
4. 删除旧名称 `userdata-video-cleanup` 的安装文件；
|
||||
5. 不停止或重启 smart-cam Docker。
|
||||
|
||||
## 清理规则
|
||||
|
||||
- 开机2分钟后首次检查；
|
||||
- 之后每5分钟检查一次；
|
||||
- `/userdata` 低于15GiB时开始安全清理；
|
||||
- 只清理 `processed_files.log` 中确认成功、且超过60分钟的数据；
|
||||
- 同时删除同一任务的：
|
||||
|
||||
```text
|
||||
/userdata/data/videos/<base>_annotated.mp4
|
||||
/userdata/data/videos_raw/<base>_raw.mp4
|
||||
/userdata/data/annotations/<base>.xml
|
||||
```
|
||||
|
||||
- 超过60分钟的0字节 raw，如果未被进程占用，也会按上述三件套删除；
|
||||
- 低于8GiB时暂停采集，Producer继续上传；
|
||||
- 恢复到15GiB后自动恢复采集；
|
||||
- 低于5GiB时输出 CRITICAL 告警；
|
||||
- 无法读取成功记录时不删除数据。
|
||||
|
||||
## 验证
|
||||
|
||||
```bash
|
||||
sudo bash -n /usr/local/sbin/cleanup-smartcam-data
|
||||
|
||||
systemctl status cleanup-smartcam-data.timer
|
||||
systemctl list-timers --all | grep cleanup-smartcam-data
|
||||
|
||||
sudo systemctl start cleanup-smartcam-data.service
|
||||
systemctl status cleanup-smartcam-data.service
|
||||
|
||||
journalctl -u cleanup-smartcam-data.service -n 100 --no-pager
|
||||
```
|
||||
|
||||
确认成功记录可读：
|
||||
|
||||
```bash
|
||||
docker exec smart-cam \
|
||||
test -r /smart-cam/kafka_sender/processed_files.log
|
||||
```
|
||||
|
||||
确认 Docker 未重启：
|
||||
|
||||
```bash
|
||||
docker inspect smart-cam --format \
|
||||
'running={{.State.Running}} restart={{.RestartCount}} started={{.State.StartedAt}}'
|
||||
```
|
||||
|
||||
## 查看日志
|
||||
|
||||
```bash
|
||||
journalctl -u cleanup-smartcam-data.service -f
|
||||
docker logs -f -t --tail 200 smart-cam
|
||||
```
|
||||
|
||||
日志标识：
|
||||
|
||||
```text
|
||||
[DISK_GUARD][INFO]
|
||||
[DISK_GUARD][WARNING]
|
||||
[DISK_GUARD][CRITICAL]
|
||||
[DISK_GUARD][RECOVERED]
|
||||
```
|
||||
|
||||
## 升级
|
||||
|
||||
在目标设备进入新版目录后重新执行：
|
||||
|
||||
```bash
|
||||
cd /home/zhangw/cleanup-smartcam-data
|
||||
sudo ./scripts/install.sh
|
||||
```
|
||||
|
||||
## 卸载
|
||||
|
||||
```bash
|
||||
cd /home/zhangw/cleanup-smartcam-data
|
||||
sudo ./scripts/uninstall.sh
|
||||
```
|
||||
|
||||
卸载后检查采集是否处于暂停状态，再决定是否删除：
|
||||
|
||||
```text
|
||||
/var/lib/cleanup-smartcam-data/
|
||||
```
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.3 MiB |
Executable
+252
@@ -0,0 +1,252 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_MOUNT="/userdata"
|
||||
DATA_ROOT="/userdata/data"
|
||||
CONTAINER="smart-cam"
|
||||
PROCESSED_FILE="/smart-cam/kafka_sender/processed_files.log"
|
||||
STATE_DIR="/var/lib/cleanup-smartcam-data"
|
||||
PAUSED_STATE="${STATE_DIR}/capture-paused"
|
||||
LOCK_FILE="/run/cleanup-smartcam-data.lock"
|
||||
|
||||
# Start deleting confirmed bundles below 15 GiB, pause capture below 8 GiB,
|
||||
# and resume only after the filesystem has recovered to 15 GiB.
|
||||
CLEAN_TARGET_BYTES="${CLEAN_TARGET_BYTES:-16106127360}"
|
||||
PAUSE_BELOW_BYTES="${PAUSE_BELOW_BYTES:-8589934592}"
|
||||
CRITICAL_BELOW_BYTES="${CRITICAL_BELOW_BYTES:-5368709120}"
|
||||
MIN_FILE_AGE_MINUTES="${MIN_FILE_AGE_MINUTES:-60}"
|
||||
SAFE_WINDOW_SECONDS="${SAFE_WINDOW_SECONDS:-10}"
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
exec 9>"$LOCK_FILE"
|
||||
flock -n 9 || exit 0
|
||||
|
||||
log() {
|
||||
local level="$1"
|
||||
shift
|
||||
local message="[DISK_GUARD][${level}] $*"
|
||||
printf '%s\n' "$message"
|
||||
logger -t cleanup-smartcam-data -- "$message" || true
|
||||
}
|
||||
|
||||
container_log() {
|
||||
local message="$1"
|
||||
if docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null | grep -qx true; then
|
||||
docker exec "$CONTAINER" sh -c \
|
||||
'printf "%s\n" "$1" > /proc/1/fd/1' sh "$message" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
alert() {
|
||||
local level="$1"
|
||||
shift
|
||||
local message="[DISK_GUARD][${level}] $*"
|
||||
log "$level" "$*"
|
||||
container_log "$message"
|
||||
}
|
||||
|
||||
available_bytes() {
|
||||
df -PB1 "$TARGET_MOUNT" | awk 'NR == 2 {print $4}'
|
||||
}
|
||||
|
||||
human_bytes() {
|
||||
numfmt --to=iec-i --suffix=B "$1"
|
||||
}
|
||||
|
||||
camera_host_pid() {
|
||||
docker top "$CONTAINER" -eo pid,args 2>/dev/null |
|
||||
awk '$0 ~ /build\/bin\/smart_camera([[:space:]]|$)/ {print $1; exit}'
|
||||
}
|
||||
|
||||
valid_camera_pid() {
|
||||
local pid="$1"
|
||||
[[ "$pid" =~ ^[0-9]+$ ]] &&
|
||||
[[ -r "/proc/${pid}/cmdline" ]] &&
|
||||
tr '\0' ' ' < "/proc/${pid}/cmdline" | grep -q 'build/bin/smart_camera'
|
||||
}
|
||||
|
||||
camera_is_stopped() {
|
||||
local pid="$1"
|
||||
ps -o stat= -p "$pid" 2>/dev/null | grep -q '^T'
|
||||
}
|
||||
|
||||
wait_for_safe_window() {
|
||||
local newest size1 size2
|
||||
newest="$(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f -name '*.mp4' \
|
||||
-printf '%T@ %p\n' 2>/dev/null | sort -n | tail -1 | cut -d' ' -f2-)"
|
||||
[[ -n "$newest" && -f "$newest" ]] || return 0
|
||||
size1="$(stat -c %s "$newest" 2>/dev/null || printf 0)"
|
||||
sleep "$SAFE_WINDOW_SECONDS"
|
||||
[[ -f "$newest" ]] || return 0
|
||||
size2="$(stat -c %s "$newest" 2>/dev/null || printf 0)"
|
||||
[[ "$size1" -eq "$size2" && "$size2" -gt 0 ]]
|
||||
}
|
||||
|
||||
pause_capture() {
|
||||
local free_bytes="$1" pid
|
||||
pid="$(camera_host_pid || true)"
|
||||
if ! valid_camera_pid "$pid"; then
|
||||
alert ERROR "cannot find smart_camera process; capture was not paused"
|
||||
return 1
|
||||
fi
|
||||
if camera_is_stopped "$pid"; then
|
||||
printf 'pid=%s\npaused_at=%s\n' "$pid" "$(date --iso-8601=seconds)" > "$PAUSED_STATE"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! wait_for_safe_window; then
|
||||
alert WARNING "no stable recording gap found; pausing capture to protect disk"
|
||||
fi
|
||||
kill -STOP "$pid"
|
||||
printf 'pid=%s\npaused_at=%s\n' "$pid" "$(date --iso-8601=seconds)" > "$PAUSED_STATE"
|
||||
alert CRITICAL "capture paused at free=$(human_bytes "$free_bytes"); SFTP/Kafka uploader remains running; automatic resume target=$(human_bytes "$CLEAN_TARGET_BYTES")"
|
||||
}
|
||||
|
||||
resume_capture_if_ready() {
|
||||
local free_bytes="$1" pid
|
||||
[[ -f "$PAUSED_STATE" ]] || return 0
|
||||
pid="$(sed -n 's/^pid=//p' "$PAUSED_STATE" | head -1)"
|
||||
|
||||
if ! valid_camera_pid "$pid"; then
|
||||
log WARNING "discarding stale capture pause state (pid=${pid:-unknown})"
|
||||
rm -f "$PAUSED_STATE"
|
||||
return 0
|
||||
fi
|
||||
if ! camera_is_stopped "$pid"; then
|
||||
log WARNING "capture is already running; clearing stale pause state"
|
||||
rm -f "$PAUSED_STATE"
|
||||
return 0
|
||||
fi
|
||||
if [[ "$free_bytes" -ge "$CLEAN_TARGET_BYTES" ]]; then
|
||||
kill -CONT "$pid"
|
||||
rm -f "$PAUSED_STATE"
|
||||
alert RECOVERED "free space reached $(human_bytes "$free_bytes"); capture resumed automatically"
|
||||
else
|
||||
log WARNING "capture remains paused: free=$(human_bytes "$free_bytes"), resume target=$(human_bytes "$CLEAN_TARGET_BYTES")"
|
||||
fi
|
||||
}
|
||||
|
||||
load_processed_files() {
|
||||
local destination="$1"
|
||||
if ! docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null | grep -qx true; then
|
||||
return 1
|
||||
fi
|
||||
docker exec "$CONTAINER" sh -c \
|
||||
'test -r "$1" && cat "$1"' sh "$PROCESSED_FILE" 2>/dev/null |
|
||||
sed -nE '/^video_[0-9]{8}_[0-9]{6}_[0-9]{3}\.xml$/p' |
|
||||
sort -u > "$destination"
|
||||
[[ -s "$destination" ]]
|
||||
}
|
||||
|
||||
build_safe_bundles() {
|
||||
local processed="$1" output="$2" xml base raw annotated newest_ts
|
||||
: > "$output"
|
||||
while IFS= read -r xml; do
|
||||
base="${xml%.xml}"
|
||||
raw="$DATA_ROOT/videos_raw/${base}_raw.mp4"
|
||||
annotated="$DATA_ROOT/videos/${base}_annotated.mp4"
|
||||
# The raw file is the authoritative large asset. Only select completed,
|
||||
# sufficiently old bundles. Missing members are harmless and skipped later.
|
||||
[[ -f "$raw" ]] || continue
|
||||
find "$raw" -mmin "+${MIN_FILE_AGE_MINUTES}" -print -quit 2>/dev/null | grep -q . || continue
|
||||
newest_ts="$(stat -c %Y "$raw" 2>/dev/null || printf 0)"
|
||||
printf '%s %s\n' "$newest_ts" "$base" >> "$output"
|
||||
done < "$processed"
|
||||
sort -n -u "$output" -o "$output"
|
||||
}
|
||||
|
||||
delete_safe_bundles() {
|
||||
local bundles="$1" free_bytes deleted_count=0 deleted_bytes=0 record base path size
|
||||
while IFS= read -r record; do
|
||||
base="${record#* }"
|
||||
for path in \
|
||||
"$DATA_ROOT/videos_raw/${base}_raw.mp4" \
|
||||
"$DATA_ROOT/videos/${base}_annotated.mp4" \
|
||||
"$DATA_ROOT/annotations/${base}.xml"; do
|
||||
[[ -f "$path" ]] || continue
|
||||
size="$(stat -c %s "$path" 2>/dev/null || printf 0)"
|
||||
if rm -f -- "$path"; then
|
||||
deleted_count=$((deleted_count + 1))
|
||||
deleted_bytes=$((deleted_bytes + size))
|
||||
log INFO "deleted confirmed file (${size} bytes): ${path}"
|
||||
else
|
||||
log ERROR "failed to delete confirmed file: ${path}"
|
||||
fi
|
||||
done
|
||||
free_bytes="$(available_bytes)"
|
||||
if [[ "$free_bytes" -ge "$CLEAN_TARGET_BYTES" ]]; then
|
||||
break
|
||||
fi
|
||||
done < "$bundles"
|
||||
log INFO "safe cleanup result: deleted=${deleted_count} files bytes=${deleted_bytes} free=$(human_bytes "$(available_bytes)")"
|
||||
}
|
||||
|
||||
delete_stale_zero_bundles() {
|
||||
local raw base path deleted_bundles=0 deleted_files=0
|
||||
while IFS= read -r -d '' raw; do
|
||||
# A newly opened recording may temporarily be zero bytes. Never touch a
|
||||
# recent or open file; only remove a stale, unusable bundle as one unit.
|
||||
find "$raw" -mmin "+${MIN_FILE_AGE_MINUTES}" -print -quit 2>/dev/null | grep -q . || continue
|
||||
if command -v fuser >/dev/null 2>&1 && fuser -s "$raw" 2>/dev/null; then
|
||||
log WARNING "stale zero-byte raw is still open; skipped: ${raw}"
|
||||
continue
|
||||
fi
|
||||
|
||||
base="$(basename "$raw" _raw.mp4)"
|
||||
for path in \
|
||||
"$DATA_ROOT/videos_raw/${base}_raw.mp4" \
|
||||
"$DATA_ROOT/videos/${base}_annotated.mp4" \
|
||||
"$DATA_ROOT/annotations/${base}.xml"; do
|
||||
[[ -f "$path" ]] || continue
|
||||
if rm -f -- "$path"; then
|
||||
deleted_files=$((deleted_files + 1))
|
||||
log INFO "deleted stale zero-byte bundle member: ${path}"
|
||||
else
|
||||
log ERROR "failed to delete stale zero-byte bundle member: ${path}"
|
||||
fi
|
||||
done
|
||||
deleted_bundles=$((deleted_bundles + 1))
|
||||
done < <(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f \
|
||||
-name '*_raw.mp4' -size 0 -print0 2>/dev/null)
|
||||
|
||||
if [[ "$deleted_bundles" -gt 0 ]]; then
|
||||
log INFO "stale zero-byte cleanup result: bundles=${deleted_bundles} files=${deleted_files}"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
local free_bytes processed bundles
|
||||
delete_stale_zero_bundles
|
||||
free_bytes="$(available_bytes)"
|
||||
resume_capture_if_ready "$free_bytes"
|
||||
|
||||
if [[ "$free_bytes" -lt "$CLEAN_TARGET_BYTES" ]]; then
|
||||
processed="$(mktemp /tmp/userdata-processed.XXXXXX)"
|
||||
bundles="$(mktemp /tmp/userdata-bundles.XXXXXX)"
|
||||
trap 'rm -f "${processed:-}" "${bundles:-}"' EXIT
|
||||
|
||||
if load_processed_files "$processed"; then
|
||||
build_safe_bundles "$processed" "$bundles"
|
||||
if [[ -s "$bundles" ]]; then
|
||||
delete_safe_bundles "$bundles"
|
||||
else
|
||||
log WARNING "free space is low, but no confirmed old bundles are safe to delete"
|
||||
fi
|
||||
else
|
||||
alert ERROR "cannot read confirmed-upload list; fail-safe mode prevents all deletion"
|
||||
fi
|
||||
else
|
||||
log INFO "free space is sufficient: $(human_bytes "$free_bytes") available"
|
||||
fi
|
||||
|
||||
free_bytes="$(available_bytes)"
|
||||
if [[ "$free_bytes" -lt "$CRITICAL_BELOW_BYTES" ]]; then
|
||||
alert CRITICAL "free space below critical threshold: $(human_bytes "$free_bytes")"
|
||||
fi
|
||||
if [[ "$free_bytes" -lt "$PAUSE_BELOW_BYTES" ]]; then
|
||||
pause_capture "$free_bytes"
|
||||
fi
|
||||
resume_capture_if_ready "$(available_bytes)"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "请使用 sudo 执行: sudo ./scripts/install.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
bash -n "$ROOT_DIR/bin/cleanup-smartcam-data"
|
||||
|
||||
# Disable the former unit name when upgrading an existing device. This does
|
||||
# not stop, restart, or modify the smart-cam Docker container.
|
||||
systemctl disable --now userdata-video-cleanup.timer 2>/dev/null || true
|
||||
|
||||
install -d -o root -g root -m 0755 /var/lib/cleanup-smartcam-data
|
||||
if [[ -f /var/lib/userdata-video-cleanup/capture-paused && \
|
||||
! -f /var/lib/cleanup-smartcam-data/capture-paused ]]; then
|
||||
mv /var/lib/userdata-video-cleanup/capture-paused \
|
||||
/var/lib/cleanup-smartcam-data/capture-paused
|
||||
fi
|
||||
|
||||
install -o root -g root -m 0755 \
|
||||
"$ROOT_DIR/bin/cleanup-smartcam-data" \
|
||||
/usr/local/sbin/cleanup-smartcam-data
|
||||
|
||||
install -o root -g root -m 0644 \
|
||||
"$ROOT_DIR/systemd/cleanup-smartcam-data.service" \
|
||||
/etc/systemd/system/cleanup-smartcam-data.service
|
||||
|
||||
install -o root -g root -m 0644 \
|
||||
"$ROOT_DIR/systemd/cleanup-smartcam-data.timer" \
|
||||
/etc/systemd/system/cleanup-smartcam-data.timer
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now cleanup-smartcam-data.timer
|
||||
|
||||
# Remove obsolete installed names only after the new timer is active.
|
||||
rm -f \
|
||||
/etc/systemd/system/userdata-video-cleanup.timer \
|
||||
/etc/systemd/system/userdata-video-cleanup.service \
|
||||
/usr/local/sbin/userdata-video-cleanup
|
||||
systemctl daemon-reload
|
||||
|
||||
echo "安装完成。"
|
||||
echo "程序: /usr/local/sbin/cleanup-smartcam-data"
|
||||
echo "服务: /etc/systemd/system/cleanup-smartcam-data.service"
|
||||
echo "定时器: /etc/systemd/system/cleanup-smartcam-data.timer"
|
||||
systemctl --no-pager status cleanup-smartcam-data.timer || true
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${EUID}" -ne 0 ]]; then
|
||||
echo "请使用 sudo 执行: sudo ./scripts/uninstall.sh" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
systemctl disable --now cleanup-smartcam-data.timer 2>/dev/null || true
|
||||
rm -f \
|
||||
/etc/systemd/system/cleanup-smartcam-data.timer \
|
||||
/etc/systemd/system/cleanup-smartcam-data.service \
|
||||
/usr/local/sbin/cleanup-smartcam-data
|
||||
systemctl daemon-reload
|
||||
|
||||
echo "cleanup-smartcam-data 已卸载。"
|
||||
echo "未删除 /var/lib/cleanup-smartcam-data；请确认采集未处于暂停状态后再人工处理该目录。"
|
||||
@@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=Clean old video files to keep /userdata free space
|
||||
Documentation=man:systemd.service(5)
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/cleanup-smartcam-data
|
||||
Nice=10
|
||||
IOSchedulingClass=best-effort
|
||||
IOSchedulingPriority=7
|
||||
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=Run safe userdata video cleanup every 5 minutes
|
||||
Documentation=man:systemd.timer(5)
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=5min
|
||||
AccuracySec=30s
|
||||
Persistent=true
|
||||
Unit=cleanup-smartcam-data.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
Reference in new issue
Block a user