Clean up 0-byte files in 60 minutes
This commit is contained in:
1 parent
2e6f47ceff
commit
c460ae8fbd
2 files changed
+93
No files matched your search
@@ -83,6 +83,11 @@ sudo ./scripts/install.sh
|
||||
```
|
||||
|
||||
- 超过60分钟的0字节 raw,如果未被进程占用,也会按上述三件套删除;
|
||||
- 对`<base>_raw.mp4.partN.mp4`录制残留执行整组安全检查:
|
||||
- 同一base的所有part都必须超过60分钟、全部为0字节且均未被进程打开;
|
||||
- 如果正式`<base>_raw.mp4`已经存在,只删除符合条件的残留part,保留正常三件套;
|
||||
- 如果正式raw不存在,删除全部残留part及对应的annotated视频和XML;
|
||||
- 任一part仍较新、非0字节或正在使用时,整组跳过;
|
||||
- 空闲低于3GiB时暂停整个 `smart_camera` 进程(持续检测、录像和识别一起暂停),Producer继续上传,并输出 CRITICAL 告警;
|
||||
- 暂停期间每2分钟检查一次;
|
||||
- 清理使空闲恢复到5GiB后自动恢复 `smart_camera`;程序重新持续检测,达到鸟类确认门控后再录像和识别,并停止2分钟快速检查;
|
||||
|
||||
@@ -224,9 +224,97 @@ delete_stale_zero_bundles() {
|
||||
fi
|
||||
}
|
||||
|
||||
delete_stale_zero_part_bundles() {
|
||||
local part filename base final_raw sibling path size
|
||||
local safe part_count deleted_bundles=0 deleted_files=0 deleted_bytes=0
|
||||
declare -A checked_bases=()
|
||||
|
||||
while IFS= read -r -d '' part; do
|
||||
[[ -f "$part" ]] || continue
|
||||
filename="$(basename "$part")"
|
||||
base="${filename%%_raw.mp4.part*}"
|
||||
[[ -n "$base" && -z "${checked_bases[$base]:-}" ]] || continue
|
||||
checked_bases["$base"]=1
|
||||
final_raw="$DATA_ROOT/videos_raw/${base}_raw.mp4"
|
||||
|
||||
# If a valid final raw exists, these are harmless recorder leftovers.
|
||||
# Delete only old, zero-byte, unopened part files; preserve the bundle.
|
||||
if [[ -f "$final_raw" ]]; then
|
||||
while IFS= read -r -d '' sibling; do
|
||||
find "$sibling" -mmin "+${MIN_FILE_AGE_MINUTES}" -print -quit 2>/dev/null |
|
||||
grep -q . || continue
|
||||
if command -v fuser >/dev/null 2>&1 && fuser -s "$sibling" 2>/dev/null; then
|
||||
log WARNING "stale zero-byte part is still open; skipped: ${sibling}"
|
||||
continue
|
||||
fi
|
||||
if rm -f -- "$sibling"; then
|
||||
deleted_files=$((deleted_files + 1))
|
||||
log INFO "deleted stale zero-byte recorder part: ${sibling}"
|
||||
fi
|
||||
done < <(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f \
|
||||
-name "${base}_raw.mp4.part*.mp4" -size 0 -print0 2>/dev/null)
|
||||
continue
|
||||
fi
|
||||
|
||||
# Without a final raw, only remove the incomplete bundle when every part
|
||||
# is old, zero bytes, and not held open by any process.
|
||||
safe=1
|
||||
part_count=0
|
||||
while IFS= read -r -d '' sibling; do
|
||||
part_count=$((part_count + 1))
|
||||
[[ "$(stat -c %s "$sibling" 2>/dev/null || printf 1)" -eq 0 ]] || {
|
||||
safe=0
|
||||
break
|
||||
}
|
||||
find "$sibling" -mmin "+${MIN_FILE_AGE_MINUTES}" -print -quit 2>/dev/null |
|
||||
grep -q . || {
|
||||
safe=0
|
||||
break
|
||||
}
|
||||
if command -v fuser >/dev/null 2>&1 && fuser -s "$sibling" 2>/dev/null; then
|
||||
log WARNING "incomplete part bundle is still open; skipped: ${sibling}"
|
||||
safe=0
|
||||
break
|
||||
fi
|
||||
done < <(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f \
|
||||
-name "${base}_raw.mp4.part*.mp4" -print0 2>/dev/null)
|
||||
|
||||
[[ "$safe" -eq 1 && "$part_count" -gt 0 ]] || continue
|
||||
|
||||
while IFS= read -r -d '' sibling; do
|
||||
size="$(stat -c %s "$sibling" 2>/dev/null || printf 0)"
|
||||
if rm -f -- "$sibling"; then
|
||||
deleted_files=$((deleted_files + 1))
|
||||
deleted_bytes=$((deleted_bytes + size))
|
||||
log INFO "deleted stale incomplete bundle part: ${sibling}"
|
||||
fi
|
||||
done < <(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f \
|
||||
-name "${base}_raw.mp4.part*.mp4" -print0 2>/dev/null)
|
||||
|
||||
for path in \
|
||||
"$DATA_ROOT/videos/${base}_annotated.mp4" \
|
||||
"$DATA_ROOT/annotations/${base}.xml"; do
|
||||
[[ -f "$path" ]] || continue
|
||||
size="$(stat -c %s "$path" 2>/dev/null || printf 0)"
|
||||
if rm -f -- "$path"; then
|
||||
deleted_files=$((deleted_files + 1))
|
||||
deleted_bytes=$((deleted_bytes + size))
|
||||
log INFO "deleted stale incomplete bundle member: ${path}"
|
||||
fi
|
||||
done
|
||||
deleted_bundles=$((deleted_bundles + 1))
|
||||
done < <(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f \
|
||||
-name '*_raw.mp4.part*.mp4' -size 0 -print0 2>/dev/null)
|
||||
|
||||
if [[ "$deleted_bundles" -gt 0 || "$deleted_files" -gt 0 ]]; then
|
||||
log INFO "stale part cleanup result: bundles=${deleted_bundles} files=${deleted_files} bytes=${deleted_bytes}"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
local free_bytes processed bundles
|
||||
delete_stale_zero_bundles
|
||||
delete_stale_zero_part_bundles
|
||||
free_bytes="$(available_bytes)"
|
||||
|
||||
# Protect the disk before cleanup work: once free space is below 3 GiB,
|
||||
|
||||
Reference in new issue
Block a user