Clean up 0-byte files in 60 minutes
This commit is contained in:
1 parent
2e6f47ceff
commit
c460ae8fbd
2 files changed
+93
No files matched your search
@@ -83,6 +83,11 @@ sudo ./scripts/install.sh
|
|||||||
```
|
```
|
||||||
|
|
||||||
- 超过60分钟的0字节 raw,如果未被进程占用,也会按上述三件套删除;
|
- 超过60分钟的0字节 raw,如果未被进程占用,也会按上述三件套删除;
|
||||||
|
- 对`<base>_raw.mp4.partN.mp4`录制残留执行整组安全检查:
|
||||||
|
- 同一base的所有part都必须超过60分钟、全部为0字节且均未被进程打开;
|
||||||
|
- 如果正式`<base>_raw.mp4`已经存在,只删除符合条件的残留part,保留正常三件套;
|
||||||
|
- 如果正式raw不存在,删除全部残留part及对应的annotated视频和XML;
|
||||||
|
- 任一part仍较新、非0字节或正在使用时,整组跳过;
|
||||||
- 空闲低于3GiB时暂停整个 `smart_camera` 进程(持续检测、录像和识别一起暂停),Producer继续上传,并输出 CRITICAL 告警;
|
- 空闲低于3GiB时暂停整个 `smart_camera` 进程(持续检测、录像和识别一起暂停),Producer继续上传,并输出 CRITICAL 告警;
|
||||||
- 暂停期间每2分钟检查一次;
|
- 暂停期间每2分钟检查一次;
|
||||||
- 清理使空闲恢复到5GiB后自动恢复 `smart_camera`;程序重新持续检测,达到鸟类确认门控后再录像和识别,并停止2分钟快速检查;
|
- 清理使空闲恢复到5GiB后自动恢复 `smart_camera`;程序重新持续检测,达到鸟类确认门控后再录像和识别,并停止2分钟快速检查;
|
||||||
|
|||||||
@@ -224,9 +224,97 @@ delete_stale_zero_bundles() {
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
delete_stale_zero_part_bundles() {
|
||||||
|
local part filename base final_raw sibling path size
|
||||||
|
local safe part_count deleted_bundles=0 deleted_files=0 deleted_bytes=0
|
||||||
|
declare -A checked_bases=()
|
||||||
|
|
||||||
|
while IFS= read -r -d '' part; do
|
||||||
|
[[ -f "$part" ]] || continue
|
||||||
|
filename="$(basename "$part")"
|
||||||
|
base="${filename%%_raw.mp4.part*}"
|
||||||
|
[[ -n "$base" && -z "${checked_bases[$base]:-}" ]] || continue
|
||||||
|
checked_bases["$base"]=1
|
||||||
|
final_raw="$DATA_ROOT/videos_raw/${base}_raw.mp4"
|
||||||
|
|
||||||
|
# If a valid final raw exists, these are harmless recorder leftovers.
|
||||||
|
# Delete only old, zero-byte, unopened part files; preserve the bundle.
|
||||||
|
if [[ -f "$final_raw" ]]; then
|
||||||
|
while IFS= read -r -d '' sibling; do
|
||||||
|
find "$sibling" -mmin "+${MIN_FILE_AGE_MINUTES}" -print -quit 2>/dev/null |
|
||||||
|
grep -q . || continue
|
||||||
|
if command -v fuser >/dev/null 2>&1 && fuser -s "$sibling" 2>/dev/null; then
|
||||||
|
log WARNING "stale zero-byte part is still open; skipped: ${sibling}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if rm -f -- "$sibling"; then
|
||||||
|
deleted_files=$((deleted_files + 1))
|
||||||
|
log INFO "deleted stale zero-byte recorder part: ${sibling}"
|
||||||
|
fi
|
||||||
|
done < <(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f \
|
||||||
|
-name "${base}_raw.mp4.part*.mp4" -size 0 -print0 2>/dev/null)
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Without a final raw, only remove the incomplete bundle when every part
|
||||||
|
# is old, zero bytes, and not held open by any process.
|
||||||
|
safe=1
|
||||||
|
part_count=0
|
||||||
|
while IFS= read -r -d '' sibling; do
|
||||||
|
part_count=$((part_count + 1))
|
||||||
|
[[ "$(stat -c %s "$sibling" 2>/dev/null || printf 1)" -eq 0 ]] || {
|
||||||
|
safe=0
|
||||||
|
break
|
||||||
|
}
|
||||||
|
find "$sibling" -mmin "+${MIN_FILE_AGE_MINUTES}" -print -quit 2>/dev/null |
|
||||||
|
grep -q . || {
|
||||||
|
safe=0
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if command -v fuser >/dev/null 2>&1 && fuser -s "$sibling" 2>/dev/null; then
|
||||||
|
log WARNING "incomplete part bundle is still open; skipped: ${sibling}"
|
||||||
|
safe=0
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done < <(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f \
|
||||||
|
-name "${base}_raw.mp4.part*.mp4" -print0 2>/dev/null)
|
||||||
|
|
||||||
|
[[ "$safe" -eq 1 && "$part_count" -gt 0 ]] || continue
|
||||||
|
|
||||||
|
while IFS= read -r -d '' sibling; do
|
||||||
|
size="$(stat -c %s "$sibling" 2>/dev/null || printf 0)"
|
||||||
|
if rm -f -- "$sibling"; then
|
||||||
|
deleted_files=$((deleted_files + 1))
|
||||||
|
deleted_bytes=$((deleted_bytes + size))
|
||||||
|
log INFO "deleted stale incomplete bundle part: ${sibling}"
|
||||||
|
fi
|
||||||
|
done < <(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f \
|
||||||
|
-name "${base}_raw.mp4.part*.mp4" -print0 2>/dev/null)
|
||||||
|
|
||||||
|
for path in \
|
||||||
|
"$DATA_ROOT/videos/${base}_annotated.mp4" \
|
||||||
|
"$DATA_ROOT/annotations/${base}.xml"; do
|
||||||
|
[[ -f "$path" ]] || continue
|
||||||
|
size="$(stat -c %s "$path" 2>/dev/null || printf 0)"
|
||||||
|
if rm -f -- "$path"; then
|
||||||
|
deleted_files=$((deleted_files + 1))
|
||||||
|
deleted_bytes=$((deleted_bytes + size))
|
||||||
|
log INFO "deleted stale incomplete bundle member: ${path}"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
deleted_bundles=$((deleted_bundles + 1))
|
||||||
|
done < <(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f \
|
||||||
|
-name '*_raw.mp4.part*.mp4' -size 0 -print0 2>/dev/null)
|
||||||
|
|
||||||
|
if [[ "$deleted_bundles" -gt 0 || "$deleted_files" -gt 0 ]]; then
|
||||||
|
log INFO "stale part cleanup result: bundles=${deleted_bundles} files=${deleted_files} bytes=${deleted_bytes}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
main() {
|
main() {
|
||||||
local free_bytes processed bundles
|
local free_bytes processed bundles
|
||||||
delete_stale_zero_bundles
|
delete_stale_zero_bundles
|
||||||
|
delete_stale_zero_part_bundles
|
||||||
free_bytes="$(available_bytes)"
|
free_bytes="$(available_bytes)"
|
||||||
|
|
||||||
# Protect the disk before cleanup work: once free space is below 3 GiB,
|
# Protect the disk before cleanup work: once free space is below 3 GiB,
|
||||||
|
|||||||
Reference in new issue
Block a user