first commit
This commit is contained in:
Ankle committed
commit
24f5b3fdfa
7 files changed
+494
No files matched your search
Executable
+252
@@ -0,0 +1,252 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_MOUNT="/userdata"
|
||||
DATA_ROOT="/userdata/data"
|
||||
CONTAINER="smart-cam"
|
||||
PROCESSED_FILE="/smart-cam/kafka_sender/processed_files.log"
|
||||
STATE_DIR="/var/lib/cleanup-smartcam-data"
|
||||
PAUSED_STATE="${STATE_DIR}/capture-paused"
|
||||
LOCK_FILE="/run/cleanup-smartcam-data.lock"
|
||||
|
||||
# Start deleting confirmed bundles below 15 GiB, pause capture below 8 GiB,
|
||||
# and resume only after the filesystem has recovered to 15 GiB.
|
||||
CLEAN_TARGET_BYTES="${CLEAN_TARGET_BYTES:-16106127360}"
|
||||
PAUSE_BELOW_BYTES="${PAUSE_BELOW_BYTES:-8589934592}"
|
||||
CRITICAL_BELOW_BYTES="${CRITICAL_BELOW_BYTES:-5368709120}"
|
||||
MIN_FILE_AGE_MINUTES="${MIN_FILE_AGE_MINUTES:-60}"
|
||||
SAFE_WINDOW_SECONDS="${SAFE_WINDOW_SECONDS:-10}"
|
||||
|
||||
mkdir -p "$STATE_DIR"
|
||||
exec 9>"$LOCK_FILE"
|
||||
flock -n 9 || exit 0
|
||||
|
||||
log() {
|
||||
local level="$1"
|
||||
shift
|
||||
local message="[DISK_GUARD][${level}] $*"
|
||||
printf '%s\n' "$message"
|
||||
logger -t cleanup-smartcam-data -- "$message" || true
|
||||
}
|
||||
|
||||
container_log() {
|
||||
local message="$1"
|
||||
if docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null | grep -qx true; then
|
||||
docker exec "$CONTAINER" sh -c \
|
||||
'printf "%s\n" "$1" > /proc/1/fd/1' sh "$message" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
alert() {
|
||||
local level="$1"
|
||||
shift
|
||||
local message="[DISK_GUARD][${level}] $*"
|
||||
log "$level" "$*"
|
||||
container_log "$message"
|
||||
}
|
||||
|
||||
available_bytes() {
|
||||
df -PB1 "$TARGET_MOUNT" | awk 'NR == 2 {print $4}'
|
||||
}
|
||||
|
||||
human_bytes() {
|
||||
numfmt --to=iec-i --suffix=B "$1"
|
||||
}
|
||||
|
||||
camera_host_pid() {
|
||||
docker top "$CONTAINER" -eo pid,args 2>/dev/null |
|
||||
awk '$0 ~ /build\/bin\/smart_camera([[:space:]]|$)/ {print $1; exit}'
|
||||
}
|
||||
|
||||
valid_camera_pid() {
|
||||
local pid="$1"
|
||||
[[ "$pid" =~ ^[0-9]+$ ]] &&
|
||||
[[ -r "/proc/${pid}/cmdline" ]] &&
|
||||
tr '\0' ' ' < "/proc/${pid}/cmdline" | grep -q 'build/bin/smart_camera'
|
||||
}
|
||||
|
||||
camera_is_stopped() {
|
||||
local pid="$1"
|
||||
ps -o stat= -p "$pid" 2>/dev/null | grep -q '^T'
|
||||
}
|
||||
|
||||
wait_for_safe_window() {
|
||||
local newest size1 size2
|
||||
newest="$(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f -name '*.mp4' \
|
||||
-printf '%T@ %p\n' 2>/dev/null | sort -n | tail -1 | cut -d' ' -f2-)"
|
||||
[[ -n "$newest" && -f "$newest" ]] || return 0
|
||||
size1="$(stat -c %s "$newest" 2>/dev/null || printf 0)"
|
||||
sleep "$SAFE_WINDOW_SECONDS"
|
||||
[[ -f "$newest" ]] || return 0
|
||||
size2="$(stat -c %s "$newest" 2>/dev/null || printf 0)"
|
||||
[[ "$size1" -eq "$size2" && "$size2" -gt 0 ]]
|
||||
}
|
||||
|
||||
pause_capture() {
|
||||
local free_bytes="$1" pid
|
||||
pid="$(camera_host_pid || true)"
|
||||
if ! valid_camera_pid "$pid"; then
|
||||
alert ERROR "cannot find smart_camera process; capture was not paused"
|
||||
return 1
|
||||
fi
|
||||
if camera_is_stopped "$pid"; then
|
||||
printf 'pid=%s\npaused_at=%s\n' "$pid" "$(date --iso-8601=seconds)" > "$PAUSED_STATE"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! wait_for_safe_window; then
|
||||
alert WARNING "no stable recording gap found; pausing capture to protect disk"
|
||||
fi
|
||||
kill -STOP "$pid"
|
||||
printf 'pid=%s\npaused_at=%s\n' "$pid" "$(date --iso-8601=seconds)" > "$PAUSED_STATE"
|
||||
alert CRITICAL "capture paused at free=$(human_bytes "$free_bytes"); SFTP/Kafka uploader remains running; automatic resume target=$(human_bytes "$CLEAN_TARGET_BYTES")"
|
||||
}
|
||||
|
||||
resume_capture_if_ready() {
|
||||
local free_bytes="$1" pid
|
||||
[[ -f "$PAUSED_STATE" ]] || return 0
|
||||
pid="$(sed -n 's/^pid=//p' "$PAUSED_STATE" | head -1)"
|
||||
|
||||
if ! valid_camera_pid "$pid"; then
|
||||
log WARNING "discarding stale capture pause state (pid=${pid:-unknown})"
|
||||
rm -f "$PAUSED_STATE"
|
||||
return 0
|
||||
fi
|
||||
if ! camera_is_stopped "$pid"; then
|
||||
log WARNING "capture is already running; clearing stale pause state"
|
||||
rm -f "$PAUSED_STATE"
|
||||
return 0
|
||||
fi
|
||||
if [[ "$free_bytes" -ge "$CLEAN_TARGET_BYTES" ]]; then
|
||||
kill -CONT "$pid"
|
||||
rm -f "$PAUSED_STATE"
|
||||
alert RECOVERED "free space reached $(human_bytes "$free_bytes"); capture resumed automatically"
|
||||
else
|
||||
log WARNING "capture remains paused: free=$(human_bytes "$free_bytes"), resume target=$(human_bytes "$CLEAN_TARGET_BYTES")"
|
||||
fi
|
||||
}
|
||||
|
||||
load_processed_files() {
|
||||
local destination="$1"
|
||||
if ! docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null | grep -qx true; then
|
||||
return 1
|
||||
fi
|
||||
docker exec "$CONTAINER" sh -c \
|
||||
'test -r "$1" && cat "$1"' sh "$PROCESSED_FILE" 2>/dev/null |
|
||||
sed -nE '/^video_[0-9]{8}_[0-9]{6}_[0-9]{3}\.xml$/p' |
|
||||
sort -u > "$destination"
|
||||
[[ -s "$destination" ]]
|
||||
}
|
||||
|
||||
build_safe_bundles() {
|
||||
local processed="$1" output="$2" xml base raw annotated newest_ts
|
||||
: > "$output"
|
||||
while IFS= read -r xml; do
|
||||
base="${xml%.xml}"
|
||||
raw="$DATA_ROOT/videos_raw/${base}_raw.mp4"
|
||||
annotated="$DATA_ROOT/videos/${base}_annotated.mp4"
|
||||
# The raw file is the authoritative large asset. Only select completed,
|
||||
# sufficiently old bundles. Missing members are harmless and skipped later.
|
||||
[[ -f "$raw" ]] || continue
|
||||
find "$raw" -mmin "+${MIN_FILE_AGE_MINUTES}" -print -quit 2>/dev/null | grep -q . || continue
|
||||
newest_ts="$(stat -c %Y "$raw" 2>/dev/null || printf 0)"
|
||||
printf '%s %s\n' "$newest_ts" "$base" >> "$output"
|
||||
done < "$processed"
|
||||
sort -n -u "$output" -o "$output"
|
||||
}
|
||||
|
||||
delete_safe_bundles() {
|
||||
local bundles="$1" free_bytes deleted_count=0 deleted_bytes=0 record base path size
|
||||
while IFS= read -r record; do
|
||||
base="${record#* }"
|
||||
for path in \
|
||||
"$DATA_ROOT/videos_raw/${base}_raw.mp4" \
|
||||
"$DATA_ROOT/videos/${base}_annotated.mp4" \
|
||||
"$DATA_ROOT/annotations/${base}.xml"; do
|
||||
[[ -f "$path" ]] || continue
|
||||
size="$(stat -c %s "$path" 2>/dev/null || printf 0)"
|
||||
if rm -f -- "$path"; then
|
||||
deleted_count=$((deleted_count + 1))
|
||||
deleted_bytes=$((deleted_bytes + size))
|
||||
log INFO "deleted confirmed file (${size} bytes): ${path}"
|
||||
else
|
||||
log ERROR "failed to delete confirmed file: ${path}"
|
||||
fi
|
||||
done
|
||||
free_bytes="$(available_bytes)"
|
||||
if [[ "$free_bytes" -ge "$CLEAN_TARGET_BYTES" ]]; then
|
||||
break
|
||||
fi
|
||||
done < "$bundles"
|
||||
log INFO "safe cleanup result: deleted=${deleted_count} files bytes=${deleted_bytes} free=$(human_bytes "$(available_bytes)")"
|
||||
}
|
||||
|
||||
delete_stale_zero_bundles() {
|
||||
local raw base path deleted_bundles=0 deleted_files=0
|
||||
while IFS= read -r -d '' raw; do
|
||||
# A newly opened recording may temporarily be zero bytes. Never touch a
|
||||
# recent or open file; only remove a stale, unusable bundle as one unit.
|
||||
find "$raw" -mmin "+${MIN_FILE_AGE_MINUTES}" -print -quit 2>/dev/null | grep -q . || continue
|
||||
if command -v fuser >/dev/null 2>&1 && fuser -s "$raw" 2>/dev/null; then
|
||||
log WARNING "stale zero-byte raw is still open; skipped: ${raw}"
|
||||
continue
|
||||
fi
|
||||
|
||||
base="$(basename "$raw" _raw.mp4)"
|
||||
for path in \
|
||||
"$DATA_ROOT/videos_raw/${base}_raw.mp4" \
|
||||
"$DATA_ROOT/videos/${base}_annotated.mp4" \
|
||||
"$DATA_ROOT/annotations/${base}.xml"; do
|
||||
[[ -f "$path" ]] || continue
|
||||
if rm -f -- "$path"; then
|
||||
deleted_files=$((deleted_files + 1))
|
||||
log INFO "deleted stale zero-byte bundle member: ${path}"
|
||||
else
|
||||
log ERROR "failed to delete stale zero-byte bundle member: ${path}"
|
||||
fi
|
||||
done
|
||||
deleted_bundles=$((deleted_bundles + 1))
|
||||
done < <(find "$DATA_ROOT/videos_raw" -maxdepth 1 -type f \
|
||||
-name '*_raw.mp4' -size 0 -print0 2>/dev/null)
|
||||
|
||||
if [[ "$deleted_bundles" -gt 0 ]]; then
|
||||
log INFO "stale zero-byte cleanup result: bundles=${deleted_bundles} files=${deleted_files}"
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
local free_bytes processed bundles
|
||||
delete_stale_zero_bundles
|
||||
free_bytes="$(available_bytes)"
|
||||
resume_capture_if_ready "$free_bytes"
|
||||
|
||||
if [[ "$free_bytes" -lt "$CLEAN_TARGET_BYTES" ]]; then
|
||||
processed="$(mktemp /tmp/userdata-processed.XXXXXX)"
|
||||
bundles="$(mktemp /tmp/userdata-bundles.XXXXXX)"
|
||||
trap 'rm -f "${processed:-}" "${bundles:-}"' EXIT
|
||||
|
||||
if load_processed_files "$processed"; then
|
||||
build_safe_bundles "$processed" "$bundles"
|
||||
if [[ -s "$bundles" ]]; then
|
||||
delete_safe_bundles "$bundles"
|
||||
else
|
||||
log WARNING "free space is low, but no confirmed old bundles are safe to delete"
|
||||
fi
|
||||
else
|
||||
alert ERROR "cannot read confirmed-upload list; fail-safe mode prevents all deletion"
|
||||
fi
|
||||
else
|
||||
log INFO "free space is sufficient: $(human_bytes "$free_bytes") available"
|
||||
fi
|
||||
|
||||
free_bytes="$(available_bytes)"
|
||||
if [[ "$free_bytes" -lt "$CRITICAL_BELOW_BYTES" ]]; then
|
||||
alert CRITICAL "free space below critical threshold: $(human_bytes "$free_bytes")"
|
||||
fi
|
||||
if [[ "$free_bytes" -lt "$PAUSE_BELOW_BYTES" ]]; then
|
||||
pause_capture "$free_bytes"
|
||||
fi
|
||||
resume_capture_if_ready "$(available_bytes)"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in new issue
Block a user