fix: recover calibration after smart-cam restart

scan_calibrator.py

@@ -21,12 +21,21 @@ HELPER_CONTAINER_BIN = "/tmp/scan-calibrator-measure"
 DEFAULT_TARGET_OVERLAP = 0.04
 MAX_GENERATED_OVERLAP = 0.05
 PHASE_MIN_RESPONSE = 0.20
+API_MAX_ATTEMPTS = 3
+API_RECOVERY_TIMEOUT = 90.0
+API_RECOVERY_POLL_INTERVAL = 2.0
+TRANSIENT_CURL_EXIT_CODES = {7, 28, 52, 55, 56}
+_HELPER_READY = False
 
 
 def fail(message: str) -> "NoReturn":
     raise SystemExit(f"错误：{message}")
 
 
+def warn(message: str) -> None:
+    print(f"警告：{message}", file=sys.stderr, flush=True)
+
+
 def run(command: list[str], *, timeout: int = 90) -> subprocess.CompletedProcess[str]:
     result = subprocess.run(command, text=True, capture_output=True, timeout=timeout)
     if result.returncode:
@@ -34,17 +43,95 @@ def run(command: list[str], *, timeout: int = 90) -> subprocess.CompletedProcess
     return result
 
 
-def api(method: str, path: str, body: dict | None = None, timeout: int = 60) -> dict:
+def api_command(method: str, path: str, body: dict | None, timeout: int) -> list[str]:
     command = ["docker", "exec", "smart-cam", "curl", "-sS", "--fail-with-body",
                "--max-time", str(timeout), "-X", method]
     if body is not None:
         command += ["-H", "Content-Type: application/json", "-d", json.dumps(body)]
     command.append(f"http://127.0.0.1:18080{path}")
-    result = run(command, timeout=timeout + 10)
+    return command
+
+
+def transient_api_failure(result: subprocess.CompletedProcess[str]) -> bool:
+    if result.returncode in TRANSIENT_CURL_EXIT_CODES:
+        return True
+    detail = f"{result.stdout}\n{result.stderr}".lower()
+    return any(marker in detail for marker in (
+        "container is not running",
+        "container is restarting",
+        "no such container",
+        "couldn't connect to server",
+        "connection reset by peer",
+        "empty reply from server",
+        "returned error: 502",
+        "returned error: 503",
+        "returned error: 504",
+    ))
+
+
+def scan_http_ready() -> bool:
     try:
-        return json.loads(result.stdout)
-    except json.JSONDecodeError:
-        fail(f"scan HTTP返回的不是JSON：{result.stdout}")
+        state = subprocess.run(
+            ["docker", "inspect", "-f", "{{.State.Running}}", "smart-cam"],
+            text=True, capture_output=True, timeout=5,
+        )
+        if state.returncode or state.stdout.strip() != "true":
+            return False
+        health = subprocess.run(
+            ["docker", "exec", "smart-cam", "curl", "-sS", "--fail",
+             "--max-time", "5", "-o", "/dev/null",
+             "http://127.0.0.1:18080/health"],
+            text=True, capture_output=True, timeout=10,
+        )
+        return health.returncode == 0
+    except (OSError, subprocess.TimeoutExpired):
+        return False
+
+
+def wait_for_scan_http(timeout: float = API_RECOVERY_TIMEOUT) -> bool:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if scan_http_ready():
+            return True
+        time.sleep(API_RECOVERY_POLL_INTERVAL)
+    return False
+
+
+def api(method: str, path: str, body: dict | None = None, timeout: int = 60) -> dict:
+    global _HELPER_READY
+    command = api_command(method, path, body, timeout)
+    last_detail = ""
+    for attempt in range(1, API_MAX_ATTEMPTS + 1):
+        try:
+            result = subprocess.run(
+                command, text=True, capture_output=True, timeout=timeout + 10,
+            )
+        except subprocess.TimeoutExpired:
+            result = subprocess.CompletedProcess(command, 28, "", "请求执行超时")
+        if result.returncode == 0:
+            try:
+                return json.loads(result.stdout)
+            except json.JSONDecodeError:
+                fail(f"scan HTTP返回的不是JSON：{result.stdout}")
+
+        last_detail = result.stderr.strip() or result.stdout.strip() or (
+            f"退出码={result.returncode}"
+        )
+        if not transient_api_failure(result) or attempt == API_MAX_ATTEMPTS:
+            break
+        warn(
+            f"scan HTTP或smart-cam容器短暂不可用（{last_detail}），"
+            f"正在等待恢复后重试当前操作 "
+            f"[{attempt}/{API_MAX_ATTEMPTS - 1}]"
+        )
+        if not wait_for_scan_http():
+            last_detail += f"；等待{API_RECOVERY_TIMEOUT:.0f}秒后scan HTTP仍未恢复"
+            break
+        # The container might have been replaced instead of restarted. Recheck
+        # the in-container overlap helper before the next image comparison.
+        _HELPER_READY = False
+
+    fail(f"命令失败：{' '.join(command)}\n{last_detail}")
 
 
 def get_ptz() -> dict[str, int]:
@@ -81,10 +168,20 @@ def save_state(state: dict) -> None:
 
 
 def ensure_helper() -> None:
+    global _HELPER_READY
+    if _HELPER_READY:
+        existing = subprocess.run(
+            ["docker", "exec", "smart-cam", "test", "-x", HELPER_CONTAINER_BIN],
+            text=True, capture_output=True, timeout=10,
+        )
+        if existing.returncode == 0:
+            return
+        _HELPER_READY = False
     run(["docker", "cp", str(HELPER_SOURCE), f"smart-cam:{HELPER_CONTAINER_SOURCE}"])
     run(["docker", "exec", "smart-cam", "sh", "-lc",
          f"g++ -O2 -std=c++17 {HELPER_CONTAINER_SOURCE} -o {HELPER_CONTAINER_BIN} "
          "$(pkg-config --cflags --libs opencv4)"], timeout=120)
+    _HELPER_READY = True
 
 
 def measure_images(first: str, second: str) -> dict:
@@ -179,9 +276,15 @@ def measure_delta(axis: str, delta: int, direction: int, wait_seconds: float,
         result.update({"axis": axis, "delta": delta, "direction": direction,
                        "origin": origin, "target": target, "actual": actual,
                        "baseline_file": baseline, "candidate_file": candidate})
-        return result
-    finally:
+    except BaseException:
+        try:
+            move(origin)
+        except BaseException as restore_error:
+            warn(f"当前样本失败后暂时无法返回标定起点：{restore_error}")
+        raise
+    else:
         move(origin)
+        return result
 
 
 def command_ptz(_: argparse.Namespace) -> None: