fix: recover calibration after smart-cam restart

README.md

@@ -1,7 +1,7 @@
 # scan参数交互式配置向导
 
 该工具用于在不同摄像头点位通过固定对话完成scan参数标定、配置替换、完整试扫和验证拼图，
-适合批量部署RK3588。详细的底层命令和原理见[手动配置README.md](./手动配置README.md)。
+适合批量部署RK3576和RK3588。详细的底层命令和原理见[手动配置README.md](./手动配置README.md)。
 
 ## 目录结构
 
@@ -25,7 +25,7 @@ scan-calibrator/
 
 ## 启动
 
-首次把本目录复制到一台RK3588后，先执行一次安装与自检：
+首次把本目录复制到一台RK3576或RK3588后，先执行一次安装与自检：
 
 ```bash
 cd /home/zhangw/scan-calibrator
@@ -201,7 +201,10 @@ smart-cam的HTTP接口读取PTZ并记录，等价于：
 11. 若pan实测与tilt透视预测相差超过15%，不强制套用比例，并明确提示镜头畸变或不同tilt行
     水平覆盖不一致；这种摄像头使用单一pan_step无法保证所有行同时零重叠且无缺口；
 12. 根据多个水平样本的带符号位移，自动识别该型号摄像头从画面左到右时pan是递增还是递减；
-13. 删除本次自动标定产生的临时截图。
+13. 每次标定会话只编译一次OpenCV重叠分析器，后续step直接复用；
+14. 如果smart-cam因RTSP采集看门狗而短暂重启，向导会等待scan HTTP恢复，
+    然后重试当前PTZ或拍照操作，不会直接丢失整个第四步；
+15. 删除本次自动标定产生的临时截图。
 
 这一步不能省略。两个角点只能确定扫描范围，无法确定当前摄像头型号和zoom下的一幅画面
 覆盖多少角度。

scan_calibrator.py

@@ -21,12 +21,21 @@ HELPER_CONTAINER_BIN = "/tmp/scan-calibrator-measure"
 DEFAULT_TARGET_OVERLAP = 0.04
 MAX_GENERATED_OVERLAP = 0.05
 PHASE_MIN_RESPONSE = 0.20
+API_MAX_ATTEMPTS = 3
+API_RECOVERY_TIMEOUT = 90.0
+API_RECOVERY_POLL_INTERVAL = 2.0
+TRANSIENT_CURL_EXIT_CODES = {7, 28, 52, 55, 56}
+_HELPER_READY = False
 
 
 def fail(message: str) -> "NoReturn":
     raise SystemExit(f"错误：{message}")
 
 
+def warn(message: str) -> None:
+    print(f"警告：{message}", file=sys.stderr, flush=True)
+
+
 def run(command: list[str], *, timeout: int = 90) -> subprocess.CompletedProcess[str]:
     result = subprocess.run(command, text=True, capture_output=True, timeout=timeout)
     if result.returncode:
@@ -34,18 +43,96 @@ def run(command: list[str], *, timeout: int = 90) -> subprocess.CompletedProcess
     return result
 
 
-def api(method: str, path: str, body: dict | None = None, timeout: int = 60) -> dict:
+def api_command(method: str, path: str, body: dict | None, timeout: int) -> list[str]:
     command = ["docker", "exec", "smart-cam", "curl", "-sS", "--fail-with-body",
                "--max-time", str(timeout), "-X", method]
     if body is not None:
         command += ["-H", "Content-Type: application/json", "-d", json.dumps(body)]
     command.append(f"http://127.0.0.1:18080{path}")
-    result = run(command, timeout=timeout + 10)
+    return command
+
+
+def transient_api_failure(result: subprocess.CompletedProcess[str]) -> bool:
+    if result.returncode in TRANSIENT_CURL_EXIT_CODES:
+        return True
+    detail = f"{result.stdout}\n{result.stderr}".lower()
+    return any(marker in detail for marker in (
+        "container is not running",
+        "container is restarting",
+        "no such container",
+        "couldn't connect to server",
+        "connection reset by peer",
+        "empty reply from server",
+        "returned error: 502",
+        "returned error: 503",
+        "returned error: 504",
+    ))
+
+
+def scan_http_ready() -> bool:
+    try:
+        state = subprocess.run(
+            ["docker", "inspect", "-f", "{{.State.Running}}", "smart-cam"],
+            text=True, capture_output=True, timeout=5,
+        )
+        if state.returncode or state.stdout.strip() != "true":
+            return False
+        health = subprocess.run(
+            ["docker", "exec", "smart-cam", "curl", "-sS", "--fail",
+             "--max-time", "5", "-o", "/dev/null",
+             "http://127.0.0.1:18080/health"],
+            text=True, capture_output=True, timeout=10,
+        )
+        return health.returncode == 0
+    except (OSError, subprocess.TimeoutExpired):
+        return False
+
+
+def wait_for_scan_http(timeout: float = API_RECOVERY_TIMEOUT) -> bool:
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if scan_http_ready():
+            return True
+        time.sleep(API_RECOVERY_POLL_INTERVAL)
+    return False
+
+
+def api(method: str, path: str, body: dict | None = None, timeout: int = 60) -> dict:
+    global _HELPER_READY
+    command = api_command(method, path, body, timeout)
+    last_detail = ""
+    for attempt in range(1, API_MAX_ATTEMPTS + 1):
+        try:
+            result = subprocess.run(
+                command, text=True, capture_output=True, timeout=timeout + 10,
+            )
+        except subprocess.TimeoutExpired:
+            result = subprocess.CompletedProcess(command, 28, "", "请求执行超时")
+        if result.returncode == 0:
             try:
                 return json.loads(result.stdout)
             except json.JSONDecodeError:
                 fail(f"scan HTTP返回的不是JSON：{result.stdout}")
 
+        last_detail = result.stderr.strip() or result.stdout.strip() or (
+            f"退出码={result.returncode}"
+        )
+        if not transient_api_failure(result) or attempt == API_MAX_ATTEMPTS:
+            break
+        warn(
+            f"scan HTTP或smart-cam容器短暂不可用（{last_detail}），"
+            f"正在等待恢复后重试当前操作 "
+            f"[{attempt}/{API_MAX_ATTEMPTS - 1}]"
+        )
+        if not wait_for_scan_http():
+            last_detail += f"；等待{API_RECOVERY_TIMEOUT:.0f}秒后scan HTTP仍未恢复"
+            break
+        # The container might have been replaced instead of restarted. Recheck
+        # the in-container overlap helper before the next image comparison.
+        _HELPER_READY = False
+
+    fail(f"命令失败：{' '.join(command)}\n{last_detail}")
+
 
 def get_ptz() -> dict[str, int]:
     payload = api("GET", "/api/v1/ptz", timeout=15)
@@ -81,10 +168,20 @@ def save_state(state: dict) -> None:
 
 
 def ensure_helper() -> None:
+    global _HELPER_READY
+    if _HELPER_READY:
+        existing = subprocess.run(
+            ["docker", "exec", "smart-cam", "test", "-x", HELPER_CONTAINER_BIN],
+            text=True, capture_output=True, timeout=10,
+        )
+        if existing.returncode == 0:
+            return
+        _HELPER_READY = False
     run(["docker", "cp", str(HELPER_SOURCE), f"smart-cam:{HELPER_CONTAINER_SOURCE}"])
     run(["docker", "exec", "smart-cam", "sh", "-lc",
          f"g++ -O2 -std=c++17 {HELPER_CONTAINER_SOURCE} -o {HELPER_CONTAINER_BIN} "
          "$(pkg-config --cflags --libs opencv4)"], timeout=120)
+    _HELPER_READY = True
 
 
 def measure_images(first: str, second: str) -> dict:
@@ -179,9 +276,15 @@ def measure_delta(axis: str, delta: int, direction: int, wait_seconds: float,
         result.update({"axis": axis, "delta": delta, "direction": direction,
                        "origin": origin, "target": target, "actual": actual,
                        "baseline_file": baseline, "candidate_file": candidate})
-        return result
-    finally:
+    except BaseException:
+        try:
             move(origin)
+        except BaseException as restore_error:
+            warn(f"当前样本失败后暂时无法返回标定起点：{restore_error}")
+        raise
+    else:
+        move(origin)
+        return result
 
 
 def command_ptz(_: argparse.Namespace) -> None:

tests/test_scan_calibrator.py

@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+from contextlib import redirect_stderr
+import io
+import subprocess
+import unittest
+from unittest import mock
+
+import scan_calibrator as core
+
+
+def completed(returncode: int, stdout: str = "", stderr: str = ""):
+    return subprocess.CompletedProcess(["docker"], returncode, stdout, stderr)
+
+
+class ApiRecoveryTests(unittest.TestCase):
+    def setUp(self) -> None:
+        core._HELPER_READY = False
+
+    @mock.patch("scan_calibrator.wait_for_scan_http", return_value=True)
+    @mock.patch("scan_calibrator.subprocess.run")
+    def test_transient_curl_failure_waits_and_retries(self, run_mock, wait_mock):
+        core._HELPER_READY = True
+        run_mock.side_effect = [
+            completed(52, stderr="curl: (52) Empty reply from server"),
+            completed(0, stdout='{"ptz": {"pan": 1, "tilt": 2, "zoom": 3}}'),
+        ]
+
+        payload = core.api("GET", "/api/v1/ptz", timeout=5)
+
+        self.assertEqual(payload["ptz"]["zoom"], 3)
+        self.assertEqual(run_mock.call_count, 2)
+        wait_mock.assert_called_once_with()
+        self.assertFalse(core._HELPER_READY)
+
+    @mock.patch("scan_calibrator.wait_for_scan_http", return_value=True)
+    @mock.patch("scan_calibrator.subprocess.run")
+    def test_connection_refused_is_retried(self, run_mock, wait_mock):
+        run_mock.side_effect = [
+            completed(7, stderr="curl: (7) Failed to connect to 127.0.0.1"),
+            completed(0, stdout='{"scan": {"state": "idle"}}'),
+        ]
+
+        payload = core.api("GET", "/api/v1/scan/status", timeout=5)
+
+        self.assertEqual(payload["scan"]["state"], "idle")
+        wait_mock.assert_called_once_with()
+
+    @mock.patch("scan_calibrator.wait_for_scan_http")
+    @mock.patch("scan_calibrator.subprocess.run")
+    def test_non_transient_http_error_is_not_retried(self, run_mock, wait_mock):
+        run_mock.return_value = completed(
+            22, stderr="curl: (22) The requested URL returned error: 404"
+        )
+
+        with self.assertRaisesRegex(SystemExit, "404"):
+            core.api("GET", "/missing", timeout=5)
+
+        self.assertEqual(run_mock.call_count, 1)
+        wait_mock.assert_not_called()
+
+    @mock.patch("scan_calibrator.time.sleep")
+    @mock.patch("scan_calibrator.scan_http_ready", side_effect=[False, True])
+    @mock.patch("scan_calibrator.time.monotonic", side_effect=[0.0, 1.0, 2.0])
+    def test_health_wait_polls_until_service_recovers(
+        self, _monotonic_mock, ready_mock, sleep_mock
+    ):
+        self.assertTrue(core.wait_for_scan_http(timeout=10.0))
+        self.assertEqual(ready_mock.call_count, 2)
+        sleep_mock.assert_called_once_with(core.API_RECOVERY_POLL_INTERVAL)
+
+    @mock.patch("scan_calibrator.run")
+    @mock.patch("scan_calibrator.subprocess.run")
+    def test_overlap_helper_compiles_only_once_per_session(
+        self, subprocess_run_mock, checked_run_mock
+    ):
+        subprocess_run_mock.return_value = completed(0)
+
+        core.ensure_helper()
+        core.ensure_helper()
+
+        self.assertEqual(checked_run_mock.call_count, 2)
+        subprocess_run_mock.assert_called_once_with(
+            ["docker", "exec", "smart-cam", "test", "-x", core.HELPER_CONTAINER_BIN],
+            text=True, capture_output=True, timeout=10,
+        )
+
+    @mock.patch("scan_calibrator.time.sleep")
+    @mock.patch("scan_calibrator.measure_images")
+    @mock.patch("scan_calibrator.snapshot", return_value="candidate.jpg")
+    @mock.patch("scan_calibrator.move")
+    def test_restore_failure_does_not_mask_measurement_failure(
+        self, move_mock, _snapshot_mock, measure_mock, _sleep_mock
+    ):
+        origin = {"pan": 100, "tilt": 200, "zoom": 300}
+        move_mock.side_effect = [dict(origin), SystemExit("恢复起点失败")]
+        measure_mock.side_effect = SystemExit("原始测量失败")
+        stderr = io.StringIO()
+
+        with redirect_stderr(stderr):
+            with self.assertRaisesRegex(SystemExit, "原始测量失败"):
+                core.measure_delta(
+                    "pan", 50, 1, 0.0, baseline="baseline.jpg", origin=origin
+                )
+
+        self.assertIn("恢复起点失败", stderr.getvalue())
+        self.assertEqual(move_mock.call_count, 2)
+
+
+if __name__ == "__main__":
+    unittest.main()