diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..f1426ed --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,44 @@ +# Copilot / AI agent instructions for this repo + +目标:帮助 AI 编码代理快速上手此环视(surround-view)项目,聚焦可验证的代码模式、运行流程和常见修改点。 + +- **大体架构**:数据流为 Capture -> Undistort -> Project -> Flip -> Stitch -> WhiteBalance -> Display。主要线程/模块: + - `surround_view.capture_thread.CaptureThread`:采集摄像头帧(USB/CSI,可传 `api_preference`)。 + - `surround_view.fisheye_camera.FisheyeCameraModel`:负责 `undistort()`, `project()`, `flip()` 和相机参数读写(yaml 文件)。 + - `surround_view.process_thread.CameraProcessingThread`:每个摄像头的处理线程,调用相机模型并写入 `ProjectedImageBuffer`。 + - `surround_view.birdview.BirdView`:融合/拼接、亮度匹配与白平衡,输出最终鸟瞰图。 + - 同步/缓冲:`surround_view.imagebuffer.Buffer`、`MultiBufferManager` 与 `ProjectedImageBuffer`,通过 `sync()`/信号等待实现多流同步。 + +- **关键运行脚本(推荐顺序)**: + 1. `python run_calibrate_camera.py`(标定/生成 camera yaml) + 2. `python run_get_projection_maps.py -camera -scale ... -shift ...`(选择投影点) + 3. `python run_get_weight_matrices.py`(生成拼接权重/掩码) + 4. `python run_live_demo.py`(车载/实时 demo) + - 调试摄像头采集:`python test_cameras.py` + +- **项目约定 / 易错点(请严格遵循)**: + - 摄像头名称固定为 `front, back, left, right`,对应 yaml/param 中的配置;device id 需在本机环境中确认(参见 `test_cameras.py`)。 + - yaml 文件(`yaml/*.yaml`)里保存的字段:`camera_matrix`, `dist_coeffs`, `resolution`, `project_matrix`, `scale_xy`, `shift_xy`。修改后用 `FisheyeCameraModel.save_data()` 写回。 + - `param_settings.py` 用 cm 为单位构造 birdview 参数(`total_w/total_h`、`xl/xr/yt/yb` 等),birdview 的切片函数(`FI, FM, ...`)依赖这些常量。 + - 镜头“翻转”策略在 `FisheyeCameraModel.flip()` 明确:`front` 不变,`back` 180°, `left/right` 用转置+翻转处理。 + - 缓冲区 `Buffer.add(..., drop_if_full=True)` 会丢帧;修 bug 时注意是否因为丢帧导致状态不一致。 + +- **拼接与亮度调整实现要点**(参见 `surround_view/birdview.py` 与 `surround_view/utils.py`): + - 权重计算:`get_weight_mask_matrix()` 提取重叠区域并基于点到多边形距离计算平滑权重 G。 + - 亮度匹配通过 `mean_luminance_ratio()` 聚合重叠区域平均亮度并以几何/经验方式融合(见 `make_luminance_balance()`)。 + - 白平衡用 `make_white_balance()` 简单按通道均值缩放。 + +- **如何修改/扩展(示例)**: + - 若加入新相机或改变布局:更新 `param_settings.py` 的 `camera_names`、`project_shapes` 与 `xl/xr/yt/yb`;更新 yaml 参数并通过 `FisheyeCameraModel` 测试 `undistort()`/`project()` 输出。 + - 若替换采集后端(gstreamer / v4l2),优先修改 `surround_view.utils.gstreamer_pipeline()` 与 `CaptureThread.connect_camera()` 的 `api_preference` 分支,确保 `cap.isOpened()` 行为一致。 + +- **测试 / 调试建议**: + - 使用 `test_cameras.py` 确认设备索引与支持分辨率。 + - 单线程走查:直接在 REPL 导入 `FisheyeCameraModel`,用已存在的 yaml 文件运行 `undistort()`、`project()` 并保存图片来验证投影矩阵。 + - 帧同步问题优先检查 `MultiBufferManager.sync()` 与 `ProjectedImageBuffer.sync()` 的 `do_sync` 与 `sync_devices` 配置。 + +- **风格/约定**: + - 代码使用 Python 3、OpenCV、PyQt5,多线程依赖 `QThread` + Qt 同步原语;避免用 `time.sleep()` 作为同步手段。 + - 尽量在修改后用现有脚本跑完整流程(calibrate -> get_projection -> get_weight -> live_demo)以捕获参数联动问题。 + +如果需要,我可以把这些点合并成更短或更详尽的版本,或者加入常见问题/故障排查列表;或者你希望我直接提交此文件为 PR?请告诉我想要的调整或遗漏的内容。 diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 0000000..66b0075 --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,12 @@ +{ + // See https://go.microsoft.com/fwlink/?LinkId=733558 + // for the documentation about the tasks.json format + "version": "2.0.0", + "tasks": [ + { + "label": "kill python", + "type": "shell", + "command": "pkill -f python3" + } + ] +} \ No newline at end of file diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..0832449 --- /dev/null +++ b/run.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# 获取当前脚本的 PID +SELF_PID=$$ + +echo "【1/3】终止其他 run.sh(跳过自身 PID=$SELF_PID)..." +# 使用 pgrep + grep -v 排除自己 +pgrep -f "run.sh" | grep -v "^$SELF_PID$" | xargs kill -9 2>/dev/null + +echo "【2/3】终止 run_live_demo.py ..." +pkill -9 -f "run_live_demo.py" 2>/dev/null + +sleep 1 + + +echo "【3/3】启动新实例..." +exec python3 ./run_live_demo.py + diff --git a/run_live_demo.py b/run_live_demo.py index 4b27637..18f85da 100644 --- a/run_live_demo.py +++ b/run_live_demo.py @@ -8,14 +8,15 @@ import surround_view.param_settings as settings yamls_dir = os.path.join(os.getcwd(), "yaml") camera_ids = [0, 1, 2, 3] -flip_methods = [0, 2, 0, 2] +flip_methods = [0, 0, 0,0] + names = settings.camera_names cameras_files = [os.path.join(yamls_dir, name + ".yaml") for name in names] camera_models = [FisheyeCameraModel(camera_file, name) for camera_file, name in zip(cameras_files, names)] def main(): - capture_tds = [CaptureThread(camera_id, flip_method) + capture_tds = [CaptureThread(camera_id, flip_method,resolution=(1920, 1080)) for camera_id, flip_method in zip(camera_ids, flip_methods)] capture_buffer_manager = MultiBufferManager() for td in capture_tds: @@ -38,17 +39,18 @@ def main(): while True: img = cv2.resize(birdview.get(), (300, 400)) cv2.imshow("birdview", img) + key = cv2.waitKey(1) & 0xFF if key == ord("q"): break - for td in capture_tds: - print("camera {} fps: {}\n".format(td.device_id, td.stat_data.average_fps), end="\r") + # for td in capture_tds: + # print("camera {} fps: {}\n".format(td.device_id, td.stat_data.average_fps), end="\r") - for td in process_tds: - print("process {} fps: {}\n".format(td.device_id, td.stat_data.average_fps), end="\r") + # for td in process_tds: + # print("process {} fps: {}\n".format(td.device_id, td.stat_data.average_fps), end="\r") - print("birdview fps: {}".format(birdview.stat_data.average_fps)) + # print("birdview fps: {}".format(birdview.stat_data.average_fps)) for td in process_tds: diff --git a/saveimg.py b/saveimg.py index 77483e1..c2fa63d 100644 --- a/saveimg.py +++ b/saveimg.py @@ -10,7 +10,7 @@ running = True def video_thread(): global frame, running - cap = cv2.VideoCapture(0, cv2.CAP_ANY) + cap = cv2.VideoCapture(1, cv2.CAP_ANY) cap.set(cv2.CAP_PROP_FOURCC,cv2.VideoWriter_fourcc(*"YUYV")) cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1920) cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1080) @@ -18,7 +18,7 @@ def video_thread(): if not cap.isOpened(): print("[ERROR] Cannot open camera", file=sys.stderr) - running = Falseq + running = False return while running: @@ -26,7 +26,13 @@ def video_thread(): if not ret: break frame = f.copy() - cv2.imshow('Live Feed (Local Display)', f) + # print(frame.shape) + # frame = cv2.resize(frame, (1280, 720)) + # print(frame.shape) + cv2.namedWindow('AHD Video', cv2.WND_PROP_FULLSCREEN) + cv2.setWindowProperty('AHD Video', cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN) + cv2.imshow('AHD Video', f) + # cv2.showFullscreen('Live Feed (Local Display)', f) if cv2.waitKey(1) & 0xFF == ord('q'): running = False break diff --git a/surround_view/__pycache__/__init__.cpython-38.pyc b/surround_view/__pycache__/__init__.cpython-38.pyc index 5706dce..fea8586 100644 Binary files a/surround_view/__pycache__/__init__.cpython-38.pyc and b/surround_view/__pycache__/__init__.cpython-38.pyc differ diff --git a/surround_view/__pycache__/base_thread.cpython-38.pyc b/surround_view/__pycache__/base_thread.cpython-38.pyc index c89bfca..2bfad32 100644 Binary files a/surround_view/__pycache__/base_thread.cpython-38.pyc and b/surround_view/__pycache__/base_thread.cpython-38.pyc differ diff --git a/surround_view/__pycache__/birdview.cpython-38.pyc b/surround_view/__pycache__/birdview.cpython-38.pyc index 4c513e5..8fd841b 100644 Binary files a/surround_view/__pycache__/birdview.cpython-38.pyc and b/surround_view/__pycache__/birdview.cpython-38.pyc differ diff --git a/surround_view/__pycache__/capture_thread.cpython-38.pyc b/surround_view/__pycache__/capture_thread.cpython-38.pyc index 27f480c..9560eb8 100644 Binary files a/surround_view/__pycache__/capture_thread.cpython-38.pyc and b/surround_view/__pycache__/capture_thread.cpython-38.pyc differ diff --git a/surround_view/__pycache__/fisheye_camera.cpython-38.pyc b/surround_view/__pycache__/fisheye_camera.cpython-38.pyc index e7ce846..531a893 100644 Binary files a/surround_view/__pycache__/fisheye_camera.cpython-38.pyc and b/surround_view/__pycache__/fisheye_camera.cpython-38.pyc differ diff --git a/surround_view/__pycache__/imagebuffer.cpython-38.pyc b/surround_view/__pycache__/imagebuffer.cpython-38.pyc index a4d8ac8..dcc9287 100644 Binary files a/surround_view/__pycache__/imagebuffer.cpython-38.pyc and b/surround_view/__pycache__/imagebuffer.cpython-38.pyc differ diff --git a/surround_view/__pycache__/param_settings.cpython-38.pyc b/surround_view/__pycache__/param_settings.cpython-38.pyc index a1e41c5..dd5a42a 100644 Binary files a/surround_view/__pycache__/param_settings.cpython-38.pyc and b/surround_view/__pycache__/param_settings.cpython-38.pyc differ diff --git a/surround_view/__pycache__/process_thread.cpython-38.pyc b/surround_view/__pycache__/process_thread.cpython-38.pyc index 4b7c32a..8492872 100644 Binary files a/surround_view/__pycache__/process_thread.cpython-38.pyc and b/surround_view/__pycache__/process_thread.cpython-38.pyc differ diff --git a/surround_view/__pycache__/simple_gui.cpython-38.pyc b/surround_view/__pycache__/simple_gui.cpython-38.pyc index 25e4148..df8b600 100644 Binary files a/surround_view/__pycache__/simple_gui.cpython-38.pyc and b/surround_view/__pycache__/simple_gui.cpython-38.pyc differ diff --git a/surround_view/__pycache__/structures.cpython-38.pyc b/surround_view/__pycache__/structures.cpython-38.pyc index ec34fb9..c50cee3 100644 Binary files a/surround_view/__pycache__/structures.cpython-38.pyc and b/surround_view/__pycache__/structures.cpython-38.pyc differ diff --git a/surround_view/__pycache__/utils.cpython-38.pyc b/surround_view/__pycache__/utils.cpython-38.pyc index 0b2264d..b0dcb0b 100644 Binary files a/surround_view/__pycache__/utils.cpython-38.pyc and b/surround_view/__pycache__/utils.cpython-38.pyc differ diff --git a/surround_view/birdview.py b/surround_view/birdview.py index 606fbd7..579d636 100644 --- a/surround_view/birdview.py +++ b/surround_view/birdview.py @@ -104,6 +104,7 @@ def LIII(left_image): def LM(left_image): + return left_image[yt:yb, :] @@ -116,6 +117,7 @@ def RIV(right_image): def RM(right_image): + return right_image[yt:yb, :] @@ -194,7 +196,10 @@ class BirdView(BaseThread): return self.image[yt:yb, xl:xr] def stitch_all_parts(self): + front, back, left, right = self.frames + print(front.shape, back.shape, left.shape, right.shape) + np.copyto(self.F, FM(front)) np.copyto(self.B, BM(back)) np.copyto(self.L, LM(left)) @@ -326,7 +331,8 @@ class BirdView(BaseThread): self.processing_mutex.lock() self.update_frames(self.proc_buffer_manager.get().values()) - self.make_luminance_balance().stitch_all_parts() + self.stitch_all_parts() # 直接拼接原始投影图像 + # self.make_luminance_balance().stitch_all_parts() self.make_white_balance() self.copy_car_image() self.buffer.add(self.image.copy(), self.drop_if_full) diff --git a/surround_view/capture_thread.py b/surround_view/capture_thread.py index cc72eda..6513263 100644 --- a/surround_view/capture_thread.py +++ b/surround_view/capture_thread.py @@ -1,9 +1,7 @@ import cv2 from PyQt5.QtCore import qDebug - from .base_thread import BaseThread from .structures import ImageFrame -from .utils import gstreamer_pipeline @@ -11,11 +9,10 @@ class CaptureThread(BaseThread): def __init__(self, device_id, - # flip_method=2, + flip_method=0, drop_if_full=True, api_preference=cv2.CAP_ANY, resolution=None, - # use_gst=None, parent=None): """ device_id: device number of the camera. @@ -27,8 +24,7 @@ class CaptureThread(BaseThread): """ super(CaptureThread, self).__init__(parent) self.device_id = device_id - # self.flip_method = flip_method - # self.use_gst = None + self.flip_method = flip_method self.drop_if_full = drop_if_full self.api_preference = api_preference self.resolution = resolution @@ -61,15 +57,25 @@ class CaptureThread(BaseThread): continue # retrieve frame and add it to buffer - _, frame = self.cap.retrieve() + _, frame = self.cap.read() + + # Skip empty frames (e.g., when camera capture times out) + if frame is None or frame.size == 0: + continue + + # Apply image flip if needed + if self.flip_method == 2: + frame = cv2.rotate(frame, cv2.ROTATE_180) + + # frame = cv2.resize(frame, (960, 640)) img_frame = ImageFrame(self.clock.msecsSinceStartOfDay(), frame) self.buffer_manager.get_device(self.device_id).add(img_frame, self.drop_if_full) # update statistics - self.update_fps(self.processing_time) - self.stat_data.frames_processed_count += 1 - # inform GUI of updated statistics - self.update_statistics_gui.emit(self.stat_data) + # self.update_fps(self.processing_time) + # self.stat_data.frames_processed_count += 1 + # # inform GUI of updated statistics + # self.update_statistics_gui.emit(self.stat_data) qDebug("Stopping capture thread...") @@ -83,7 +89,6 @@ class CaptureThread(BaseThread): # try to set camera resolution if self.resolution is not None: width, height = self.resolution - self.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"YUYV")) self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, width) self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height) diff --git a/surround_view/fisheye_camera.py b/surround_view/fisheye_camera.py index b8cb0f2..e5f9d26 100644 --- a/surround_view/fisheye_camera.py +++ b/surround_view/fisheye_camera.py @@ -92,7 +92,7 @@ class FisheyeCameraModel(object): return cv2.transpose(image)[::-1] else: - return np.flip(cv2.transpose(image), 1) + return cv2.transpose(image)[::-1] def save_data(self): fs = cv2.FileStorage(self.camera_file, cv2.FILE_STORAGE_WRITE) diff --git a/surround_view/process_thread.py b/surround_view/process_thread.py index 17a0295..c5dcc3c 100644 --- a/surround_view/process_thread.py +++ b/surround_view/process_thread.py @@ -2,6 +2,7 @@ import cv2 from PyQt5.QtCore import qDebug, QMutex from .base_thread import BaseThread +from . import param_settings as settings class CameraProcessingThread(BaseThread): @@ -47,10 +48,28 @@ class CameraProcessingThread(BaseThread): self.processing_mutex.lock() raw_frame = self.capture_buffer_manager.get_device(self.device_id).get() + + # Validate frame before processing + if raw_frame.image is None or raw_frame.image.size == 0: + self.processing_mutex.unlock() + continue + und_frame = self.camera_model.undistort(raw_frame.image) pro_frame = self.camera_model.project(und_frame) flip_frame = self.camera_model.flip(pro_frame) self.processing_mutex.unlock() + + # Check if the processed frame has valid dimensions + name = self.camera_model.camera_name + if name in settings.project_shapes: + # For left and right cameras, the flip operation changes the shape + if name in ['left', 'right']: + expected_shape = settings.project_shapes[name] + (3,) + else: + expected_shape = settings.project_shapes[name][::-1] + (3,) + if flip_frame.shape != expected_shape: + print(f"Warning: {name} camera frame has unexpected shape {flip_frame.shape}, expected {expected_shape}") + continue self.proc_buffer_manager.sync(self.device_id) self.proc_buffer_manager.set_frame_for_device(self.device_id, flip_frame) diff --git a/yaml/left.yaml b/yaml/left.yaml index 0d01196..30ed65c 100644 --- a/yaml/left.yaml +++ b/yaml/left.yaml @@ -16,7 +16,7 @@ resolution: !!opencv-matrix rows: 2 cols: 1 dt: i - data: [ 960, 640 ] + data: [ 1920, 1080 ] project_matrix: !!opencv-matrix rows: 3 cols: 3 diff --git a/yaml/right.yaml b/yaml/right.yaml index 3112d03..1c9404e 100644 --- a/yaml/right.yaml +++ b/yaml/right.yaml @@ -1,19 +1,37 @@ %YAML:1.0 --- +camera_matrix: !!opencv-matrix + rows: 3 + cols: 3 + dt: d + data: [ 3.0245305983229298e+02, 0., 4.9664001463163459e+02, 0., + 3.2074618594392325e+02, 3.3119980984361649e+02, 0., 0., 1. ] +dist_coeffs: !!opencv-matrix + rows: 4 + cols: 1 + dt: d + data: [ -4.3735601598704078e-02, 2.1692522970939803e-02, + -2.6388839028513571e-02, 8.4123126605702321e-03 ] resolution: !!opencv-matrix rows: 2 cols: 1 dt: i data: [ 1920, 1080 ] -camera_matrix: !!opencv-matrix +project_matrix: !!opencv-matrix rows: 3 cols: 3 dt: d - data: [ 5.3426649626056496e+02, 0., 9.4035331422643389e+02, 0., - 5.3438087838176853e+02, 5.6780561206232505e+02, 0., 0., 1. ] -dist_coeffs: !!opencv-matrix - rows: 4 + data: [ -7.0390891066994388e-01, -2.5544083216952904e+00, + 7.0809808916259806e+02, -2.9600383808093766e-01, + -2.4971504395791286e+00, 6.3578234365104447e+02, + -5.6872782515522376e-04, -4.4482832729892769e-03, 1. ] +scale_xy: !!opencv-matrix + rows: 2 cols: 1 - dt: d - data: [ -1.3928766380167932e-02, 5.8925693280706653e-04, - -1.1389190533024249e-03, -9.6700186038395686e-05 ] + dt: f + data: [ 6.99999988e-01, 8.00000012e-01 ] +shift_xy: !!opencv-matrix + rows: 2 + cols: 1 + dt: f + data: [ -150., -100. ]