From 1cac67fa470d02043f158fa0d602fc2b981bc444 Mon Sep 17 00:00:00 2001 From: Hendrik Date: Thu, 18 Jul 2024 13:14:35 +0200 Subject: [PATCH] Draw segments in line detection overlay (#1319) --- crates/types/src/image_segments.rs | 34 ++++++++++++++++++- crates/vision/src/line_detection.rs | 20 +++++------ crates/vision/src/line_detection/checks.rs | 17 +++++----- .../vision/src/line_detection/map_segments.rs | 8 ++--- crates/vision/src/line_detection/segment.rs | 20 ----------- .../src/line_detection/segment_merger.rs | 12 +++---- .../panels/image/overlays/line_detection.rs | 18 +++++----- 7 files changed, 72 insertions(+), 57 deletions(-) delete mode 100644 crates/vision/src/line_detection/segment.rs diff --git a/crates/types/src/image_segments.rs b/crates/types/src/image_segments.rs index 0675a52479..63304a054d 100644 --- a/crates/types/src/image_segments.rs +++ b/crates/types/src/image_segments.rs @@ -1,3 +1,5 @@ +use coordinate_systems::Pixel; +use linear_algebra::{point, Point2}; use path_serde::{PathDeserialize, PathIntrospect, PathSerialize}; use serde::{Deserialize, Serialize}; @@ -44,7 +46,18 @@ impl Segment { } } -#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[derive( + Clone, + Copy, + Debug, + Deserialize, + Eq, + PartialEq, + Serialize, + PathSerialize, + PathDeserialize, + PathIntrospect, +)] pub enum EdgeType { Rising, Falling, @@ -57,3 +70,22 @@ pub enum Direction { Horizontal, Vertical, } + +#[derive( + Clone, Copy, Debug, Deserialize, Serialize, PathSerialize, PathDeserialize, PathIntrospect, +)] +pub struct GenericSegment { + pub start: Point2, + pub end: Point2, + pub start_edge_type: EdgeType, + pub end_edge_type: EdgeType, +} + +impl GenericSegment { + pub fn center(&self) -> Point2 { + point![ + self.start.x() + (self.end.x() - self.start.x()) / 2, + self.start.y() + (self.end.y() - self.start.y()) / 2, + ] + } +} diff --git a/crates/vision/src/line_detection.rs b/crates/vision/src/line_detection.rs index 0238f8d4cd..6e5e985c97 100644 --- a/crates/vision/src/line_detection.rs +++ b/crates/vision/src/line_detection.rs @@ -1,7 +1,6 @@ mod checks; mod iter_if; mod map_segments; -mod segment; mod segment_merger; use std::{collections::HashSet, ops::Range}; @@ -24,6 +23,7 @@ use projection::{camera_matrix::CameraMatrix, Projection}; use ransac::{Ransac, RansacResult}; use types::{ filtered_segments::FilteredSegments, + image_segments::GenericSegment, line_data::{LineData, LineDiscardReason}, ycbcr422_image::YCbCr422Image, }; @@ -41,7 +41,8 @@ pub struct CycleContext { lines_in_image: AdditionalOutput>, "lines_in_image">, discarded_lines: AdditionalOutput, LineDiscardReason)>, "discarded_lines">, - ransac_input: AdditionalOutput>, "ransac_input">, + filtered_segments_output: + AdditionalOutput, "line_detection.filtered_segments">, use_horizontal_segments: Parameter, @@ -138,10 +139,16 @@ impl LineDetection { *context.gradient_alignment, *context.gradient_sobel_stride, ) - }); + }) + .collect::>(); + + context + .filtered_segments_output + .fill_if_subscribed(|| filtered_segments.clone()); let (line_points, used_segments): (Vec>, HashSet>) = filtered_segments + .into_iter() .filter_map(|segment| { Some(( context @@ -153,13 +160,6 @@ impl LineDetection { }) .unzip(); - context.ransac_input.fill_if_subscribed(|| { - line_points - .iter() - .map(|point| context.camera_matrix.ground_to_pixel(*point).unwrap()) - .collect() - }); - let mut ransac = Ransac::new(line_points); let mut lines_in_ground = Vec::new(); for _ in 0..*context.maximum_number_of_lines { diff --git a/crates/vision/src/line_detection/checks.rs b/crates/vision/src/line_detection/checks.rs index f87aa37169..bd5e9cbc1c 100644 --- a/crates/vision/src/line_detection/checks.rs +++ b/crates/vision/src/line_detection/checks.rs @@ -3,16 +3,17 @@ use std::ops::Range; use coordinate_systems::Pixel; use linear_algebra::{distance, vector, Point2, Vector2}; use projection::{camera_matrix::CameraMatrix, Projection}; -use types::{image_segments::EdgeType, ycbcr422_image::YCbCr422Image}; +use types::{ + image_segments::{EdgeType, GenericSegment}, + ycbcr422_image::YCbCr422Image, +}; -use super::segment::Segment; - -pub fn is_non_field_segment(segment: &Segment) -> bool { +pub fn is_non_field_segment(segment: &GenericSegment) -> bool { segment.start_edge_type == EdgeType::Rising && segment.end_edge_type == EdgeType::Falling } pub fn is_in_length_range( - segment: &Segment, + segment: &GenericSegment, camera_matrix: &CameraMatrix, allowed_projected_segment_length: &Range, ) -> bool { @@ -26,7 +27,7 @@ pub fn is_in_length_range( } pub fn has_opposite_gradients( - segment: &Segment, + segment: &GenericSegment, image: &YCbCr422Image, gradient_alignment: f32, gradient_sobel_stride: u32, @@ -129,7 +130,7 @@ mod tests { Isometry3::identity().framed_transform(), ); - let segment = Segment { + let segment = GenericSegment { start: point![40, 2], end: point![40, 202], start_edge_type: EdgeType::ImageBorder, @@ -137,7 +138,7 @@ mod tests { }; assert!(!is_in_length_range(&segment, &camera_matrix, &(0.0..0.3))); - let segment = Segment { + let segment = GenericSegment { start: point![40, 364], end: point![40, 366], start_edge_type: EdgeType::ImageBorder, diff --git a/crates/vision/src/line_detection/map_segments.rs b/crates/vision/src/line_detection/map_segments.rs index c23a88b7c4..5b942f0d88 100644 --- a/crates/vision/src/line_detection/map_segments.rs +++ b/crates/vision/src/line_detection/map_segments.rs @@ -2,17 +2,17 @@ use std::slice::Iter; use coordinate_systems::Pixel; use linear_algebra::{point, Point2}; -use types::image_segments::ScanLine; +use types::image_segments::{GenericSegment, ScanLine}; -use super::{segment::Segment, segment_merger::SegmentMerger}; +use super::segment_merger::SegmentMerger; pub fn map_segments( scan_lines: Iter<'_, ScanLine>, maximum_merge_gap: u16, -) -> impl Iterator + '_ { +) -> impl Iterator + '_ { scan_lines.flat_map(move |scan_line| { SegmentMerger::new( - scan_line.segments.iter().map(|segment| Segment { + scan_line.segments.iter().map(|segment| GenericSegment { start: Mapping::map(scan_line.position, segment.start), end: Mapping::map(scan_line.position, segment.end), start_edge_type: segment.start_edge_type, diff --git a/crates/vision/src/line_detection/segment.rs b/crates/vision/src/line_detection/segment.rs deleted file mode 100644 index 9dfb899298..0000000000 --- a/crates/vision/src/line_detection/segment.rs +++ /dev/null @@ -1,20 +0,0 @@ -use coordinate_systems::Pixel; -use linear_algebra::{point, Point2}; -use types::image_segments::EdgeType; - -#[derive(Clone, Copy)] -pub struct Segment { - pub start: Point2, - pub end: Point2, - pub start_edge_type: EdgeType, - pub end_edge_type: EdgeType, -} - -impl Segment { - pub fn center(&self) -> Point2 { - point![ - self.start.x() + (self.end.x() - self.start.x()) / 2, - self.start.y() + (self.end.y() - self.start.y()) / 2, - ] - } -} diff --git a/crates/vision/src/line_detection/segment_merger.rs b/crates/vision/src/line_detection/segment_merger.rs index f4952389dd..ad5c34b945 100644 --- a/crates/vision/src/line_detection/segment_merger.rs +++ b/crates/vision/src/line_detection/segment_merger.rs @@ -1,15 +1,15 @@ use std::iter::Peekable; -use super::segment::Segment; +use types::image_segments::GenericSegment; -pub struct SegmentMerger> { +pub struct SegmentMerger> { iterator: Peekable, maximum_merge_gap: u16, } impl SegmentMerger where - T: Iterator, + T: Iterator, { pub fn new(iterator: T, maximum_merge_gap: u16) -> Self { Self { @@ -21,9 +21,9 @@ where impl Iterator for SegmentMerger where - T: Iterator, + T: Iterator, { - type Item = Segment; + type Item = GenericSegment; fn next(&mut self) -> Option { let mut current = self.iterator.next()?; @@ -42,6 +42,6 @@ where } } -fn distance_between_segments(first: Segment, second: Segment) -> u16 { +fn distance_between_segments(first: GenericSegment, second: GenericSegment) -> u16 { (second.start.x() - first.end.x()) + (second.start.y() - first.end.y()) } diff --git a/tools/twix/src/panels/image/overlays/line_detection.rs b/tools/twix/src/panels/image/overlays/line_detection.rs index 695634bfcb..b9780af921 100644 --- a/tools/twix/src/panels/image/overlays/line_detection.rs +++ b/tools/twix/src/panels/image/overlays/line_detection.rs @@ -3,8 +3,7 @@ use eframe::epaint::{Color32, Stroke}; use coordinate_systems::Pixel; use geometry::line_segment::LineSegment; -use linear_algebra::Point2; -use types::line_data::LineDiscardReason; +use types::{image_segments::GenericSegment, line_data::LineDiscardReason}; use crate::{ panels::image::{cycler_selector::VisionCycler, overlay::Overlay}, @@ -17,7 +16,7 @@ type DiscardedLines = Vec<(LineSegment, LineDiscardReason)>; pub struct LineDetection { lines_in_image: BufferHandle>>>, discarded_lines: BufferHandle>, - ransac_input: BufferHandle>>>, + filtered_segments: BufferHandle>>, } impl Overlay for LineDetection { @@ -30,8 +29,9 @@ impl Overlay for LineDetection { .subscribe_value(format!("{cycler_path}.additional_outputs.lines_in_image")), discarded_lines: nao .subscribe_value(format!("{cycler_path}.additional_outputs.discarded_lines")), - ransac_input: nao - .subscribe_value(format!("{cycler_path}.additional_outputs.ransac_input")), + filtered_segments: nao.subscribe_value(format!( + "{cycler_path}.additional_outputs.line_detection.filtered_segments" + )), } } @@ -42,11 +42,13 @@ impl Overlay for LineDetection { let Some(discarded_lines) = self.discarded_lines.get_last_value()?.flatten() else { return Ok(()); }; - let Some(ransac_input) = self.ransac_input.get_last_value()?.flatten() else { + let Some(filtered_segments) = self.filtered_segments.get_last_value()?.flatten() else { return Ok(()); }; - for point in ransac_input { - painter.circle_stroke(point, 3.0, Stroke::new(1.0, Color32::RED)) + for segment in filtered_segments { + let stroke = Stroke::new(1.0, Color32::RED); + painter.line_segment(segment.start.cast(), segment.end.cast(), stroke); + painter.circle_stroke(segment.center().cast(), 3.0, stroke); } for (line, reason) in discarded_lines { let color = match reason {