From 551c73068f43da1dece0feae8d8a795f0f3726b7 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Fri, 19 Jun 2015 10:02:23 +0200 Subject: pt: remove hysteresis No one used it, and we have a better solution for stable extraction, pending commit. --- ftnoir_tracker_pt/FTNoIR_PT_Controls.ui | 96 ++++++++------------------ ftnoir_tracker_pt/ftnoir_tracker_pt_dialog.cpp | 1 - ftnoir_tracker_pt/ftnoir_tracker_pt_settings.h | 2 - ftnoir_tracker_pt/point_extractor.cpp | 40 +---------- ftnoir_tracker_pt/point_extractor.h | 1 - 5 files changed, 31 insertions(+), 109 deletions(-) (limited to 'ftnoir_tracker_pt') diff --git a/ftnoir_tracker_pt/FTNoIR_PT_Controls.ui b/ftnoir_tracker_pt/FTNoIR_PT_Controls.ui index 0e6048c3..39fb0686 100644 --- a/ftnoir_tracker_pt/FTNoIR_PT_Controls.ui +++ b/ftnoir_tracker_pt/FTNoIR_PT_Controls.ui @@ -259,19 +259,6 @@ Point extraction - - - - Minimum point diameter - - - px - - - 1024 - - - @@ -297,81 +284,59 @@ - - - - Maximum point diameter - - - px + + + + Min size - - 1024 + + mindiam_spin - - + + - Max size + Threshold - maxdiam_spin + threshold_slider - - + + - Per pixel hysteresis width (leave left if there is little difference between dot and non-dot, move right for increased stability against pixel noise) - - - 255 - - - 1 - - - 100 - - - Qt::Horizontal + Minimum point diameter - - QSlider::TicksBothSides + + px - - 25 + + 1024 - - + + - Threshold + Max size - threshold_slider + maxdiam_spin - - - - Hysteresis - - - threshold_secondary_slider + + + + Maximum point diameter - - - - - - Min size + + px - - mindiam_spin + + 1024 @@ -1150,9 +1115,6 @@ fps_spin fov threshold_slider - mindiam_spin - threshold_secondary_slider - maxdiam_spin model_tabs clip_tlength_spin clip_theight_spin diff --git a/ftnoir_tracker_pt/ftnoir_tracker_pt_dialog.cpp b/ftnoir_tracker_pt/ftnoir_tracker_pt_dialog.cpp index 86178cb3..e7f1384d 100644 --- a/ftnoir_tracker_pt/ftnoir_tracker_pt_dialog.cpp +++ b/ftnoir_tracker_pt/ftnoir_tracker_pt_dialog.cpp @@ -44,7 +44,6 @@ TrackerDialog_PT::TrackerDialog_PT() tie_setting(s.cam_res_y, ui.res_y_spin); tie_setting(s.cam_fps, ui.fps_spin); - tie_setting(s.threshold_secondary, ui.threshold_secondary_slider); tie_setting(s.threshold, ui.threshold_slider); tie_setting(s.min_point_size, ui.mindiam_spin); diff --git a/ftnoir_tracker_pt/ftnoir_tracker_pt_settings.h b/ftnoir_tracker_pt/ftnoir_tracker_pt_settings.h index 14786013..4e241bc3 100644 --- a/ftnoir_tracker_pt/ftnoir_tracker_pt_settings.h +++ b/ftnoir_tracker_pt/ftnoir_tracker_pt_settings.h @@ -18,7 +18,6 @@ struct settings_pt : opts cam_res_y, cam_fps, threshold, - threshold_secondary, min_point_size, max_point_size; @@ -41,7 +40,6 @@ struct settings_pt : opts cam_res_y(b, "camera-res-height", 480), cam_fps(b, "camera-fps", 30), threshold(b, "threshold-primary", 128), - threshold_secondary(b, "threshold-secondary", 128), min_point_size(b, "min-point-size", 10), max_point_size(b, "max-point-size", 50), m01_x(b, "m_01-x", 0), diff --git a/ftnoir_tracker_pt/point_extractor.cpp b/ftnoir_tracker_pt/point_extractor.cpp index 5f7f6829..dab1d967 100644 --- a/ftnoir_tracker_pt/point_extractor.cpp +++ b/ftnoir_tracker_pt/point_extractor.cpp @@ -25,51 +25,15 @@ std::vector PointExtractor::extract_points(Mat& frame) const int W = frame.cols; const int H = frame.rows; - if (frame_last.cols != W || frame_last.rows != H) - { - frame_last = cv::Mat(); - } - // convert to grayscale Mat frame_gray; cvtColor(frame, frame_gray, cv::COLOR_RGB2GRAY); - int secondary = s.threshold_secondary; - int primary = s.threshold; - // mask for everything that passes the threshold (or: the upper threshold of the hysteresis) Mat frame_bin; - // only used if draw_output - Mat frame_bin_copy; - // mask for everything that passes - Mat frame_bin_low; - // mask for lower-threshold && combined result of last, needs to remain in scope until drawing, but is only used if secondary != 0 - Mat frame_last_and_low; - - if(secondary==0){ - threshold(frame_gray, frame_bin, primary, 255, THRESH_BINARY); - }else{ - // we recombine a number of buffers, this might be slower than a single loop of per-pixel logic - // but it might as well be faster if openCV makes good use of SIMD - float t = primary; - //float hyst = float(threshold_secondary_val)/512.; - //threshold(frame_gray, frame_bin, (t + ((255.-t)*hyst)), 255, THRESH_BINARY); - float hyst = float(primary)/(256.*8.); - threshold(frame_gray, frame_bin, t, 255, THRESH_BINARY); - threshold(frame_gray, frame_bin_low,std::max(float(1), t - (t*hyst)), 255, THRESH_BINARY); - frame_bin.copyTo(frame_bin_copy); - if(frame_last.empty()){ - frame_bin.copyTo(frame_last); - }else{ - // keep pixels from last if they are above lower threshold - bitwise_and(frame_last, frame_bin_low, frame_last_and_low); - // union of pixels >= higher threshold and pixels >= lower threshold - bitwise_or(frame_bin, frame_last_and_low, frame_last); - frame_last.copyTo(frame_bin); - } - } - + threshold(frame_gray, frame_bin, s.threshold, 255, THRESH_BINARY); + int min_size = s.min_point_size; int max_size = s.max_point_size; diff --git a/ftnoir_tracker_pt/point_extractor.h b/ftnoir_tracker_pt/point_extractor.h index 3f6cfb72..b9368ab6 100644 --- a/ftnoir_tracker_pt/point_extractor.h +++ b/ftnoir_tracker_pt/point_extractor.h @@ -28,7 +28,6 @@ public: settings_pt s; private: std::vector points; - cv::Mat frame_last; }; #endif //POINTEXTRACTOR_H -- cgit v1.2.3 From e033465ceb37c727ede335e5832a4b884cf72376 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Fri, 19 Jun 2015 12:38:53 +0200 Subject: pt: extractor algorithm based on OpenCV's SimpleBlobDetector Tunables are probably wrong, we'll get to that later. --- ftnoir_tracker_pt/point_extractor.cpp | 166 +++++++++++++++++++++++++++++----- 1 file changed, 144 insertions(+), 22 deletions(-) (limited to 'ftnoir_tracker_pt') diff --git a/ftnoir_tracker_pt/point_extractor.cpp b/ftnoir_tracker_pt/point_extractor.cpp index dab1d967..86454729 100644 --- a/ftnoir_tracker_pt/point_extractor.cpp +++ b/ftnoir_tracker_pt/point_extractor.cpp @@ -28,11 +28,6 @@ std::vector PointExtractor::extract_points(Mat& frame) // convert to grayscale Mat frame_gray; cvtColor(frame, frame_gray, cv::COLOR_RGB2GRAY); - - // mask for everything that passes the threshold (or: the upper threshold of the hysteresis) - Mat frame_bin; - - threshold(frame_gray, frame_bin, s.threshold, 255, THRESH_BINARY); int min_size = s.min_point_size; int max_size = s.max_point_size; @@ -40,30 +35,157 @@ std::vector PointExtractor::extract_points(Mat& frame) unsigned int region_size_min = 3.14*min_size*min_size/4.0; unsigned int region_size_max = 3.14*max_size*max_size/4.0; - std::vector> contours; - cv::findContours(frame_bin, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE); + // testing indicates threshold difference of 45 from lowest to highest + // that's applicable to poor lighting conditions. + + static constexpr int diff = 20; + static constexpr int steps = 10; + static constexpr int successes = 8; + + int thres = s.threshold; + + struct blob { + double max_radius; + std::vector pos; + std::vector confids; + + cv::Vec2d effective_pos() const + { + double x = 0; + double y = 0; + double norm = 0; + for (unsigned i = 0; i < pos.size(); i++) + { + x += pos[i][0] * confids[i]; + y += pos[i][1] * confids[i]; + norm += confids[i]; + } + cv::Vec2d ret(x, y); + ret *= 1./norm; + //qDebug() << "ret" << ret[0] << ret[1] << "norm" << norm << "count" << pos.size(); + return ret; + } + }; + + struct simple_blob + { + double radius; + cv::Vec2d pos; + double confid; + bool taken; + simple_blob(double radius, const cv::Vec2d& pos, double confid) : radius(radius), pos(pos), confid(confid), taken(false) + { + //qDebug() << "radius" << radius << "pos" << pos[0] << pos[1] << "confid" << confid; + } + bool inside(const simple_blob& other) + { + cv::Vec2d tmp = pos - other.pos; + double p = sqrt(1e-4 + tmp.dot(tmp)); + return p < radius; + } + static std::vector merge(std::vector& blobs) + { + std::vector ret; + for (unsigned i = 0; i < blobs.size(); i++) + { + auto& b = blobs[i]; + if (b.taken) + continue; + b.taken = true; + blob b_; + b_.pos.push_back(b.pos); + b_.confids.push_back(b.confid); + b_.max_radius = b.radius; + + for (unsigned j = i+1; j < blobs.size(); j++) + { + auto& b2 = blobs[j]; + if (b2.taken) + continue; + if (b.inside(b2) || b2.inside(b)) + { + b2.taken = true; + b_.pos.push_back(b2.pos); + b_.confids.push_back(b2.confid); + b_.max_radius = std::max(b_.max_radius, b2.radius); + } + } + if (b_.pos.size() >= successes) + ret.push_back(b_); + } + return ret; + } + }; + + // mask for everything that passes the threshold (or: the upper threshold of the hysteresis) + Mat frame_bin = cv::Mat::zeros(H, W, CV_8U); + + const int min = std::max(0, thres - diff/2); + const int max = std::min(255, thres + diff/2); + const int step = std::max(1, diff / steps); + + std::vector blobs; + + // this code is based on OpenCV SimpleBlobDetector + for (int i = min; i < max; i += step) + { + Mat frame_bin_; + threshold(frame_gray, frame_bin_, i, 255, THRESH_BINARY); + frame_bin.setTo(170, frame_bin_); + + std::vector> contours; + cv::findContours(frame_bin_, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE); + + for (auto& c : contours) + { + auto m = cv::moments(cv::Mat(c)); + const double area = m.m00; + if (area == 0.) + continue; + cv::Vec2d pos(m.m10 / m.m00, m.m01 / m.m00); + if (area < region_size_min || area > region_size_max) + continue; + + double radius = 0; + + for (auto& k : c) + { + cv::Vec2d pos_(k.x, k.y); + cv::Vec2d tmp = pos_ - pos; + radius = std::max(radius, sqrt(1e-2 + tmp.dot(tmp))); + } + double confid = 1; + { + double denominator = std::sqrt(std::pow(2 * m.mu11, 2) + std::pow(m.mu20 - m.mu02, 2)); + const double eps = 1e-2; + if (denominator > eps) + { + double cosmin = (m.mu20 - m.mu02) / denominator; + double sinmin = 2 * m.mu11 / denominator; + double cosmax = -cosmin; + double sinmax = -sinmin; + + double imin = 0.5 * (m.mu20 + m.mu02) - 0.5 * (m.mu20 - m.mu02) * cosmin - m.mu11 * sinmin; + double imax = 0.5 * (m.mu20 + m.mu02) - 0.5 * (m.mu20 - m.mu02) * cosmax - m.mu11 * sinmax; + confid = imin / imax; + } + } + blobs.push_back(simple_blob(radius, pos, confid)); + } + } // clear old points points.clear(); - for (auto& c : contours) + for (auto& b : simple_blob::merge(blobs)) { - auto m = cv::moments(cv::Mat(c)); - const double area = m.m00; - if (area == 0.) - continue; - cv::Vec2f pos(m.m10 / m.m00, m.m01 / m.m00); - if (area < region_size_min || area > region_size_max) - continue; - pos[0] = (pos[0] - W/2)/W; - pos[1] = -(pos[1] - H/2)/W; - - points.push_back(pos); + auto pos = b.effective_pos(); + Vec2f p((pos[0] - W/2)/W, -(pos[1] - H/2)/W); + points.push_back(p); } - - // draw output image + + // draw output image vector channels; - frame_bin.setTo(170, frame_bin); channels.push_back(frame_gray + frame_bin); channels.push_back(frame_gray - frame_bin); channels.push_back(frame_gray - frame_bin); -- cgit v1.2.3 From 66bcf1db948ca409b7474a4efc9d447f61353060 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Wed, 1 Jul 2015 13:16:48 +0200 Subject: pt: use iterative contour detector --- ftnoir_tracker_pt/point_extractor.cpp | 54 ++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 11 deletions(-) (limited to 'ftnoir_tracker_pt') diff --git a/ftnoir_tracker_pt/point_extractor.cpp b/ftnoir_tracker_pt/point_extractor.cpp index 86454729..71023a38 100644 --- a/ftnoir_tracker_pt/point_extractor.cpp +++ b/ftnoir_tracker_pt/point_extractor.cpp @@ -8,6 +8,9 @@ #include "point_extractor.h" #include +#ifdef DEBUG_EXTRACTION +# include "opentrack/timer.hpp" +#endif using namespace cv; using namespace std; @@ -39,15 +42,15 @@ std::vector PointExtractor::extract_points(Mat& frame) // that's applicable to poor lighting conditions. static constexpr int diff = 20; - static constexpr int steps = 10; - static constexpr int successes = 8; + static constexpr int steps = 5; + static constexpr int successes = 5; int thres = s.threshold; struct blob { - double max_radius; std::vector pos; std::vector confids; + std::vector areas; cv::Vec2d effective_pos() const { @@ -56,13 +59,13 @@ std::vector PointExtractor::extract_points(Mat& frame) double norm = 0; for (unsigned i = 0; i < pos.size(); i++) { - x += pos[i][0] * confids[i]; - y += pos[i][1] * confids[i]; - norm += confids[i]; + const double w = confids[i] * areas[i]; + x += pos[i][0] * w; + y += pos[i][1] * w; + norm += w; } cv::Vec2d ret(x, y); ret *= 1./norm; - //qDebug() << "ret" << ret[0] << ret[1] << "norm" << norm << "count" << pos.size(); return ret; } }; @@ -73,7 +76,8 @@ std::vector PointExtractor::extract_points(Mat& frame) cv::Vec2d pos; double confid; bool taken; - simple_blob(double radius, const cv::Vec2d& pos, double confid) : radius(radius), pos(pos), confid(confid), taken(false) + double area; + simple_blob(double radius, const cv::Vec2d& pos, double confid, double area) : radius(radius), pos(pos), confid(confid), taken(false), area(area) { //qDebug() << "radius" << radius << "pos" << pos[0] << pos[1] << "confid" << confid; } @@ -85,6 +89,12 @@ std::vector PointExtractor::extract_points(Mat& frame) } static std::vector merge(std::vector& blobs) { +#ifdef DEBUG_EXTRACTION + static Timer t; + bool debug = t.elapsed_ms() > 100; + if (debug) t.start(); +#endif + std::vector ret; for (unsigned i = 0; i < blobs.size(); i++) { @@ -95,7 +105,7 @@ std::vector PointExtractor::extract_points(Mat& frame) blob b_; b_.pos.push_back(b.pos); b_.confids.push_back(b.confid); - b_.max_radius = b.radius; + b_.areas.push_back(b.area); for (unsigned j = i+1; j < blobs.size(); j++) { @@ -107,12 +117,29 @@ std::vector PointExtractor::extract_points(Mat& frame) b2.taken = true; b_.pos.push_back(b2.pos); b_.confids.push_back(b2.confid); - b_.max_radius = std::max(b_.max_radius, b2.radius); + b_.areas.push_back(b2.area); } } if (b_.pos.size() >= successes) ret.push_back(b_); } +#ifdef DEBUG_EXTRACTION + if (debug) + { + double diff = 0; + for (unsigned j = 0; j < ret.size(); j++) + { + auto& b = ret[j]; + cv::Vec2d pos = b.effective_pos(); + for (unsigned i = 0; i < b.pos.size(); i++) + { + auto tmp = pos - b.pos[i]; + diff += std::abs(tmp.dot(tmp)); + } + } + qDebug() << "diff" << diff; + } +#endif return ret; } }; @@ -136,8 +163,13 @@ std::vector PointExtractor::extract_points(Mat& frame) std::vector> contours; cv::findContours(frame_bin_, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE); + int cnt = 0; + for (auto& c : contours) { + if (cnt++ > 30) + break; + auto m = cv::moments(cv::Mat(c)); const double area = m.m00; if (area == 0.) @@ -170,7 +202,7 @@ std::vector PointExtractor::extract_points(Mat& frame) confid = imin / imax; } } - blobs.push_back(simple_blob(radius, pos, confid)); + blobs.push_back(simple_blob(radius, pos, confid, area)); } } -- cgit v1.2.3 From 8c5a34f9002f45bdda2f3e7240faf63766bc729a Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Wed, 1 Jul 2015 13:35:10 +0200 Subject: pt: fix sqrt eps --- ftnoir_tracker_pt/point_extractor.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'ftnoir_tracker_pt') diff --git a/ftnoir_tracker_pt/point_extractor.cpp b/ftnoir_tracker_pt/point_extractor.cpp index 71023a38..dd75852c 100644 --- a/ftnoir_tracker_pt/point_extractor.cpp +++ b/ftnoir_tracker_pt/point_extractor.cpp @@ -72,20 +72,19 @@ std::vector PointExtractor::extract_points(Mat& frame) struct simple_blob { - double radius; + double radius_2; cv::Vec2d pos; double confid; bool taken; double area; - simple_blob(double radius, const cv::Vec2d& pos, double confid, double area) : radius(radius), pos(pos), confid(confid), taken(false), area(area) + simple_blob(double radius, const cv::Vec2d& pos, double confid, double area) : radius_2(radius*radius), pos(pos), confid(confid), taken(false), area(area) { //qDebug() << "radius" << radius << "pos" << pos[0] << pos[1] << "confid" << confid; } bool inside(const simple_blob& other) { cv::Vec2d tmp = pos - other.pos; - double p = sqrt(1e-4 + tmp.dot(tmp)); - return p < radius; + return tmp.dot(tmp) < radius_2; } static std::vector merge(std::vector& blobs) { -- cgit v1.2.3 From e94be88e28b41610bab983a1cbf8f31133a4ced8 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Wed, 1 Jul 2015 14:25:34 +0200 Subject: pt: show color frame in widget --- ftnoir_tracker_pt/point_extractor.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'ftnoir_tracker_pt') diff --git a/ftnoir_tracker_pt/point_extractor.cpp b/ftnoir_tracker_pt/point_extractor.cpp index dd75852c..e81e3aa0 100644 --- a/ftnoir_tracker_pt/point_extractor.cpp +++ b/ftnoir_tracker_pt/point_extractor.cpp @@ -215,11 +215,14 @@ std::vector PointExtractor::extract_points(Mat& frame) points.push_back(p); } + vector channels_; + cv::split(frame, channels_); // draw output image + Mat frame_bin_ = frame_bin * .5; vector channels; - channels.push_back(frame_gray + frame_bin); - channels.push_back(frame_gray - frame_bin); - channels.push_back(frame_gray - frame_bin); + channels.push_back(channels_[0] + frame_bin_); + channels.push_back(channels_[1] - frame_bin_); + channels.push_back(channels_[2] - frame_bin_); merge(channels, frame); return points; -- cgit v1.2.3