From 143ffce5486739368a9b35c10f2e8a7ad22a2236 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Tue, 8 Dec 2015 02:34:46 +0100 Subject: tracker/pt: reduce auto thresholding histogram bucket size Previously it was too slow to 640x480@75. --- tracker-pt/point_extractor.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'tracker-pt/point_extractor.cpp') diff --git a/tracker-pt/point_extractor.cpp b/tracker-pt/point_extractor.cpp index ec37dd00..e1d55b8d 100644 --- a/tracker-pt/point_extractor.cpp +++ b/tracker-pt/point_extractor.cpp @@ -67,12 +67,13 @@ std::vector PointExtractor::extract_points(cv::Mat& frame) else { cv::Mat hist; + constexpr int hist_c = 6; cv::calcHist(std::vector { frame_gray }, std::vector { 0 }, cv::Mat(), hist, - std::vector { 256 }, - std::vector { 0, 256 }, + std::vector { 256/hist_c }, + std::vector { 0, 256/hist_c }, false); const int sz = hist.rows*hist.cols; int val = 0; @@ -88,6 +89,7 @@ std::vector PointExtractor::extract_points(cv::Mat& frame) break; } } + val *= hist_c; val *= 240./256.; //qDebug() << "val" << val; -- cgit v1.2.3 From f38541f1956133be9b093d5439128d3481b7e5fa Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Thu, 17 Dec 2015 10:26:47 +0100 Subject: pt: histogram more granular 6 -> 8 256 is divisible by 8, also less cpu usage --- tracker-pt/point_extractor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tracker-pt/point_extractor.cpp') diff --git a/tracker-pt/point_extractor.cpp b/tracker-pt/point_extractor.cpp index e1d55b8d..c90310bb 100644 --- a/tracker-pt/point_extractor.cpp +++ b/tracker-pt/point_extractor.cpp @@ -67,7 +67,7 @@ std::vector PointExtractor::extract_points(cv::Mat& frame) else { cv::Mat hist; - constexpr int hist_c = 6; + constexpr int hist_c = 8; cv::calcHist(std::vector { frame_gray }, std::vector { 0 }, cv::Mat(), -- cgit v1.2.3 From 2ef49b8f192b7856e7e6b54064e5981a7a334d44 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Fri, 18 Dec 2015 03:56:56 +0100 Subject: tracker/pt: don't fill mask on frame Saves few % of cpu load --- tracker-pt/point_extractor.cpp | 27 ++++++--------------------- 1 file changed, 6 insertions(+), 21 deletions(-) (limited to 'tracker-pt/point_extractor.cpp') diff --git a/tracker-pt/point_extractor.cpp b/tracker-pt/point_extractor.cpp index c90310bb..bd33ffdd 100644 --- a/tracker-pt/point_extractor.cpp +++ b/tracker-pt/point_extractor.cpp @@ -51,7 +51,6 @@ std::vector PointExtractor::extract_points(cv::Mat& frame) }; // mask for everything that passes the threshold (or: the upper threshold of the hysteresis) - cv::Mat frame_bin = cv::Mat::zeros(H, W, CV_8U); std::vector blobs; std::vector> contours; @@ -59,10 +58,9 @@ std::vector PointExtractor::extract_points(cv::Mat& frame) const int thres = s.threshold; if (!s.auto_threshold) { - cv::Mat frame_bin_; - cv::threshold(frame_gray, frame_bin_, thres, 255, cv::THRESH_BINARY); - frame_bin.setTo(170, frame_bin_); - cv::findContours(frame_bin_, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE); + cv::Mat frame_bin; + cv::threshold(frame_gray, frame_bin, thres, 255, cv::THRESH_BINARY); + cv::findContours(frame_bin, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE); } else { @@ -93,10 +91,9 @@ std::vector PointExtractor::extract_points(cv::Mat& frame) val *= 240./256.; //qDebug() << "val" << val; - cv::Mat frame_bin_; - cv::threshold(frame_gray, frame_bin_, val, 255, CV_THRESH_BINARY); - frame_bin.setTo(170, frame_bin_); - cv::findContours(frame_bin_, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE); + cv::Mat frame_bin; + cv::threshold(frame_gray, frame_bin, val, 255, CV_THRESH_BINARY); + cv::findContours(frame_bin, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE); } int cnt = 0; @@ -166,17 +163,5 @@ std::vector PointExtractor::extract_points(cv::Mat& frame) points.push_back(p); } - // draw output image - std::vector channels_; - cv::split(frame, channels_); - std::vector channels; - { - cv::Mat frame_bin__ = frame_bin * .5; - channels.push_back(channels_[0] + frame_bin__); - channels.push_back(channels_[1] - frame_bin__); - channels.push_back(channels_[2] - frame_bin__); - cv::merge(channels, frame); - } - return points; } -- cgit v1.2.3 From f34dca33bc07680d89591ca5a8ac1e5801bf2950 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Fri, 18 Dec 2015 19:46:05 +0100 Subject: tracker/pt: remove krap --- tracker-pt/point_extractor.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'tracker-pt/point_extractor.cpp') diff --git a/tracker-pt/point_extractor.cpp b/tracker-pt/point_extractor.cpp index bd33ffdd..dd36c57f 100644 --- a/tracker-pt/point_extractor.cpp +++ b/tracker-pt/point_extractor.cpp @@ -13,13 +13,9 @@ # include "opentrack-compat/timer.hpp" #endif -PointExtractor::PointExtractor(){ - //if (!AllocConsole()){} - //else SetConsoleTitle("debug"); - //freopen("CON", "w", stdout); - //freopen("CON", "w", stderr); +PointExtractor::PointExtractor() +{ } -// ---------------------------------------------------------------------------- std::vector PointExtractor::extract_points(cv::Mat& frame) { const int W = frame.cols; -- cgit v1.2.3 From 8a60f1c9613d396b50aa6adec4b1f5ffdb9e9806 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Fri, 18 Dec 2015 19:57:42 +0100 Subject: tracker/pt: reduce mutex contention --- tracker-pt/ftnoir_tracker_pt.cpp | 12 ++++-------- tracker-pt/ftnoir_tracker_pt.h | 5 ++--- tracker-pt/point_extractor.cpp | 7 ++++--- tracker-pt/point_extractor.h | 7 ++++--- tracker-pt/point_tracker.cpp | 1 + tracker-pt/point_tracker.h | 6 +++--- 6 files changed, 18 insertions(+), 20 deletions(-) (limited to 'tracker-pt/point_extractor.cpp') diff --git a/tracker-pt/ftnoir_tracker_pt.cpp b/tracker-pt/ftnoir_tracker_pt.cpp index 877b58fd..80227a39 100644 --- a/tracker-pt/ftnoir_tracker_pt.cpp +++ b/tracker-pt/ftnoir_tracker_pt.cpp @@ -19,8 +19,7 @@ //----------------------------------------------------------------------------- Tracker_PT::Tracker_PT() - : mutex(QMutex::Recursive), - commands(0), + : commands(0), video_widget(NULL), video_frame(NULL), ever_success(false) @@ -94,29 +93,26 @@ void Tracker_PT::run() if (new_frame && !frame.empty()) { - QMutexLocker lock(&mutex); - std::vector points = point_extractor.extract_points(frame); // blobs are sorted in order of circularity if (points.size() > PointModel::N_POINTS) points.resize(PointModel::N_POINTS); - bool success = points.size() == PointModel::N_POINTS; - float fx; if (!get_focal_length(fx)) continue; + + const bool success = points.size() == PointModel::N_POINTS; if (success) { point_tracker.track(points, PointModel(s), fx, s.dynamic_pose, s.init_phase_timeout); + ever_success = true; } Affine X_CM = pose(); - ever_success |= success; - { Affine X_MH(cv::Matx33f::eye(), cv::Vec3f(s.t_MH_x, s.t_MH_y, s.t_MH_z)); // just copy pasted these lines from below Affine X_GH = X_CM * X_MH; diff --git a/tracker-pt/ftnoir_tracker_pt.h b/tracker-pt/ftnoir_tracker_pt.h index f73d106b..5ec7ea91 100644 --- a/tracker-pt/ftnoir_tracker_pt.h +++ b/tracker-pt/ftnoir_tracker_pt.h @@ -40,15 +40,14 @@ public: void start_tracker(QFrame* parent_window) override; void data(double* data) override; - Affine pose() { QMutexLocker lock(&mutex); return point_tracker.pose(); } - int get_n_points() { QMutexLocker lock(&mutex); return point_extractor.get_points().size(); } + Affine pose() { return point_tracker.pose(); } + int get_n_points() { return point_extractor.get_points().size(); } bool get_cam_info(CamInfo* info) { QMutexLocker lock(&camera_mtx); return camera.get_info(*info); } public slots: void apply_settings(); protected: void run() override; private: - QMutex mutex; // thread commands enum Command { ABORT = 1<<0 diff --git a/tracker-pt/point_extractor.cpp b/tracker-pt/point_extractor.cpp index dd36c57f..6eb255a5 100644 --- a/tracker-pt/point_extractor.cpp +++ b/tracker-pt/point_extractor.cpp @@ -147,12 +147,13 @@ std::vector PointExtractor::extract_points(cv::Mat& frame) blobs.push_back(blob(radius, pos, confid, area)); } - // clear old points - points.clear(); - using b = const blob; std::sort(blobs.begin(), blobs.end(), [](b& b1, b& b2) {return b1.confid > b2.confid;}); + QMutexLocker l(&mtx); + + points.clear(); + for (auto& b : blobs) { cv::Vec2f p((b.pos[0] - W/2)/W, -(b.pos[1] - H/2)/W); diff --git a/tracker-pt/point_extractor.h b/tracker-pt/point_extractor.h index b9368ab6..80c1897d 100644 --- a/tracker-pt/point_extractor.h +++ b/tracker-pt/point_extractor.h @@ -13,8 +13,8 @@ #include "ftnoir_tracker_pt_settings.h" -// ---------------------------------------------------------------------------- -// Extracts points from an opencv image +#include + class PointExtractor { public: @@ -22,12 +22,13 @@ public: // dt: time since last call in seconds // WARNING: returned reference is valid as long as object std::vector extract_points(cv::Mat &frame); - const std::vector& get_points() { return points; } + const std::vector& get_points() { QMutexLocker l(&mtx); return points; } PointExtractor(); settings_pt s; private: std::vector points; + QMutex mtx; }; #endif //POINTEXTRACTOR_H diff --git a/tracker-pt/point_tracker.cpp b/tracker-pt/point_tracker.cpp index 924b75de..aa6feb5b 100644 --- a/tracker-pt/point_tracker.cpp +++ b/tracker-pt/point_tracker.cpp @@ -249,6 +249,7 @@ int PointTracker::POSIT(const PointModel& model, const PointOrder& order_, float old_epsilon_2 = epsilon_2; } + QMutexLocker l(&mtx); // apply results X_CM.R = *R_current; X_CM.t[0] = order[0][0] * Z0/focal_length; diff --git a/tracker-pt/point_tracker.h b/tracker-pt/point_tracker.h index cdcf2998..2757f22c 100644 --- a/tracker-pt/point_tracker.h +++ b/tracker-pt/point_tracker.h @@ -15,9 +15,8 @@ #include "ftnoir_tracker_pt_settings.h" #include +#include -// ---------------------------------------------------------------------------- -// Affine frame trafo class Affine { public: @@ -116,7 +115,7 @@ public: // f : (focal length)/(sensor width) // dt : time since last call void track(const std::vector& projected_points, const PointModel& model, float f, bool dynamic_pose, int init_phase_timeout); - Affine pose() const { return X_CM; } + Affine pose() { QMutexLocker l(&mtx); return X_CM; } cv::Vec2f project(const cv::Vec3f& v_M, float f); private: // the points in model order @@ -138,6 +137,7 @@ private: Timer t; bool init_phase; + QMutex mtx; }; #endif //POINTTRACKER_H -- cgit v1.2.3 From a2ceed01e32fd941e20e1362b6c44af8db05b5b2 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Sat, 19 Dec 2015 19:48:55 +0100 Subject: tracker/pt: cv::Mat::at is slow, use cv::Mat::ptr --- tracker-pt/point_extractor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tracker-pt/point_extractor.cpp') diff --git a/tracker-pt/point_extractor.cpp b/tracker-pt/point_extractor.cpp index 6eb255a5..6118f3d2 100644 --- a/tracker-pt/point_extractor.cpp +++ b/tracker-pt/point_extractor.cpp @@ -74,9 +74,10 @@ std::vector PointExtractor::extract_points(cv::Mat& frame) int cnt = 0; constexpr int min_pixels = 250; const auto pixels_to_include = std::max(0, min_pixels * s.threshold/100.); + auto ptr = reinterpret_cast(hist.ptr(0)); for (int i = sz-1; i >= 0; i--) { - cnt += hist.at(i); + cnt += ptr[i]; if (cnt >= pixels_to_include) { val = i; -- cgit v1.2.3 From fba00166c94f066bf0d8d2174b508d2f849abe53 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Sat, 19 Dec 2015 20:44:41 +0100 Subject: tracker/pt: don't copy points array needlessly --- tracker-pt/ftnoir_tracker_pt.cpp | 47 +++++++++++++++++++--------------------- tracker-pt/point_extractor.cpp | 13 +++++++++-- tracker-pt/point_extractor.h | 2 +- 3 files changed, 34 insertions(+), 28 deletions(-) (limited to 'tracker-pt/point_extractor.cpp') diff --git a/tracker-pt/ftnoir_tracker_pt.cpp b/tracker-pt/ftnoir_tracker_pt.cpp index 9a4fa037..88356771 100644 --- a/tracker-pt/ftnoir_tracker_pt.cpp +++ b/tracker-pt/ftnoir_tracker_pt.cpp @@ -97,17 +97,13 @@ void Tracker_PT::run() if (new_frame && !frame_.empty()) { - std::vector points = point_extractor.extract_points(frame); - - // blobs are sorted in order of circularity - if (points.size() > PointModel::N_POINTS) - points.resize(PointModel::N_POINTS); + const auto& points = point_extractor.extract_points(frame_); float fx; if (!get_focal_length(fx)) continue; - const bool success = points.size() == PointModel::N_POINTS; + const bool success = points.size() >= PointModel::N_POINTS; if (success) { @@ -116,35 +112,36 @@ void Tracker_PT::run() } Affine X_CM = pose(); - - { - Affine X_MH(cv::Matx33f::eye(), cv::Vec3f(s.t_MH_x, s.t_MH_y, s.t_MH_z)); // just copy pasted these lines from below - Affine X_GH = X_CM * X_MH; - cv::Vec3f p = X_GH.t; // head (center?) position in global space - cv::Vec2f p_(p[0] / p[2] * fx, p[1] / p[2] * fx); // projected to screen - points.push_back(p_); - } - - for (unsigned i = 0; i < points.size(); i++) + + std::function fun = [&](const cv::Vec2f& p, const cv::Scalar color) { - auto& p = points[i]; - auto p2 = cv::Point(p[0] * frame.cols + frame.cols/2, -p[1] * frame.cols + frame.rows/2); - cv::Scalar color(0, 255, 0); - if (i == points.size()-1) - color = cv::Scalar(0, 0, 255); - cv::line(frame, + auto p2 = cv::Point(p[0] * frame_.cols + frame_.cols/2, -p[1] * frame_.cols + frame_.rows/2); + cv::line(frame_, cv::Point(p2.x - 20, p2.y), cv::Point(p2.x + 20, p2.y), color, 4); - cv::line(frame, + cv::line(frame_, cv::Point(p2.x, p2.y - 20), cv::Point(p2.x, p2.y + 20), color, - 4); + 4); + }; + + for (unsigned i = 0; i < points.size(); i++) + { + fun(points[i], cv::Scalar(0, 255, 0)); + } + + { + Affine X_MH(cv::Matx33f::eye(), cv::Vec3f(s.t_MH_x, s.t_MH_y, s.t_MH_z)); // just copy pasted these lines from below + Affine X_GH = X_CM * X_MH; + cv::Vec3f p = X_GH.t; // head (center?) position in global space + cv::Vec2f p_(p[0] / p[2] * fx, p[1] / p[2] * fx); // projected to screen + fun(p_, cv::Scalar(0, 0, 255)); } - video_widget->update_image(frame); + video_widget->update_image(frame_); } } qDebug()<<"Tracker:: Thread stopping"; diff --git a/tracker-pt/point_extractor.cpp b/tracker-pt/point_extractor.cpp index 6118f3d2..3808c408 100644 --- a/tracker-pt/point_extractor.cpp +++ b/tracker-pt/point_extractor.cpp @@ -16,10 +16,17 @@ PointExtractor::PointExtractor() { } -std::vector PointExtractor::extract_points(cv::Mat& frame) + +const std::vector& PointExtractor::extract_points(cv::Mat& frame) { const int W = frame.cols; const int H = frame.rows; + + if (frame_gray.rows != frame.rows || frame_gray.cols != frame.cols) + { + frame_gray = cv::Mat(frame.rows, frame.cols, CV_8U); + frame_bin = cv::Mat(frame.rows, frame.cols, CV_8U);; + } // convert to grayscale cv::Mat frame_gray; @@ -151,9 +158,11 @@ std::vector PointExtractor::extract_points(cv::Mat& frame) using b = const blob; std::sort(blobs.begin(), blobs.end(), [](b& b1, b& b2) {return b1.confid > b2.confid;}); + points.reserve(blobs.size()); + QMutexLocker l(&mtx); - points.clear(); + points.clear(); for (auto& b : blobs) { diff --git a/tracker-pt/point_extractor.h b/tracker-pt/point_extractor.h index 80c1897d..979cc8b6 100644 --- a/tracker-pt/point_extractor.h +++ b/tracker-pt/point_extractor.h @@ -21,7 +21,7 @@ public: // extracts points from frame and draws some processing info into frame, if draw_output is set // dt: time since last call in seconds // WARNING: returned reference is valid as long as object - std::vector extract_points(cv::Mat &frame); + const std::vector &extract_points(cv::Mat &frame); const std::vector& get_points() { QMutexLocker l(&mtx); return points; } PointExtractor(); -- cgit v1.2.3 From f4647a9960f531829f1add40554442a7c84d82a6 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Sat, 19 Dec 2015 20:46:10 +0100 Subject: tracker/pt: don't allocate temporary dynamic size arrays --- tracker-pt/point_extractor.cpp | 7 +------ tracker-pt/point_extractor.h | 4 ++++ 2 files changed, 5 insertions(+), 6 deletions(-) (limited to 'tracker-pt/point_extractor.cpp') diff --git a/tracker-pt/point_extractor.cpp b/tracker-pt/point_extractor.cpp index 3808c408..655e1412 100644 --- a/tracker-pt/point_extractor.cpp +++ b/tracker-pt/point_extractor.cpp @@ -29,7 +29,6 @@ const std::vector& PointExtractor::extract_points(cv::Mat& frame) } // convert to grayscale - cv::Mat frame_gray; cv::cvtColor(frame, frame_gray, cv::COLOR_RGB2GRAY); const double region_size_min = s.min_point_size; @@ -61,14 +60,11 @@ const std::vector& PointExtractor::extract_points(cv::Mat& frame) const int thres = s.threshold; if (!s.auto_threshold) { - cv::Mat frame_bin; cv::threshold(frame_gray, frame_bin, thres, 255, cv::THRESH_BINARY); cv::findContours(frame_bin, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE); } else { - cv::Mat hist; - constexpr int hist_c = 8; cv::calcHist(std::vector { frame_gray }, std::vector { 0 }, cv::Mat(), @@ -76,7 +72,7 @@ const std::vector& PointExtractor::extract_points(cv::Mat& frame) std::vector { 256/hist_c }, std::vector { 0, 256/hist_c }, false); - const int sz = hist.rows*hist.cols; + const int sz = hist.cols * hist.rows; int val = 0; int cnt = 0; constexpr int min_pixels = 250; @@ -95,7 +91,6 @@ const std::vector& PointExtractor::extract_points(cv::Mat& frame) val *= 240./256.; //qDebug() << "val" << val; - cv::Mat frame_bin; cv::threshold(frame_gray, frame_bin, val, 255, CV_THRESH_BINARY); cv::findContours(frame_bin, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE); } diff --git a/tracker-pt/point_extractor.h b/tracker-pt/point_extractor.h index 979cc8b6..ad8955fc 100644 --- a/tracker-pt/point_extractor.h +++ b/tracker-pt/point_extractor.h @@ -27,8 +27,12 @@ public: settings_pt s; private: + enum { hist_c = 8 }; std::vector points; QMutex mtx; + cv::Mat frame_gray; + cv::Mat frame_bin; + cv::Mat hist; }; #endif //POINTEXTRACTOR_H -- cgit v1.2.3 From ccdf021f5d1a8daed729369cc078c5756a3a5bee Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Thu, 24 Dec 2015 19:24:59 +0100 Subject: tracker/pt: limit max amount of extracted blobs --- tracker-pt/point_extractor.cpp | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'tracker-pt/point_extractor.cpp') diff --git a/tracker-pt/point_extractor.cpp b/tracker-pt/point_extractor.cpp index 655e1412..0208b11d 100644 --- a/tracker-pt/point_extractor.cpp +++ b/tracker-pt/point_extractor.cpp @@ -148,6 +148,11 @@ const std::vector& PointExtractor::extract_points(cv::Mat& frame) } blobs.push_back(blob(radius, pos, confid, area)); + + enum { max_blobs = 16 }; + + if (blobs.size() == max_blobs) + break; } using b = const blob; -- cgit v1.2.3