summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--tracker-aruco/ftnoir_tracker_aruco.cpp429
-rw-r--r--tracker-aruco/ftnoir_tracker_aruco.h49
2 files changed, 276 insertions, 202 deletions
diff --git a/tracker-aruco/ftnoir_tracker_aruco.cpp b/tracker-aruco/ftnoir_tracker_aruco.cpp
index b222ebbb..65056a63 100644
--- a/tracker-aruco/ftnoir_tracker_aruco.cpp
+++ b/tracker-aruco/ftnoir_tracker_aruco.cpp
@@ -5,12 +5,6 @@
* copyright notice and this permission notice appear in all copies.
*/
-#include <vector>
-#include <cstdio>
-#include <cmath>
-#include <algorithm>
-#include <QMutexLocker>
-#include "./include/markerdetector.h"
#include "ftnoir_tracker_aruco.h"
#include "opentrack/plugin-api.hpp"
#include <opencv2/core/core.hpp>
@@ -19,10 +13,20 @@
#include "opentrack-compat/camera-names.hpp"
#include "opentrack-compat/sleep.hpp"
-typedef struct {
+#include <QMutexLocker>
+#include <QDebug>
+
+#include <vector>
+#include <cstdio>
+#include <cmath>
+#include <algorithm>
+#include <iterator>
+
+struct resolution_tuple
+{
int width;
int height;
-} resolution_tuple;
+};
static resolution_tuple resolution_choices[] =
{
@@ -35,8 +39,21 @@ static resolution_tuple resolution_choices[] =
Tracker::Tracker() :
stop(false),
layout(nullptr),
- videoWidget(nullptr)
+ videoWidget(nullptr),
+ obj_points(4),
+ intrinsics(decltype(intrinsics)::eye()),
+ dist_coeffs(decltype(dist_coeffs)::zeros()),
+ centroid { cv::Point3f(0, 0, 0) },
+ rmat(decltype(rmat)::eye()),
+ roi_points(4),
+ last_roi(65535, 65535, 0, 0),
+ freq(cv::getTickFrequency()), // XXX change to timer.hpp
+ cur_fps(0)
{
+ // param 2 ignored for Otsu thresholding. it's required to use our fork of Aruco.
+ detector.setThresholdParams(5, -1);
+ detector.setDesiredSpeed(3);
+ detector._thresMethod = aruco::MarkerDetector::FIXED_THRES;
}
Tracker::~Tracker()
@@ -79,10 +96,40 @@ void Tracker::getRT(cv::Matx33d& r_, cv::Vec3d& t_)
t_ = t;
}
-void Tracker::run()
+bool Tracker::detect_with_roi()
{
- cv::setNumThreads(0);
+ if (last_roi.width > 0)
+ {
+ detector.setMinMaxSize(std::min(1.f, std::max(0.01f, size_min * grayscale.cols / last_roi.width)),
+ std::max(.01f, std::min(1.0f, size_max * grayscale.cols / last_roi.width)));
+
+ detector.detect(grayscale(last_roi), markers, cv::Mat(), cv::Mat(), -1, false);
+
+ if (markers.size() == 1 && markers[0].size() == 4)
+ {
+ auto& m = markers.at(0);
+ for (unsigned i = 0; i < 4; i++)
+ {
+ auto& p = m.at(i);
+ p.x += last_roi.x;
+ p.y += last_roi.y;
+ }
+ return true;
+ }
+ }
+ last_roi = cv::Rect(65535, 65535, 0, 0);
+ return false;
+}
+bool Tracker::detect_without_roi()
+{
+ detector.setMinMaxSize(size_min, size_max);
+ detector.detect(grayscale, markers, cv::Mat(), cv::Mat(), -1, false);
+ return markers.size() == 1 && markers[0].size() == 4;
+}
+
+bool Tracker::open_camera()
+{
int rint = s.resolution;
if (rint < 0 || rint >= (int)(sizeof(resolution_choices) / sizeof(resolution_tuple)))
rint = 0;
@@ -108,238 +155,226 @@ void Tracker::run()
break;
}
- {
- QMutexLocker l(&camera_mtx);
+ QMutexLocker l(&camera_mtx);
- camera = cv::VideoCapture(camera_name_to_index(s.camera_name));
- if (res.width)
- {
- camera.set(cv::CAP_PROP_FRAME_WIDTH, res.width);
- camera.set(cv::CAP_PROP_FRAME_HEIGHT, res.height);
- }
- if (fps)
- camera.set(cv::CAP_PROP_FPS, fps);
+ camera = cv::VideoCapture(camera_name_to_index(s.camera_name));
+ if (res.width)
+ {
+ camera.set(cv::CAP_PROP_FRAME_WIDTH, res.width);
+ camera.set(cv::CAP_PROP_FRAME_HEIGHT, res.height);
+ }
+ if (fps)
+ camera.set(cv::CAP_PROP_FPS, fps);
- if (!camera.isOpened())
- {
- qDebug() << "aruco tracker: can't open camera";
- return;
- }
+ if (!camera.isOpened())
+ {
+ qDebug() << "aruco tracker: can't open camera";
+ return false;
}
+ return true;
+}
- aruco::MarkerDetector detector;
- detector.setDesiredSpeed(3);
- detector._thresMethod = aruco::MarkerDetector::FIXED_THRES;
+void Tracker::set_intrinsics()
+{
+ static constexpr double pi = 3.1415926f;
+ const int w = grayscale.cols, h = grayscale.rows;
+ const double diag_fov = static_cast<int>(s.fov) * pi / 180.;
+ const double fov_w = 2.*atan(tan(diag_fov/2.)/sqrt(1. + h/(double)w * h/(double)w));
+ const double fov_h = 2.*atan(tan(diag_fov/2.)/sqrt(1. + w/(double)h * w/(double)h));
+ const double focal_length_w = .5 * w / tan(.5 * fov_w);
+ const double focal_length_h = .5 * h / tan(.5 * fov_h);
+
+ intrinsics(0, 0) = focal_length_w;
+ intrinsics(0, 2) = grayscale.cols/2;
+ intrinsics(1, 1) = focal_length_h;
+ intrinsics(1, 2) = grayscale.rows/2;
+}
- cv::Rect last_roi(65535, 65535, 0, 0);
+void Tracker::update_fps(double alpha)
+{
+ std::uint64_t time = std::uint64_t(cv::getTickCount());
+ const double dt = std::max(0., (time - last_time) / freq);
+ last_time = time;
- // XXX change to timer.hpp
- auto freq = cv::getTickFrequency();
- auto last_time = cv::getTickCount();
- double cur_fps = 0;
- cv::Vec3d rvec, tvec;
- cv::Mat intrinsics = cv::Mat::eye(3, 3, CV_32FC1);
- cv::Mat dist_coeffs = cv::Mat::zeros(5, 1, CV_32FC1);
+ cur_fps = cur_fps * alpha + (1-alpha) * (fabs(dt) < 1e-6 ? 0 : 1./dt);
+}
- while (!stop)
+void Tracker::draw_ar(bool ok)
+{
+ if (ok)
{
- cv::Mat color;
- {
- QMutexLocker l(&camera_mtx);
+ const auto& m = markers.at(0);
+ for (unsigned i = 0; i < 4; i++)
+ cv::line(frame, m[i], m[(i+1)%4], cv::Scalar(0, 0, 255), 2, 8);
+ }
- if (!camera.read(color))
- continue;
- }
- cv::Mat grayscale;
- cv::cvtColor(color, grayscale, cv::COLOR_RGB2GRAY);
+ char buf[32];
+ ::sprintf(buf, "Hz: %d", (unsigned short)cur_fps);
+ cv::putText(frame, buf, cv::Point(10, 32), cv::FONT_HERSHEY_PLAIN, 2, cv::Scalar(0, 255, 0), 1);
+}
- const int scale = grayscale.cols > 480 ? 2 : 1;
- // param 2 ignored for Otsu thresholding. it's required to use our fork of Aruco.
- detector.setThresholdParams(5, -1);
+void Tracker::clamp_last_roi()
+{
+ if (last_roi.x < 0)
+ last_roi.x = 0;
+ if (last_roi.y < 0)
+ last_roi.y = 0;
+ if (last_roi.width < 0)
+ last_roi.width = 0;
+ if (last_roi.height < 0)
+ last_roi.height = 0;
+ if (last_roi.x >= color.cols-1)
+ last_roi.x = color.cols-1;
+ if (last_roi.width >= color.cols-1)
+ last_roi.width = color.cols-1;
+ if (last_roi.y >= color.rows-1)
+ last_roi.y = color.rows-1;
+ if (last_roi.height >= color.rows-1)
+ last_roi.height = color.rows-1;
+}
- static constexpr double pi = 3.1415926f;
- const int w = grayscale.cols, h = grayscale.rows;
- const double diag_fov = static_cast<int>(s.fov) * pi / 180.;
- const double fov_w = 2.*atan(tan(diag_fov/2.)/sqrt(1. + h/(double)w * h/(double)w));
- const double fov_h = 2.*atan(tan(diag_fov/2.)/sqrt(1. + w/(double)h * w/(double)h));
- const float focal_length_w = .5 * w / tan(.5 * fov_w);
- const float focal_length_h = .5 * h / tan(.5 * fov_h);
+void Tracker::set_points()
+{
+ using f = float;
+ const f hx = f(s.headpos_x), hy = f(s.headpos_y), hz = f(s.headpos_z);
- intrinsics.at<float> (0, 0) = focal_length_w;
- intrinsics.at<float> (1, 1) = focal_length_h;
- intrinsics.at<float> (0, 2) = grayscale.cols/2;
- intrinsics.at<float> (1, 2) = grayscale.rows/2;
+ static constexpr float size = 40;
- std::vector< aruco::Marker > markers;
+ const int x1=1, x2=2, x3=3, x4=0;
- const double size_min = 0.05;
- const double size_max = 0.3;
+ obj_points[x1] = cv::Point3f(-size + hx, -size + hy, 0 + hz);
+ obj_points[x2] = cv::Point3f(size + hx, -size + hy, 0 + hz);
+ obj_points[x3] = cv::Point3f(size + hx, size + hy, 0 + hz);
+ obj_points[x4] = cv::Point3f(-size + hx, size + hy, 0 + hz);
+}
- bool roi_valid = false;
+void Tracker::draw_centroid()
+{
+ repr2.clear();
+ cv::projectPoints(centroid, rvec, tvec, intrinsics, dist_coeffs, repr2);
- auto time = cv::getTickCount();
+ auto s = cv::Scalar(255, 0, 255);
+ cv::circle(frame, repr2.at(0), 4, s, -1);
+}
- const double dt = (time - last_time) / freq;
- last_time = time;
- cur_fps = cur_fps * 0.97 + 0.03 * (dt == 0 ? 0 : 1./dt);
+bool Tracker::set_last_roi()
+{
+ roi_projection.clear();
- if (last_roi.width > 0 && last_roi.height)
- {
- detector.setMinMaxSize(std::min(1., std::max(0.01, size_min * grayscale.cols / last_roi.width)),
- std::max(0.01, std::min(1.0, size_max * grayscale.cols / last_roi.width)));
+ if (!cv::solvePnP(obj_points, markers[0], intrinsics, dist_coeffs, rvec_, tvec_, false, cv::SOLVEPNP_ITERATIVE))
+ return false;
- cv::Mat grayscale_ = grayscale(last_roi).clone();
- if (detector.detect(grayscale_, markers, cv::Mat(), cv::Mat(), -1, false),
- markers.size() == 1 && markers[0].size() == 4)
- {
- auto& m = markers.at(0);
- for (unsigned i = 0; i < 4; i++)
- {
- auto& p = m.at(i);
- p.x += last_roi.x;
- p.y += last_roi.y;
- }
- roi_valid = true;
- }
- }
+ for (unsigned i = 0; i < 4; i++)
+ {
+ using f = float;
+ cv::Point3f pt(obj_points[i] - cv::Point3f(f(s.headpos_x), f(s.headpos_y), f(s.headpos_z)));
+ roi_points[i] = pt * c_search_window;
+ }
+ cv::projectPoints(roi_points, rvec_, tvec_, intrinsics, dist_coeffs, roi_projection);
- if (!roi_valid)
- {
- detector.setMinMaxSize(size_min, size_max);
- detector.detect(grayscale, markers, cv::Mat(), cv::Mat(), -1, false);
- }
+ set_roi_from_projection();
- if (markers.size() == 1 && markers[0].size() == 4) {
- const auto& m = markers.at(0);
- for (int i = 0; i < 4; i++)
- cv::line(color, m[i], m[(i+1)%4], cv::Scalar(0, 0, 255), scale, 8);
- }
+ return true;
+}
- char buf[128];
+void Tracker::set_rmat()
+{
+ cv::Rodrigues(rvec, rmat);
- frame = color.clone();
+ euler = cv::RQDecomp3x3(rmat, m_r, m_q);
- ::sprintf(buf, "Hz: %d", (int)cur_fps);
- cv::putText(frame, buf, cv::Point(10, 32), cv::FONT_HERSHEY_PLAIN, scale, cv::Scalar(0, 255, 0), scale*2);
+ QMutexLocker lck(&mtx);
- const float hx = s.headpos_x, hy = s.headpos_y, hz = s.headpos_z;
+ for (int i = 0; i < 3; i++)
+ pose[i] = tvec(i) * .1;
- if (markers.size() == 1 && markers[0].size() == 4) {
- const auto& m = markers.at(0);
- constexpr float size = 40;
+ pose[Yaw] = euler[1];
+ pose[Pitch] = -euler[0];
+ pose[Roll] = euler[2];
- cv::Mat obj_points(4,3,CV_32FC1);
- const int x1=1, x2=2, x3=3, x4=0;
- obj_points.at<float>(x1,0)=-size + hx;
- obj_points.at<float>(x1,1)=-size + hy;
- obj_points.at<float>(x1,2)= 0 + hz;
+ r = rmat;
+ t = cv::Vec3d(tvec[0], -tvec[1], tvec[2]);
+}
- obj_points.at<float>(x2,0)=size + hx;
- obj_points.at<float>(x2,1)=-size + hy;
- obj_points.at<float>(x2,2)= 0 + hz;
+void Tracker::set_roi_from_projection()
+{
+ last_roi = cv::Rect(color.cols-1, color.rows-1, 0, 0);
- obj_points.at<float>(x3,0)=size + hx;
- obj_points.at<float>(x3,1)=size + hy;
- obj_points.at<float>(x3,2)= 0 + hz;
+ for (unsigned i = 0; i < 4; i++)
+ {
+ const auto& proj = roi_projection[i];
+ int min_x = std::min(int(proj.x), last_roi.x),
+ min_y = std::min(int(proj.y), last_roi.y);
- obj_points.at<float>(x4,0)= -size + hx;
- obj_points.at<float>(x4,1)= size + hy;
- obj_points.at<float>(x4,2)= 0 + hz;
+ int max_x = std::max(int(proj.x), last_roi.width),
+ max_y = std::max(int(proj.y), last_roi.height);
- std::vector<cv::Point2f> img_points = m;
- if (!cv::solvePnP(obj_points, img_points, intrinsics, dist_coeffs, rvec, tvec, false, cv::SOLVEPNP_ITERATIVE))
- goto fail;
+ last_roi.x = min_x;
+ last_roi.y = min_y;
- {
- std::vector<cv::Point2f> repr2;
- std::vector<cv::Point3f> centroid;
- centroid.push_back(cv::Point3f(0, 0, 0));
- cv::projectPoints(centroid, rvec, tvec, intrinsics, dist_coeffs, repr2);
-
- {
- auto s = cv::Scalar(255, 0, 255);
- cv::circle(frame, repr2.at(0), 4, s, -1);
- }
- }
+ last_roi.width = max_x;
+ last_roi.height = max_y;
+ }
- for (int i = 0; i < 4; i++)
- {
- obj_points.at<float>(i, 0) -= hx;
- obj_points.at<float>(i, 1) -= hy;
- obj_points.at<float>(i, 2) -= hz;
- }
+ last_roi.width -= last_roi.x;
+ last_roi.height -= last_roi.y;
- cv::Mat rvec_, tvec_;
- cv::solvePnP(obj_points, m, intrinsics, dist_coeffs, rvec_, tvec_, false, cv::SOLVEPNP_ITERATIVE);
+ clamp_last_roi();
+}
- cv::Mat roi_points = obj_points * c_search_window;
- std::vector<cv::Point2f> roi_projection(4);
- cv::projectPoints(roi_points, rvec_, tvec_, intrinsics, dist_coeffs, roi_projection);
+void Tracker::run()
+{
+ cv::setNumThreads(0);
- last_roi = cv::Rect(color.cols-1, color.rows-1, 0, 0);
+ using std::fabs;
+ using std::atan;
+ using std::tan;
+ using std::sqrt;
- for (int i = 0; i < 4; i++)
- {
- auto proj = roi_projection[i];
- int min_x = std::min<int>(proj.x, last_roi.x),
- min_y = std::min<int>(proj.y, last_roi.y);
+ if (!open_camera())
+ return;
- int max_x = std::max<int>(proj.x, last_roi.width),
- max_y = std::max<int>(proj.y, last_roi.height);
+ last_time = std::uint64_t(cv::getTickCount());
- last_roi.x = min_x;
- last_roi.y = min_y;
+ while (!stop)
+ {
+ {
+ QMutexLocker l(&camera_mtx);
- last_roi.width = max_x;
- last_roi.height = max_y;
- }
+ if (!camera.read(color))
+ continue;
+ }
- if (last_roi.x < 0)
- last_roi.x = 0;
- if (last_roi.y < 0)
- last_roi.y = 0;
- if (last_roi.width < 0)
- last_roi.width = 0;
- if (last_roi.height < 0)
- last_roi.height = 0;
- if (last_roi.x >= color.cols-1)
- last_roi.x = color.cols-1;
- if (last_roi.width >= color.cols-1)
- last_roi.width = color.cols-1;
- if (last_roi.y >= color.rows-1)
- last_roi.y= color.rows-1;
- if (last_roi.height >= color.rows-1)
- last_roi.height = color.rows-1;
-
- last_roi.width -= last_roi.x;
- last_roi.height -= last_roi.y;
-
- auto rmat = cv::Matx33d::zeros();
- cv::Matx33d m_r(3, 3, CV_64FC1), m_q(3, 3, CV_64FC1);
- cv::Rodrigues(rvec, rmat);
+ cv::cvtColor(color, grayscale, cv::COLOR_RGB2GRAY);
+ color.copyTo(frame);
- {
- cv::Vec3d euler = cv::RQDecomp3x3(rmat, m_r, m_q);
+ set_intrinsics();
+ update_fps(alpha_);
- QMutexLocker lck(&mtx);
+ markers.clear();
- for (int i = 0; i < 3; i++)
- pose[i] = tvec(i);
- pose[Yaw] = euler[1];
- pose[Pitch] = -euler[0];
- pose[Roll] = euler[2];
+ const bool ok = detect_with_roi() || detect_without_roi();
- r = rmat;
- t = cv::Vec3d(tvec[0], -tvec[1], tvec[2]);
- }
+ if (ok)
+ {
+ set_points();
- if (roi_valid)
- cv::rectangle(frame, last_roi, cv::Scalar(255, 0, 255), 1);
+ if (!cv::solvePnP(obj_points, markers[0], intrinsics, dist_coeffs, rvec, tvec, false, cv::SOLVEPNP_ITERATIVE))
+ goto fail;
+
+ if (!set_last_roi())
+ goto fail;
+
+ draw_centroid();
+ set_rmat();
}
else
fail:
+ // no marker found, reset search region
last_roi = cv::Rect(65535, 65535, 0, 0);
+ draw_ar(ok);
+
if (frame.rows > 0)
videoWidget->update_image(frame);
}
@@ -355,9 +390,9 @@ void Tracker::data(double *data)
data[Yaw] = pose[Yaw];
data[Pitch] = pose[Pitch];
data[Roll] = pose[Roll];
- data[TX] = pose[TX] * .1;
- data[TY] = pose[TY] * .1;
- data[TZ] = pose[TZ] * .1;
+ data[TX] = pose[TX];
+ data[TY] = pose[TY];
+ data[TZ] = pose[TZ];
}
TrackerControls::TrackerControls()
@@ -391,7 +426,9 @@ void TrackerControls::toggleCalibrate()
s.headpos_z = 0;
calibrator.reset();
calib_timer.start();
- } else {
+ }
+ else
+ {
cleanupCalib();
auto pos = calibrator.get_estimate();
diff --git a/tracker-aruco/ftnoir_tracker_aruco.h b/tracker-aruco/ftnoir_tracker_aruco.h
index bef27ac7..fb207f6f 100644
--- a/tracker-aruco/ftnoir_tracker_aruco.h
+++ b/tracker-aruco/ftnoir_tracker_aruco.h
@@ -9,16 +9,20 @@
#include "ui_aruco-trackercontrols.h"
#include "ar_video_widget.h"
+#include "opentrack-compat/options.hpp"
+#include "trans_calib.h"
+#include "opentrack/plugin-api.hpp"
+#include "opentrack/opencv-camera-dialog.hpp"
+#include "include/markerdetector.h"
+
#include <QObject>
#include <QThread>
#include <QMutex>
#include <QHBoxLayout>
#include <QDialog>
#include <QTimer>
-#include "opentrack-compat/options.hpp"
-#include "trans_calib.h"
-#include "opentrack/plugin-api.hpp"
-#include "opentrack/opencv-camera-dialog.hpp"
+
+#include <cinttypes>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
@@ -48,7 +52,7 @@ class Tracker : protected QThread, public ITracker
{
Q_OBJECT
friend class TrackerControls;
- static constexpr double c_search_window = 1.3;
+ static constexpr float c_search_window = 1.3f;
public:
Tracker();
~Tracker() override;
@@ -57,6 +61,19 @@ public:
void run() override;
void getRT(cv::Matx33d &r, cv::Vec3d &t);
private:
+ bool detect_with_roi();
+ bool detect_without_roi();
+ bool open_camera();
+ void set_intrinsics();
+ void update_fps(double dt);
+ void draw_ar(bool ok);
+ void clamp_last_roi();
+ void set_points();
+ void draw_centroid();
+ bool set_last_roi();
+ void set_rmat();
+ void set_roi_from_projection();
+
cv::VideoCapture camera;
QMutex camera_mtx;
QMutex mtx;
@@ -65,9 +82,29 @@ private:
ArucoVideoWidget* videoWidget;
settings s;
double pose[6];
- cv::Mat frame;
+ cv::Mat frame, grayscale, color;
cv::Matx33d r;
+ std::vector<cv::Point3f> obj_points;
+ cv::Matx33d intrinsics;
+ cv::Matx14f dist_coeffs;
+ aruco::MarkerDetector detector;
+ std::vector<aruco::Marker> markers;
cv::Vec3d t;
+ cv::Vec3d rvec, tvec, rvec_, tvec_;
+ std::vector<cv::Point2f> roi_projection;
+ std::vector<cv::Point2f> repr2;
+ std::vector<cv::Point3f> centroid;
+ cv::Matx33d m_r, m_q, rmat;
+ cv::Vec3d euler;
+ std::vector<cv::Point3f> roi_points;
+ cv::Rect last_roi;
+ double freq, cur_fps;
+ std::uint64_t last_time;
+
+ static constexpr float size_min = 0.05f;
+ static constexpr float size_max = 0.3f;
+
+ static constexpr double alpha_ = .985;
};
class TrackerControls : public ITrackerDialog, protected camera_dialog