diff options
Diffstat (limited to 'tracker-pt/module')
| -rw-r--r-- | tracker-pt/module/CMakeLists.txt | 5 | ||||
| -rw-r--r-- | tracker-pt/module/camera.cpp | 21 | ||||
| -rw-r--r-- | tracker-pt/module/camera.h | 4 | ||||
| -rw-r--r-- | tracker-pt/module/frame.cpp | 53 | ||||
| -rw-r--r-- | tracker-pt/module/frame.hpp | 8 | ||||
| -rw-r--r-- | tracker-pt/module/lang/de_DE.ts | 11 | ||||
| -rw-r--r-- | tracker-pt/module/lang/zh_CN.ts | 2 | ||||
| -rw-r--r-- | tracker-pt/module/point_extractor.cpp | 178 | ||||
| -rw-r--r-- | tracker-pt/module/point_extractor.h | 17 |
9 files changed, 197 insertions, 102 deletions
diff --git a/tracker-pt/module/CMakeLists.txt b/tracker-pt/module/CMakeLists.txt index f7a6cc7f..b7fc974f 100644 --- a/tracker-pt/module/CMakeLists.txt +++ b/tracker-pt/module/CMakeLists.txt @@ -1,7 +1,10 @@ include(opentrack-opencv) find_package(OpenCV QUIET) if(OpenCV_FOUND) + foreach(k core imgproc) + otr_install_lib("opencv_${k}" "${opentrack-libexec}") + endforeach() otr_module(tracker-pt) - target_link_libraries(${self} opentrack-tracker-pt-base) + target_link_libraries(${self} opentrack-video opencv_imgproc opentrack-tracker-pt-base) target_include_directories(${self} PUBLIC "${CMAKE_SOURCE_DIR}/tracker-pt") endif() diff --git a/tracker-pt/module/camera.cpp b/tracker-pt/module/camera.cpp index a70698de..1beba474 100644 --- a/tracker-pt/module/camera.cpp +++ b/tracker-pt/module/camera.cpp @@ -7,10 +7,7 @@ #include "camera.h" #include "frame.hpp" - -#include "compat/math-imports.hpp" - -#include <opencv2/core.hpp> +#include <opencv2/core/mat.hpp> namespace pt_module { @@ -73,23 +70,29 @@ Camera::result Camera::get_frame(pt_frame& frame_) return { false, {} }; } -bool Camera::start(const QString& name, int fps, int res_x, int res_y) +bool Camera::start(const pt_settings& s) { + int fps = s.cam_fps, res_x = s.cam_res_x, res_y = s.cam_res_y; + QString name = s.camera_name; + bool use_mjpeg = s.use_mjpeg; + if (fps >= 0 && res_x >= 0 && res_y >= 0) { if (cam_desired.name != name || (int)cam_desired.fps != fps || cam_desired.res_x != res_x || cam_desired.res_y != res_y || + cam_desired.use_mjpeg != use_mjpeg || !cap || !cap->is_open()) { stop(); cam_desired.name = name; - cam_desired.fps = fps; + cam_desired.fps = (f)fps; cam_desired.res_x = res_x; cam_desired.res_y = res_y; cam_desired.fov = fov; + cam_desired.use_mjpeg = use_mjpeg; cap = video::make_camera(name); @@ -100,12 +103,16 @@ bool Camera::start(const QString& name, int fps, int res_x, int res_y) info.fps = fps; info.width = res_x; info.height = res_y; + info.use_mjpeg = use_mjpeg; + info.num_channels = s.blob_color == pt_color_hardware ? 1 : 3; if (!cap->start(info)) goto fail; cam_info = pt_camera_info(); cam_info.name = name; + cam_info.use_mjpeg = use_mjpeg; + cam_info.fov = (f)s.fov; dt_mean = 0; cv::Mat tmp; @@ -141,7 +148,7 @@ bool Camera::get_frame_(cv::Mat& img) int stride = frame.stride; if (stride == 0) stride = cv::Mat::AUTO_STEP; - img = cv::Mat(frame.height, frame.width, CV_8UC(frame.channels), (void*)frame.data, stride); + img = cv::Mat(frame.height, frame.width, CV_8UC(frame.channels), (void*)frame.data, (size_t)stride); return true; } } diff --git a/tracker-pt/module/camera.h b/tracker-pt/module/camera.h index e2ba159f..e4772178 100644 --- a/tracker-pt/module/camera.h +++ b/tracker-pt/module/camera.h @@ -13,8 +13,6 @@ #include <memory> -#include <opencv2/core.hpp> - #include <QString> namespace pt_module { @@ -23,7 +21,7 @@ struct Camera final : pt_camera { Camera(const QString& module_name); - bool start(const QString& name, int fps, int res_x, int res_y) override; + bool start(const pt_settings& s) override; void stop() override; result get_frame(pt_frame& Frame) override; diff --git a/tracker-pt/module/frame.cpp b/tracker-pt/module/frame.cpp index ab110871..1a276f16 100644 --- a/tracker-pt/module/frame.cpp +++ b/tracker-pt/module/frame.cpp @@ -1,43 +1,52 @@ #include "frame.hpp" - #include "compat/math.hpp" - #include <opencv2/imgproc.hpp> namespace pt_module { -Preview& Preview::operator=(const pt_frame& frame_) +void Preview::set_last_frame(const pt_frame& frame_) { const cv::Mat& frame = frame_.as_const<const Frame>()->mat; + const bool need_resize = frame.size != frame_copy.size; - if (frame.channels() != 3) + if (frame.channels() == 1) { - eval_once(qDebug() << "tracker/pt: camera frame depth: 3 !=" << frame.channels()); - return *this; + if (need_resize) + { + frame_tmp.create(frame.size(), CV_8UC3); + cv::cvtColor(frame, frame_tmp, cv::COLOR_GRAY2BGR); + cv::resize(frame_tmp, frame_copy, frame_copy.size(), 0, 0, cv::INTER_NEAREST); + } + else + cv::cvtColor(frame, frame_copy, cv::COLOR_GRAY2BGR); + } + else if (frame.channels() == 3) + { + if (need_resize) + cv::resize(frame, frame_copy, frame_copy.size(), 0, 0, cv::INTER_NEAREST); + else + frame.copyTo(frame_copy); } - - const bool need_resize = frame.cols != frame_out.cols || frame.rows != frame_out.rows; - if (need_resize) - cv::resize(frame, frame_copy, cv::Size(frame_out.cols, frame_out.rows), 0, 0, cv::INTER_NEAREST); else - frame.copyTo(frame_copy); - - return *this; + { + eval_once(qDebug() << "tracker/pt: camera frame depth" << frame.channels() << "!= 3"); + frame_copy.create(frame_copy.size(), CV_8UC3); + frame_copy.setTo({0}); + } } Preview::Preview(int w, int h) { - ensure_size(frame_out, w, h, CV_8UC4); - ensure_size(frame_copy, w, h, CV_8UC3); - - frame_copy.setTo(cv::Scalar(0, 0, 0)); + frame_out.create(h, w, CV_8UC4); + frame_copy.create(h, w, CV_8UC3); + frame_copy.setTo({0}); } QImage Preview::get_bitmap() { - int stride = frame_out.step.p[0]; + int stride = (int)frame_out.step.p[0]; - if (stride < 64 || stride < frame_out.cols * 4) + if (stride < frame_out.cols * 4) { eval_once(qDebug() << "bad stride" << stride << "for bitmap size" << frame_copy.cols << frame_copy.rows); @@ -71,10 +80,4 @@ void Preview::draw_head_center(f x, f y) color, 1); } -void Preview::ensure_size(cv::Mat& frame, int w, int h, int type) -{ - if (frame.cols != w || frame.rows != h || frame.type() != type) - frame = cv::Mat(h, w, type); -} - } // ns pt_module diff --git a/tracker-pt/module/frame.hpp b/tracker-pt/module/frame.hpp index 89334599..0569a323 100644 --- a/tracker-pt/module/frame.hpp +++ b/tracker-pt/module/frame.hpp @@ -2,7 +2,7 @@ #include "pt-api.hpp" -#include <opencv2/core.hpp> +#include <opencv2/core/mat.hpp> #include <QImage> #ifdef __clang__ @@ -24,7 +24,7 @@ struct Preview final : pt_preview { Preview(int w, int h); - Preview& operator=(const pt_frame& frame) override; + void set_last_frame(const pt_frame& frame) override; QImage get_bitmap() override; void draw_head_center(f x, f y) override; @@ -32,9 +32,7 @@ struct Preview final : pt_preview operator cv::Mat const&() const { return frame_copy; } private: - static void ensure_size(cv::Mat& frame, int w, int h, int type); - - cv::Mat frame_copy, frame_out; + cv::Mat frame_copy, frame_out, frame_tmp; }; } // ns pt_module diff --git a/tracker-pt/module/lang/de_DE.ts b/tracker-pt/module/lang/de_DE.ts new file mode 100644 index 00000000..6c548aba --- /dev/null +++ b/tracker-pt/module/lang/de_DE.ts @@ -0,0 +1,11 @@ +<?xml version="1.0" encoding="utf-8"?> +<!DOCTYPE TS> +<TS version="2.1" language="de_DE"> +<context> + <name>pt_module::metadata_pt</name> + <message> + <source>PointTracker 1.1</source> + <translation>PointTracker 1.1</translation> + </message> +</context> +</TS> diff --git a/tracker-pt/module/lang/zh_CN.ts b/tracker-pt/module/lang/zh_CN.ts index 03d19f4e..c39728a1 100644 --- a/tracker-pt/module/lang/zh_CN.ts +++ b/tracker-pt/module/lang/zh_CN.ts @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="utf-8"?> <!DOCTYPE TS> -<TS version="2.1"> +<TS version="2.1" language="zh_CN"> <context> <name>pt_module::metadata_pt</name> <message> diff --git a/tracker-pt/module/point_extractor.cpp b/tracker-pt/module/point_extractor.cpp index 7ad9925a..3329fafc 100644 --- a/tracker-pt/module/point_extractor.cpp +++ b/tracker-pt/module/point_extractor.cpp @@ -9,9 +9,11 @@ #include "point_extractor.h" #include "point_tracker.h" #include "frame.hpp" - #include "cv/numeric.hpp" #include "compat/math.hpp" +#include "compat/math-imports.hpp" + +#include <opencv2/imgproc.hpp> #undef PREVIEW //#define PREVIEW @@ -33,7 +35,7 @@ using namespace numeric_types; /* http://en.wikipedia.org/wiki/Mean-shift -In this application the idea, is to eliminate any bias of the point estimate +In this application the idea, is to eliminate any bias of the point estimate which is introduced by the rather arbitrary thresholded area. One must recognize that the thresholded area can only move in one pixel increments since it is binary. Thus, its center of mass might make "jumps" as pixels are added/removed @@ -42,9 +44,9 @@ With mean-shift, a moving "window" or kernel is multiplied with the gray-scale image, and the COM is calculated of the result. This is iterated where the kernel center is set the previously computed COM. Thus, peaks in the image intensity distribution "pull" the kernel towards themselves. Eventually it stops moving, i.e. -then the computed COM coincides with the kernel center. We hope that the +then the computed COM coincides with the kernel center. We hope that the corresponding location is a good candidate for the extracted point. -The idea similar to the window scaling suggested in Berglund et al. "Fast, bias-free +The idea similar to the window scaling suggested in Berglund et al. "Fast, bias-free algorithm for tracking single particles with variable size and shape." (2008). */ static vec2 MeanShiftIteration(const cv::Mat1b &frame_gray, const vec2 ¤t_center, f filter_width) @@ -87,21 +89,16 @@ PointExtractor::PointExtractor(const QString& module_name) : s(module_name) void PointExtractor::ensure_channel_buffers(const cv::Mat& orig_frame) { - if (ch[0].rows != orig_frame.rows || ch[0].cols != orig_frame.cols) - for (cv::Mat1b& x : ch) - x = cv::Mat1b(orig_frame.rows, orig_frame.cols); + for (cv::Mat1b& x : ch) + x.create(orig_frame.rows, orig_frame.cols); } void PointExtractor::ensure_buffers(const cv::Mat& frame) { const int W = frame.cols, H = frame.rows; - if (frame_gray.rows != W || frame_gray.cols != H) - { - frame_gray = cv::Mat1b(H, W); - frame_bin = cv::Mat1b(H, W); - frame_gray_unmasked = cv::Mat1b(H, W); - } + frame_gray.create(H, W); + frame_bin.create(H, W); } void PointExtractor::extract_single_channel(const cv::Mat& orig_frame, int idx, cv::Mat1b& dest) @@ -115,8 +112,43 @@ void PointExtractor::extract_single_channel(const cv::Mat& orig_frame, int idx, cv::mixChannels(&orig_frame, 1, &dest, 1, from_to, 1); } +void PointExtractor::filter_single_channel(const cv::Mat& orig_frame, float r, float g, float b, bool overexp, cv::Mat1b& dest) +{ + ensure_channel_buffers(orig_frame); + + // just filter for colour or also include overexposed regions? + if (!overexp) + cv::transform(orig_frame, dest, cv::Mat(cv::Matx13f(b, g, r))); + else + { + for (int i = 0; i < orig_frame.rows; i++) + { + cv::Vec3b const* const __restrict orig_ptr = orig_frame.ptr<cv::Vec3b>(i); + uint8_t* const __restrict dest_ptr = dest.ptr(i); + for (int j = 0; j < orig_frame.cols; j++) + { + // get the intensity of the key color (i.e. +ve coefficients) + uchar blue = orig_ptr[j][0], green = orig_ptr[j][1], red = orig_ptr[j][2]; + float key = std::max(b, 0.0f) * blue + std::max(g, 0.0f) * green + std::max(r, 0.0f) * red; + // get the intensity of the non-key color (i.e. -ve coefficients) + float nonkey = std::max(-b, 0.0f) * blue + std::max(-g, 0.0f) * green + std::max(-r, 0.0f) * red; + // the result is key color minus non-key color inversely weighted by key colour intensity + dest_ptr[j] = std::max(0.0f, std::min(255.0f, key - (255.0f - key) / 255.0f * nonkey)); + } + } + } +} + void PointExtractor::color_to_grayscale(const cv::Mat& frame, cv::Mat1b& output) { + if (frame.channels() == 1) + { + output.create(frame.rows, frame.cols); + frame.copyTo(output); + return; + } + + const float half_chr_key_str = *s.chroma_key_strength * 0.5; switch (s.blob_color) { case pt_color_green_only: @@ -134,18 +166,44 @@ void PointExtractor::color_to_grayscale(const cv::Mat& frame, cv::Mat1b& output) extract_single_channel(frame, 2, output); break; } - case pt_color_average: + case pt_color_red_chromakey: + { + filter_single_channel(frame, 1, -half_chr_key_str, -half_chr_key_str, s.chroma_key_overexposed, output); + break; + } + case pt_color_green_chromakey: + { + filter_single_channel(frame, -half_chr_key_str, 1, -half_chr_key_str, s.chroma_key_overexposed, output); + break; + } + case pt_color_blue_chromakey: + { + filter_single_channel(frame, -half_chr_key_str, -half_chr_key_str, 1, s.chroma_key_overexposed, output); + break; + } + case pt_color_cyan_chromakey: + { + filter_single_channel(frame, -*s.chroma_key_strength, 0.5, 0.5, s.chroma_key_overexposed, output); + break; + } + case pt_color_yellow_chromakey: + { + filter_single_channel(frame, 0.5, 0.5, -*s.chroma_key_strength, s.chroma_key_overexposed, output); + break; + } + case pt_color_magenta_chromakey: { - const int W = frame.cols, H = frame.rows, sz = W*H; - cv::reduce(frame.reshape(1, sz), - output.reshape(1, sz), - 1, cv::REDUCE_AVG); + filter_single_channel(frame, 0.5, -*s.chroma_key_strength, 0.5, s.chroma_key_overexposed, output); break; } + case pt_color_hardware: + eval_once(qDebug() << "camera driver doesn't support grayscale"); + goto do_grayscale; default: eval_once(qDebug() << "wrong pt_color_type enum value" << int(s.blob_color)); [[fallthrough]]; - case pt_color_natural: + case pt_color_bt709: +do_grayscale: cv::cvtColor(frame, output, cv::COLOR_BGR2GRAY); break; } @@ -177,11 +235,10 @@ void PointExtractor::threshold_image(const cv::Mat& frame_gray, cv::Mat1b& outpu const f radius = threshold_radius_value(frame_gray.cols, frame_gray.rows, threshold_slider_value); float const* const __restrict ptr = hist.ptr<float>(0); - const unsigned area = uround(3 * pi * radius*radius); + const unsigned area = unsigned(iround(3 * pi * radius*radius)); const unsigned sz = unsigned(hist.cols * hist.rows); - constexpr unsigned min_thres = 64; - unsigned thres = min_thres; - for (unsigned i = sz-1, cnt = 0; i > 32; i--) + unsigned thres = 1; + for (unsigned i = sz-1, cnt = 0; i > 1; i--) { cnt += (unsigned)ptr[i]; if (cnt >= area) @@ -209,20 +266,15 @@ static void draw_blobs(cv::Mat& preview_frame, const blob* blobs, unsigned nblob cy = preview_frame.rows / f(size.height), c = std::fmax(f(1), cx+cy)/2; - constexpr unsigned fract_bits = 8; - constexpr int c_fract(1 << fract_bits); - - cv::Point p(iround(b.pos[0] * cx * c_fract), iround(b.pos[1] * cy * c_fract)); + cv::Point p(iround(b.pos[0] * cx), iround(b.pos[1] * cy)); - auto circle_color = k >= PointModel::N_POINTS - ? cv::Scalar(192, 192, 192) - : cv::Scalar(255, 255, 0); + auto outline_color = k >= PointModel::N_POINTS + ? cv::Scalar(192, 192, 192) + : cv::Scalar(255, 255, 0); - const int overlay_size = iround(dpi); - - cv::circle(preview_frame, p, iround((b.radius + f(3.3) * c) * c_fract), - circle_color, overlay_size, - cv::LINE_AA, fract_bits); + cv::ellipse(preview_frame, p, + {iround(b.rect.width/(f)2+2*c), iround(b.rect.height/(f)2+2*c)}, + 0, 0, 360, outline_color, iround(dpi), cv::LINE_AA); char buf[16]; std::snprintf(buf, sizeof(buf), "%.2fpx", (double)b.radius); @@ -233,25 +285,51 @@ static void draw_blobs(cv::Mat& preview_frame, const blob* blobs, unsigned nblob cv::Point pos(iround(b.pos[0]*cx+offx), iround(b.pos[1]*cy+offy)); cv::putText(preview_frame, buf, pos, - cv::FONT_HERSHEY_PLAIN, overlay_size, text_color, + cv::FONT_HERSHEY_PLAIN, iround(dpi), text_color, 1); } } -void PointExtractor::extract_points(const pt_frame& frame_, pt_preview& preview_frame_, std::vector<vec2>& points) +static vec2 meanshift_initial_guess(const cv::Rect rect, cv::Mat& frame_roi) +{ + vec2 ret = {rect.width/(f)2, rect.height/(f)2}; + + // compute center initial guess + double ynorm = 0, xnorm = 0, y = 0, x = 0; + for (int j = 0; j < rect.height; j++) + { + const unsigned char* __restrict ptr = frame_roi.ptr<unsigned char>(j); + for (int i = 0; i < rect.width; i++) + { + double val = ptr[i] * 1./255; + x += i * val; + y += j * val; + xnorm += val; + ynorm += val; + } + } + constexpr double eps = 1e-4; + if (xnorm > eps && ynorm > eps) + ret = { (f)(x / xnorm), (f)(y / ynorm) }; + return ret; +} + +void PointExtractor::extract_points(const pt_frame& frame_, + pt_preview& preview_frame_, + bool preview_visible, + std::vector<vec2>& points) { const cv::Mat& frame = frame_.as_const<Frame>()->mat; ensure_buffers(frame); - color_to_grayscale(frame, frame_gray_unmasked); + color_to_grayscale(frame, frame_gray); #if defined PREVIEW cv::imshow("capture", frame_gray); cv::waitKey(1); #endif - threshold_image(frame_gray_unmasked, frame_bin); - frame_gray_unmasked.copyTo(frame_gray, frame_bin); + threshold_image(frame_gray, frame_bin); const f region_size_min = (f)s.min_point_size; const f region_size_max = (f)s.max_point_size; @@ -299,7 +377,7 @@ void PointExtractor::extract_points(const pt_frame& frame_, pt_preview& preview_ } } - const f radius = std::sqrt(cnt / pi); + const f radius = std::sqrt((f)cnt) / std::sqrt(pi); if (radius > region_size_max || radius < region_size_min) continue; @@ -329,21 +407,14 @@ end: for (idx = 0; idx < sz; ++idx) { blob& b = blobs[idx]; - cv::Rect rect = b.rect; - - rect.x -= rect.width / 2; - rect.y -= rect.height / 2; - rect.width *= 2; - rect.height *= 2; - rect &= cv::Rect(0, 0, W, H); // crop at frame boundaries - + cv::Rect rect = b.rect & cv::Rect(0, 0, W, H); // crop at frame boundaries cv::Mat frame_roi = frame_gray(rect); // smaller values mean more changes. 1 makes too many changes while 1.5 makes about .1 static constexpr f radius_c = f(1.75); const f kernel_radius = b.radius * radius_c; - vec2 pos(rect.width/f(2), rect.height/f(2)); // position relative to ROI. + vec2 pos = meanshift_initial_guess(rect, frame_roi); // position relative to ROI. for (int iter = 0; iter < 10; ++iter) { @@ -358,9 +429,10 @@ end: b.pos[1] = pos[1] + rect.y; } - draw_blobs(preview_frame_.as<Frame>()->mat, - blobs.data(), blobs.size(), - frame_gray.size()); + if (preview_visible) + draw_blobs(preview_frame_.as<Frame>()->mat, + blobs.data(), blobs.size(), + frame_gray.size()); // End of mean shift code. At this point, blob positions are updated with hopefully less noisy less biased values. diff --git a/tracker-pt/module/point_extractor.h b/tracker-pt/module/point_extractor.h index a6103667..fbfdbb0b 100644 --- a/tracker-pt/module/point_extractor.h +++ b/tracker-pt/module/point_extractor.h @@ -9,12 +9,10 @@ #pragma once #include "pt-api.hpp" - +#include <opencv2/core/mat.hpp> +#include <opencv2/core/types.hpp> #include <vector> -#include <opencv2/core.hpp> -#include <opencv2/imgproc.hpp> - namespace pt_module { using namespace numeric_types; @@ -33,14 +31,18 @@ class PointExtractor final : public pt_point_extractor public: // extracts points from frame and draws some processing info into frame, if draw_output is set // dt: time since last call in seconds - void extract_points(const pt_frame& frame, pt_preview& preview_frame, std::vector<vec2>& points) override; - PointExtractor(const QString& module_name); + void extract_points(const pt_frame& frame, + pt_preview& preview_frame, bool preview_visible, + std::vector<vec2>& points) override; + + explicit PointExtractor(const QString& module_name); + private: static constexpr int max_blobs = 16; pt_settings s; - cv::Mat1b frame_gray_unmasked, frame_bin, frame_gray; + cv::Mat1b frame_bin, frame_gray; cv::Mat1f hist; std::vector<blob> blobs; cv::Mat1b ch[3]; @@ -49,6 +51,7 @@ private: void ensure_buffers(const cv::Mat& frame); void extract_single_channel(const cv::Mat& orig_frame, int idx, cv::Mat1b& dest); + void filter_single_channel(const cv::Mat& orig_frame, float r, float g, float b, bool overexp, cv::Mat1b& dest); void color_to_grayscale(const cv::Mat& frame, cv::Mat1b& output); void threshold_image(const cv::Mat& frame_gray, cv::Mat1b& output); |