From 47bc9e57ad55128ae0a5bf206f3931a3aa4f5dc1 Mon Sep 17 00:00:00 2001 From: Stanislaw Halik Date: Sun, 11 Feb 2024 15:01:26 +0100 Subject: tracker/nn: update from master --- cmake/FindONNXRuntime.cmake | 1 + tracker-neuralnet/CMakeLists.txt | 4 +- tracker-neuralnet/ftnoir_tracker_neuralnet.cpp | 71 ++++++++++++++++++++-- tracker-neuralnet/ftnoir_tracker_neuralnet.h | 5 +- tracker-neuralnet/lang/nl_NL.ts | 34 +++++++---- tracker-neuralnet/lang/ru_RU.ts | 8 +++ tracker-neuralnet/lang/stub.ts | 34 +++++++---- tracker-neuralnet/lang/zh_CN.ts | 61 +++++++++++-------- tracker-neuralnet/models/head-pose-0.2-small.onnx | Bin 0 -> 12975846 bytes tracker-neuralnet/models/head-pose.onnx | Bin 18370187 -> 0 bytes 10 files changed, 158 insertions(+), 60 deletions(-) create mode 100644 tracker-neuralnet/models/head-pose-0.2-small.onnx delete mode 100644 tracker-neuralnet/models/head-pose.onnx diff --git a/cmake/FindONNXRuntime.cmake b/cmake/FindONNXRuntime.cmake index af5093f3..60ddfc2b 100644 --- a/cmake/FindONNXRuntime.cmake +++ b/cmake/FindONNXRuntime.cmake @@ -74,6 +74,7 @@ find_path(ONNXRuntime_INCLUDE_DIR onnxruntime_cxx_api.h "build/native/include" # For when the directory structure of the onnx source repo is preserved "include/onnxruntime/core/session" + "include/onnxruntime" # For when we copy the files somewhere "include" ) diff --git a/tracker-neuralnet/CMakeLists.txt b/tracker-neuralnet/CMakeLists.txt index db568fae..35071f27 100644 --- a/tracker-neuralnet/CMakeLists.txt +++ b/tracker-neuralnet/CMakeLists.txt @@ -32,8 +32,8 @@ if(OpenCV_FOUND AND ONNXRuntime_FOUND AND OpenMP_FOUND) endif() install( - FILES "models/head-localizer.onnx" - "models/head-pose.onnx" + FILES "models/head-localizer.onnx" + "models/head-pose-0.2-small.onnx" DESTINATION "${opentrack-libexec}/models" PERMISSIONS ${opentrack-perms-file} ) diff --git a/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp b/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp index 34368478..dfee19c8 100644 --- a/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp +++ b/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp @@ -28,6 +28,8 @@ #include #include #include +#include +#include #include #include @@ -57,6 +59,12 @@ std::string convert(const QString &s) { return s.toStdString(); } #endif +QDir get_default_model_directory() +{ + return QDir(OPENTRACK_BASE_PATH+ "/" OPENTRACK_LIBRARY_PATH "models"); +} + + int enum_to_fps(int value) { switch (value) @@ -465,8 +473,7 @@ bool NeuralNetTracker::load_and_initialize_model() { const QString localizer_model_path_enc = OPENTRACK_BASE_PATH+"/" OPENTRACK_LIBRARY_PATH "/models/head-localizer.onnx"; - const QString poseestimator_model_path_enc = - OPENTRACK_BASE_PATH+"/" OPENTRACK_LIBRARY_PATH "/models/head-pose.onnx"; + const QString poseestimator_model_path_enc = get_posenet_filename(); try { @@ -486,6 +493,7 @@ bool NeuralNetTracker::load_and_initialize_model() allocator_info_, Ort::Session{env_, convert(localizer_model_path_enc).c_str(), opts}); + qDebug() << "Loading pose net " << poseestimator_model_path_enc; poseestimator_.emplace( allocator_info_, Ort::Session{env_, convert(poseestimator_model_path_enc).c_str(), opts}); @@ -496,6 +504,11 @@ bool NeuralNetTracker::load_and_initialize_model() << e.what(); return false; } + catch (const std::exception &e) + { + qDebug() << "Failed to initialize the neural network models. Error message: " << e.what(); + return false; + } return true; } @@ -682,7 +695,7 @@ void NeuralNetTracker::data(double *data) const auto& mx = tmp.R.col(0); const auto& my = tmp.R.col(1); - const auto& mz = -tmp.R.col(2); + const auto& mz = tmp.R.col(2); const float yaw = std::atan2(mx(2), mx(0)); const float pitch = -std::atan2(-mx(1), std::sqrt(mx(2)*mx(2)+mx(0)*mx(0))); @@ -707,12 +720,31 @@ Affine NeuralNetTracker::pose() return last_pose_affine_ ? *last_pose_affine_ : Affine{}; } + std::tuple NeuralNetTracker::stats() const { QMutexLocker lck(&stats_mtx_); return { resolution_, fps_, inference_time_ }; } + +QString NeuralNetTracker::get_posenet_filename() const +{ + QString filename = settings_.posenet_file; + if (QFileInfo(filename).isRelative()) + filename = get_default_model_directory().absoluteFilePath(filename); + return filename; +} + + + + + + + + + + void NeuralNetDialog::make_fps_combobox() { #if 0 @@ -789,8 +821,7 @@ NeuralNetDialog::NeuralNetDialog() : connect(ui_.buttonBox, SIGNAL(accepted()), this, SLOT(doOK())); connect(ui_.buttonBox, SIGNAL(rejected()), this, SLOT(doCancel())); connect(ui_.camera_settings, SIGNAL(clicked()), this, SLOT(camera_settings())); - - connect(&settings_.camera_name, value_::value_changed(), this, &NeuralNetDialog::update_camera_settings_state); + //connect(ui_.posenetSelectButton, SIGNAL(clicked()), this, SLOT(onSelectPoseNetFile())); update_camera_settings_state(settings_.camera_name); @@ -798,6 +829,11 @@ NeuralNetDialog::NeuralNetDialog() : calib_timer_.setInterval(35); connect(ui_.tcalib_button,SIGNAL(toggled(bool)), this, SLOT(startstop_trans_calib(bool))); + connect(&cs_.exposure, value_::value_changed(), this, [this](int value) { + ui_.camera_settings->setEnabled(value == (int)exposure_preset::ignored); + }); + ui_.camera_settings->setEnabled(cs_.exposure == exposure_preset::ignored); + connect(&tracker_status_poll_timer_, &QTimer::timeout, this, &NeuralNetDialog::status_poll); tracker_status_poll_timer_.setInterval(250); tracker_status_poll_timer_.start(); @@ -958,6 +994,31 @@ void NeuralNetDialog::startstop_trans_calib(bool start) } +void NeuralNetDialog::onSelectPoseNetFile() +{ + const auto root = get_default_model_directory(); + // Start with the current setting + QString filename = settings_.posenet_file; + // If the filename is relative then assume that the file is located under the + // model directory. Under regular use this should always be the case. + if (QFileInfo(filename).isRelative()) + filename = root.absoluteFilePath(filename); + filename = QFileDialog::getOpenFileName(this, + tr("Select Pose Net ONNX"), filename, tr("ONNX Files (*.onnx)")); + // In case the user aborted. + if (filename.isEmpty()) + return; + // When a file under the model directory was selected we can get rid of the + // directory prefix. This is more robust than storing absolute paths, e.g. + // in case the user moves the opentrack install folder / reuses old settings. + // When the file is not in the model directory, we have to use the absolute path, + // which is also fine as developer feature. + if (filename.startsWith(root.absolutePath())) + filename = root.relativeFilePath(filename); + settings_.posenet_file = filename; +} + + Settings::Settings() : opts("neuralnet-tracker") {} } // neuralnet_tracker_ns diff --git a/tracker-neuralnet/ftnoir_tracker_neuralnet.h b/tracker-neuralnet/ftnoir_tracker_neuralnet.h index d44a9953..e5696cc9 100644 --- a/tracker-neuralnet/ftnoir_tracker_neuralnet.h +++ b/tracker-neuralnet/ftnoir_tracker_neuralnet.h @@ -86,6 +86,7 @@ struct Settings : opts { value resolution { b, "force-resolution", 0 }; value deadzone_size { b, "deadzone-size", 1. }; value deadzone_hardness { b, "deadzone-hardness", 1.5 }; + value posenet_file { b, "posenet-file", "head-pose-0.2-small.onnx" }; Settings(); }; @@ -128,6 +129,7 @@ private: QuatPose compute_filtered_pose(const PoseEstimator::Face &face); // Compute the pose in 3d space taking the network outputs QuatPose transform_to_world_pose(const cv::Quatf &face_rotation, const cv::Point2f& face_xy, const float face_size) const; + QString get_posenet_filename() const; Settings settings_; std::optional localizer_; @@ -146,7 +148,7 @@ private: double fps_ = 0; double inference_time_ = 0; cv::Size resolution_ = {}; - + static constexpr double RC = .25; int num_threads_ = 1; bool is_visible_ = true; @@ -196,6 +198,7 @@ private Q_SLOTS: void startstop_trans_calib(bool start); void trans_calib_step(); void status_poll(); + void onSelectPoseNetFile(); }; diff --git a/tracker-neuralnet/lang/nl_NL.ts b/tracker-neuralnet/lang/nl_NL.ts index 52a7c174..8972e35b 100644 --- a/tracker-neuralnet/lang/nl_NL.ts +++ b/tracker-neuralnet/lang/nl_NL.ts @@ -11,6 +11,10 @@ Camera settings Camera-instellingen + + Camera Configuration + + Head Center Offset @@ -41,21 +45,13 @@ Don't roll or change position. - Exposure preset - - - - Camera Configuration + Show Network Input Tuning / Debug - - Number of threads. Can be used to balance the CPU load between the game and the tracker. - - ROI Smoothing Alpha @@ -65,19 +61,19 @@ Don't roll or change position. - Show the image patch that the pose estimation model sees. + Thread Count - Show Network Input + Number of threads. Can be used to balance the CPU load between the game and the tracker. - Amount of smoothing of the face region coordinates. Can help stabilize the pose. + Show the image patch that the pose estimation model sees. - Thread Count + Amount of smoothing of the face region coordinates. Can help stabilize the pose. @@ -88,6 +84,10 @@ Don't roll or change position. Camera override + + Exposure preset + + neuralnet_tracker_ns::NeuralNetDialog @@ -123,5 +123,13 @@ Don't roll or change position. Start calibration + + Select Pose Net ONNX + + + + ONNX Files (*.onnx) + + diff --git a/tracker-neuralnet/lang/ru_RU.ts b/tracker-neuralnet/lang/ru_RU.ts index cbfce1d5..f88e0529 100644 --- a/tracker-neuralnet/lang/ru_RU.ts +++ b/tracker-neuralnet/lang/ru_RU.ts @@ -126,5 +126,13 @@ Don't roll or change position. Start calibration Начать калибровку + + Select Pose Net ONNX + + + + ONNX Files (*.onnx) + + diff --git a/tracker-neuralnet/lang/stub.ts b/tracker-neuralnet/lang/stub.ts index 5936d630..df883d3f 100644 --- a/tracker-neuralnet/lang/stub.ts +++ b/tracker-neuralnet/lang/stub.ts @@ -11,6 +11,10 @@ Camera settings + + Camera Configuration + + Head Center Offset @@ -41,21 +45,13 @@ Don't roll or change position. - Exposure preset - - - - Camera Configuration + Show Network Input Tuning / Debug - - Number of threads. Can be used to balance the CPU load between the game and the tracker. - - ROI Smoothing Alpha @@ -65,19 +61,19 @@ Don't roll or change position. - Show the image patch that the pose estimation model sees. + Thread Count - Show Network Input + Number of threads. Can be used to balance the CPU load between the game and the tracker. - Amount of smoothing of the face region coordinates. Can help stabilize the pose. + Show the image patch that the pose estimation model sees. - Thread Count + Amount of smoothing of the face region coordinates. Can help stabilize the pose. @@ -88,6 +84,10 @@ Don't roll or change position. Camera override + + Exposure preset + + neuralnet_tracker_ns::NeuralNetDialog @@ -123,5 +123,13 @@ Don't roll or change position. Start calibration + + Select Pose Net ONNX + + + + ONNX Files (*.onnx) + + diff --git a/tracker-neuralnet/lang/zh_CN.ts b/tracker-neuralnet/lang/zh_CN.ts index 13dc34cf..efac5204 100644 --- a/tracker-neuralnet/lang/zh_CN.ts +++ b/tracker-neuralnet/lang/zh_CN.ts @@ -5,15 +5,19 @@ Form Tracker settings - + 追踪器设置 Camera settings - + 相机设置 + + + Camera Configuration + 相机配置 Head Center Offset - + 头部归中补偿 mm @@ -22,40 +26,33 @@ Use only yaw and pitch while calibrating. Don't roll or change position. - + 在校准时只使用偏航和俯仰, +不要滚转或是改变位置. Start calibration - + 开始校准 Right - + 向右 Forward - + 向前 Up - + 向上 - Exposure preset - - - - Camera Configuration - + Show Network Input + 展示神经网络输入 Tuning / Debug - - Number of threads. Can be used to balance the CPU load between the game and the tracker. - - ROI Smoothing Alpha @@ -65,19 +62,19 @@ Don't roll or change position. - Show the image patch that the pose estimation model sees. + Thread Count - Show Network Input + Number of threads. Can be used to balance the CPU load between the game and the tracker. - Amount of smoothing of the face region coordinates. Can help stabilize the pose. + Show the image patch that the pose estimation model sees. - Thread Count + Amount of smoothing of the face region coordinates. Can help stabilize the pose. @@ -88,20 +85,24 @@ Don't roll or change position. Camera override + + Exposure preset + + neuralnet_tracker_ns::NeuralNetDialog Default - + 默认 Tracker Offline - + 追踪器离线 %1x%2 @ %3 FPS / Inference: %4 ms - + %1x%2 @ %3 FPS / 推理: %4 ms %1 yaw samples. Yaw more to %2 samples for stable calibration. @@ -117,10 +118,18 @@ Don't roll or change position. Stop calibration - + 结束校准 Start calibration + 开始校准 + + + Select Pose Net ONNX + + + + ONNX Files (*.onnx) diff --git a/tracker-neuralnet/models/head-pose-0.2-small.onnx b/tracker-neuralnet/models/head-pose-0.2-small.onnx new file mode 100644 index 00000000..4da6bb6a Binary files /dev/null and b/tracker-neuralnet/models/head-pose-0.2-small.onnx differ diff --git a/tracker-neuralnet/models/head-pose.onnx b/tracker-neuralnet/models/head-pose.onnx deleted file mode 100644 index d70dfa49..00000000 Binary files a/tracker-neuralnet/models/head-pose.onnx and /dev/null differ -- cgit v1.2.3