tracker/nn: update from master

9 files changed, 158 insertions, 60 deletions

diff --git a/cmake/FindONNXRuntime.cmake b/cmake/FindONNXRuntime.cmake
index af5093f3..60ddfc2b 100644
--- a/cmake/FindONNXRuntime.cmake
+++ b/cmake/FindONNXRuntime.cmake
@@ -74,6 +74,7 @@ find_path(ONNXRuntime_INCLUDE_DIR onnxruntime_cxx_api.h
 		"build/native/include"
 		# For when the directory structure of the onnx source repo is preserved
 		"include/onnxruntime/core/session" 
+		"include/onnxruntime"
 		# For when we copy the files somewhere
 		"include"
 		)
diff --git a/tracker-neuralnet/CMakeLists.txt b/tracker-neuralnet/CMakeLists.txt
index db568fae..35071f27 100644
--- a/tracker-neuralnet/CMakeLists.txt
+++ b/tracker-neuralnet/CMakeLists.txt
@@ -32,8 +32,8 @@ if(OpenCV_FOUND AND ONNXRuntime_FOUND AND OpenMP_FOUND)
     endif()
 
     install(
-        FILES "models/head-localizer.onnx" 
-              "models/head-pose.onnx"
+        FILES "models/head-localizer.onnx"
+              "models/head-pose-0.2-small.onnx"
         DESTINATION "${opentrack-libexec}/models"
         PERMISSIONS ${opentrack-perms-file}
         )
diff --git a/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp b/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp
index 34368478..dfee19c8 100644
--- a/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp
+++ b/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp
@@ -28,6 +28,8 @@
 #include <QMutexLocker>
 #include <QDebug>
 #include <QFile>
+#include <QFileDialog>
+#include <QFileInfo>
 
 #include <cstdio>
 #include <cmath>
@@ -57,6 +59,12 @@ std::string convert(const QString &s) { return s.toStdString(); }
 #endif
 
 
+QDir get_default_model_directory()
+{
+    return QDir(OPENTRACK_BASE_PATH+ "/" OPENTRACK_LIBRARY_PATH "models");
+}
+
+
 int enum_to_fps(int value)
 {
     switch (value)
@@ -465,8 +473,7 @@ bool NeuralNetTracker::load_and_initialize_model()
 {
     const QString localizer_model_path_enc =
         OPENTRACK_BASE_PATH+"/" OPENTRACK_LIBRARY_PATH "/models/head-localizer.onnx";
-    const QString poseestimator_model_path_enc =
-        OPENTRACK_BASE_PATH+"/" OPENTRACK_LIBRARY_PATH "/models/head-pose.onnx";
+    const QString poseestimator_model_path_enc = get_posenet_filename();
 
     try
     {
@@ -486,6 +493,7 @@ bool NeuralNetTracker::load_and_initialize_model()
             allocator_info_, 
             Ort::Session{env_, convert(localizer_model_path_enc).c_str(), opts});
 
+        qDebug() << "Loading pose net " << poseestimator_model_path_enc;
         poseestimator_.emplace(
             allocator_info_,
             Ort::Session{env_, convert(poseestimator_model_path_enc).c_str(), opts});
@@ -496,6 +504,11 @@ bool NeuralNetTracker::load_and_initialize_model()
             << e.what();
         return false;
     }
+    catch (const std::exception &e)
+    {
+        qDebug() << "Failed to initialize the neural network models. Error message: " << e.what();
+        return false;
+    }
 
     return true;
 }
@@ -682,7 +695,7 @@ void NeuralNetTracker::data(double *data)
 
     const auto& mx = tmp.R.col(0);
     const auto& my = tmp.R.col(1);
-    const auto& mz = -tmp.R.col(2);
+    const auto& mz = tmp.R.col(2);
 
     const float yaw = std::atan2(mx(2), mx(0));
     const float pitch = -std::atan2(-mx(1), std::sqrt(mx(2)*mx(2)+mx(0)*mx(0)));
@@ -707,12 +720,31 @@ Affine NeuralNetTracker::pose()
     return last_pose_affine_ ? *last_pose_affine_ : Affine{};
 }
 
+
 std::tuple<cv::Size,double, double> NeuralNetTracker::stats() const
 {
     QMutexLocker lck(&stats_mtx_);
     return { resolution_, fps_, inference_time_ };
 }
 
+
+QString NeuralNetTracker::get_posenet_filename() const
+{
+    QString filename = settings_.posenet_file;
+    if (QFileInfo(filename).isRelative())
+        filename = get_default_model_directory().absoluteFilePath(filename);
+    return filename;
+}
+
+
+
+
+
+
+
+
+
+
 void NeuralNetDialog::make_fps_combobox()
 {
 #if 0
@@ -789,8 +821,7 @@ NeuralNetDialog::NeuralNetDialog() :
     connect(ui_.buttonBox, SIGNAL(accepted()), this, SLOT(doOK()));
     connect(ui_.buttonBox, SIGNAL(rejected()), this, SLOT(doCancel()));
     connect(ui_.camera_settings, SIGNAL(clicked()), this, SLOT(camera_settings()));
-
-    connect(&settings_.camera_name, value_::value_changed<QString>(), this, &NeuralNetDialog::update_camera_settings_state);
+    //connect(ui_.posenetSelectButton, SIGNAL(clicked()), this, SLOT(onSelectPoseNetFile()));
 
     update_camera_settings_state(settings_.camera_name);
 
@@ -798,6 +829,11 @@ NeuralNetDialog::NeuralNetDialog() :
     calib_timer_.setInterval(35);
     connect(ui_.tcalib_button,SIGNAL(toggled(bool)), this, SLOT(startstop_trans_calib(bool)));
 
+    connect(&cs_.exposure, value_::value_changed<int>(), this, [this](int value) {
+        ui_.camera_settings->setEnabled(value == (int)exposure_preset::ignored);
+    });
+    ui_.camera_settings->setEnabled(cs_.exposure == exposure_preset::ignored);
+
     connect(&tracker_status_poll_timer_, &QTimer::timeout, this, &NeuralNetDialog::status_poll);
     tracker_status_poll_timer_.setInterval(250);
     tracker_status_poll_timer_.start();
@@ -958,6 +994,31 @@ void NeuralNetDialog::startstop_trans_calib(bool start)
 }
 
 
+void NeuralNetDialog::onSelectPoseNetFile()
+{
+    const auto root = get_default_model_directory();
+    // Start with the current setting
+    QString filename = settings_.posenet_file;
+    // If the filename is relative then assume that the file is located under the
+    // model directory. Under regular use this should always be the case.
+    if (QFileInfo(filename).isRelative())
+        filename = root.absoluteFilePath(filename);
+    filename = QFileDialog::getOpenFileName(this,
+        tr("Select Pose Net ONNX"), filename, tr("ONNX Files (*.onnx)"));
+    // In case the user aborted.
+    if (filename.isEmpty())
+        return;
+    // When a file under the model directory was selected we can get rid of the
+    // directory prefix. This is more robust than storing absolute paths, e.g.
+    // in case the user moves the opentrack install folder / reuses old settings.
+    // When the file is not in the model directory, we have to use the absolute path,
+    // which is also fine as developer feature.
+    if (filename.startsWith(root.absolutePath()))
+        filename = root.relativeFilePath(filename);
+    settings_.posenet_file = filename;
+}
+
+
 Settings::Settings() : opts("neuralnet-tracker") {}
 
 } // neuralnet_tracker_ns
diff --git a/tracker-neuralnet/ftnoir_tracker_neuralnet.h b/tracker-neuralnet/ftnoir_tracker_neuralnet.h
index d44a9953..e5696cc9 100644
--- a/tracker-neuralnet/ftnoir_tracker_neuralnet.h
+++ b/tracker-neuralnet/ftnoir_tracker_neuralnet.h
@@ -86,6 +86,7 @@ struct Settings : opts {
     value<int> resolution { b, "force-resolution", 0 };
     value<double> deadzone_size { b, "deadzone-size", 1. };
     value<double> deadzone_hardness { b, "deadzone-hardness", 1.5 };
+    value<QString> posenet_file { b, "posenet-file", "head-pose-0.2-small.onnx" };
     Settings();
 };
 
@@ -128,6 +129,7 @@ private:
     QuatPose compute_filtered_pose(const PoseEstimator::Face &face);
     // Compute the pose in 3d space taking the network outputs
     QuatPose transform_to_world_pose(const cv::Quatf &face_rotation, const cv::Point2f& face_xy, const float face_size) const;
+    QString get_posenet_filename() const;
 
     Settings settings_;
     std::optional<Localizer> localizer_;
@@ -146,7 +148,7 @@ private:
     double fps_ = 0;
     double inference_time_ = 0;
     cv::Size resolution_ = {};
-    
+
     static constexpr double RC = .25;
     int num_threads_ = 1;
     bool is_visible_ = true;
@@ -196,6 +198,7 @@ private Q_SLOTS:
     void startstop_trans_calib(bool start);
     void trans_calib_step();
     void status_poll();
+    void onSelectPoseNetFile();
 };
 
 
diff --git a/tracker-neuralnet/lang/nl_NL.ts b/tracker-neuralnet/lang/nl_NL.ts
index 52a7c174..8972e35b 100644
--- a/tracker-neuralnet/lang/nl_NL.ts
+++ b/tracker-neuralnet/lang/nl_NL.ts
@@ -12,6 +12,10 @@
         <translation>Camera-instellingen</translation>
     </message>
     <message>
+        <source>Camera Configuration</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
         <source>Head Center Offset</source>
         <translation type="unfinished"></translation>
     </message>
@@ -41,11 +45,7 @@ Don&apos;t roll or change position.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Exposure preset</source>
-        <translation type="unfinished"></translation>
-    </message>
-    <message>
-        <source>Camera Configuration</source>
+        <source>Show Network Input</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
@@ -53,10 +53,6 @@ Don&apos;t roll or change position.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Number of threads. Can be used to balance the CPU load between the game and the tracker.</source>
-        <translation type="unfinished"></translation>
-    </message>
-    <message>
         <source>ROI Smoothing Alpha</source>
         <translation type="unfinished"></translation>
     </message>
@@ -65,19 +61,19 @@ Don&apos;t roll or change position.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Show the image patch that the pose estimation model sees.</source>
+        <source>Thread Count</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Show Network Input</source>
+        <source>Number of threads. Can be used to balance the CPU load between the game and the tracker.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Amount of smoothing of the face region coordinates. Can help stabilize the pose.</source>
+        <source>Show the image patch that the pose estimation model sees.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Thread Count</source>
+        <source>Amount of smoothing of the face region coordinates. Can help stabilize the pose.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
@@ -88,6 +84,10 @@ Don&apos;t roll or change position.</source>
         <source>Camera override</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Exposure preset</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>neuralnet_tracker_ns::NeuralNetDialog</name>
@@ -123,5 +123,13 @@ Don&apos;t roll or change position.</source>
         <source>Start calibration</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Select Pose Net ONNX</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <source>ONNX Files (*.onnx)</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 </TS>
diff --git a/tracker-neuralnet/lang/ru_RU.ts b/tracker-neuralnet/lang/ru_RU.ts
index cbfce1d5..f88e0529 100644
--- a/tracker-neuralnet/lang/ru_RU.ts
+++ b/tracker-neuralnet/lang/ru_RU.ts
@@ -126,5 +126,13 @@ Don&apos;t roll or change position.</source>
         <source>Start calibration</source>
         <translation>Начать калибровку</translation>
     </message>
+    <message>
+        <source>Select Pose Net ONNX</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <source>ONNX Files (*.onnx)</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 </TS>
diff --git a/tracker-neuralnet/lang/stub.ts b/tracker-neuralnet/lang/stub.ts
index 5936d630..df883d3f 100644
--- a/tracker-neuralnet/lang/stub.ts
+++ b/tracker-neuralnet/lang/stub.ts
@@ -12,6 +12,10 @@
         <translation type="unfinished"></translation>
     </message>
     <message>
+        <source>Camera Configuration</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
         <source>Head Center Offset</source>
         <translation type="unfinished"></translation>
     </message>
@@ -41,11 +45,7 @@ Don&apos;t roll or change position.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Exposure preset</source>
-        <translation type="unfinished"></translation>
-    </message>
-    <message>
-        <source>Camera Configuration</source>
+        <source>Show Network Input</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
@@ -53,10 +53,6 @@ Don&apos;t roll or change position.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Number of threads. Can be used to balance the CPU load between the game and the tracker.</source>
-        <translation type="unfinished"></translation>
-    </message>
-    <message>
         <source>ROI Smoothing Alpha</source>
         <translation type="unfinished"></translation>
     </message>
@@ -65,19 +61,19 @@ Don&apos;t roll or change position.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Show the image patch that the pose estimation model sees.</source>
+        <source>Thread Count</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Show Network Input</source>
+        <source>Number of threads. Can be used to balance the CPU load between the game and the tracker.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Amount of smoothing of the face region coordinates. Can help stabilize the pose.</source>
+        <source>Show the image patch that the pose estimation model sees.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Thread Count</source>
+        <source>Amount of smoothing of the face region coordinates. Can help stabilize the pose.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
@@ -88,6 +84,10 @@ Don&apos;t roll or change position.</source>
         <source>Camera override</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Exposure preset</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>neuralnet_tracker_ns::NeuralNetDialog</name>
@@ -123,5 +123,13 @@ Don&apos;t roll or change position.</source>
         <source>Start calibration</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Select Pose Net ONNX</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <source>ONNX Files (*.onnx)</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 </TS>
diff --git a/tracker-neuralnet/lang/zh_CN.ts b/tracker-neuralnet/lang/zh_CN.ts
index 13dc34cf..efac5204 100644
--- a/tracker-neuralnet/lang/zh_CN.ts
+++ b/tracker-neuralnet/lang/zh_CN.ts
@@ -5,15 +5,19 @@
     <name>Form</name>
     <message>
         <source>Tracker settings</source>
-        <translation type="unfinished"></translation>
+        <translation>追踪器设置</translation>
     </message>
     <message>
         <source>Camera settings</source>
-        <translation type="unfinished"></translation>
+        <translation>相机设置</translation>
+    </message>
+    <message>
+        <source>Camera Configuration</source>
+        <translation>相机配置</translation>
     </message>
     <message>
         <source>Head Center Offset</source>
-        <translation type="unfinished"></translation>
+        <translation>头部归中补偿</translation>
     </message>
     <message>
         <source> mm</source>
@@ -22,41 +26,34 @@
     <message>
         <source>Use only yaw and pitch while calibrating.
 Don&apos;t roll or change position.</source>
-        <translation type="unfinished"></translation>
+        <translation>在校准时只使用偏航和俯仰,
+不要滚转或是改变位置. </translation>
     </message>
     <message>
         <source>Start calibration</source>
-        <translation type="unfinished"></translation>
+        <translation>开始校准</translation>
     </message>
     <message>
         <source>Right</source>
-        <translation type="unfinished"></translation>
+        <translation>向右</translation>
     </message>
     <message>
         <source>Forward</source>
-        <translation type="unfinished"></translation>
+        <translation>向前</translation>
     </message>
     <message>
         <source>Up</source>
-        <translation type="unfinished"></translation>
+        <translation>向上</translation>
     </message>
     <message>
-        <source>Exposure preset</source>
-        <translation type="unfinished"></translation>
-    </message>
-    <message>
-        <source>Camera Configuration</source>
-        <translation type="unfinished"></translation>
+        <source>Show Network Input</source>
+        <translation>展示神经网络输入</translation>
     </message>
     <message>
         <source>Tuning / Debug</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Number of threads. Can be used to balance the CPU load between the game and the tracker.</source>
-        <translation type="unfinished"></translation>
-    </message>
-    <message>
         <source>ROI Smoothing Alpha</source>
         <translation type="unfinished"></translation>
     </message>
@@ -65,19 +62,19 @@ Don&apos;t roll or change position.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Show the image patch that the pose estimation model sees.</source>
+        <source>Thread Count</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Show Network Input</source>
+        <source>Number of threads. Can be used to balance the CPU load between the game and the tracker.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Amount of smoothing of the face region coordinates. Can help stabilize the pose.</source>
+        <source>Show the image patch that the pose estimation model sees.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
-        <source>Thread Count</source>
+        <source>Amount of smoothing of the face region coordinates. Can help stabilize the pose.</source>
         <translation type="unfinished"></translation>
     </message>
     <message>
@@ -88,20 +85,24 @@ Don&apos;t roll or change position.</source>
         <source>Camera override</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Exposure preset</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>neuralnet_tracker_ns::NeuralNetDialog</name>
     <message>
         <source>Default</source>
-        <translation type="unfinished"></translation>
+        <translation>默认</translation>
     </message>
     <message>
         <source>Tracker Offline</source>
-        <translation type="unfinished"></translation>
+        <translation>追踪器离线</translation>
     </message>
     <message>
         <source>%1x%2 @ %3 FPS / Inference: %4 ms</source>
-        <translation type="unfinished"></translation>
+        <translation>%1x%2 @ %3 FPS / 推理: %4 ms</translation>
     </message>
     <message>
         <source>%1 yaw samples. Yaw more to %2 samples for stable calibration.</source>
@@ -117,10 +118,18 @@ Don&apos;t roll or change position.</source>
     </message>
     <message>
         <source>Stop calibration</source>
-        <translation type="unfinished"></translation>
+        <translation>结束校准</translation>
     </message>
     <message>
         <source>Start calibration</source>
+        <translation>开始校准</translation>
+    </message>
+    <message>
+        <source>Select Pose Net ONNX</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <source>ONNX Files (*.onnx)</source>
         <translation type="unfinished"></translation>
     </message>
 </context>
diff --git a/tracker-neuralnet/models/head-pose.onnx b/tracker-neuralnet/models/head-pose-0.2-small.onnx
index d70dfa49..4da6bb6a 100644
--- a/tracker-neuralnet/models/head-pose.onnx
+++ b/tracker-neuralnet/models/head-pose-0.2-small.onnx