From 8b0d87ff815b7ac6370e6123b6344c2a9a12fd22 Mon Sep 17 00:00:00 2001 From: Michael Welter Date: Sun, 1 Oct 2023 14:39:10 +0200 Subject: tracker/nn: Add button to choose pose net file --- tracker-neuralnet/ftnoir_tracker_neuralnet.cpp | 59 +- tracker-neuralnet/ftnoir_tracker_neuralnet.h | 3 + tracker-neuralnet/lang/nl_NL.ts | 20 + tracker-neuralnet/lang/ru_RU.ts | 20 + tracker-neuralnet/lang/stub.ts | 20 + tracker-neuralnet/lang/zh_CN.ts | 61 +- tracker-neuralnet/neuralnet-trackercontrols.ui | 760 +++++++++++++------------ 7 files changed, 561 insertions(+), 382 deletions(-) (limited to 'tracker-neuralnet') diff --git a/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp b/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp index 4ee8de1e..2243c143 100644 --- a/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp +++ b/tracker-neuralnet/ftnoir_tracker_neuralnet.cpp @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include #include @@ -56,6 +58,12 @@ std::string convert(const QString &s) { return s.toStdString(); } #endif +QDir get_default_model_directory() +{ + return QDir(OPENTRACK_BASE_PATH+ "/" OPENTRACK_LIBRARY_PATH "models"); +} + + int enum_to_fps(int value) { switch (value) @@ -463,8 +471,7 @@ bool NeuralNetTracker::load_and_initialize_model() { const QString localizer_model_path_enc = OPENTRACK_BASE_PATH+"/" OPENTRACK_LIBRARY_PATH "/models/head-localizer.onnx"; - const QString poseestimator_model_path_enc = - OPENTRACK_BASE_PATH+"/" OPENTRACK_LIBRARY_PATH "/models/head-pose.onnx"; + const QString poseestimator_model_path_enc = get_posenet_filename(); try { @@ -484,6 +491,7 @@ bool NeuralNetTracker::load_and_initialize_model() allocator_info_, Ort::Session{env_, convert(localizer_model_path_enc).c_str(), opts}); + qDebug() << "Loading pose net " << poseestimator_model_path_enc; poseestimator_.emplace( allocator_info_, Ort::Session{env_, convert(poseestimator_model_path_enc).c_str(), opts}); @@ -677,12 +685,31 @@ Affine NeuralNetTracker::pose() return last_pose_affine_; } + std::tuple NeuralNetTracker::stats() const { QMutexLocker lck(&stats_mtx_); return { resolution_, fps_, inference_time_ }; } + +QString NeuralNetTracker::get_posenet_filename() const +{ + QString filename = settings_.posenet_file; + if (QFileInfo(filename).isRelative()) + filename = get_default_model_directory().absoluteFilePath(filename); + return filename; +} + + + + + + + + + + void NeuralNetDialog::make_fps_combobox() { for (int k = 0; k < fps_MAX; k++) @@ -729,11 +756,12 @@ NeuralNetDialog::NeuralNetDialog() : tie_setting(settings_.num_threads, ui_.threadCount); tie_setting(settings_.resolution, ui_.resolution); tie_setting(settings_.force_fps, ui_.cameraFPS); + tie_setting(settings_.posenet_file, ui_.posenetFileDisplay); connect(ui_.buttonBox, SIGNAL(accepted()), this, SLOT(doOK())); connect(ui_.buttonBox, SIGNAL(rejected()), this, SLOT(doCancel())); connect(ui_.camera_settings, SIGNAL(clicked()), this, SLOT(camera_settings())); - + connect(ui_.posenetSelectButton, SIGNAL(clicked()), this, SLOT(onSelectPoseNetFile())); connect(&settings_.camera_name, value_::value_changed(), this, &NeuralNetDialog::update_camera_settings_state); update_camera_settings_state(settings_.camera_name); @@ -900,6 +928,31 @@ void NeuralNetDialog::startstop_trans_calib(bool start) } +void NeuralNetDialog::onSelectPoseNetFile() +{ + const auto root = get_default_model_directory(); + // Start with the current setting + QString filename = settings_.posenet_file; + // If the filename is relative then assume that the file is located under the + // model directory. Under regular use this should always be the case. + if (QFileInfo(filename).isRelative()) + filename = root.absoluteFilePath(filename); + filename = QFileDialog::getOpenFileName(this, + tr("Select Pose Net ONNX"), filename, tr("ONNX Files (*.onnx)")); + // In case the user aborted. + if (filename.isEmpty()) + return; + // When a file under the model directory was selected we can get rid of the + // directory prefix. This is more robust than storing absolute paths, e.g. + // in case the user moves the opentrack install folder / reuses old settings. + // When the file is not in the model directory, we have to use the absolute path, + // which is also fine as developer feature. + if (filename.startsWith(root.absolutePath())) + filename = root.relativeFilePath(filename); + settings_.posenet_file = filename; +} + + Settings::Settings() : opts("neuralnet-tracker") {} } // neuralnet_tracker_ns diff --git a/tracker-neuralnet/ftnoir_tracker_neuralnet.h b/tracker-neuralnet/ftnoir_tracker_neuralnet.h index 3548335e..bafaa6e5 100644 --- a/tracker-neuralnet/ftnoir_tracker_neuralnet.h +++ b/tracker-neuralnet/ftnoir_tracker_neuralnet.h @@ -84,6 +84,7 @@ struct Settings : opts { value resolution { b, "force-resolution", 0 }; value deadzone_size { b, "deadzone-size", 1. }; value deadzone_hardness { b, "deadzone-hardness", 1.5 }; + value posenet_file { b, "posenet-file", "head-pose.onnx" }; Settings(); }; @@ -126,6 +127,7 @@ private: QuatPose compute_filtered_pose(const PoseEstimator::Face &face); // Compute the pose in 3d space taking the network outputs QuatPose transform_to_world_pose(const cv::Quatf &face_rotation, const cv::Point2f& face_xy, const float face_size) const; + QString get_posenet_filename() const; Settings settings_; std::optional localizer_; @@ -192,6 +194,7 @@ private Q_SLOTS: void startstop_trans_calib(bool start); void trans_calib_step(); void status_poll(); + void onSelectPoseNetFile(); }; diff --git a/tracker-neuralnet/lang/nl_NL.ts b/tracker-neuralnet/lang/nl_NL.ts index dbcd3c8c..27da4f5a 100644 --- a/tracker-neuralnet/lang/nl_NL.ts +++ b/tracker-neuralnet/lang/nl_NL.ts @@ -112,6 +112,18 @@ Don't roll or change position. Zoom factor for the face region. Applied before the patch is fed into the pose estimation model. There is a sweet spot near 1. + + Select Pose Net ONNX + + + + <the pose net file> + + + + Select the pose network. Changes take affect on the next tracker start + + neuralnet_tracker_ns::NeuralNetDialog @@ -147,5 +159,13 @@ Don't roll or change position. Start calibration + + Select Pose Net ONNX + + + + ONNX Files (*.onnx) + + diff --git a/tracker-neuralnet/lang/ru_RU.ts b/tracker-neuralnet/lang/ru_RU.ts index b191e769..c32d4fa7 100644 --- a/tracker-neuralnet/lang/ru_RU.ts +++ b/tracker-neuralnet/lang/ru_RU.ts @@ -113,6 +113,18 @@ Don't roll or change position. Zoom factor for the face region. Applied before the patch is fed into the pose estimation model. There is a sweet spot near 1. Фактор масштабирования области лица. Применяется перед передачей кадра в модель определения позиции. Наилучшие результаты близки к 1 + + Select Pose Net ONNX + + + + <the pose net file> + + + + Select the pose network. Changes take affect on the next tracker start + + neuralnet_tracker_ns::NeuralNetDialog @@ -150,5 +162,13 @@ Don't roll or change position. Start calibration Начать калибровку + + Select Pose Net ONNX + + + + ONNX Files (*.onnx) + + diff --git a/tracker-neuralnet/lang/stub.ts b/tracker-neuralnet/lang/stub.ts index 4cde86a9..9609f05e 100644 --- a/tracker-neuralnet/lang/stub.ts +++ b/tracker-neuralnet/lang/stub.ts @@ -112,6 +112,18 @@ Don't roll or change position. Zoom factor for the face region. Applied before the patch is fed into the pose estimation model. There is a sweet spot near 1. + + Select Pose Net ONNX + + + + <the pose net file> + + + + Select the pose network. Changes take affect on the next tracker start + + neuralnet_tracker_ns::NeuralNetDialog @@ -147,5 +159,13 @@ Don't roll or change position. Start calibration + + Select Pose Net ONNX + + + + ONNX Files (*.onnx) + + diff --git a/tracker-neuralnet/lang/zh_CN.ts b/tracker-neuralnet/lang/zh_CN.ts index cf12f304..53da04ae 100644 --- a/tracker-neuralnet/lang/zh_CN.ts +++ b/tracker-neuralnet/lang/zh_CN.ts @@ -1,36 +1,35 @@ - - + Form Tracker settings - 追踪器设置 + 追踪器设置 Diagonal FOV - 对角FOV + 对角FOV Camera name - 相机名 + 相机名 Frames per second - FPS + FPS Camera settings - 相机设置 + 相机设置 Camera Configuration - 相机配置 + 相机配置 Head Center Offset - 头部归中补偿 + 头部归中补偿 mm @@ -39,28 +38,28 @@ Use only yaw and pitch while calibrating. Don't roll or change position. - 在校准时只使用偏航和俯仰, + 在校准时只使用偏航和俯仰, 不要滚转或是改变位置. Start calibration - 开始校准 + 开始校准 Right - 向右 + 向右 Forward - 向前 + 向前 Up - 向上 + 向上 Show Network Input - 展示神经网络输入 + 展示神经网络输入 MJPEG @@ -114,20 +113,32 @@ Don't roll or change position. Zoom factor for the face region. Applied before the patch is fed into the pose estimation model. There is a sweet spot near 1. - + + Select the pose network. Changes take affect on the next tracker start + + + + Select Pose Net ONNX + + + + <the pose net file> + + + neuralnet_tracker_ns::NeuralNetDialog Default - 默认 + 默认 Tracker Offline - 追踪器离线 + 追踪器离线 %1x%2 @ %3 FPS / Inference: %4 ms - %1x%2 @ %3 FPS / 推理: %4 ms + %1x%2 @ %3 FPS / 推理: %4 ms %1 yaw samples. Yaw more to %2 samples for stable calibration. @@ -143,11 +154,19 @@ Don't roll or change position. Stop calibration - 结束校准 + 结束校准 Start calibration - 开始校准 + 开始校准 + + + Select Pose Net ONNX + + + + ONNX Files (*.onnx) + diff --git a/tracker-neuralnet/neuralnet-trackercontrols.ui b/tracker-neuralnet/neuralnet-trackercontrols.ui index 750e6ef3..ae2450b4 100644 --- a/tracker-neuralnet/neuralnet-trackercontrols.ui +++ b/tracker-neuralnet/neuralnet-trackercontrols.ui @@ -9,14 +9,241 @@ 0 0 - 671 - 357 + 651 + 432 Tracker settings + + + + + 0 + 0 + + + + true + + + Head Center Offset + + + + + + + 0 + 0 + + + + + 16777215 + 16777215 + + + + QFrame::NoFrame + + + QFrame::Raised + + + + QLayout::SetDefaultConstraint + + + 0 + + + + + + 0 + 0 + + + + Right + + + + + + + + 0 + 0 + + + + Forward + + + + + + + + 0 + 0 + + + + Up + + + + + + + + 150 + 16777215 + + + + mm + + + -65535 + + + 65536 + + + + + + + + 150 + 16777215 + + + + mm + + + -65535 + + + 65536 + + + + + + + + 150 + 16777215 + + + + mm + + + -65535 + + + 65536 + + + + + + + + + + + 0 + 0 + + + + + 260 + 0 + + + + QFrame::NoFrame + + + QFrame::Raised + + + + + + Use only yaw and pitch while calibrating. +Don't roll or change position. + + + Qt::AlignCenter + + + true + + + false + + + + + + + + 0 + 0 + + + + QFrame::Panel + + + QFrame::Sunken + + + + + + true + + + + + + + false + + + Start calibration + + + true + + + + + + + + + + + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + @@ -219,10 +446,19 @@ - - - - QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + true + + + QFrame::Panel + + + QFrame::Sunken + + + @@ -246,364 +482,188 @@ Tuning / Debug - - - - - Qt::Vertical - - - - - - - Number of threads. Can be used to balance the CPU load between the game and the tracker. - - - 1 - - - 32 - - - - - - - Qt::Vertical - - - - - - - - 0 - 0 - - - - ROI Smoothing Alpha - - - - - - - ROI Zoom - - - - - - - Qt::Vertical - - - - - - - - 0 - 0 - - - - Show the image patch that the pose estimation model sees. - - - Show Network Input - - - - - - - - 0 - 0 - - - - - 150 - 16777215 - - - - Amount of smoothing of the face region coordinates. Can help stabilize the pose. - - - false - - - 2 - - - 1.000000000000000 - - - 0.010000000000000 - - - 1.000000000000000 - - - - - - - Thread Count - - - - - - - Zoom factor for the face region. Applied before the patch is fed into the pose estimation model. There is a sweet spot near 1. - - - 0.100000000000000 - - - 2.000000000000000 - - - 0.010000000000000 - - - 1.000000000000000 - - - - - - - Qt::Horizontal - - - - 40 - 20 - - - + + + + + + + Thread Count + + + + + + + Number of threads. Can be used to balance the CPU load between the game and the tracker. + + + 1 + + + 32 + + + + + + + Qt::Vertical + + + + + + + + 0 + 0 + + + + Show the image patch that the pose estimation model sees. + + + Show Network Input + + + + + + + Qt::Vertical + + + + + + + + 0 + 0 + + + + ROI Smoothing Alpha + + + + + + + + 0 + 0 + + + + + 150 + 16777215 + + + + Amount of smoothing of the face region coordinates. Can help stabilize the pose. + + + false + + + 2 + + + 1.000000000000000 + + + 0.010000000000000 + + + 1.000000000000000 + + + + + + + ROI Zoom + + + + + + + Zoom factor for the face region. Applied before the patch is fed into the pose estimation model. There is a sweet spot near 1. + + + 0.100000000000000 + + + 2.000000000000000 + + + 0.010000000000000 + + + 1.000000000000000 + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + - - - - - - - - 0 - 0 - - - - true - - - Head Center Offset - - - - - - - 0 - 0 - - - - - 16777215 - 16777215 - - + + QFrame::NoFrame QFrame::Raised - - - QLayout::SetDefaultConstraint + + + 0 - + + 0 + + + 0 + + 0 - - - - - 150 - 16777215 - - - - mm - - - -65535 - - - 65536 - - - - - - - - 0 - 0 - - - - Right - - - - - - - - 150 - 16777215 - - - - mm - - - -65535 - - - 65536 - - - - - - - - 0 - 0 - - - - Forward - - - - - - - - 150 - 16777215 - - - - mm - - - -65535 - - - 65536 - - - - - - - - 0 - 0 - - - - Up - - - - - - - - - - - 0 - 0 - - - - - 260 - 0 - - - - QFrame::NoFrame - - - QFrame::Raised - - - - - Use only yaw and pitch while calibrating. -Don't roll or change position. + + + Select the pose network. Changes take affect on the next tracker start - - Qt::AlignCenter - - - true - - - false + + Select Pose Net ONNX - + - + 0 0 - - QFrame::Panel - - - QFrame::Sunken - - - - - true - - - - - - - false - - - Start calibration - - - true + <the pose net file> @@ -613,22 +673,6 @@ Don't roll or change position. - - - - true - - - QFrame::Panel - - - QFrame::Sunken - - - - - - -- cgit v1.2.3