diff options
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | cmake/msvc.cmake | 6 | ||||
| -rw-r--r-- | gui/init.cpp | 78 | ||||
| -rw-r--r-- | sdk-paths-sthalik@MSVC-windows.cmake | 9 | ||||
| -rw-r--r-- | tracker-eyeware-beam/CMakeLists.txt | 10 | ||||
| -rw-r--r-- | tracker-neuralnet/CMakeLists.txt | 9 | ||||
| -rw-r--r-- | tracker-neuralnet/model_adapters.cpp | 41 | ||||
| -rw-r--r-- | tracker-neuralnet/model_adapters.h | 13 |
8 files changed, 111 insertions, 59 deletions
@@ -1,6 +1,6 @@ ## Intro -[<img src="https://ci.appveyor.com/api/projects/status/n0j9h38jnif5qbe9/branch/unstable?svg=true"/>](https://ci.appveyor.com/project/sthalik/opentrack/branch/unstable) +[<img src="https://github.com/opentrack/opentrack/actions/workflows/cmake.yml/badge.svg">](https://github.com/opentrack/opentrack/actions/workflows/cmake.yml) opentrack project home is located at <<http://github.com/opentrack/opentrack>>. @@ -40,6 +40,7 @@ Don't be afraid to submit an **issue/feature request** if you have any problems! - BBC micro:bit, LEGO, sensortag support via Smalltalk<sup>[(1)](https://en.wikipedia.org/wiki/Smalltalk)[(2)](https://en.wikipedia.org/wiki/Alan_Kay)</sup> [S2Bot](http://www.picaxe.com/Teaching/Other-Software/Scratch-Helper-Apps/) - Wiimote (Windows) +- Eyeware Beam<sup>[[1](https://beam.eyeware.tech/)]</sup> ## Output protocols @@ -69,6 +70,7 @@ Don't be afraid to submit an **issue/feature request** if you have any problems! - Wei Shuai (Wiimote tracker) - Stéphane Lenclud (Kinect Face Tracker, Easy Tracker) - GO63-samara (Hamilton Filter, Pose-widget improvement) +- Davide Mameli (Eyeware Beam tracker) ## Thanks diff --git a/cmake/msvc.cmake b/cmake/msvc.cmake index e7e3c329..cf3bb0cb 100644 --- a/cmake/msvc.cmake +++ b/cmake/msvc.cmake @@ -37,7 +37,7 @@ add_definitions(-D_HAS_EXCEPTIONS=0) if(DEFINED CMAKE_TOOLCHAIN_FILE) # ignore cmake warning: Manually-specified variable not used by the project - set(CMAKE_TOOLCHAIN_FILE "${CMAKE_TOOLCHAIN_FILE}}") + set(CMAKE_TOOLCHAIN_FILE "${CMAKE_TOOLCHAIN_FILE}") endif() include("${CMAKE_CURRENT_LIST_DIR}/opentrack-policy.cmake" NO_POLICY_SCOPE) @@ -106,6 +106,10 @@ if(CMAKE_PROJECT_NAME STREQUAL "OpenCV") set(BUILD_opencv_gapi OFF) endif() +if(CMAKE_PROJECT_NAME STREQUAL "TestOscpack") + add_compile_definitions(OSC_HOST_LITTLE_ENDIAN) +endif() + set(opentrack-simd "SSE2") if(CMAKE_PROJECT_NAME STREQUAL "onnxruntime") diff --git a/gui/init.cpp b/gui/init.cpp index b54a085e..5984269c 100644 --- a/gui/init.cpp +++ b/gui/init.cpp @@ -193,51 +193,56 @@ static void apply_dark_windows_theme_if_needed() static void add_win32_path() { - // see https://software.intel.com/en-us/articles/limitation-to-the-length-of-the-system-path-variable - static char env_path[4096] { '\0', }; + // see https://web.archive.org/web/20180924055536/https://software.intel.com/en-us/articles/limitation-to-the-length-of-the-system-path-variable { QString lib_path = OPENTRACK_BASE_PATH; lib_path.replace("/", "\\"); - const QByteArray lib_path_ = QFile::encodeName(lib_path); - QString mod_path = OPENTRACK_BASE_PATH + OPENTRACK_LIBRARY_PATH; mod_path.replace("/", "\\"); - const QByteArray mod_path_ = QFile::encodeName(mod_path); - - const char* contents[] { - "PATH=", - lib_path_.constData(), - ";", - mod_path_.constData(), - ";", - getenv("PATH"), - }; - bool ok = true; + const QString orig_path = qgetenv("PATH"); - for (const char* ptr : contents) - { - if (ptr) - strcat_s(env_path, sizeof(env_path), ptr); + QString env_path; env_path.reserve(4096); - if (!ptr || ptr[0] == '\0' || env_path[0] == '\0') - { - qDebug() << "bad path element" - << (ptr == nullptr ? "<null>" : ptr); - ok = false; - break; - } - } +#if 0 + qDebug() << "orig" << orig_path; + qDebug() << "libpath" << lib_path; + qDebug() << "modpath" << mod_path; +#endif - if (ok) + if (lib_path.isEmpty()) + qDebug() << "env: empty lib_path!"; + else { - const int error = _putenv(env_path); - - if (error) - qDebug() << "can't _putenv win32 path"; + if (!QFile(lib_path).exists()) + qDebug() << "env: lib_path doesn't exist, this shouldn't happen!"; + env_path += lib_path; + env_path += ';'; } + if (mod_path.isEmpty()) + qDebug() << "env: can't add mod_path to env PATH"; else - qDebug() << "can't set win32 path"; + { + if (!QFile(mod_path).exists()) + qDebug() << "env: mod_path doesn't exist, did you install it correctly?"; + env_path += mod_path; + env_path += ';'; + } + + if (orig_path.isEmpty()) + qDebug() << "env: empty PATH"; + else + env_path += orig_path; + +#if 0 + qDebug() << "data" << env_path.constData(); +#endif + + // better length limit than putenv() and SetEnvironmentVariableA + bool ret = SetEnvironmentVariableW(L"PATH", (const wchar_t*)env_path.constData()); + + if (!ret) + qDebug() << "_putenv() failed with" << (void*)GetLastError(); } } @@ -300,11 +305,13 @@ int otr_main(int argc, char** argv, std::function<std::unique_ptr<QWidget>()> co QApplication app(argc, argv); #ifdef _WIN32 - apply_dark_windows_theme_if_needed(); - add_win32_path(); attach_parent_console(); #endif (void)qInstallMessageHandler(qdebug_to_console); +#ifdef _WIN32 + apply_dark_windows_theme_if_needed(); + add_win32_path(); +#endif QDir::setCurrent(OPENTRACK_BASE_PATH); @@ -345,4 +352,3 @@ int otr_main(int argc, char** argv, std::function<std::unique_ptr<QWidget>()> co return ret; } - diff --git a/sdk-paths-sthalik@MSVC-windows.cmake b/sdk-paths-sthalik@MSVC-windows.cmake index 8f185695..340b78ad 100644 --- a/sdk-paths-sthalik@MSVC-windows.cmake +++ b/sdk-paths-sthalik@MSVC-windows.cmake @@ -28,22 +28,23 @@ setq(SDK_REALSENSE "RSSDK-R2") setq(SDK_VALVE_STEAMVR "steamvr") setq(SDK_FSUIPC "fsuipc") setq(SDK_HYDRA "SixenseSDK") +setq(SDK_EYEWARE_BEAM "eyeware-beam-sdk") if(CMAKE_SIZEOF_VOID_P EQUAL 8) setq(Qt5_DIR "../qt-5.15-kde-amd64/lib/cmake/Qt5") setq(OpenCV_DIR "opencv/build-amd64/install") setq(SDK_ARUCO_LIBPATH "aruco/build-amd64/src/aruco.lib") setq(SDK_LIBUSB "libusb-msvc-amd64") -setq(ONNXRuntime_DIR "onnxruntime-1.12.1-amd64") +setq(ONNXRuntime_DIR "onnxruntime-1.14.1-amd64") setq(SDK_TRACKHAT_SENSOR "trackhat-c-library-driver/build-amd64/install") -setq(SDK_EYEWARE_BEAM "BeamSDK-Windows64-1.1.0") +setq(SDK_OSCPACK "oscpack/build-amd64") elseif(CMAKE_SIZEOF_VOID_P EQUAL 4) setq(Qt5_DIR "../qt-5.15-kde/lib/cmake/Qt5") setq(OpenCV_DIR "opencv/build/install") setq(SDK_ARUCO_LIBPATH "aruco/build/src/aruco.lib") setq(SDK_LIBUSB "libusb-msvc-x86") -setq(ONNXRuntime_DIR "onnxruntime-1.12.1") +setq(ONNXRuntime_DIR "onnxruntime-1.14.1") setq(SDK_TRACKHAT_SENSOR "trackhat-c-library-driver/build/install") -set(SDK_EYEWARE_BEAM "" CACHE PATH "" FORCE) +setq(SDK_OSCPACK "oscpack/build") else() message(FATAL_ERROR "unknown word size ${CMAKE_SIZEOF_VOID_P}b") endif() diff --git a/tracker-eyeware-beam/CMakeLists.txt b/tracker-eyeware-beam/CMakeLists.txt index 73ea231a..e041c131 100644 --- a/tracker-eyeware-beam/CMakeLists.txt +++ b/tracker-eyeware-beam/CMakeLists.txt @@ -7,9 +7,15 @@ if(WIN32 AND SDK_EYEWARE_BEAM) endif() otr_module(tracker-eyeware-beam) + if("${CMAKE_SIZEOF_VOID_P}" STREQUAL "4") + set(arch "x86") + else() + set(arch "x64") + endif() + target_include_directories(${self} SYSTEM PRIVATE "${SDK_EYEWARE_BEAM}/API/cpp/include") - target_link_directories(${self} PRIVATE "${SDK_EYEWARE_BEAM}/API/cpp/lib") - set(dll "${SDK_EYEWARE_BEAM}/API/cpp/lib/tracker_client.dll" "${SDK_EYEWARE_BEAM}/API/cpp/lib/libsodium.dll") + target_link_directories(${self} PRIVATE "${SDK_EYEWARE_BEAM}/API/cpp/lib/${arch}") + set(dll "${SDK_EYEWARE_BEAM}/API/cpp/lib/${arch}/tracker_client.dll") set(lib tracker_client.lib) #message(${self}) diff --git a/tracker-neuralnet/CMakeLists.txt b/tracker-neuralnet/CMakeLists.txt index 4689dba1..db568fae 100644 --- a/tracker-neuralnet/CMakeLists.txt +++ b/tracker-neuralnet/CMakeLists.txt @@ -16,16 +16,21 @@ if(OpenCV_FOUND AND ONNXRuntime_FOUND AND OpenMP_FOUND) otr_module(tracker-neuralnet) - target_link_libraries(${self} + target_link_libraries(${self} opentrack-cv onnxruntime::onnxruntime opencv_calib3d opencv_imgproc opencv_imgcodecs opencv_core - OpenMP::OpenMP_C + OpenMP::OpenMP_CXX ) + # OpenMP::OpenMP_CXX doesn't set up the -fopenmp linking option, so set it up ourselves. + if(NOT MSVC) + target_link_options(${self} PUBLIC ${OpenMP_CXX_FLAGS}) + endif() + install( FILES "models/head-localizer.onnx" "models/head-pose.onnx" diff --git a/tracker-neuralnet/model_adapters.cpp b/tracker-neuralnet/model_adapters.cpp index af599321..a8580a89 100644 --- a/tracker-neuralnet/model_adapters.cpp +++ b/tracker-neuralnet/model_adapters.cpp @@ -8,7 +8,6 @@ #include <QDebug> - namespace neuralnet_tracker_ns { @@ -165,6 +164,24 @@ double Localizer::last_inference_time_millis() const } +std::string PoseEstimator::get_network_input_name(size_t i) const +{ +#if ORT_API_VERSION >= 12 + return std::string(&*session_.GetInputNameAllocated(i, allocator_)); +#else + return std::string(session_.GetInputName(i, allocator_)); +#endif +} + +std::string PoseEstimator::get_network_output_name(size_t i) const +{ +#if ORT_API_VERSION >= 12 + return std::string(&*session_.GetOutputNameAllocated(i, allocator_)); +#else + return std::string(session_.GetOutputName(i, allocator_)); +#endif +} + PoseEstimator::PoseEstimator(Ort::MemoryInfo &allocator_info, Ort::Session &&session) : model_version_{session.GetModelMetadata().GetVersion()} , session_{std::move(session)} @@ -215,14 +232,16 @@ PoseEstimator::PoseEstimator(Ort::MemoryInfo &allocator_info, Ort::Session &&ses qDebug() << "Pose model inputs (" << session_.GetInputCount() << ")"; qDebug() << "Pose model outputs (" << session_.GetOutputCount() << "):"; + output_names_.resize(session_.GetOutputCount()); + output_c_names_.resize(session_.GetOutputCount()); for (size_t i=0; i<session_.GetOutputCount(); ++i) { - const char* name = session_.GetOutputName(i, allocator_); + std::string name = get_network_output_name(i); const auto& output_info = session_.GetOutputTypeInfo(i); const auto& onnx_tensor_spec = output_info.GetTensorTypeAndShapeInfo(); auto my_tensor_spec = understood_outputs.find(name); - qDebug() << "\t" << name << " (" << onnx_tensor_spec.GetShape() << ") dtype: " << onnx_tensor_spec.GetElementType() << " " << + qDebug() << "\t" << name.c_str() << " (" << onnx_tensor_spec.GetShape() << ") dtype: " << onnx_tensor_spec.GetElementType() << " " << (my_tensor_spec != understood_outputs.end() ? "ok" : "unknown"); if (my_tensor_spec != understood_outputs.end()) @@ -240,7 +259,8 @@ PoseEstimator::PoseEstimator(Ort::MemoryInfo &allocator_info, Ort::Session &&ses // Create tensor regardless and ignore output output_val_.push_back(create_tensor(output_info, allocator_)); } - output_names_.push_back(name); + output_names_[i] = name; + output_c_names_[i] = output_names_[i].c_str(); } has_uncertainty_ = understood_outputs.at("rotaxis_scales_tril").available || @@ -270,9 +290,12 @@ PoseEstimator::PoseEstimator(Ort::MemoryInfo &allocator_info, Ort::Session &&ses // output_val_.push_back(create_tensor(output_info, allocator_)); // } + input_names_.resize(session_.GetInputCount()); + input_c_names_.resize(session_.GetInputCount()); for (size_t i = 0; i < session_.GetInputCount(); ++i) { - input_names_.push_back(session_.GetInputName(i, allocator_)); + input_names_[i] = get_network_input_name(i); + input_c_names_[i] = input_names_[i].c_str(); } assert (input_names_.size() == input_val_.size()); @@ -312,11 +335,11 @@ std::optional<PoseEstimator::Face> PoseEstimator::run( { session_.Run( Ort::RunOptions{ nullptr }, - input_names_.data(), + input_c_names_.data(), input_val_.data(), input_val_.size(), - output_names_.data(), - output_val_.data(), + output_c_names_.data(), + output_val_.data(), output_val_.size()); } catch (const Ort::Exception &e) @@ -430,4 +453,4 @@ double PoseEstimator::last_inference_time_millis() const -} // namespace neuralnet_tracker_ns
\ No newline at end of file +} // namespace neuralnet_tracker_ns diff --git a/tracker-neuralnet/model_adapters.h b/tracker-neuralnet/model_adapters.h index 3fbfb861..820330cf 100644 --- a/tracker-neuralnet/model_adapters.h +++ b/tracker-neuralnet/model_adapters.h @@ -3,6 +3,7 @@ #include <optional> #include <array> #include <vector> +#include <string> #include <onnxruntime_cxx_api.h> #include <opencv2/core.hpp> @@ -21,7 +22,7 @@ class Localizer public: Localizer(Ort::MemoryInfo &allocator_info, Ort::Session &&session); - + // Returns bounding wrt image coordinate of the input image // The preceeding float is the score for being a face normalized to [0,1]. std::pair<float, cv::Rect2f> run( @@ -68,13 +69,16 @@ class PoseEstimator bool has_uncertainty() const { return has_uncertainty_; } private: + std::string get_network_input_name(size_t i) const; + std::string get_network_output_name(size_t i) const; int64_t model_version_ = 0; // Queried meta data from the ONNX file Ort::Session session_{nullptr}; // ONNX's runtime context for running the model Ort::Allocator allocator_; // Memory allocator for tensors // Inputs cv::Mat scaled_frame_{}, input_mat_{}; // Input. One is the original crop, the other is rescaled (?) std::vector<Ort::Value> input_val_; // Tensors to put into the model - std::vector<const char*> input_names_; // Refers to the names in the onnx model. + std::vector<std::string> input_names_; // Refers to the names in the onnx model. + std::vector<const char *> input_c_names_; // Refers to the C names in the onnx model. // Outputs cv::Vec<float, 3> output_coord_{}; // 2d Coordinate and head size output. cv::Vec<float, 4> output_quat_{}; // Quaternion output @@ -83,7 +87,8 @@ class PoseEstimator cv::Vec<float, 2> output_eyes_{}; cv::Vec<float, 3> output_coord_scales_{}; std::vector<Ort::Value> output_val_; // Tensors to put the model outputs in. - std::vector<const char*> output_names_; // Refers to the names in the onnx model. + std::vector<std::string> output_names_; // Refers to the names in the onnx model. + std::vector<const char *> output_c_names_; // Refers to the C names in the onnx model. // More bookkeeping size_t num_recurrent_states_ = 0; double last_inference_time_ = 0; @@ -99,4 +104,4 @@ int find_input_intensity_quantile(const cv::Mat& frame, float percentage); void normalize_brightness(const cv::Mat& frame, cv::Mat& out); -} // namespace neuralnet_tracker_ns
\ No newline at end of file +} // namespace neuralnet_tracker_ns |