From a774441fde00f0ce6c8387339c786e9e2f42f088 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Thu, 31 Jan 2019 10:32:17 +0100 Subject: Kinect: First draft. --- tracker-kinect-face/CMakeLists.txt | 6 + tracker-kinect-face/lang/nl_NL.ts | 22 ++ tracker-kinect-face/lang/ru_RU.ts | 22 ++ tracker-kinect-face/lang/stub.ts | 22 ++ tracker-kinect-face/lang/zh_CN.ts | 22 ++ tracker-kinect-face/test.cpp | 45 ++++ tracker-kinect-face/test.h | 28 +++ tracker-kinect-face/test.ui | 65 ++++++ tracker-kinect-face/test_dialog.cpp | 3 + tracker-kinect-face/tracker.cpp | 451 ++++++++++++++++++++++++++++++++++++ tracker-kinect-face/tracker.h | 90 +++++++ 11 files changed, 776 insertions(+) create mode 100644 tracker-kinect-face/CMakeLists.txt create mode 100644 tracker-kinect-face/lang/nl_NL.ts create mode 100644 tracker-kinect-face/lang/ru_RU.ts create mode 100644 tracker-kinect-face/lang/stub.ts create mode 100644 tracker-kinect-face/lang/zh_CN.ts create mode 100644 tracker-kinect-face/test.cpp create mode 100644 tracker-kinect-face/test.h create mode 100644 tracker-kinect-face/test.ui create mode 100644 tracker-kinect-face/test_dialog.cpp create mode 100644 tracker-kinect-face/tracker.cpp create mode 100644 tracker-kinect-face/tracker.h (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/CMakeLists.txt b/tracker-kinect-face/CMakeLists.txt new file mode 100644 index 00000000..d139b445 --- /dev/null +++ b/tracker-kinect-face/CMakeLists.txt @@ -0,0 +1,6 @@ + +if(DEFINED ENV{KINECTSDK20_DIR}) + otr_module(tracker-kinect-face) + target_include_directories(opentrack-tracker-kinect-face SYSTEM PUBLIC $ENV{KINECTSDK20_DIR}inc) + target_link_libraries(opentrack-tracker-kinect-face $ENV{KINECTSDK20_DIR}lib/x86/kinect20.lib $ENV{KINECTSDK20_DIR}lib/x86/kinect20.face.lib) +endif() diff --git a/tracker-kinect-face/lang/nl_NL.ts b/tracker-kinect-face/lang/nl_NL.ts new file mode 100644 index 00000000..b6139882 --- /dev/null +++ b/tracker-kinect-face/lang/nl_NL.ts @@ -0,0 +1,22 @@ + + + + + test_metadata + + Kinect Face + + + + + test_ui + + Sine wave + + + + Pressing "Abort" will immediately crash the application. + + + + diff --git a/tracker-kinect-face/lang/ru_RU.ts b/tracker-kinect-face/lang/ru_RU.ts new file mode 100644 index 00000000..52aab9f3 --- /dev/null +++ b/tracker-kinect-face/lang/ru_RU.ts @@ -0,0 +1,22 @@ + + + + + test_metadata + + Kinect Face + + + + + test_ui + + Sine wave + + + + Pressing "Abort" will immediately crash the application. + + + + diff --git a/tracker-kinect-face/lang/stub.ts b/tracker-kinect-face/lang/stub.ts new file mode 100644 index 00000000..84a3c6af --- /dev/null +++ b/tracker-kinect-face/lang/stub.ts @@ -0,0 +1,22 @@ + + + + + test_metadata + + Kinect Face + + + + + test_ui + + Sine wave + + + + Pressing "Abort" will immediately crash the application. + + + + diff --git a/tracker-kinect-face/lang/zh_CN.ts b/tracker-kinect-face/lang/zh_CN.ts new file mode 100644 index 00000000..84a3c6af --- /dev/null +++ b/tracker-kinect-face/lang/zh_CN.ts @@ -0,0 +1,22 @@ + + + + + test_metadata + + Kinect Face + + + + + test_ui + + Sine wave + + + + Pressing "Abort" will immediately crash the application. + + + + diff --git a/tracker-kinect-face/test.cpp b/tracker-kinect-face/test.cpp new file mode 100644 index 00000000..c1ada177 --- /dev/null +++ b/tracker-kinect-face/test.cpp @@ -0,0 +1,45 @@ +/* Copyright (c) 2014, Stanislaw Halik + + * Permission to use, copy, modify, and/or distribute this + * software for any purpose with or without fee is hereby granted, + * provided that the above copyright notice and this permission + * notice appear in all copies. + */ + +#include "test.h" +#include "tracker.h" +#include "api/plugin-api.hpp" +#include "compat/math-imports.hpp" + +#include + +#include +#include + + +test_dialog::test_dialog() +{ + ui.setupUi(this); + + connect(ui.buttonBox, &QDialogButtonBox::clicked, [this](QAbstractButton* btn) { + if (btn == ui.buttonBox->button(QDialogButtonBox::Abort)) + // NOLINTNEXTLINE + *(volatile int*)nullptr = 0; + }); + + connect(ui.buttonBox, SIGNAL(accepted()), this, SLOT(doOK())); + connect(ui.buttonBox, SIGNAL(rejected()), this, SLOT(doCancel())); +} + +void test_dialog::doOK() +{ + //s.b->save(); + close(); +} + +void test_dialog::doCancel() +{ + close(); +} + +OPENTRACK_DECLARE_TRACKER(KinectFaceTracker, test_dialog, test_metadata) diff --git a/tracker-kinect-face/test.h b/tracker-kinect-face/test.h new file mode 100644 index 00000000..2e105ae3 --- /dev/null +++ b/tracker-kinect-face/test.h @@ -0,0 +1,28 @@ +#pragma once +#include "ui_test.h" +#include "compat/macros.hpp" +#include "api/plugin-api.hpp" + + +class test_dialog : public ITrackerDialog +{ + Q_OBJECT + + Ui::test_ui ui; +public: + test_dialog(); + void register_tracker(ITracker *) override {} + void unregister_tracker() override {} +private slots: + void doOK(); + void doCancel(); +}; + +class test_metadata : public Metadata +{ + Q_OBJECT + + QString name() { return tr("Kinect Face"); } + QIcon icon() { return QIcon(":/images/opentrack.png"); } +}; + diff --git a/tracker-kinect-face/test.ui b/tracker-kinect-face/test.ui new file mode 100644 index 00000000..509eb819 --- /dev/null +++ b/tracker-kinect-face/test.ui @@ -0,0 +1,65 @@ + + + test_ui + + + Qt::NonModal + + + + 0 + 0 + 278 + 58 + + + + Sine wave + + + + ../gui/images/opentrack.png../gui/images/opentrack.png + + + Qt::LeftToRight + + + false + + + + + + + 0 + 0 + + + + Pressing "Abort" will immediately crash the application. + + + + + + + + 0 + 0 + + + + QDialogButtonBox::Abort|QDialogButtonBox::Close + + + + + + + + + startEngineClicked() + stopEngineClicked() + cameraSettingsClicked() + + diff --git a/tracker-kinect-face/test_dialog.cpp b/tracker-kinect-face/test_dialog.cpp new file mode 100644 index 00000000..5d33555b --- /dev/null +++ b/tracker-kinect-face/test_dialog.cpp @@ -0,0 +1,3 @@ +#include "test.h" +#include "api/plugin-api.hpp" + diff --git a/tracker-kinect-face/tracker.cpp b/tracker-kinect-face/tracker.cpp new file mode 100644 index 00000000..3fe3b8cc --- /dev/null +++ b/tracker-kinect-face/tracker.cpp @@ -0,0 +1,451 @@ + + +#include "tracker.h" + + +const double KinectFaceTracker::incr[6] = +{ + 50, 40, 80, + 70, 5, 3 +}; + +KinectFaceTracker::KinectFaceTracker(): + m_pKinectSensor(nullptr), + m_pCoordinateMapper(nullptr), + m_pColorFrameReader(nullptr), + m_pColorRGBX(nullptr), + m_pBodyFrameReader(nullptr) + +{ + for (int i = 0; i < BODY_COUNT; i++) + { + m_pFaceFrameSources[i] = nullptr; + m_pFaceFrameReaders[i] = nullptr; + } + + // create heap storage for color pixel data in RGBX format + m_pColorRGBX = new RGBQUAD[cColorWidth * cColorHeight]; +} + +KinectFaceTracker::~KinectFaceTracker() +{ + if (m_pColorRGBX) + { + delete[] m_pColorRGBX; + m_pColorRGBX = nullptr; + } + + // clean up Direct2D + //SafeRelease(m_pD2DFactory); + + // done with face sources and readers + for (int i = 0; i < BODY_COUNT; i++) + { + SafeRelease(m_pFaceFrameSources[i]); + SafeRelease(m_pFaceFrameReaders[i]); + } + + // done with body frame reader + SafeRelease(m_pBodyFrameReader); + + // done with color frame reader + SafeRelease(m_pColorFrameReader); + + // done with coordinate mapper + SafeRelease(m_pCoordinateMapper); + + // close the Kinect Sensor + if (m_pKinectSensor) + { + m_pKinectSensor->Close(); + } + + SafeRelease(m_pKinectSensor); +} + +module_status KinectFaceTracker::start_tracker(QFrame*) +{ + t.start(); + + if (SUCCEEDED(InitializeDefaultSensor())) + { + return status_ok(); + } + + return error("Kinect init failed!"); +} + +#ifdef EMIT_NAN +# include +#endif + +void KinectFaceTracker::data(double *data) +{ + const double dt = t.elapsed_seconds(); + t.start(); + +#ifdef EMIT_NAN + if ((rand() % 4) == 0) + { + for (int i = 0; i < 6; i++) + data[i] = 0. / 0.; + } + else +#endif + for (int i = 0; i < 6; i++) + { + double x = last_x[i] + incr[i] * dt; + if (x > 180) + x = -360 + x; + else if (x < -180) + x = 360 + x; + x = copysign(fmod(fabs(x), 360), x); + last_x[i] = x; + + if (i >= 3) + { + data[i] = x; + } + else + { + data[i] = x * 100 / 180.; + } + } + + + Update(); + //TODO: check if data is valid + data[0] = 0; + data[1] = 0; + data[2] = 0; + ExtractFaceRotationInDegrees(&faceRotation,&data[3], &data[4], &data[5]); + +} + + +/// +/// Converts rotation quaternion to Euler angles +/// And then maps them to a specified range of values to control the refresh rate +/// +/// face rotation quaternion +/// rotation about the X-axis +/// rotation about the Y-axis +/// rotation about the Z-axis +void KinectFaceTracker::ExtractFaceRotationInDegrees(const Vector4* pQuaternion, double* pYaw, double* pPitch, double* pRoll) +{ + double x = pQuaternion->x; + double y = pQuaternion->y; + double z = pQuaternion->z; + double w = pQuaternion->w; + + // convert face rotation quaternion to Euler angles in degrees + double dPitch, dYaw, dRoll; + dPitch = atan2(2 * (y * z + w * x), w * w - x * x - y * y + z * z) / M_PI * 180.0; + dYaw = asin(2 * (w * y - x * z)) / M_PI * 180.0; + dRoll = atan2(2 * (x * y + w * z), w * w + x * x - y * y - z * z) / M_PI * 180.0; + + // clamp rotation values in degrees to a specified range of values to control the refresh rate + /* + double increment = c_FaceRotationIncrementInDegrees; + *pPitch = static_cast(floor((dPitch + increment/2.0 * (dPitch > 0 ? 1.0 : -1.0)) / increment) * increment); + *pYaw = static_cast(floor((dYaw + increment/2.0 * (dYaw > 0 ? 1.0 : -1.0)) / increment) * increment); + *pRoll = static_cast(floor((dRoll + increment/2.0 * (dRoll > 0 ? 1.0 : -1.0)) / increment) * increment); + */ + + *pPitch = dPitch; + *pYaw = dYaw; + *pRoll = dRoll; + + +} + + +/// +/// Initializes the default Kinect sensor +/// +/// S_OK on success else the failure code +HRESULT KinectFaceTracker::InitializeDefaultSensor() +{ + HRESULT hr; + + hr = GetDefaultKinectSensor(&m_pKinectSensor); + if (FAILED(hr)) + { + return hr; + } + + if (m_pKinectSensor) + { + // Initialize Kinect and get color, body and face readers + IColorFrameSource* pColorFrameSource = nullptr; + IBodyFrameSource* pBodyFrameSource = nullptr; + + hr = m_pKinectSensor->Open(); + + if (SUCCEEDED(hr)) + { + hr = m_pKinectSensor->get_CoordinateMapper(&m_pCoordinateMapper); + } + + if (SUCCEEDED(hr)) + { + hr = m_pKinectSensor->get_ColorFrameSource(&pColorFrameSource); + } + + if (SUCCEEDED(hr)) + { + hr = pColorFrameSource->OpenReader(&m_pColorFrameReader); + } + + if (SUCCEEDED(hr)) + { + hr = m_pKinectSensor->get_BodyFrameSource(&pBodyFrameSource); + } + + if (SUCCEEDED(hr)) + { + hr = pBodyFrameSource->OpenReader(&m_pBodyFrameReader); + } + + if (SUCCEEDED(hr)) + { + // create a face frame source + reader to track each body in the fov + for (int i = 0; i < BODY_COUNT; i++) + { + if (SUCCEEDED(hr)) + { + // create the face frame source by specifying the required face frame features + hr = CreateFaceFrameSource(m_pKinectSensor, 0, c_FaceFrameFeatures, &m_pFaceFrameSources[i]); + } + if (SUCCEEDED(hr)) + { + // open the corresponding reader + hr = m_pFaceFrameSources[i]->OpenReader(&m_pFaceFrameReaders[i]); + } + } + } + + SafeRelease(pColorFrameSource); + SafeRelease(pBodyFrameSource); + } + + if (!m_pKinectSensor || FAILED(hr)) + { + //SetStatusMessage(L"No ready Kinect found!", 10000, true); + return E_FAIL; + } + + return hr; +} + +/// +/// Main processing function +/// +void KinectFaceTracker::Update() +{ + if (!m_pColorFrameReader || !m_pBodyFrameReader) + { + return; + } + + IColorFrame* pColorFrame = nullptr; + HRESULT hr = m_pColorFrameReader->AcquireLatestFrame(&pColorFrame); + + if (SUCCEEDED(hr)) + { + INT64 nTime = 0; + IFrameDescription* pFrameDescription = nullptr; + int nWidth = 0; + int nHeight = 0; + ColorImageFormat imageFormat = ColorImageFormat_None; + UINT nBufferSize = 0; + RGBQUAD *pBuffer = nullptr; + + hr = pColorFrame->get_RelativeTime(&nTime); + + if (SUCCEEDED(hr)) + { + hr = pColorFrame->get_FrameDescription(&pFrameDescription); + } + + if (SUCCEEDED(hr)) + { + hr = pFrameDescription->get_Width(&nWidth); + } + + if (SUCCEEDED(hr)) + { + hr = pFrameDescription->get_Height(&nHeight); + } + + if (SUCCEEDED(hr)) + { + hr = pColorFrame->get_RawColorImageFormat(&imageFormat); + } + + if (SUCCEEDED(hr)) + { + if (imageFormat == ColorImageFormat_Bgra) + { + hr = pColorFrame->AccessRawUnderlyingBuffer(&nBufferSize, reinterpret_cast(&pBuffer)); + } + else if (m_pColorRGBX) + { + pBuffer = m_pColorRGBX; + nBufferSize = cColorWidth * cColorHeight * sizeof(RGBQUAD); + hr = pColorFrame->CopyConvertedFrameDataToArray(nBufferSize, reinterpret_cast(pBuffer), ColorImageFormat_Bgra); + } + else + { + hr = E_FAIL; + } + } + + if (SUCCEEDED(hr)) + { + //DrawStreams(nTime, pBuffer, nWidth, nHeight); + ProcessFaces(); + } + + SafeRelease(pFrameDescription); + } + + SafeRelease(pColorFrame); +} + + +/// +/// Updates body data +/// +/// pointer to the body data storage +/// indicates success or failure +HRESULT KinectFaceTracker::UpdateBodyData(IBody** ppBodies) +{ + HRESULT hr = E_FAIL; + + if (m_pBodyFrameReader != nullptr) + { + IBodyFrame* pBodyFrame = nullptr; + hr = m_pBodyFrameReader->AcquireLatestFrame(&pBodyFrame); + if (SUCCEEDED(hr)) + { + hr = pBodyFrame->GetAndRefreshBodyData(BODY_COUNT, ppBodies); + } + SafeRelease(pBodyFrame); + } + + return hr; +} + + + +/// +/// Processes new face frames +/// +void KinectFaceTracker::ProcessFaces() +{ + HRESULT hr; + IBody* ppBodies[BODY_COUNT] = { 0 }; + bool bHaveBodyData = SUCCEEDED(UpdateBodyData(ppBodies)); + + // iterate through each face reader + for (int iFace = 0; iFace < BODY_COUNT; ++iFace) + { + // retrieve the latest face frame from this reader + IFaceFrame* pFaceFrame = nullptr; + hr = m_pFaceFrameReaders[iFace]->AcquireLatestFrame(&pFaceFrame); + + BOOLEAN bFaceTracked = false; + if (SUCCEEDED(hr) && nullptr != pFaceFrame) + { + // check if a valid face is tracked in this face frame + hr = pFaceFrame->get_IsTrackingIdValid(&bFaceTracked); + } + + if (SUCCEEDED(hr)) + { + if (bFaceTracked) + { + IFaceFrameResult* pFaceFrameResult = nullptr; + RectI faceBox = { 0 }; + PointF facePoints[FacePointType::FacePointType_Count]; + DetectionResult faceProperties[FaceProperty::FaceProperty_Count]; + //D2D1_POINT_2F faceTextLayout; + + hr = pFaceFrame->get_FaceFrameResult(&pFaceFrameResult); + + // need to verify if pFaceFrameResult contains data before trying to access it + if (SUCCEEDED(hr) && pFaceFrameResult != nullptr) + { + hr = pFaceFrameResult->get_FaceBoundingBoxInColorSpace(&faceBox); + + if (SUCCEEDED(hr)) + { + hr = pFaceFrameResult->GetFacePointsInColorSpace(FacePointType::FacePointType_Count, facePoints); + } + + if (SUCCEEDED(hr)) + { + hr = pFaceFrameResult->get_FaceRotationQuaternion(&faceRotation); + } + + if (SUCCEEDED(hr)) + { + hr = pFaceFrameResult->GetFaceProperties(FaceProperty::FaceProperty_Count, faceProperties); + } + + if (SUCCEEDED(hr)) + { + //hr = GetFaceTextPositionInColorSpace(ppBodies[iFace], &faceTextLayout); + } + + if (SUCCEEDED(hr)) + { + // draw face frame results + //m_pDrawDataStreams->DrawFaceFrameResults(iFace, &faceBox, facePoints, &faceRotation, faceProperties, &faceTextLayout); + } + } + + SafeRelease(pFaceFrameResult); + } + else + { + // face tracking is not valid - attempt to fix the issue + // a valid body is required to perform this step + if (bHaveBodyData) + { + // check if the corresponding body is tracked + // if this is true then update the face frame source to track this body + IBody* pBody = ppBodies[iFace]; + if (pBody != nullptr) + { + BOOLEAN bTracked = false; + hr = pBody->get_IsTracked(&bTracked); + + UINT64 bodyTId; + if (SUCCEEDED(hr) && bTracked) + { + // get the tracking ID of this body + hr = pBody->get_TrackingId(&bodyTId); + if (SUCCEEDED(hr)) + { + // update the face frame source with the tracking ID + m_pFaceFrameSources[iFace]->put_TrackingId(bodyTId); + } + } + } + } + } + } + + SafeRelease(pFaceFrame); + } + + if (bHaveBodyData) + { + for (int i = 0; i < _countof(ppBodies); ++i) + { + SafeRelease(ppBodies[i]); + } + } +} diff --git a/tracker-kinect-face/tracker.h b/tracker-kinect-face/tracker.h new file mode 100644 index 00000000..88243b1f --- /dev/null +++ b/tracker-kinect-face/tracker.h @@ -0,0 +1,90 @@ + + + +#include + +#include "api/plugin-api.hpp" +#include "compat/timer.hpp" +#include "compat/macros.hpp" + +// Kinect Header files +#include +#include + +#pragma once + +// Safe release for interfaces +template +inline void SafeRelease(Interface *& pInterfaceToRelease) +{ + if (pInterfaceToRelease != nullptr) + { + pInterfaceToRelease->Release(); + pInterfaceToRelease = nullptr; + } +} + + +class KinectFaceTracker : public ITracker +{ +public: + KinectFaceTracker(); + ~KinectFaceTracker() override; + module_status start_tracker(QFrame *) override; + void data(double *data) override; + +private: + static const double incr[6]; + double last_x[6]{}; + Timer t; + + // Kinect stuff + + // define the face frame features required to be computed by this application + static const DWORD c_FaceFrameFeatures = + FaceFrameFeatures::FaceFrameFeatures_BoundingBoxInColorSpace + | FaceFrameFeatures::FaceFrameFeatures_PointsInColorSpace + | FaceFrameFeatures::FaceFrameFeatures_RotationOrientation + | FaceFrameFeatures::FaceFrameFeatures_Happy + | FaceFrameFeatures::FaceFrameFeatures_RightEyeClosed + | FaceFrameFeatures::FaceFrameFeatures_LeftEyeClosed + | FaceFrameFeatures::FaceFrameFeatures_MouthOpen + | FaceFrameFeatures::FaceFrameFeatures_MouthMoved + | FaceFrameFeatures::FaceFrameFeatures_LookingAway + | FaceFrameFeatures::FaceFrameFeatures_Glasses + | FaceFrameFeatures::FaceFrameFeatures_FaceEngagement; + + static const int cColorWidth = 1920; + static const int cColorHeight = 1080; + + + void Update(); + HRESULT InitializeDefaultSensor(); + void ProcessFaces(); + HRESULT UpdateBodyData(IBody** ppBodies); + void ExtractFaceRotationInDegrees(const Vector4* pQuaternion, double* pPitch, double* pYaw, double* pRoll); + + // Current Kinect + IKinectSensor* m_pKinectSensor; + + // Coordinate mapper + ICoordinateMapper* m_pCoordinateMapper; + + // Color reader + IColorFrameReader* m_pColorFrameReader; + + // Body reader + IBodyFrameReader* m_pBodyFrameReader; + + // Face sources + IFaceFrameSource* m_pFaceFrameSources[BODY_COUNT]; + + // Face readers + IFaceFrameReader* m_pFaceFrameReaders[BODY_COUNT]; + + // + RGBQUAD* m_pColorRGBX; + + Vector4 faceRotation; + +}; -- cgit v1.2.3 From af286a3fa5e1460faad85b816149ad1309ef3584 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Thu, 31 Jan 2019 16:59:14 +0100 Subject: Kinect: Now using HD API --- tracker-kinect-face/tracker.cpp | 229 +++++++++++++++++++++++++++++----------- tracker-kinect-face/tracker.h | 18 ++-- 2 files changed, 177 insertions(+), 70 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/tracker.cpp b/tracker-kinect-face/tracker.cpp index 3fe3b8cc..9b03f492 100644 --- a/tracker-kinect-face/tracker.cpp +++ b/tracker-kinect-face/tracker.cpp @@ -3,11 +3,81 @@ #include "tracker.h" -const double KinectFaceTracker::incr[6] = + +/// +bool IsValidRect(const RectI& aRect) +{ + if (aRect.Bottom != 0) + { + return true; + } + + if (aRect.Left != 0) + { + return true; + } + + if (aRect.Right != 0) + { + return true; + } + + if (aRect.Top != 0) + { + return true; + } + + return false; +} + +/// +bool IsNullVetor(const Vector4& aVector) +{ + if (aVector.w != 0) + { + return false; + } + + if (aVector.y != 0) + { + return false; + } + + if (aVector.y != 0) + { + return false; + } + + if (aVector.z != 0) + { + return false; + } + + return true; +} + +/// +bool IsNullPoint(const CameraSpacePoint& aPoint) { - 50, 40, 80, - 70, 5, 3 -}; + if (aPoint.X != 0) + { + return false; + } + + if (aPoint.Y != 0) + { + return false; + } + + if (aPoint.Z != 0) + { + return false; + } + + + return true; +} + KinectFaceTracker::KinectFaceTracker(): m_pKinectSensor(nullptr), @@ -17,11 +87,8 @@ KinectFaceTracker::KinectFaceTracker(): m_pBodyFrameReader(nullptr) { - for (int i = 0; i < BODY_COUNT; i++) - { - m_pFaceFrameSources[i] = nullptr; - m_pFaceFrameReaders[i] = nullptr; - } + m_pFaceFrameSource = nullptr; + m_pFaceFrameReader = nullptr; // create heap storage for color pixel data in RGBX format m_pColorRGBX = new RGBQUAD[cColorWidth * cColorHeight]; @@ -39,11 +106,8 @@ KinectFaceTracker::~KinectFaceTracker() //SafeRelease(m_pD2DFactory); // done with face sources and readers - for (int i = 0; i < BODY_COUNT; i++) - { - SafeRelease(m_pFaceFrameSources[i]); - SafeRelease(m_pFaceFrameReaders[i]); - } + SafeRelease(m_pFaceFrameSource); + SafeRelease(m_pFaceFrameReader); // done with body frame reader SafeRelease(m_pBodyFrameReader); @@ -79,46 +143,43 @@ module_status KinectFaceTracker::start_tracker(QFrame*) # include #endif +bool KinectFaceTracker::center() +{ + iFacePositionCenter = iFacePosition; + iFaceRotationCenter = iFaceRotation; + //TODO: Rotation center too + return true; +} + + void KinectFaceTracker::data(double *data) { const double dt = t.elapsed_seconds(); t.start(); -#ifdef EMIT_NAN - if ((rand() % 4) == 0) + + Update(); + //TODO: check if data is valid + + ExtractFaceRotationInDegrees(&iFaceRotationQuaternion, &iFaceRotation.X, &iFaceRotation.Y, &iFaceRotation.Z); + + if (!IsNullPoint(iFacePosition) && !IsNullPoint(iFaceRotation)) { - for (int i = 0; i < 6; i++) - data[i] = 0. / 0.; + // We have valid tracking retain position and rotation + iLastFacePosition = iFacePosition; + iLastFaceRotation = iFaceRotation; } - else -#endif - for (int i = 0; i < 6; i++) - { - double x = last_x[i] + incr[i] * dt; - if (x > 180) - x = -360 + x; - else if (x < -180) - x = 360 + x; - x = copysign(fmod(fabs(x), 360), x); - last_x[i] = x; - - if (i >= 3) - { - data[i] = x; - } - else - { - data[i] = x * 100 / 180.; - } - } + + // Feed our framework our last valid position and rotation + data[0] = (iLastFacePosition.X - iFacePositionCenter.X) * 100; // Convert to centimer to be in a range that suites OpenTrack. + data[1] = (iLastFacePosition.Y - iFacePositionCenter.Y) * 100; + data[2] = (iLastFacePosition.Z - iFacePositionCenter.Z) * 100; + // Yaw, picth, Roll + data[3] = 0-(iLastFaceRotation.X - iFaceRotationCenter.X); // Invert to be compatible with ED out-of-the-box + data[4] = (iLastFaceRotation.Y - iFaceRotationCenter.Y); + data[5] = (iLastFaceRotation.Z - iFaceRotationCenter.Z); - Update(); - //TODO: check if data is valid - data[0] = 0; - data[1] = 0; - data[2] = 0; - ExtractFaceRotationInDegrees(&faceRotation,&data[3], &data[4], &data[5]); } @@ -131,7 +192,7 @@ void KinectFaceTracker::data(double *data) /// rotation about the X-axis /// rotation about the Y-axis /// rotation about the Z-axis -void KinectFaceTracker::ExtractFaceRotationInDegrees(const Vector4* pQuaternion, double* pYaw, double* pPitch, double* pRoll) +void KinectFaceTracker::ExtractFaceRotationInDegrees(const Vector4* pQuaternion, float* pYaw, float* pPitch, float* pRoll) { double x = pQuaternion->x; double y = pQuaternion->y; @@ -215,12 +276,12 @@ HRESULT KinectFaceTracker::InitializeDefaultSensor() if (SUCCEEDED(hr)) { // create the face frame source by specifying the required face frame features - hr = CreateFaceFrameSource(m_pKinectSensor, 0, c_FaceFrameFeatures, &m_pFaceFrameSources[i]); + hr = CreateHighDefinitionFaceFrameSource(m_pKinectSensor, &m_pFaceFrameSource); } if (SUCCEEDED(hr)) { // open the corresponding reader - hr = m_pFaceFrameSources[i]->OpenReader(&m_pFaceFrameReaders[i]); + hr = m_pFaceFrameSource->OpenReader(&m_pFaceFrameReader); } } } @@ -348,12 +409,43 @@ void KinectFaceTracker::ProcessFaces() IBody* ppBodies[BODY_COUNT] = { 0 }; bool bHaveBodyData = SUCCEEDED(UpdateBodyData(ppBodies)); + if (!bHaveBodyData) + { + return; + } + + // TODO: Select closest body + // Just use the first body we find + BOOLEAN tracked; + int i = 0; + while (i < BODY_COUNT) + { + hr = ppBodies[i]->get_IsTracked(&tracked); + UINT64 trackingId = 0; + + if (SUCCEEDED(hr) && tracked) + { + hr = ppBodies[i]->get_TrackingId(&trackingId); + + if (SUCCEEDED(hr)) + { + // Tell our face source to use the given body id + hr = m_pFaceFrameSource->put_TrackingId(trackingId); + break; + } + + } + + i++; + } + + // iterate through each face reader for (int iFace = 0; iFace < BODY_COUNT; ++iFace) { // retrieve the latest face frame from this reader - IFaceFrame* pFaceFrame = nullptr; - hr = m_pFaceFrameReaders[iFace]->AcquireLatestFrame(&pFaceFrame); + IHighDefinitionFaceFrame* pFaceFrame = nullptr; + hr = m_pFaceFrameReader->AcquireLatestFrame(&pFaceFrame); BOOLEAN bFaceTracked = false; if (SUCCEEDED(hr) && nullptr != pFaceFrame) @@ -366,47 +458,54 @@ void KinectFaceTracker::ProcessFaces() { if (bFaceTracked) { - IFaceFrameResult* pFaceFrameResult = nullptr; + OutputDebugStringA("Tracking face!\n"); + + //IFaceFrameResult* pFaceFrameResult = nullptr; + IFaceAlignment* pFaceAlignment = nullptr; + CreateFaceAlignment(&pFaceAlignment); // TODO: check return? RectI faceBox = { 0 }; - PointF facePoints[FacePointType::FacePointType_Count]; - DetectionResult faceProperties[FaceProperty::FaceProperty_Count]; - //D2D1_POINT_2F faceTextLayout; + //D2D1_POINT_2F faceTextLayout; + + //hr = pFaceFrame->get_FaceFrameResult(&pFaceFrameResult); - hr = pFaceFrame->get_FaceFrameResult(&pFaceFrameResult); + hr = pFaceFrame->GetAndRefreshFaceAlignmentResult(pFaceAlignment); // need to verify if pFaceFrameResult contains data before trying to access it - if (SUCCEEDED(hr) && pFaceFrameResult != nullptr) + if (SUCCEEDED(hr) && pFaceAlignment != nullptr) { - hr = pFaceFrameResult->get_FaceBoundingBoxInColorSpace(&faceBox); + hr = pFaceAlignment->get_FaceBoundingBox(&faceBox); + //pFaceFrameResult->get_FaceBoundingBoxInColorSpace(); if (SUCCEEDED(hr)) { - hr = pFaceFrameResult->GetFacePointsInColorSpace(FacePointType::FacePointType_Count, facePoints); + //hr = pFaceFrameResult->GetFacePointsInColorSpace(FacePointType::FacePointType_Count, facePoints); + hr = pFaceAlignment->get_HeadPivotPoint(&iFacePosition); } if (SUCCEEDED(hr)) { - hr = pFaceFrameResult->get_FaceRotationQuaternion(&faceRotation); + //hr = pFaceFrameResult->get_FaceRotationQuaternion(&faceRotation); + hr = pFaceAlignment->get_FaceOrientation(&iFaceRotationQuaternion); } if (SUCCEEDED(hr)) { - hr = pFaceFrameResult->GetFaceProperties(FaceProperty::FaceProperty_Count, faceProperties); + //hr = pFaceFrameResult->GetFaceProperties(FaceProperty::FaceProperty_Count, faceProperties); } if (SUCCEEDED(hr)) { - //hr = GetFaceTextPositionInColorSpace(ppBodies[iFace], &faceTextLayout); + //hr = GetFaceTextPositionInColorSpace(ppBodies[0], &faceTextLayout); } if (SUCCEEDED(hr)) { // draw face frame results - //m_pDrawDataStreams->DrawFaceFrameResults(iFace, &faceBox, facePoints, &faceRotation, faceProperties, &faceTextLayout); + //m_pDrawDataStreams->DrawFaceFrameResults(0, &faceBox, facePoints, &faceRotation, faceProperties, &faceTextLayout); } } - SafeRelease(pFaceFrameResult); + SafeRelease(pFaceAlignment); } else { @@ -430,7 +529,7 @@ void KinectFaceTracker::ProcessFaces() if (SUCCEEDED(hr)) { // update the face frame source with the tracking ID - m_pFaceFrameSources[iFace]->put_TrackingId(bodyTId); + m_pFaceFrameSource->put_TrackingId(bodyTId); } } } @@ -449,3 +548,5 @@ void KinectFaceTracker::ProcessFaces() } } } + + diff --git a/tracker-kinect-face/tracker.h b/tracker-kinect-face/tracker.h index 88243b1f..cfadf997 100644 --- a/tracker-kinect-face/tracker.h +++ b/tracker-kinect-face/tracker.h @@ -32,10 +32,9 @@ public: ~KinectFaceTracker() override; module_status start_tracker(QFrame *) override; void data(double *data) override; + bool center() override; private: - static const double incr[6]; - double last_x[6]{}; Timer t; // Kinect stuff @@ -62,7 +61,7 @@ private: HRESULT InitializeDefaultSensor(); void ProcessFaces(); HRESULT UpdateBodyData(IBody** ppBodies); - void ExtractFaceRotationInDegrees(const Vector4* pQuaternion, double* pPitch, double* pYaw, double* pRoll); + void ExtractFaceRotationInDegrees(const Vector4* pQuaternion, float* pPitch, float* pYaw, float* pRoll); // Current Kinect IKinectSensor* m_pKinectSensor; @@ -77,14 +76,21 @@ private: IBodyFrameReader* m_pBodyFrameReader; // Face sources - IFaceFrameSource* m_pFaceFrameSources[BODY_COUNT]; + IHighDefinitionFaceFrameSource* m_pFaceFrameSource; // Face readers - IFaceFrameReader* m_pFaceFrameReaders[BODY_COUNT]; + IHighDefinitionFaceFrameReader* m_pFaceFrameReader; // RGBQUAD* m_pColorRGBX; - Vector4 faceRotation; + CameraSpacePoint iLastFacePosition; + CameraSpacePoint iFacePosition; + CameraSpacePoint iFacePositionCenter; + Vector4 iFaceRotationQuaternion; + // As Yaw, Pitch, Roll + CameraSpacePoint iLastFaceRotation; + CameraSpacePoint iFaceRotation; + CameraSpacePoint iFaceRotationCenter; }; -- cgit v1.2.3 From e419e8056cd883d0c5465b20343d69eba8c255ae Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Fri, 1 Feb 2019 17:11:43 +0100 Subject: Kinect: Initializing positions and rotations to zero. --- tracker-kinect-face/tracker.cpp | 11 ++++++++--- tracker-kinect-face/tracker.h | 15 --------------- 2 files changed, 8 insertions(+), 18 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/tracker.cpp b/tracker-kinect-face/tracker.cpp index 9b03f492..1ab14c87 100644 --- a/tracker-kinect-face/tracker.cpp +++ b/tracker-kinect-face/tracker.cpp @@ -84,8 +84,13 @@ KinectFaceTracker::KinectFaceTracker(): m_pCoordinateMapper(nullptr), m_pColorFrameReader(nullptr), m_pColorRGBX(nullptr), - m_pBodyFrameReader(nullptr) - + m_pBodyFrameReader(nullptr), + iLastFacePosition{0,0,0}, + iFacePositionCenter{ 0,0,0 }, + iFacePosition{ 0,0,0 }, + iLastFaceRotation{ 0,0,0 }, + iFaceRotationCenter{ 0,0,0 }, + iFaceRotation{ 0,0,0 } { m_pFaceFrameSource = nullptr; m_pFaceFrameReader = nullptr; @@ -175,7 +180,7 @@ void KinectFaceTracker::data(double *data) data[1] = (iLastFacePosition.Y - iFacePositionCenter.Y) * 100; data[2] = (iLastFacePosition.Z - iFacePositionCenter.Z) * 100; - // Yaw, picth, Roll + // Yaw, Picth, Roll data[3] = 0-(iLastFaceRotation.X - iFaceRotationCenter.X); // Invert to be compatible with ED out-of-the-box data[4] = (iLastFaceRotation.Y - iFaceRotationCenter.Y); data[5] = (iLastFaceRotation.Z - iFaceRotationCenter.Z); diff --git a/tracker-kinect-face/tracker.h b/tracker-kinect-face/tracker.h index cfadf997..162e1cff 100644 --- a/tracker-kinect-face/tracker.h +++ b/tracker-kinect-face/tracker.h @@ -38,21 +38,6 @@ private: Timer t; // Kinect stuff - - // define the face frame features required to be computed by this application - static const DWORD c_FaceFrameFeatures = - FaceFrameFeatures::FaceFrameFeatures_BoundingBoxInColorSpace - | FaceFrameFeatures::FaceFrameFeatures_PointsInColorSpace - | FaceFrameFeatures::FaceFrameFeatures_RotationOrientation - | FaceFrameFeatures::FaceFrameFeatures_Happy - | FaceFrameFeatures::FaceFrameFeatures_RightEyeClosed - | FaceFrameFeatures::FaceFrameFeatures_LeftEyeClosed - | FaceFrameFeatures::FaceFrameFeatures_MouthOpen - | FaceFrameFeatures::FaceFrameFeatures_MouthMoved - | FaceFrameFeatures::FaceFrameFeatures_LookingAway - | FaceFrameFeatures::FaceFrameFeatures_Glasses - | FaceFrameFeatures::FaceFrameFeatures_FaceEngagement; - static const int cColorWidth = 1920; static const int cColorHeight = 1080; -- cgit v1.2.3 From d528f282305b24484861d8913c96c222621cf18c Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sat, 2 Feb 2019 02:12:30 +0100 Subject: Kinect: Improved CMake configuration for Kinect SDK. --- tracker-kinect-face/CMakeLists.txt | 27 +++++++++++++++++++++++++-- tracker-kinect-face/tracker.cpp | 2 +- tracker-kinect-face/tracker.h | 2 +- 3 files changed, 27 insertions(+), 4 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/CMakeLists.txt b/tracker-kinect-face/CMakeLists.txt index d139b445..9a37f2d8 100644 --- a/tracker-kinect-face/CMakeLists.txt +++ b/tracker-kinect-face/CMakeLists.txt @@ -1,6 +1,29 @@ +# Setup cache variable to Kinect SDK path if(DEFINED ENV{KINECTSDK20_DIR}) + set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH $ENV{KINECTSDK20_DIR}) +else() + set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH "") +endif() + +# If we have a valid SDK path, try build that tracker +if(EXISTS ${KINECTSDK20_DIR}) + # Register our module otr_module(tracker-kinect-face) - target_include_directories(opentrack-tracker-kinect-face SYSTEM PUBLIC $ENV{KINECTSDK20_DIR}inc) - target_link_libraries(opentrack-tracker-kinect-face $ENV{KINECTSDK20_DIR}lib/x86/kinect20.lib $ENV{KINECTSDK20_DIR}lib/x86/kinect20.face.lib) + + # Add include path to Kinect SDK + target_include_directories(opentrack-tracker-kinect-face SYSTEM PUBLIC ${KINECTSDK20_DIR}/inc) + + # Check processor architecture + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + # 32 bits + set (kinect-arch-dir "x86") + elseif(CMAKE_SIZEOF_VOID_P EQUAL 8) + # 64 bits + set (kinect-arch-dir "x64") + endif() + + # Link against Kinect SDK libraries + target_link_libraries(opentrack-tracker-kinect-face ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/kinect20.lib ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/kinect20.face.lib) + endif() diff --git a/tracker-kinect-face/tracker.cpp b/tracker-kinect-face/tracker.cpp index 1ab14c87..407557f0 100644 --- a/tracker-kinect-face/tracker.cpp +++ b/tracker-kinect-face/tracker.cpp @@ -132,7 +132,7 @@ KinectFaceTracker::~KinectFaceTracker() SafeRelease(m_pKinectSensor); } -module_status KinectFaceTracker::start_tracker(QFrame*) +module_status KinectFaceTracker::start_tracker(QFrame* aFrame) { t.start(); diff --git a/tracker-kinect-face/tracker.h b/tracker-kinect-face/tracker.h index 162e1cff..1f986e22 100644 --- a/tracker-kinect-face/tracker.h +++ b/tracker-kinect-face/tracker.h @@ -30,7 +30,7 @@ class KinectFaceTracker : public ITracker public: KinectFaceTracker(); ~KinectFaceTracker() override; - module_status start_tracker(QFrame *) override; + module_status start_tracker(QFrame* aFrame) override; void data(double *data) override; bool center() override; -- cgit v1.2.3 From 388a876b68a83956b687100007aab7afafbb08ab Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sat, 2 Feb 2019 02:50:04 +0100 Subject: Kinect: Adding tracker icon. --- tracker-kinect-face/images/kinect.png | Bin 0 -> 217 bytes tracker-kinect-face/kinect.qrc | 5 +++++ tracker-kinect-face/lang/nl_NL.ts | 2 +- tracker-kinect-face/lang/ru_RU.ts | 2 +- tracker-kinect-face/lang/stub.ts | 2 +- tracker-kinect-face/lang/zh_CN.ts | 2 +- tracker-kinect-face/test.h | 4 ++-- 7 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 tracker-kinect-face/images/kinect.png create mode 100644 tracker-kinect-face/kinect.qrc (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/images/kinect.png b/tracker-kinect-face/images/kinect.png new file mode 100644 index 00000000..fd8f5f77 Binary files /dev/null and b/tracker-kinect-face/images/kinect.png differ diff --git a/tracker-kinect-face/kinect.qrc b/tracker-kinect-face/kinect.qrc new file mode 100644 index 00000000..8b27c81e --- /dev/null +++ b/tracker-kinect-face/kinect.qrc @@ -0,0 +1,5 @@ + + + images/kinect.png + + diff --git a/tracker-kinect-face/lang/nl_NL.ts b/tracker-kinect-face/lang/nl_NL.ts index b6139882..dec7a177 100644 --- a/tracker-kinect-face/lang/nl_NL.ts +++ b/tracker-kinect-face/lang/nl_NL.ts @@ -4,7 +4,7 @@ test_metadata - Kinect Face + Kinect Face - v0.1 diff --git a/tracker-kinect-face/lang/ru_RU.ts b/tracker-kinect-face/lang/ru_RU.ts index 52aab9f3..1634c0c2 100644 --- a/tracker-kinect-face/lang/ru_RU.ts +++ b/tracker-kinect-face/lang/ru_RU.ts @@ -4,7 +4,7 @@ test_metadata - Kinect Face + Kinect Face - v0.1 diff --git a/tracker-kinect-face/lang/stub.ts b/tracker-kinect-face/lang/stub.ts index 84a3c6af..bb36fb1e 100644 --- a/tracker-kinect-face/lang/stub.ts +++ b/tracker-kinect-face/lang/stub.ts @@ -4,7 +4,7 @@ test_metadata - Kinect Face + Kinect Face - v0.1 diff --git a/tracker-kinect-face/lang/zh_CN.ts b/tracker-kinect-face/lang/zh_CN.ts index 84a3c6af..bb36fb1e 100644 --- a/tracker-kinect-face/lang/zh_CN.ts +++ b/tracker-kinect-face/lang/zh_CN.ts @@ -4,7 +4,7 @@ test_metadata - Kinect Face + Kinect Face - v0.1 diff --git a/tracker-kinect-face/test.h b/tracker-kinect-face/test.h index 2e105ae3..1af0409b 100644 --- a/tracker-kinect-face/test.h +++ b/tracker-kinect-face/test.h @@ -22,7 +22,7 @@ class test_metadata : public Metadata { Q_OBJECT - QString name() { return tr("Kinect Face"); } - QIcon icon() { return QIcon(":/images/opentrack.png"); } + QString name() { return tr("Kinect Face - v0.1"); } + QIcon icon() { return QIcon(":/images/kinect.png"); } }; -- cgit v1.2.3 From ba8796acda06c9ae5f528dbbf655c714844d5429 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sat, 2 Feb 2019 17:36:05 +0100 Subject: Various fixes from code review. --- tracker-kinect-face/CMakeLists.txt | 51 ++++++++++++++++++++------------------ tracker-kinect-face/lang/nl_NL.ts | 2 +- tracker-kinect-face/lang/ru_RU.ts | 2 +- tracker-kinect-face/lang/stub.ts | 2 +- tracker-kinect-face/lang/zh_CN.ts | 2 +- tracker-kinect-face/test.h | 2 +- tracker-kinect-face/tracker.cpp | 11 +++----- 7 files changed, 36 insertions(+), 36 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/CMakeLists.txt b/tracker-kinect-face/CMakeLists.txt index 9a37f2d8..fca1e495 100644 --- a/tracker-kinect-face/CMakeLists.txt +++ b/tracker-kinect-face/CMakeLists.txt @@ -1,29 +1,32 @@ +# Kinect SDK is Windows only +if (WIN32) + # Setup cache variable to Kinect SDK path + if(DEFINED ENV{KINECTSDK20_DIR}) + set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH $ENV{KINECTSDK20_DIR}) + else() + set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH "") + endif() -# Setup cache variable to Kinect SDK path -if(DEFINED ENV{KINECTSDK20_DIR}) - set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH $ENV{KINECTSDK20_DIR}) -else() - set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH "") -endif() + # If we have a valid SDK path, try build that tracker + if(EXISTS ${KINECTSDK20_DIR}) + # Register our module + otr_module(tracker-kinect-face) -# If we have a valid SDK path, try build that tracker -if(EXISTS ${KINECTSDK20_DIR}) - # Register our module - otr_module(tracker-kinect-face) - - # Add include path to Kinect SDK - target_include_directories(opentrack-tracker-kinect-face SYSTEM PUBLIC ${KINECTSDK20_DIR}/inc) + # Add include path to Kinect SDK + target_include_directories(opentrack-tracker-kinect-face SYSTEM PUBLIC ${KINECTSDK20_DIR}/inc) - # Check processor architecture - if(CMAKE_SIZEOF_VOID_P EQUAL 4) - # 32 bits - set (kinect-arch-dir "x86") - elseif(CMAKE_SIZEOF_VOID_P EQUAL 8) - # 64 bits - set (kinect-arch-dir "x64") - endif() + # Check processor architecture + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + # 32 bits + set (kinect-arch-dir "x86") + elseif(CMAKE_SIZEOF_VOID_P EQUAL 8) + # 64 bits + set (kinect-arch-dir "x64") + endif() - # Link against Kinect SDK libraries - target_link_libraries(opentrack-tracker-kinect-face ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/kinect20.lib ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/kinect20.face.lib) + # Link against Kinect SDK libraries + target_link_libraries(opentrack-tracker-kinect-face ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/kinect20.lib ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/kinect20.face.lib) + + endif() -endif() +endif() \ No newline at end of file diff --git a/tracker-kinect-face/lang/nl_NL.ts b/tracker-kinect-face/lang/nl_NL.ts index dec7a177..f4fe07d8 100644 --- a/tracker-kinect-face/lang/nl_NL.ts +++ b/tracker-kinect-face/lang/nl_NL.ts @@ -4,7 +4,7 @@ test_metadata - Kinect Face - v0.1 + Kinect Face 0.1 diff --git a/tracker-kinect-face/lang/ru_RU.ts b/tracker-kinect-face/lang/ru_RU.ts index 1634c0c2..d4108ed3 100644 --- a/tracker-kinect-face/lang/ru_RU.ts +++ b/tracker-kinect-face/lang/ru_RU.ts @@ -4,7 +4,7 @@ test_metadata - Kinect Face - v0.1 + Kinect Face 0.1 diff --git a/tracker-kinect-face/lang/stub.ts b/tracker-kinect-face/lang/stub.ts index bb36fb1e..f82903a3 100644 --- a/tracker-kinect-face/lang/stub.ts +++ b/tracker-kinect-face/lang/stub.ts @@ -4,7 +4,7 @@ test_metadata - Kinect Face - v0.1 + Kinect Face 0.1 diff --git a/tracker-kinect-face/lang/zh_CN.ts b/tracker-kinect-face/lang/zh_CN.ts index bb36fb1e..f82903a3 100644 --- a/tracker-kinect-face/lang/zh_CN.ts +++ b/tracker-kinect-face/lang/zh_CN.ts @@ -4,7 +4,7 @@ test_metadata - Kinect Face - v0.1 + Kinect Face 0.1 diff --git a/tracker-kinect-face/test.h b/tracker-kinect-face/test.h index 1af0409b..0f8c170b 100644 --- a/tracker-kinect-face/test.h +++ b/tracker-kinect-face/test.h @@ -22,7 +22,7 @@ class test_metadata : public Metadata { Q_OBJECT - QString name() { return tr("Kinect Face - v0.1"); } + QString name() { return tr("Kinect Face 0.1"); } QIcon icon() { return QIcon(":/images/kinect.png"); } }; diff --git a/tracker-kinect-face/tracker.cpp b/tracker-kinect-face/tracker.cpp index 407557f0..c5517f89 100644 --- a/tracker-kinect-face/tracker.cpp +++ b/tracker-kinect-face/tracker.cpp @@ -38,7 +38,7 @@ bool IsNullVetor(const Vector4& aVector) return false; } - if (aVector.y != 0) + if (aVector.x != 0) { return false; } @@ -74,7 +74,6 @@ bool IsNullPoint(const CameraSpacePoint& aPoint) return false; } - return true; } @@ -150,9 +149,9 @@ module_status KinectFaceTracker::start_tracker(QFrame* aFrame) bool KinectFaceTracker::center() { + // Mark our center iFacePositionCenter = iFacePosition; iFaceRotationCenter = iFaceRotation; - //TODO: Rotation center too return true; } @@ -181,11 +180,9 @@ void KinectFaceTracker::data(double *data) data[2] = (iLastFacePosition.Z - iFacePositionCenter.Z) * 100; // Yaw, Picth, Roll - data[3] = 0-(iLastFaceRotation.X - iFaceRotationCenter.X); // Invert to be compatible with ED out-of-the-box + data[3] = -(iLastFaceRotation.X - iFaceRotationCenter.X); // Invert to be compatible with ED out-of-the-box data[4] = (iLastFaceRotation.Y - iFaceRotationCenter.Y); data[5] = (iLastFaceRotation.Z - iFaceRotationCenter.Z); - - } @@ -463,7 +460,7 @@ void KinectFaceTracker::ProcessFaces() { if (bFaceTracked) { - OutputDebugStringA("Tracking face!\n"); + //OutputDebugStringA("Tracking face!\n"); //IFaceFrameResult* pFaceFrameResult = nullptr; IFaceAlignment* pFaceAlignment = nullptr; -- cgit v1.2.3 From 32c809f5e0c6a9b6a5f202fffe5d034125957551 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sat, 2 Feb 2019 18:17:05 +0100 Subject: Kinect: Installing SDK binaries. --- tracker-kinect-face/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/CMakeLists.txt b/tracker-kinect-face/CMakeLists.txt index fca1e495..cbdc2a97 100644 --- a/tracker-kinect-face/CMakeLists.txt +++ b/tracker-kinect-face/CMakeLists.txt @@ -25,7 +25,12 @@ if (WIN32) endif() # Link against Kinect SDK libraries - target_link_libraries(opentrack-tracker-kinect-face ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/kinect20.lib ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/kinect20.face.lib) + target_link_libraries(opentrack-tracker-kinect-face ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.lib ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.Face.lib) + + # Install Kinect Face DLL + install(FILES ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/Kinect20.Face.dll DESTINATION ./modules/ PERMISSIONS ${opentrack-perms-exec}) + # Install Kinect Face Database + install(DIRECTORY ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/NuiDatabase DESTINATION ./modules/) endif() -- cgit v1.2.3 From 997df9f3cbc9ef04c666e404b96ac4b02097b4f3 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sat, 2 Feb 2019 21:51:27 +0100 Subject: Kinect: now with basic video preview. We have a massive memory leak, could have been there before though. --- tracker-kinect-face/CMakeLists.txt | 63 ++++++++++++++++++++------------------ tracker-kinect-face/tracker.cpp | 44 +++++++++++++++++++++----- tracker-kinect-face/tracker.h | 6 ++++ 3 files changed, 75 insertions(+), 38 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/CMakeLists.txt b/tracker-kinect-face/CMakeLists.txt index cbdc2a97..e12534b2 100644 --- a/tracker-kinect-face/CMakeLists.txt +++ b/tracker-kinect-face/CMakeLists.txt @@ -1,37 +1,40 @@ # Kinect SDK is Windows only if (WIN32) - # Setup cache variable to Kinect SDK path - if(DEFINED ENV{KINECTSDK20_DIR}) - set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH $ENV{KINECTSDK20_DIR}) - else() - set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH "") - endif() + find_package(OpenCV QUIET) + if(OpenCV_FOUND) + # Setup cache variable to Kinect SDK path + if(DEFINED ENV{KINECTSDK20_DIR}) + set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH $ENV{KINECTSDK20_DIR}) + else() + set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH "") + endif() - # If we have a valid SDK path, try build that tracker - if(EXISTS ${KINECTSDK20_DIR}) - # Register our module - otr_module(tracker-kinect-face) + # If we have a valid SDK path, try build that tracker + if(EXISTS ${KINECTSDK20_DIR}) + # Register our module + otr_module(tracker-kinect-face) - # Add include path to Kinect SDK - target_include_directories(opentrack-tracker-kinect-face SYSTEM PUBLIC ${KINECTSDK20_DIR}/inc) + # Add include path to Kinect SDK + target_include_directories(opentrack-tracker-kinect-face SYSTEM PUBLIC ${KINECTSDK20_DIR}/inc) - # Check processor architecture - if(CMAKE_SIZEOF_VOID_P EQUAL 4) - # 32 bits - set (kinect-arch-dir "x86") - elseif(CMAKE_SIZEOF_VOID_P EQUAL 8) - # 64 bits - set (kinect-arch-dir "x64") - endif() + # Check processor architecture + if(CMAKE_SIZEOF_VOID_P EQUAL 4) + # 32 bits + set (kinect-arch-dir "x86") + elseif(CMAKE_SIZEOF_VOID_P EQUAL 8) + # 64 bits + set (kinect-arch-dir "x64") + endif() - # Link against Kinect SDK libraries - target_link_libraries(opentrack-tracker-kinect-face ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.lib ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.Face.lib) - - # Install Kinect Face DLL - install(FILES ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/Kinect20.Face.dll DESTINATION ./modules/ PERMISSIONS ${opentrack-perms-exec}) - # Install Kinect Face Database - install(DIRECTORY ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/NuiDatabase DESTINATION ./modules/) - - endif() + # Link against Kinect SDK libraries + target_link_libraries(opentrack-tracker-kinect-face ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.lib ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.Face.lib) + # Link against OpenCV stuff, needed for video preview + target_link_libraries(opentrack-tracker-kinect-face opencv_imgproc opentrack-cv opencv_core) -endif() \ No newline at end of file + # Install Kinect Face DLL + install(FILES ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/Kinect20.Face.dll DESTINATION ./modules/ PERMISSIONS ${opentrack-perms-exec}) + # Install Kinect Face Database + install(DIRECTORY ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/NuiDatabase DESTINATION ./modules/) + endif(EXISTS ${KINECTSDK20_DIR}) + endif(OpenCV_FOUND) +endif(WIN32) \ No newline at end of file diff --git a/tracker-kinect-face/tracker.cpp b/tracker-kinect-face/tracker.cpp index c5517f89..073cf0be 100644 --- a/tracker-kinect-face/tracker.cpp +++ b/tracker-kinect-face/tracker.cpp @@ -2,6 +2,9 @@ #include "tracker.h" +#include +#include + /// @@ -137,6 +140,15 @@ module_status KinectFaceTracker::start_tracker(QFrame* aFrame) if (SUCCEEDED(InitializeDefaultSensor())) { + // Setup our video preview widget + iVideoWidget = std::make_unique(aFrame); + iLayout = std::make_unique(aFrame); + iLayout->setContentsMargins(0, 0, 0, 0); + iLayout->addWidget(iVideoWidget.get()); + aFrame->setLayout(iLayout.get()); + //video_widget->resize(video_frame->width(), video_frame->height()); + aFrame->show(); + return status_ok(); } @@ -161,12 +173,11 @@ void KinectFaceTracker::data(double *data) const double dt = t.elapsed_seconds(); t.start(); - Update(); - //TODO: check if data is valid ExtractFaceRotationInDegrees(&iFaceRotationQuaternion, &iFaceRotation.X, &iFaceRotation.Y, &iFaceRotation.Z); + //Check if data is valid if (!IsNullPoint(iFacePosition) && !IsNullPoint(iFaceRotation)) { // We have valid tracking retain position and rotation @@ -218,8 +229,6 @@ void KinectFaceTracker::ExtractFaceRotationInDegrees(const Vector4* pQuaternion, *pPitch = dPitch; *pYaw = dYaw; *pRoll = dRoll; - - } @@ -348,7 +357,8 @@ void KinectFaceTracker::Update() if (SUCCEEDED(hr)) { - if (imageFormat == ColorImageFormat_Bgra) + // Fetch color buffer + if (imageFormat == ColorImageFormat_Rgba) { hr = pColorFrame->AccessRawUnderlyingBuffer(&nBufferSize, reinterpret_cast(&pBuffer)); } @@ -356,12 +366,13 @@ void KinectFaceTracker::Update() { pBuffer = m_pColorRGBX; nBufferSize = cColorWidth * cColorHeight * sizeof(RGBQUAD); - hr = pColorFrame->CopyConvertedFrameDataToArray(nBufferSize, reinterpret_cast(pBuffer), ColorImageFormat_Bgra); + hr = pColorFrame->CopyConvertedFrameDataToArray(nBufferSize, reinterpret_cast(pBuffer), ColorImageFormat_Rgba); } else { hr = E_FAIL; } + } if (SUCCEEDED(hr)) @@ -370,6 +381,24 @@ void KinectFaceTracker::Update() ProcessFaces(); } + if (SUCCEEDED(hr)) + { + // Setup our image + QImage image((const unsigned char*)pBuffer, cColorWidth, cColorHeight, sizeof(RGBQUAD)*cColorWidth, QImage::Format_RGBA8888); + if (IsValidRect(iFaceBox)) + { + // Draw our face bounding box + QPainter painter(&image); + painter.setBrush(Qt::NoBrush); + painter.setPen(QPen(Qt::red, 8)); + painter.drawRect(iFaceBox.Left, iFaceBox.Top, iFaceBox.Right - iFaceBox.Left, iFaceBox.Bottom - iFaceBox.Top); + bool bEnd = painter.end(); + } + + // Update our video preview + iVideoWidget->update_image(image); + } + SafeRelease(pFrameDescription); } @@ -465,7 +494,6 @@ void KinectFaceTracker::ProcessFaces() //IFaceFrameResult* pFaceFrameResult = nullptr; IFaceAlignment* pFaceAlignment = nullptr; CreateFaceAlignment(&pFaceAlignment); // TODO: check return? - RectI faceBox = { 0 }; //D2D1_POINT_2F faceTextLayout; //hr = pFaceFrame->get_FaceFrameResult(&pFaceFrameResult); @@ -475,7 +503,7 @@ void KinectFaceTracker::ProcessFaces() // need to verify if pFaceFrameResult contains data before trying to access it if (SUCCEEDED(hr) && pFaceAlignment != nullptr) { - hr = pFaceAlignment->get_FaceBoundingBox(&faceBox); + hr = pFaceAlignment->get_FaceBoundingBox(&iFaceBox); //pFaceFrameResult->get_FaceBoundingBoxInColorSpace(); if (SUCCEEDED(hr)) diff --git a/tracker-kinect-face/tracker.h b/tracker-kinect-face/tracker.h index 1f986e22..5068f185 100644 --- a/tracker-kinect-face/tracker.h +++ b/tracker-kinect-face/tracker.h @@ -6,6 +6,7 @@ #include "api/plugin-api.hpp" #include "compat/timer.hpp" #include "compat/macros.hpp" +#include "cv/video-widget.hpp" // Kinect Header files #include @@ -69,6 +70,8 @@ private: // RGBQUAD* m_pColorRGBX; + RectI iFaceBox = { 0 }; + CameraSpacePoint iLastFacePosition; CameraSpacePoint iFacePosition; CameraSpacePoint iFacePositionCenter; @@ -78,4 +81,7 @@ private: CameraSpacePoint iLastFaceRotation; CameraSpacePoint iFaceRotation; CameraSpacePoint iFaceRotationCenter; + // + std::unique_ptr iVideoWidget; + std::unique_ptr iLayout; }; -- cgit v1.2.3 From fdc70ac7039d633b26f659e476681d99d45c685a Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sun, 3 Feb 2019 09:13:19 +0100 Subject: Kinect: Adding class to encapsulate Microsoft interface pointer. --- tracker-kinect-face/tracker.cpp | 26 +++++++++++++++++++------- tracker-kinect-face/tracker.h | 28 +++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 8 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/tracker.cpp b/tracker-kinect-face/tracker.cpp index 073cf0be..8c2c164d 100644 --- a/tracker-kinect-face/tracker.cpp +++ b/tracker-kinect-face/tracker.cpp @@ -232,6 +232,7 @@ void KinectFaceTracker::ExtractFaceRotationInDegrees(const Vector4* pQuaternion, } + /// /// Initializes the default Kinect sensor /// @@ -248,9 +249,9 @@ HRESULT KinectFaceTracker::InitializeDefaultSensor() if (m_pKinectSensor) { - // Initialize Kinect and get color, body and face readers - IColorFrameSource* pColorFrameSource = nullptr; - IBodyFrameSource* pBodyFrameSource = nullptr; + // Initialize Kinect and get color, body and face readers + UniqueInterface pBodyFrameSource; + UniqueInterface pColorFrameSource; hr = m_pKinectSensor->Open(); @@ -261,17 +262,23 @@ HRESULT KinectFaceTracker::InitializeDefaultSensor() if (SUCCEEDED(hr)) { - hr = m_pKinectSensor->get_ColorFrameSource(&pColorFrameSource); + hr = m_pKinectSensor->get_ColorFrameSource(pColorFrameSource.PtrPtr()); + pColorFrameSource.Reset(); } + + //std::unique_ptr colorFrameSource(pColorFrameSource, SafeRelease); if (SUCCEEDED(hr)) { + //hr = (*pColorFrameSource.PtrPtr())->OpenReader(&m_pColorFrameReader); hr = pColorFrameSource->OpenReader(&m_pColorFrameReader); } + if (SUCCEEDED(hr)) { - hr = m_pKinectSensor->get_BodyFrameSource(&pBodyFrameSource); + hr = m_pKinectSensor->get_BodyFrameSource(pBodyFrameSource.PtrPtr()); + pBodyFrameSource.Reset(); } if (SUCCEEDED(hr)) @@ -297,8 +304,13 @@ HRESULT KinectFaceTracker::InitializeDefaultSensor() } } - SafeRelease(pColorFrameSource); - SafeRelease(pBodyFrameSource); + //pColorFrameSource.Free(); + //pBodyFrameSource.Free(); + //SafeRelease(pColorFrameSource); + //SafeRelease(pBodyFrameSource); + + //SafeRelease(*pBodyFrameSource.PtrPtr()); + //SafeRelease(*pColorFrameSource.PtrPtr()); } if (!m_pKinectSensor || FAILED(hr)) diff --git a/tracker-kinect-face/tracker.h b/tracker-kinect-face/tracker.h index 5068f185..220a126f 100644 --- a/tracker-kinect-face/tracker.h +++ b/tracker-kinect-face/tracker.h @@ -14,7 +14,7 @@ #pragma once -// Safe release for interfaces +// @deprecated Use UniqueInterface instead. Remove it at some point. template inline void SafeRelease(Interface *& pInterfaceToRelease) { @@ -25,6 +25,32 @@ inline void SafeRelease(Interface *& pInterfaceToRelease) } } +template +inline void ReleaseInterface(Interface* pInterfaceToRelease) +{ + if (pInterfaceToRelease != nullptr) + { + pInterfaceToRelease->Release(); + } +} + +// Safely use Microsoft interfaces. +template +class UniqueInterface : public std::unique_ptr)> ///**/ +{ +public: + UniqueInterface() : std::unique_ptr)>(nullptr, ReleaseInterface){} + // Access pointer, typically for creation + T** PtrPtr() { return &iPtr; }; + // Called this once the pointer was created + void Reset() { std::unique_ptr)>::reset(iPtr); } + // If ever you want to release that interface before the object is deleted + void Free() { iPtr = nullptr; Reset(); } +private: + T* iPtr = nullptr; +}; + + class KinectFaceTracker : public ITracker { -- cgit v1.2.3 From 70156963a8f65d21231de88460e6559bd78f770c Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sun, 3 Feb 2019 12:13:53 +0100 Subject: Kinect: Fixing massive memory leak after start/stop cycle. --- tracker-kinect-face/tracker.cpp | 89 +++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 57 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/tracker.cpp b/tracker-kinect-face/tracker.cpp index 8c2c164d..54a716ac 100644 --- a/tracker-kinect-face/tracker.cpp +++ b/tracker-kinect-face/tracker.cpp @@ -241,82 +241,57 @@ HRESULT KinectFaceTracker::InitializeDefaultSensor() { HRESULT hr; + // Get and open Kinect sensor hr = GetDefaultKinectSensor(&m_pKinectSensor); - if (FAILED(hr)) + if (SUCCEEDED(hr)) { - return hr; + hr = m_pKinectSensor->Open(); } - if (m_pKinectSensor) + // TODO: check if we still need that guy + if (SUCCEEDED(hr)) { - // Initialize Kinect and get color, body and face readers - UniqueInterface pBodyFrameSource; - UniqueInterface pColorFrameSource; - - hr = m_pKinectSensor->Open(); + hr = m_pKinectSensor->get_CoordinateMapper(&m_pCoordinateMapper); + } - if (SUCCEEDED(hr)) - { - hr = m_pKinectSensor->get_CoordinateMapper(&m_pCoordinateMapper); - } + // Create color frame reader + if (SUCCEEDED(hr)) + { + UniqueInterface colorFrameSource; + hr = m_pKinectSensor->get_ColorFrameSource(colorFrameSource.PtrPtr()); + colorFrameSource.Reset(); if (SUCCEEDED(hr)) { - hr = m_pKinectSensor->get_ColorFrameSource(pColorFrameSource.PtrPtr()); - pColorFrameSource.Reset(); + hr = colorFrameSource->OpenReader(&m_pColorFrameReader); } + } - //std::unique_ptr colorFrameSource(pColorFrameSource, SafeRelease); - - if (SUCCEEDED(hr)) - { - //hr = (*pColorFrameSource.PtrPtr())->OpenReader(&m_pColorFrameReader); - hr = pColorFrameSource->OpenReader(&m_pColorFrameReader); - } - - - if (SUCCEEDED(hr)) - { - hr = m_pKinectSensor->get_BodyFrameSource(pBodyFrameSource.PtrPtr()); - pBodyFrameSource.Reset(); - } + // Create body frame reader + if (SUCCEEDED(hr)) + { + UniqueInterface bodyFrameSource; + hr = m_pKinectSensor->get_BodyFrameSource(bodyFrameSource.PtrPtr()); + bodyFrameSource.Reset(); if (SUCCEEDED(hr)) { - hr = pBodyFrameSource->OpenReader(&m_pBodyFrameReader); - } - - if (SUCCEEDED(hr)) - { - // create a face frame source + reader to track each body in the fov - for (int i = 0; i < BODY_COUNT; i++) - { - if (SUCCEEDED(hr)) - { - // create the face frame source by specifying the required face frame features - hr = CreateHighDefinitionFaceFrameSource(m_pKinectSensor, &m_pFaceFrameSource); - } - if (SUCCEEDED(hr)) - { - // open the corresponding reader - hr = m_pFaceFrameSource->OpenReader(&m_pFaceFrameReader); - } - } + hr = bodyFrameSource->OpenReader(&m_pBodyFrameReader); } + } - //pColorFrameSource.Free(); - //pBodyFrameSource.Free(); - //SafeRelease(pColorFrameSource); - //SafeRelease(pBodyFrameSource); - - //SafeRelease(*pBodyFrameSource.PtrPtr()); - //SafeRelease(*pColorFrameSource.PtrPtr()); + // Create HD face frame source + if (SUCCEEDED(hr)) + { + // create the face frame source by specifying the required face frame features + hr = CreateHighDefinitionFaceFrameSource(m_pKinectSensor, &m_pFaceFrameSource); } - if (!m_pKinectSensor || FAILED(hr)) + // Create HD face frame reader + if (SUCCEEDED(hr)) { - //SetStatusMessage(L"No ready Kinect found!", 10000, true); - return E_FAIL; + // open the corresponding reader + hr = m_pFaceFrameSource->OpenReader(&m_pFaceFrameReader); } return hr; -- cgit v1.2.3 From 1da2d0e71a03296a40b49316b5e23986de823a75 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sun, 3 Feb 2019 12:28:12 +0100 Subject: Kinect: Renaming a few files. --- tracker-kinect-face/kinect_face_settings.cpp | 45 +++ tracker-kinect-face/kinect_face_settings.h | 28 ++ tracker-kinect-face/kinect_face_tracker.cpp | 569 +++++++++++++++++++++++++++ tracker-kinect-face/kinect_face_tracker.h | 113 ++++++ tracker-kinect-face/test.cpp | 45 --- tracker-kinect-face/test.h | 28 -- tracker-kinect-face/test_dialog.cpp | 2 +- tracker-kinect-face/tracker.cpp | 569 --------------------------- tracker-kinect-face/tracker.h | 113 ------ 9 files changed, 756 insertions(+), 756 deletions(-) create mode 100644 tracker-kinect-face/kinect_face_settings.cpp create mode 100644 tracker-kinect-face/kinect_face_settings.h create mode 100644 tracker-kinect-face/kinect_face_tracker.cpp create mode 100644 tracker-kinect-face/kinect_face_tracker.h delete mode 100644 tracker-kinect-face/test.cpp delete mode 100644 tracker-kinect-face/test.h delete mode 100644 tracker-kinect-face/tracker.cpp delete mode 100644 tracker-kinect-face/tracker.h (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/kinect_face_settings.cpp b/tracker-kinect-face/kinect_face_settings.cpp new file mode 100644 index 00000000..53aada06 --- /dev/null +++ b/tracker-kinect-face/kinect_face_settings.cpp @@ -0,0 +1,45 @@ +/* Copyright (c) 2014, Stanislaw Halik + + * Permission to use, copy, modify, and/or distribute this + * software for any purpose with or without fee is hereby granted, + * provided that the above copyright notice and this permission + * notice appear in all copies. + */ + +#include "kinect_face_settings.h" +#include "kinect_face_tracker.h" +#include "api/plugin-api.hpp" +#include "compat/math-imports.hpp" + +#include + +#include +#include + + +test_dialog::test_dialog() +{ + ui.setupUi(this); + + connect(ui.buttonBox, &QDialogButtonBox::clicked, [this](QAbstractButton* btn) { + if (btn == ui.buttonBox->button(QDialogButtonBox::Abort)) + // NOLINTNEXTLINE + *(volatile int*)nullptr = 0; + }); + + connect(ui.buttonBox, SIGNAL(accepted()), this, SLOT(doOK())); + connect(ui.buttonBox, SIGNAL(rejected()), this, SLOT(doCancel())); +} + +void test_dialog::doOK() +{ + //s.b->save(); + close(); +} + +void test_dialog::doCancel() +{ + close(); +} + +OPENTRACK_DECLARE_TRACKER(KinectFaceTracker, test_dialog, test_metadata) diff --git a/tracker-kinect-face/kinect_face_settings.h b/tracker-kinect-face/kinect_face_settings.h new file mode 100644 index 00000000..0f8c170b --- /dev/null +++ b/tracker-kinect-face/kinect_face_settings.h @@ -0,0 +1,28 @@ +#pragma once +#include "ui_test.h" +#include "compat/macros.hpp" +#include "api/plugin-api.hpp" + + +class test_dialog : public ITrackerDialog +{ + Q_OBJECT + + Ui::test_ui ui; +public: + test_dialog(); + void register_tracker(ITracker *) override {} + void unregister_tracker() override {} +private slots: + void doOK(); + void doCancel(); +}; + +class test_metadata : public Metadata +{ + Q_OBJECT + + QString name() { return tr("Kinect Face 0.1"); } + QIcon icon() { return QIcon(":/images/kinect.png"); } +}; + diff --git a/tracker-kinect-face/kinect_face_tracker.cpp b/tracker-kinect-face/kinect_face_tracker.cpp new file mode 100644 index 00000000..ccac8aa9 --- /dev/null +++ b/tracker-kinect-face/kinect_face_tracker.cpp @@ -0,0 +1,569 @@ + + +#include "kinect_face_tracker.h" + +#include +#include + + + +/// +bool IsValidRect(const RectI& aRect) +{ + if (aRect.Bottom != 0) + { + return true; + } + + if (aRect.Left != 0) + { + return true; + } + + if (aRect.Right != 0) + { + return true; + } + + if (aRect.Top != 0) + { + return true; + } + + return false; +} + +/// +bool IsNullVetor(const Vector4& aVector) +{ + if (aVector.w != 0) + { + return false; + } + + if (aVector.x != 0) + { + return false; + } + + if (aVector.y != 0) + { + return false; + } + + if (aVector.z != 0) + { + return false; + } + + return true; +} + +/// +bool IsNullPoint(const CameraSpacePoint& aPoint) +{ + if (aPoint.X != 0) + { + return false; + } + + if (aPoint.Y != 0) + { + return false; + } + + if (aPoint.Z != 0) + { + return false; + } + + return true; +} + + +KinectFaceTracker::KinectFaceTracker(): + m_pKinectSensor(nullptr), + m_pCoordinateMapper(nullptr), + m_pColorFrameReader(nullptr), + m_pColorRGBX(nullptr), + m_pBodyFrameReader(nullptr), + iLastFacePosition{0,0,0}, + iFacePositionCenter{ 0,0,0 }, + iFacePosition{ 0,0,0 }, + iLastFaceRotation{ 0,0,0 }, + iFaceRotationCenter{ 0,0,0 }, + iFaceRotation{ 0,0,0 } +{ + m_pFaceFrameSource = nullptr; + m_pFaceFrameReader = nullptr; + + // create heap storage for color pixel data in RGBX format + m_pColorRGBX = new RGBQUAD[cColorWidth * cColorHeight]; +} + +KinectFaceTracker::~KinectFaceTracker() +{ + if (m_pColorRGBX) + { + delete[] m_pColorRGBX; + m_pColorRGBX = nullptr; + } + + // clean up Direct2D + //SafeRelease(m_pD2DFactory); + + // done with face sources and readers + SafeRelease(m_pFaceFrameSource); + SafeRelease(m_pFaceFrameReader); + + // done with body frame reader + SafeRelease(m_pBodyFrameReader); + + // done with color frame reader + SafeRelease(m_pColorFrameReader); + + // done with coordinate mapper + SafeRelease(m_pCoordinateMapper); + + // close the Kinect Sensor + if (m_pKinectSensor) + { + m_pKinectSensor->Close(); + } + + SafeRelease(m_pKinectSensor); +} + +module_status KinectFaceTracker::start_tracker(QFrame* aFrame) +{ + t.start(); + + if (SUCCEEDED(InitializeDefaultSensor())) + { + // Setup our video preview widget + iVideoWidget = std::make_unique(aFrame); + iLayout = std::make_unique(aFrame); + iLayout->setContentsMargins(0, 0, 0, 0); + iLayout->addWidget(iVideoWidget.get()); + aFrame->setLayout(iLayout.get()); + //video_widget->resize(video_frame->width(), video_frame->height()); + aFrame->show(); + + return status_ok(); + } + + return error("Kinect init failed!"); +} + +#ifdef EMIT_NAN +# include +#endif + +bool KinectFaceTracker::center() +{ + // Mark our center + iFacePositionCenter = iFacePosition; + iFaceRotationCenter = iFaceRotation; + return true; +} + + +void KinectFaceTracker::data(double *data) +{ + const double dt = t.elapsed_seconds(); + t.start(); + + Update(); + + ExtractFaceRotationInDegrees(&iFaceRotationQuaternion, &iFaceRotation.X, &iFaceRotation.Y, &iFaceRotation.Z); + + //Check if data is valid + if (!IsNullPoint(iFacePosition) && !IsNullPoint(iFaceRotation)) + { + // We have valid tracking retain position and rotation + iLastFacePosition = iFacePosition; + iLastFaceRotation = iFaceRotation; + } + + // Feed our framework our last valid position and rotation + data[0] = (iLastFacePosition.X - iFacePositionCenter.X) * 100; // Convert to centimer to be in a range that suites OpenTrack. + data[1] = (iLastFacePosition.Y - iFacePositionCenter.Y) * 100; + data[2] = (iLastFacePosition.Z - iFacePositionCenter.Z) * 100; + + // Yaw, Picth, Roll + data[3] = -(iLastFaceRotation.X - iFaceRotationCenter.X); // Invert to be compatible with ED out-of-the-box + data[4] = (iLastFaceRotation.Y - iFaceRotationCenter.Y); + data[5] = (iLastFaceRotation.Z - iFaceRotationCenter.Z); +} + + +/// +/// Converts rotation quaternion to Euler angles +/// And then maps them to a specified range of values to control the refresh rate +/// +/// face rotation quaternion +/// rotation about the X-axis +/// rotation about the Y-axis +/// rotation about the Z-axis +void KinectFaceTracker::ExtractFaceRotationInDegrees(const Vector4* pQuaternion, float* pYaw, float* pPitch, float* pRoll) +{ + double x = pQuaternion->x; + double y = pQuaternion->y; + double z = pQuaternion->z; + double w = pQuaternion->w; + + // convert face rotation quaternion to Euler angles in degrees + double dPitch, dYaw, dRoll; + dPitch = atan2(2 * (y * z + w * x), w * w - x * x - y * y + z * z) / M_PI * 180.0; + dYaw = asin(2 * (w * y - x * z)) / M_PI * 180.0; + dRoll = atan2(2 * (x * y + w * z), w * w + x * x - y * y - z * z) / M_PI * 180.0; + + // clamp rotation values in degrees to a specified range of values to control the refresh rate + /* + double increment = c_FaceRotationIncrementInDegrees; + *pPitch = static_cast(floor((dPitch + increment/2.0 * (dPitch > 0 ? 1.0 : -1.0)) / increment) * increment); + *pYaw = static_cast(floor((dYaw + increment/2.0 * (dYaw > 0 ? 1.0 : -1.0)) / increment) * increment); + *pRoll = static_cast(floor((dRoll + increment/2.0 * (dRoll > 0 ? 1.0 : -1.0)) / increment) * increment); + */ + + *pPitch = dPitch; + *pYaw = dYaw; + *pRoll = dRoll; +} + + + +/// +/// Initializes the default Kinect sensor +/// +/// S_OK on success else the failure code +HRESULT KinectFaceTracker::InitializeDefaultSensor() +{ + HRESULT hr; + + // Get and open Kinect sensor + hr = GetDefaultKinectSensor(&m_pKinectSensor); + if (SUCCEEDED(hr)) + { + hr = m_pKinectSensor->Open(); + } + + // TODO: check if we still need that guy + if (SUCCEEDED(hr)) + { + hr = m_pKinectSensor->get_CoordinateMapper(&m_pCoordinateMapper); + } + + // Create color frame reader + if (SUCCEEDED(hr)) + { + UniqueInterface colorFrameSource; + hr = m_pKinectSensor->get_ColorFrameSource(colorFrameSource.PtrPtr()); + colorFrameSource.Reset(); + + if (SUCCEEDED(hr)) + { + hr = colorFrameSource->OpenReader(&m_pColorFrameReader); + } + } + + // Create body frame reader + if (SUCCEEDED(hr)) + { + UniqueInterface bodyFrameSource; + hr = m_pKinectSensor->get_BodyFrameSource(bodyFrameSource.PtrPtr()); + bodyFrameSource.Reset(); + + if (SUCCEEDED(hr)) + { + hr = bodyFrameSource->OpenReader(&m_pBodyFrameReader); + } + } + + // Create HD face frame source + if (SUCCEEDED(hr)) + { + // create the face frame source by specifying the required face frame features + hr = CreateHighDefinitionFaceFrameSource(m_pKinectSensor, &m_pFaceFrameSource); + } + + // Create HD face frame reader + if (SUCCEEDED(hr)) + { + // open the corresponding reader + hr = m_pFaceFrameSource->OpenReader(&m_pFaceFrameReader); + } + + return hr; +} + +/// +/// Main processing function +/// +void KinectFaceTracker::Update() +{ + if (!m_pColorFrameReader || !m_pBodyFrameReader) + { + return; + } + + IColorFrame* pColorFrame = nullptr; + HRESULT hr = m_pColorFrameReader->AcquireLatestFrame(&pColorFrame); + + if (SUCCEEDED(hr)) + { + INT64 nTime = 0; + IFrameDescription* pFrameDescription = nullptr; + int nWidth = 0; + int nHeight = 0; + ColorImageFormat imageFormat = ColorImageFormat_None; + UINT nBufferSize = 0; + RGBQUAD *pBuffer = nullptr; + + hr = pColorFrame->get_RelativeTime(&nTime); + + if (SUCCEEDED(hr)) + { + hr = pColorFrame->get_FrameDescription(&pFrameDescription); + } + + if (SUCCEEDED(hr)) + { + hr = pFrameDescription->get_Width(&nWidth); + } + + if (SUCCEEDED(hr)) + { + hr = pFrameDescription->get_Height(&nHeight); + } + + if (SUCCEEDED(hr)) + { + hr = pColorFrame->get_RawColorImageFormat(&imageFormat); + } + + if (SUCCEEDED(hr)) + { + // Fetch color buffer + if (imageFormat == ColorImageFormat_Rgba) + { + hr = pColorFrame->AccessRawUnderlyingBuffer(&nBufferSize, reinterpret_cast(&pBuffer)); + } + else if (m_pColorRGBX) + { + pBuffer = m_pColorRGBX; + nBufferSize = cColorWidth * cColorHeight * sizeof(RGBQUAD); + hr = pColorFrame->CopyConvertedFrameDataToArray(nBufferSize, reinterpret_cast(pBuffer), ColorImageFormat_Rgba); + } + else + { + hr = E_FAIL; + } + + } + + if (SUCCEEDED(hr)) + { + //DrawStreams(nTime, pBuffer, nWidth, nHeight); + ProcessFaces(); + } + + if (SUCCEEDED(hr)) + { + // Setup our image + QImage image((const unsigned char*)pBuffer, cColorWidth, cColorHeight, sizeof(RGBQUAD)*cColorWidth, QImage::Format_RGBA8888); + if (IsValidRect(iFaceBox)) + { + // Draw our face bounding box + QPainter painter(&image); + painter.setBrush(Qt::NoBrush); + painter.setPen(QPen(Qt::red, 8)); + painter.drawRect(iFaceBox.Left, iFaceBox.Top, iFaceBox.Right - iFaceBox.Left, iFaceBox.Bottom - iFaceBox.Top); + bool bEnd = painter.end(); + } + + // Update our video preview + iVideoWidget->update_image(image); + } + + SafeRelease(pFrameDescription); + } + + SafeRelease(pColorFrame); +} + + +/// +/// Updates body data +/// +/// pointer to the body data storage +/// indicates success or failure +HRESULT KinectFaceTracker::UpdateBodyData(IBody** ppBodies) +{ + HRESULT hr = E_FAIL; + + if (m_pBodyFrameReader != nullptr) + { + IBodyFrame* pBodyFrame = nullptr; + hr = m_pBodyFrameReader->AcquireLatestFrame(&pBodyFrame); + if (SUCCEEDED(hr)) + { + hr = pBodyFrame->GetAndRefreshBodyData(BODY_COUNT, ppBodies); + } + SafeRelease(pBodyFrame); + } + + return hr; +} + + + +/// +/// Processes new face frames +/// +void KinectFaceTracker::ProcessFaces() +{ + HRESULT hr; + IBody* ppBodies[BODY_COUNT] = { 0 }; + bool bHaveBodyData = SUCCEEDED(UpdateBodyData(ppBodies)); + + if (!bHaveBodyData) + { + return; + } + + // TODO: Select closest body + // Just use the first body we find + BOOLEAN tracked; + int i = 0; + while (i < BODY_COUNT) + { + hr = ppBodies[i]->get_IsTracked(&tracked); + UINT64 trackingId = 0; + + if (SUCCEEDED(hr) && tracked) + { + hr = ppBodies[i]->get_TrackingId(&trackingId); + + if (SUCCEEDED(hr)) + { + // Tell our face source to use the given body id + hr = m_pFaceFrameSource->put_TrackingId(trackingId); + break; + } + + } + + i++; + } + + + // iterate through each face reader + for (int iFace = 0; iFace < BODY_COUNT; ++iFace) + { + // retrieve the latest face frame from this reader + IHighDefinitionFaceFrame* pFaceFrame = nullptr; + hr = m_pFaceFrameReader->AcquireLatestFrame(&pFaceFrame); + + BOOLEAN bFaceTracked = false; + if (SUCCEEDED(hr) && nullptr != pFaceFrame) + { + // check if a valid face is tracked in this face frame + hr = pFaceFrame->get_IsTrackingIdValid(&bFaceTracked); + } + + if (SUCCEEDED(hr)) + { + if (bFaceTracked) + { + //OutputDebugStringA("Tracking face!\n"); + + //IFaceFrameResult* pFaceFrameResult = nullptr; + IFaceAlignment* pFaceAlignment = nullptr; + CreateFaceAlignment(&pFaceAlignment); // TODO: check return? + //D2D1_POINT_2F faceTextLayout; + + //hr = pFaceFrame->get_FaceFrameResult(&pFaceFrameResult); + + hr = pFaceFrame->GetAndRefreshFaceAlignmentResult(pFaceAlignment); + + // need to verify if pFaceFrameResult contains data before trying to access it + if (SUCCEEDED(hr) && pFaceAlignment != nullptr) + { + hr = pFaceAlignment->get_FaceBoundingBox(&iFaceBox); + //pFaceFrameResult->get_FaceBoundingBoxInColorSpace(); + + if (SUCCEEDED(hr)) + { + //hr = pFaceFrameResult->GetFacePointsInColorSpace(FacePointType::FacePointType_Count, facePoints); + hr = pFaceAlignment->get_HeadPivotPoint(&iFacePosition); + } + + if (SUCCEEDED(hr)) + { + //hr = pFaceFrameResult->get_FaceRotationQuaternion(&faceRotation); + hr = pFaceAlignment->get_FaceOrientation(&iFaceRotationQuaternion); + } + + if (SUCCEEDED(hr)) + { + //hr = pFaceFrameResult->GetFaceProperties(FaceProperty::FaceProperty_Count, faceProperties); + } + + if (SUCCEEDED(hr)) + { + //hr = GetFaceTextPositionInColorSpace(ppBodies[0], &faceTextLayout); + } + + if (SUCCEEDED(hr)) + { + // draw face frame results + //m_pDrawDataStreams->DrawFaceFrameResults(0, &faceBox, facePoints, &faceRotation, faceProperties, &faceTextLayout); + } + } + + SafeRelease(pFaceAlignment); + } + else + { + // face tracking is not valid - attempt to fix the issue + // a valid body is required to perform this step + if (bHaveBodyData) + { + // check if the corresponding body is tracked + // if this is true then update the face frame source to track this body + IBody* pBody = ppBodies[iFace]; + if (pBody != nullptr) + { + BOOLEAN bTracked = false; + hr = pBody->get_IsTracked(&bTracked); + + UINT64 bodyTId; + if (SUCCEEDED(hr) && bTracked) + { + // get the tracking ID of this body + hr = pBody->get_TrackingId(&bodyTId); + if (SUCCEEDED(hr)) + { + // update the face frame source with the tracking ID + m_pFaceFrameSource->put_TrackingId(bodyTId); + } + } + } + } + } + } + + SafeRelease(pFaceFrame); + } + + if (bHaveBodyData) + { + for (int i = 0; i < _countof(ppBodies); ++i) + { + SafeRelease(ppBodies[i]); + } + } +} + + diff --git a/tracker-kinect-face/kinect_face_tracker.h b/tracker-kinect-face/kinect_face_tracker.h new file mode 100644 index 00000000..220a126f --- /dev/null +++ b/tracker-kinect-face/kinect_face_tracker.h @@ -0,0 +1,113 @@ + + + +#include + +#include "api/plugin-api.hpp" +#include "compat/timer.hpp" +#include "compat/macros.hpp" +#include "cv/video-widget.hpp" + +// Kinect Header files +#include +#include + +#pragma once + +// @deprecated Use UniqueInterface instead. Remove it at some point. +template +inline void SafeRelease(Interface *& pInterfaceToRelease) +{ + if (pInterfaceToRelease != nullptr) + { + pInterfaceToRelease->Release(); + pInterfaceToRelease = nullptr; + } +} + +template +inline void ReleaseInterface(Interface* pInterfaceToRelease) +{ + if (pInterfaceToRelease != nullptr) + { + pInterfaceToRelease->Release(); + } +} + +// Safely use Microsoft interfaces. +template +class UniqueInterface : public std::unique_ptr)> ///**/ +{ +public: + UniqueInterface() : std::unique_ptr)>(nullptr, ReleaseInterface){} + // Access pointer, typically for creation + T** PtrPtr() { return &iPtr; }; + // Called this once the pointer was created + void Reset() { std::unique_ptr)>::reset(iPtr); } + // If ever you want to release that interface before the object is deleted + void Free() { iPtr = nullptr; Reset(); } +private: + T* iPtr = nullptr; +}; + + + +class KinectFaceTracker : public ITracker +{ +public: + KinectFaceTracker(); + ~KinectFaceTracker() override; + module_status start_tracker(QFrame* aFrame) override; + void data(double *data) override; + bool center() override; + +private: + Timer t; + + // Kinect stuff + static const int cColorWidth = 1920; + static const int cColorHeight = 1080; + + + void Update(); + HRESULT InitializeDefaultSensor(); + void ProcessFaces(); + HRESULT UpdateBodyData(IBody** ppBodies); + void ExtractFaceRotationInDegrees(const Vector4* pQuaternion, float* pPitch, float* pYaw, float* pRoll); + + // Current Kinect + IKinectSensor* m_pKinectSensor; + + // Coordinate mapper + ICoordinateMapper* m_pCoordinateMapper; + + // Color reader + IColorFrameReader* m_pColorFrameReader; + + // Body reader + IBodyFrameReader* m_pBodyFrameReader; + + // Face sources + IHighDefinitionFaceFrameSource* m_pFaceFrameSource; + + // Face readers + IHighDefinitionFaceFrameReader* m_pFaceFrameReader; + + // + RGBQUAD* m_pColorRGBX; + + RectI iFaceBox = { 0 }; + + CameraSpacePoint iLastFacePosition; + CameraSpacePoint iFacePosition; + CameraSpacePoint iFacePositionCenter; + + Vector4 iFaceRotationQuaternion; + // As Yaw, Pitch, Roll + CameraSpacePoint iLastFaceRotation; + CameraSpacePoint iFaceRotation; + CameraSpacePoint iFaceRotationCenter; + // + std::unique_ptr iVideoWidget; + std::unique_ptr iLayout; +}; diff --git a/tracker-kinect-face/test.cpp b/tracker-kinect-face/test.cpp deleted file mode 100644 index c1ada177..00000000 --- a/tracker-kinect-face/test.cpp +++ /dev/null @@ -1,45 +0,0 @@ -/* Copyright (c) 2014, Stanislaw Halik - - * Permission to use, copy, modify, and/or distribute this - * software for any purpose with or without fee is hereby granted, - * provided that the above copyright notice and this permission - * notice appear in all copies. - */ - -#include "test.h" -#include "tracker.h" -#include "api/plugin-api.hpp" -#include "compat/math-imports.hpp" - -#include - -#include -#include - - -test_dialog::test_dialog() -{ - ui.setupUi(this); - - connect(ui.buttonBox, &QDialogButtonBox::clicked, [this](QAbstractButton* btn) { - if (btn == ui.buttonBox->button(QDialogButtonBox::Abort)) - // NOLINTNEXTLINE - *(volatile int*)nullptr = 0; - }); - - connect(ui.buttonBox, SIGNAL(accepted()), this, SLOT(doOK())); - connect(ui.buttonBox, SIGNAL(rejected()), this, SLOT(doCancel())); -} - -void test_dialog::doOK() -{ - //s.b->save(); - close(); -} - -void test_dialog::doCancel() -{ - close(); -} - -OPENTRACK_DECLARE_TRACKER(KinectFaceTracker, test_dialog, test_metadata) diff --git a/tracker-kinect-face/test.h b/tracker-kinect-face/test.h deleted file mode 100644 index 0f8c170b..00000000 --- a/tracker-kinect-face/test.h +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once -#include "ui_test.h" -#include "compat/macros.hpp" -#include "api/plugin-api.hpp" - - -class test_dialog : public ITrackerDialog -{ - Q_OBJECT - - Ui::test_ui ui; -public: - test_dialog(); - void register_tracker(ITracker *) override {} - void unregister_tracker() override {} -private slots: - void doOK(); - void doCancel(); -}; - -class test_metadata : public Metadata -{ - Q_OBJECT - - QString name() { return tr("Kinect Face 0.1"); } - QIcon icon() { return QIcon(":/images/kinect.png"); } -}; - diff --git a/tracker-kinect-face/test_dialog.cpp b/tracker-kinect-face/test_dialog.cpp index 5d33555b..fadf6084 100644 --- a/tracker-kinect-face/test_dialog.cpp +++ b/tracker-kinect-face/test_dialog.cpp @@ -1,3 +1,3 @@ -#include "test.h" +#include "kinect_face_settings.h" #include "api/plugin-api.hpp" diff --git a/tracker-kinect-face/tracker.cpp b/tracker-kinect-face/tracker.cpp deleted file mode 100644 index 54a716ac..00000000 --- a/tracker-kinect-face/tracker.cpp +++ /dev/null @@ -1,569 +0,0 @@ - - -#include "tracker.h" - -#include -#include - - - -/// -bool IsValidRect(const RectI& aRect) -{ - if (aRect.Bottom != 0) - { - return true; - } - - if (aRect.Left != 0) - { - return true; - } - - if (aRect.Right != 0) - { - return true; - } - - if (aRect.Top != 0) - { - return true; - } - - return false; -} - -/// -bool IsNullVetor(const Vector4& aVector) -{ - if (aVector.w != 0) - { - return false; - } - - if (aVector.x != 0) - { - return false; - } - - if (aVector.y != 0) - { - return false; - } - - if (aVector.z != 0) - { - return false; - } - - return true; -} - -/// -bool IsNullPoint(const CameraSpacePoint& aPoint) -{ - if (aPoint.X != 0) - { - return false; - } - - if (aPoint.Y != 0) - { - return false; - } - - if (aPoint.Z != 0) - { - return false; - } - - return true; -} - - -KinectFaceTracker::KinectFaceTracker(): - m_pKinectSensor(nullptr), - m_pCoordinateMapper(nullptr), - m_pColorFrameReader(nullptr), - m_pColorRGBX(nullptr), - m_pBodyFrameReader(nullptr), - iLastFacePosition{0,0,0}, - iFacePositionCenter{ 0,0,0 }, - iFacePosition{ 0,0,0 }, - iLastFaceRotation{ 0,0,0 }, - iFaceRotationCenter{ 0,0,0 }, - iFaceRotation{ 0,0,0 } -{ - m_pFaceFrameSource = nullptr; - m_pFaceFrameReader = nullptr; - - // create heap storage for color pixel data in RGBX format - m_pColorRGBX = new RGBQUAD[cColorWidth * cColorHeight]; -} - -KinectFaceTracker::~KinectFaceTracker() -{ - if (m_pColorRGBX) - { - delete[] m_pColorRGBX; - m_pColorRGBX = nullptr; - } - - // clean up Direct2D - //SafeRelease(m_pD2DFactory); - - // done with face sources and readers - SafeRelease(m_pFaceFrameSource); - SafeRelease(m_pFaceFrameReader); - - // done with body frame reader - SafeRelease(m_pBodyFrameReader); - - // done with color frame reader - SafeRelease(m_pColorFrameReader); - - // done with coordinate mapper - SafeRelease(m_pCoordinateMapper); - - // close the Kinect Sensor - if (m_pKinectSensor) - { - m_pKinectSensor->Close(); - } - - SafeRelease(m_pKinectSensor); -} - -module_status KinectFaceTracker::start_tracker(QFrame* aFrame) -{ - t.start(); - - if (SUCCEEDED(InitializeDefaultSensor())) - { - // Setup our video preview widget - iVideoWidget = std::make_unique(aFrame); - iLayout = std::make_unique(aFrame); - iLayout->setContentsMargins(0, 0, 0, 0); - iLayout->addWidget(iVideoWidget.get()); - aFrame->setLayout(iLayout.get()); - //video_widget->resize(video_frame->width(), video_frame->height()); - aFrame->show(); - - return status_ok(); - } - - return error("Kinect init failed!"); -} - -#ifdef EMIT_NAN -# include -#endif - -bool KinectFaceTracker::center() -{ - // Mark our center - iFacePositionCenter = iFacePosition; - iFaceRotationCenter = iFaceRotation; - return true; -} - - -void KinectFaceTracker::data(double *data) -{ - const double dt = t.elapsed_seconds(); - t.start(); - - Update(); - - ExtractFaceRotationInDegrees(&iFaceRotationQuaternion, &iFaceRotation.X, &iFaceRotation.Y, &iFaceRotation.Z); - - //Check if data is valid - if (!IsNullPoint(iFacePosition) && !IsNullPoint(iFaceRotation)) - { - // We have valid tracking retain position and rotation - iLastFacePosition = iFacePosition; - iLastFaceRotation = iFaceRotation; - } - - // Feed our framework our last valid position and rotation - data[0] = (iLastFacePosition.X - iFacePositionCenter.X) * 100; // Convert to centimer to be in a range that suites OpenTrack. - data[1] = (iLastFacePosition.Y - iFacePositionCenter.Y) * 100; - data[2] = (iLastFacePosition.Z - iFacePositionCenter.Z) * 100; - - // Yaw, Picth, Roll - data[3] = -(iLastFaceRotation.X - iFaceRotationCenter.X); // Invert to be compatible with ED out-of-the-box - data[4] = (iLastFaceRotation.Y - iFaceRotationCenter.Y); - data[5] = (iLastFaceRotation.Z - iFaceRotationCenter.Z); -} - - -/// -/// Converts rotation quaternion to Euler angles -/// And then maps them to a specified range of values to control the refresh rate -/// -/// face rotation quaternion -/// rotation about the X-axis -/// rotation about the Y-axis -/// rotation about the Z-axis -void KinectFaceTracker::ExtractFaceRotationInDegrees(const Vector4* pQuaternion, float* pYaw, float* pPitch, float* pRoll) -{ - double x = pQuaternion->x; - double y = pQuaternion->y; - double z = pQuaternion->z; - double w = pQuaternion->w; - - // convert face rotation quaternion to Euler angles in degrees - double dPitch, dYaw, dRoll; - dPitch = atan2(2 * (y * z + w * x), w * w - x * x - y * y + z * z) / M_PI * 180.0; - dYaw = asin(2 * (w * y - x * z)) / M_PI * 180.0; - dRoll = atan2(2 * (x * y + w * z), w * w + x * x - y * y - z * z) / M_PI * 180.0; - - // clamp rotation values in degrees to a specified range of values to control the refresh rate - /* - double increment = c_FaceRotationIncrementInDegrees; - *pPitch = static_cast(floor((dPitch + increment/2.0 * (dPitch > 0 ? 1.0 : -1.0)) / increment) * increment); - *pYaw = static_cast(floor((dYaw + increment/2.0 * (dYaw > 0 ? 1.0 : -1.0)) / increment) * increment); - *pRoll = static_cast(floor((dRoll + increment/2.0 * (dRoll > 0 ? 1.0 : -1.0)) / increment) * increment); - */ - - *pPitch = dPitch; - *pYaw = dYaw; - *pRoll = dRoll; -} - - - -/// -/// Initializes the default Kinect sensor -/// -/// S_OK on success else the failure code -HRESULT KinectFaceTracker::InitializeDefaultSensor() -{ - HRESULT hr; - - // Get and open Kinect sensor - hr = GetDefaultKinectSensor(&m_pKinectSensor); - if (SUCCEEDED(hr)) - { - hr = m_pKinectSensor->Open(); - } - - // TODO: check if we still need that guy - if (SUCCEEDED(hr)) - { - hr = m_pKinectSensor->get_CoordinateMapper(&m_pCoordinateMapper); - } - - // Create color frame reader - if (SUCCEEDED(hr)) - { - UniqueInterface colorFrameSource; - hr = m_pKinectSensor->get_ColorFrameSource(colorFrameSource.PtrPtr()); - colorFrameSource.Reset(); - - if (SUCCEEDED(hr)) - { - hr = colorFrameSource->OpenReader(&m_pColorFrameReader); - } - } - - // Create body frame reader - if (SUCCEEDED(hr)) - { - UniqueInterface bodyFrameSource; - hr = m_pKinectSensor->get_BodyFrameSource(bodyFrameSource.PtrPtr()); - bodyFrameSource.Reset(); - - if (SUCCEEDED(hr)) - { - hr = bodyFrameSource->OpenReader(&m_pBodyFrameReader); - } - } - - // Create HD face frame source - if (SUCCEEDED(hr)) - { - // create the face frame source by specifying the required face frame features - hr = CreateHighDefinitionFaceFrameSource(m_pKinectSensor, &m_pFaceFrameSource); - } - - // Create HD face frame reader - if (SUCCEEDED(hr)) - { - // open the corresponding reader - hr = m_pFaceFrameSource->OpenReader(&m_pFaceFrameReader); - } - - return hr; -} - -/// -/// Main processing function -/// -void KinectFaceTracker::Update() -{ - if (!m_pColorFrameReader || !m_pBodyFrameReader) - { - return; - } - - IColorFrame* pColorFrame = nullptr; - HRESULT hr = m_pColorFrameReader->AcquireLatestFrame(&pColorFrame); - - if (SUCCEEDED(hr)) - { - INT64 nTime = 0; - IFrameDescription* pFrameDescription = nullptr; - int nWidth = 0; - int nHeight = 0; - ColorImageFormat imageFormat = ColorImageFormat_None; - UINT nBufferSize = 0; - RGBQUAD *pBuffer = nullptr; - - hr = pColorFrame->get_RelativeTime(&nTime); - - if (SUCCEEDED(hr)) - { - hr = pColorFrame->get_FrameDescription(&pFrameDescription); - } - - if (SUCCEEDED(hr)) - { - hr = pFrameDescription->get_Width(&nWidth); - } - - if (SUCCEEDED(hr)) - { - hr = pFrameDescription->get_Height(&nHeight); - } - - if (SUCCEEDED(hr)) - { - hr = pColorFrame->get_RawColorImageFormat(&imageFormat); - } - - if (SUCCEEDED(hr)) - { - // Fetch color buffer - if (imageFormat == ColorImageFormat_Rgba) - { - hr = pColorFrame->AccessRawUnderlyingBuffer(&nBufferSize, reinterpret_cast(&pBuffer)); - } - else if (m_pColorRGBX) - { - pBuffer = m_pColorRGBX; - nBufferSize = cColorWidth * cColorHeight * sizeof(RGBQUAD); - hr = pColorFrame->CopyConvertedFrameDataToArray(nBufferSize, reinterpret_cast(pBuffer), ColorImageFormat_Rgba); - } - else - { - hr = E_FAIL; - } - - } - - if (SUCCEEDED(hr)) - { - //DrawStreams(nTime, pBuffer, nWidth, nHeight); - ProcessFaces(); - } - - if (SUCCEEDED(hr)) - { - // Setup our image - QImage image((const unsigned char*)pBuffer, cColorWidth, cColorHeight, sizeof(RGBQUAD)*cColorWidth, QImage::Format_RGBA8888); - if (IsValidRect(iFaceBox)) - { - // Draw our face bounding box - QPainter painter(&image); - painter.setBrush(Qt::NoBrush); - painter.setPen(QPen(Qt::red, 8)); - painter.drawRect(iFaceBox.Left, iFaceBox.Top, iFaceBox.Right - iFaceBox.Left, iFaceBox.Bottom - iFaceBox.Top); - bool bEnd = painter.end(); - } - - // Update our video preview - iVideoWidget->update_image(image); - } - - SafeRelease(pFrameDescription); - } - - SafeRelease(pColorFrame); -} - - -/// -/// Updates body data -/// -/// pointer to the body data storage -/// indicates success or failure -HRESULT KinectFaceTracker::UpdateBodyData(IBody** ppBodies) -{ - HRESULT hr = E_FAIL; - - if (m_pBodyFrameReader != nullptr) - { - IBodyFrame* pBodyFrame = nullptr; - hr = m_pBodyFrameReader->AcquireLatestFrame(&pBodyFrame); - if (SUCCEEDED(hr)) - { - hr = pBodyFrame->GetAndRefreshBodyData(BODY_COUNT, ppBodies); - } - SafeRelease(pBodyFrame); - } - - return hr; -} - - - -/// -/// Processes new face frames -/// -void KinectFaceTracker::ProcessFaces() -{ - HRESULT hr; - IBody* ppBodies[BODY_COUNT] = { 0 }; - bool bHaveBodyData = SUCCEEDED(UpdateBodyData(ppBodies)); - - if (!bHaveBodyData) - { - return; - } - - // TODO: Select closest body - // Just use the first body we find - BOOLEAN tracked; - int i = 0; - while (i < BODY_COUNT) - { - hr = ppBodies[i]->get_IsTracked(&tracked); - UINT64 trackingId = 0; - - if (SUCCEEDED(hr) && tracked) - { - hr = ppBodies[i]->get_TrackingId(&trackingId); - - if (SUCCEEDED(hr)) - { - // Tell our face source to use the given body id - hr = m_pFaceFrameSource->put_TrackingId(trackingId); - break; - } - - } - - i++; - } - - - // iterate through each face reader - for (int iFace = 0; iFace < BODY_COUNT; ++iFace) - { - // retrieve the latest face frame from this reader - IHighDefinitionFaceFrame* pFaceFrame = nullptr; - hr = m_pFaceFrameReader->AcquireLatestFrame(&pFaceFrame); - - BOOLEAN bFaceTracked = false; - if (SUCCEEDED(hr) && nullptr != pFaceFrame) - { - // check if a valid face is tracked in this face frame - hr = pFaceFrame->get_IsTrackingIdValid(&bFaceTracked); - } - - if (SUCCEEDED(hr)) - { - if (bFaceTracked) - { - //OutputDebugStringA("Tracking face!\n"); - - //IFaceFrameResult* pFaceFrameResult = nullptr; - IFaceAlignment* pFaceAlignment = nullptr; - CreateFaceAlignment(&pFaceAlignment); // TODO: check return? - //D2D1_POINT_2F faceTextLayout; - - //hr = pFaceFrame->get_FaceFrameResult(&pFaceFrameResult); - - hr = pFaceFrame->GetAndRefreshFaceAlignmentResult(pFaceAlignment); - - // need to verify if pFaceFrameResult contains data before trying to access it - if (SUCCEEDED(hr) && pFaceAlignment != nullptr) - { - hr = pFaceAlignment->get_FaceBoundingBox(&iFaceBox); - //pFaceFrameResult->get_FaceBoundingBoxInColorSpace(); - - if (SUCCEEDED(hr)) - { - //hr = pFaceFrameResult->GetFacePointsInColorSpace(FacePointType::FacePointType_Count, facePoints); - hr = pFaceAlignment->get_HeadPivotPoint(&iFacePosition); - } - - if (SUCCEEDED(hr)) - { - //hr = pFaceFrameResult->get_FaceRotationQuaternion(&faceRotation); - hr = pFaceAlignment->get_FaceOrientation(&iFaceRotationQuaternion); - } - - if (SUCCEEDED(hr)) - { - //hr = pFaceFrameResult->GetFaceProperties(FaceProperty::FaceProperty_Count, faceProperties); - } - - if (SUCCEEDED(hr)) - { - //hr = GetFaceTextPositionInColorSpace(ppBodies[0], &faceTextLayout); - } - - if (SUCCEEDED(hr)) - { - // draw face frame results - //m_pDrawDataStreams->DrawFaceFrameResults(0, &faceBox, facePoints, &faceRotation, faceProperties, &faceTextLayout); - } - } - - SafeRelease(pFaceAlignment); - } - else - { - // face tracking is not valid - attempt to fix the issue - // a valid body is required to perform this step - if (bHaveBodyData) - { - // check if the corresponding body is tracked - // if this is true then update the face frame source to track this body - IBody* pBody = ppBodies[iFace]; - if (pBody != nullptr) - { - BOOLEAN bTracked = false; - hr = pBody->get_IsTracked(&bTracked); - - UINT64 bodyTId; - if (SUCCEEDED(hr) && bTracked) - { - // get the tracking ID of this body - hr = pBody->get_TrackingId(&bodyTId); - if (SUCCEEDED(hr)) - { - // update the face frame source with the tracking ID - m_pFaceFrameSource->put_TrackingId(bodyTId); - } - } - } - } - } - } - - SafeRelease(pFaceFrame); - } - - if (bHaveBodyData) - { - for (int i = 0; i < _countof(ppBodies); ++i) - { - SafeRelease(ppBodies[i]); - } - } -} - - diff --git a/tracker-kinect-face/tracker.h b/tracker-kinect-face/tracker.h deleted file mode 100644 index 220a126f..00000000 --- a/tracker-kinect-face/tracker.h +++ /dev/null @@ -1,113 +0,0 @@ - - - -#include - -#include "api/plugin-api.hpp" -#include "compat/timer.hpp" -#include "compat/macros.hpp" -#include "cv/video-widget.hpp" - -// Kinect Header files -#include -#include - -#pragma once - -// @deprecated Use UniqueInterface instead. Remove it at some point. -template -inline void SafeRelease(Interface *& pInterfaceToRelease) -{ - if (pInterfaceToRelease != nullptr) - { - pInterfaceToRelease->Release(); - pInterfaceToRelease = nullptr; - } -} - -template -inline void ReleaseInterface(Interface* pInterfaceToRelease) -{ - if (pInterfaceToRelease != nullptr) - { - pInterfaceToRelease->Release(); - } -} - -// Safely use Microsoft interfaces. -template -class UniqueInterface : public std::unique_ptr)> ///**/ -{ -public: - UniqueInterface() : std::unique_ptr)>(nullptr, ReleaseInterface){} - // Access pointer, typically for creation - T** PtrPtr() { return &iPtr; }; - // Called this once the pointer was created - void Reset() { std::unique_ptr)>::reset(iPtr); } - // If ever you want to release that interface before the object is deleted - void Free() { iPtr = nullptr; Reset(); } -private: - T* iPtr = nullptr; -}; - - - -class KinectFaceTracker : public ITracker -{ -public: - KinectFaceTracker(); - ~KinectFaceTracker() override; - module_status start_tracker(QFrame* aFrame) override; - void data(double *data) override; - bool center() override; - -private: - Timer t; - - // Kinect stuff - static const int cColorWidth = 1920; - static const int cColorHeight = 1080; - - - void Update(); - HRESULT InitializeDefaultSensor(); - void ProcessFaces(); - HRESULT UpdateBodyData(IBody** ppBodies); - void ExtractFaceRotationInDegrees(const Vector4* pQuaternion, float* pPitch, float* pYaw, float* pRoll); - - // Current Kinect - IKinectSensor* m_pKinectSensor; - - // Coordinate mapper - ICoordinateMapper* m_pCoordinateMapper; - - // Color reader - IColorFrameReader* m_pColorFrameReader; - - // Body reader - IBodyFrameReader* m_pBodyFrameReader; - - // Face sources - IHighDefinitionFaceFrameSource* m_pFaceFrameSource; - - // Face readers - IHighDefinitionFaceFrameReader* m_pFaceFrameReader; - - // - RGBQUAD* m_pColorRGBX; - - RectI iFaceBox = { 0 }; - - CameraSpacePoint iLastFacePosition; - CameraSpacePoint iFacePosition; - CameraSpacePoint iFacePositionCenter; - - Vector4 iFaceRotationQuaternion; - // As Yaw, Pitch, Roll - CameraSpacePoint iLastFaceRotation; - CameraSpacePoint iFaceRotation; - CameraSpacePoint iFaceRotationCenter; - // - std::unique_ptr iVideoWidget; - std::unique_ptr iLayout; -}; -- cgit v1.2.3 From e2582c127addd7a3eaafb8cf8a63a5221c7aac20 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sun, 3 Feb 2019 12:32:16 +0100 Subject: Kinect: Renaming more files. --- tracker-kinect-face/kinect.qrc | 5 --- tracker-kinect-face/kinect_face.qrc | 5 +++ tracker-kinect-face/kinect_face_settings.ui | 65 +++++++++++++++++++++++++++++ tracker-kinect-face/test.ui | 65 ----------------------------- tracker-kinect-face/test_dialog.cpp | 3 -- 5 files changed, 70 insertions(+), 73 deletions(-) delete mode 100644 tracker-kinect-face/kinect.qrc create mode 100644 tracker-kinect-face/kinect_face.qrc create mode 100644 tracker-kinect-face/kinect_face_settings.ui delete mode 100644 tracker-kinect-face/test.ui delete mode 100644 tracker-kinect-face/test_dialog.cpp (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/kinect.qrc b/tracker-kinect-face/kinect.qrc deleted file mode 100644 index 8b27c81e..00000000 --- a/tracker-kinect-face/kinect.qrc +++ /dev/null @@ -1,5 +0,0 @@ - - - images/kinect.png - - diff --git a/tracker-kinect-face/kinect_face.qrc b/tracker-kinect-face/kinect_face.qrc new file mode 100644 index 00000000..8b27c81e --- /dev/null +++ b/tracker-kinect-face/kinect_face.qrc @@ -0,0 +1,5 @@ + + + images/kinect.png + + diff --git a/tracker-kinect-face/kinect_face_settings.ui b/tracker-kinect-face/kinect_face_settings.ui new file mode 100644 index 00000000..509eb819 --- /dev/null +++ b/tracker-kinect-face/kinect_face_settings.ui @@ -0,0 +1,65 @@ + + + test_ui + + + Qt::NonModal + + + + 0 + 0 + 278 + 58 + + + + Sine wave + + + + ../gui/images/opentrack.png../gui/images/opentrack.png + + + Qt::LeftToRight + + + false + + + + + + + 0 + 0 + + + + Pressing "Abort" will immediately crash the application. + + + + + + + + 0 + 0 + + + + QDialogButtonBox::Abort|QDialogButtonBox::Close + + + + + + + + + startEngineClicked() + stopEngineClicked() + cameraSettingsClicked() + + diff --git a/tracker-kinect-face/test.ui b/tracker-kinect-face/test.ui deleted file mode 100644 index 509eb819..00000000 --- a/tracker-kinect-face/test.ui +++ /dev/null @@ -1,65 +0,0 @@ - - - test_ui - - - Qt::NonModal - - - - 0 - 0 - 278 - 58 - - - - Sine wave - - - - ../gui/images/opentrack.png../gui/images/opentrack.png - - - Qt::LeftToRight - - - false - - - - - - - 0 - 0 - - - - Pressing "Abort" will immediately crash the application. - - - - - - - - 0 - 0 - - - - QDialogButtonBox::Abort|QDialogButtonBox::Close - - - - - - - - - startEngineClicked() - stopEngineClicked() - cameraSettingsClicked() - - diff --git a/tracker-kinect-face/test_dialog.cpp b/tracker-kinect-face/test_dialog.cpp deleted file mode 100644 index fadf6084..00000000 --- a/tracker-kinect-face/test_dialog.cpp +++ /dev/null @@ -1,3 +0,0 @@ -#include "kinect_face_settings.h" -#include "api/plugin-api.hpp" - -- cgit v1.2.3 From 57285c2269f53bc7ae61f40289edc9083432bc8a Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sun, 3 Feb 2019 13:39:45 +0100 Subject: Kinect: Better body look-up algorithm that should allow someone to look over your shoulder without the tracker getting confused. --- tracker-kinect-face/kinect_face_tracker.cpp | 239 ++++++++++++++++------------ tracker-kinect-face/kinect_face_tracker.h | 5 + 2 files changed, 143 insertions(+), 101 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/kinect_face_tracker.cpp b/tracker-kinect-face/kinect_face_tracker.cpp index ccac8aa9..b7ccfecf 100644 --- a/tracker-kinect-face/kinect_face_tracker.cpp +++ b/tracker-kinect-face/kinect_face_tracker.cpp @@ -417,141 +417,178 @@ HRESULT KinectFaceTracker::UpdateBodyData(IBody** ppBodies) } +float VectorLengthSquared(CameraSpacePoint point) +{ + float lenghtSquared = pow(point.X, 2) + pow(point.Y, 2) + pow(point.Z, 2); + + //result = Math.Sqrt(result); + return lenghtSquared; +} + +// +// Finds the closest body from the sensor if any +// +IBody* KinectFaceTracker::FindClosestBody(IBody** aBodies) +{ + IBody* result = nullptr; + float closestBodyDistance = std::numeric_limits::max(); + + for(int i=0;iget_IsTracked(&tracked); + + if (tracked) + { + Joint joints[JointType_Count]; + HRESULT hr = aBodies[i]->GetJoints(JointType_Count,joints); + if (FAILED(hr)) + { + continue; + } + + auto currentLocation = joints[JointType_SpineBase].Position; + auto currentDistance = VectorLengthSquared(currentLocation); + + if (result == nullptr || currentDistance < closestBodyDistance) + { + result = aBodies[i]; + closestBodyDistance = currentDistance; + } + } + } + + return result; +} + +// +// Search our list of body for the one matching our id +// +IBody* KinectFaceTracker::FindTrackedBodyById(IBody** aBodies, UINT64 aTrackingId) +{ + float closestBodyDistance = std::numeric_limits::max(); + + for (int i = 0; i < BODY_COUNT; i++) + { + BOOLEAN tracked; + HRESULT hr = aBodies[i]->get_IsTracked(&tracked); + + if (tracked) + { + if (SUCCEEDED(hr) && tracked) + { + UINT64 trackingId = 0; + hr = aBodies[i]->get_TrackingId(&trackingId); + + if (SUCCEEDED(hr) && aTrackingId == trackingId) + { + return aBodies[i]; + } + } + } + } + + return nullptr; +} + /// /// Processes new face frames /// void KinectFaceTracker::ProcessFaces() { - HRESULT hr; - IBody* ppBodies[BODY_COUNT] = { 0 }; - bool bHaveBodyData = SUCCEEDED(UpdateBodyData(ppBodies)); - + HRESULT hr=0; + IBody* bodies[BODY_COUNT] = { 0 }; // Each bodies will need to be released + bool bHaveBodyData = SUCCEEDED(UpdateBodyData(bodies)); if (!bHaveBodyData) { return; } - // TODO: Select closest body - // Just use the first body we find - BOOLEAN tracked; - int i = 0; - while (i < BODY_COUNT) + // Try keep tracking the same body + IBody* body = FindTrackedBodyById(bodies, iTrackingId); + if (body == nullptr) { - hr = ppBodies[i]->get_IsTracked(&tracked); - UINT64 trackingId = 0; - - if (SUCCEEDED(hr) && tracked) + // The body we were tracking is gone, try tracking the closest body if any + body = FindClosestBody(bodies); + if (body != nullptr) { - hr = ppBodies[i]->get_TrackingId(&trackingId); - + // Update our face source with our new body id + hr = body->get_TrackingId(&iTrackingId); if (SUCCEEDED(hr)) { // Tell our face source to use the given body id - hr = m_pFaceFrameSource->put_TrackingId(trackingId); - break; + hr = m_pFaceFrameSource->put_TrackingId(iTrackingId); + //OutputDebugStringA("Tracking new body!\n"); } - } - - i++; } - - // iterate through each face reader - for (int iFace = 0; iFace < BODY_COUNT; ++iFace) + // retrieve the latest face frame from this reader + IHighDefinitionFaceFrame* pFaceFrame = nullptr; + if (SUCCEEDED(hr)) { - // retrieve the latest face frame from this reader - IHighDefinitionFaceFrame* pFaceFrame = nullptr; hr = m_pFaceFrameReader->AcquireLatestFrame(&pFaceFrame); + } - BOOLEAN bFaceTracked = false; - if (SUCCEEDED(hr) && nullptr != pFaceFrame) - { - // check if a valid face is tracked in this face frame - hr = pFaceFrame->get_IsTrackingIdValid(&bFaceTracked); - } + BOOLEAN bFaceTracked = false; + if (SUCCEEDED(hr) && nullptr != pFaceFrame) + { + // check if a valid face is tracked in this face frame + hr = pFaceFrame->get_IsTrackingIdValid(&bFaceTracked); + } - if (SUCCEEDED(hr)) + if (SUCCEEDED(hr)) + { + if (bFaceTracked) { - if (bFaceTracked) + //OutputDebugStringA("Tracking face!\n"); + + //IFaceFrameResult* pFaceFrameResult = nullptr; + IFaceAlignment* pFaceAlignment = nullptr; + CreateFaceAlignment(&pFaceAlignment); // TODO: check return? + //D2D1_POINT_2F faceTextLayout; + + //hr = pFaceFrame->get_FaceFrameResult(&pFaceFrameResult); + + hr = pFaceFrame->GetAndRefreshFaceAlignmentResult(pFaceAlignment); + + // need to verify if pFaceFrameResult contains data before trying to access it + if (SUCCEEDED(hr) && pFaceAlignment != nullptr) { - //OutputDebugStringA("Tracking face!\n"); + hr = pFaceAlignment->get_FaceBoundingBox(&iFaceBox); + //pFaceFrameResult->get_FaceBoundingBoxInColorSpace(); - //IFaceFrameResult* pFaceFrameResult = nullptr; - IFaceAlignment* pFaceAlignment = nullptr; - CreateFaceAlignment(&pFaceAlignment); // TODO: check return? - //D2D1_POINT_2F faceTextLayout; + if (SUCCEEDED(hr)) + { + //hr = pFaceFrameResult->GetFacePointsInColorSpace(FacePointType::FacePointType_Count, facePoints); + hr = pFaceAlignment->get_HeadPivotPoint(&iFacePosition); + } - //hr = pFaceFrame->get_FaceFrameResult(&pFaceFrameResult); + if (SUCCEEDED(hr)) + { + //hr = pFaceFrameResult->get_FaceRotationQuaternion(&faceRotation); + hr = pFaceAlignment->get_FaceOrientation(&iFaceRotationQuaternion); + } - hr = pFaceFrame->GetAndRefreshFaceAlignmentResult(pFaceAlignment); + if (SUCCEEDED(hr)) + { + //hr = pFaceFrameResult->GetFaceProperties(FaceProperty::FaceProperty_Count, faceProperties); + } - // need to verify if pFaceFrameResult contains data before trying to access it - if (SUCCEEDED(hr) && pFaceAlignment != nullptr) + if (SUCCEEDED(hr)) { - hr = pFaceAlignment->get_FaceBoundingBox(&iFaceBox); - //pFaceFrameResult->get_FaceBoundingBoxInColorSpace(); - - if (SUCCEEDED(hr)) - { - //hr = pFaceFrameResult->GetFacePointsInColorSpace(FacePointType::FacePointType_Count, facePoints); - hr = pFaceAlignment->get_HeadPivotPoint(&iFacePosition); - } - - if (SUCCEEDED(hr)) - { - //hr = pFaceFrameResult->get_FaceRotationQuaternion(&faceRotation); - hr = pFaceAlignment->get_FaceOrientation(&iFaceRotationQuaternion); - } - - if (SUCCEEDED(hr)) - { - //hr = pFaceFrameResult->GetFaceProperties(FaceProperty::FaceProperty_Count, faceProperties); - } - - if (SUCCEEDED(hr)) - { - //hr = GetFaceTextPositionInColorSpace(ppBodies[0], &faceTextLayout); - } - - if (SUCCEEDED(hr)) - { - // draw face frame results - //m_pDrawDataStreams->DrawFaceFrameResults(0, &faceBox, facePoints, &faceRotation, faceProperties, &faceTextLayout); - } + //hr = GetFaceTextPositionInColorSpace(ppBodies[0], &faceTextLayout); } - SafeRelease(pFaceAlignment); - } - else - { - // face tracking is not valid - attempt to fix the issue - // a valid body is required to perform this step - if (bHaveBodyData) + if (SUCCEEDED(hr)) { - // check if the corresponding body is tracked - // if this is true then update the face frame source to track this body - IBody* pBody = ppBodies[iFace]; - if (pBody != nullptr) - { - BOOLEAN bTracked = false; - hr = pBody->get_IsTracked(&bTracked); - - UINT64 bodyTId; - if (SUCCEEDED(hr) && bTracked) - { - // get the tracking ID of this body - hr = pBody->get_TrackingId(&bodyTId); - if (SUCCEEDED(hr)) - { - // update the face frame source with the tracking ID - m_pFaceFrameSource->put_TrackingId(bodyTId); - } - } - } + // draw face frame results + //m_pDrawDataStreams->DrawFaceFrameResults(0, &faceBox, facePoints, &faceRotation, faceProperties, &faceTextLayout); } } + + SafeRelease(pFaceAlignment); } SafeRelease(pFaceFrame); @@ -559,9 +596,9 @@ void KinectFaceTracker::ProcessFaces() if (bHaveBodyData) { - for (int i = 0; i < _countof(ppBodies); ++i) + for (int i = 0; i < _countof(bodies); ++i) { - SafeRelease(ppBodies[i]); + SafeRelease(bodies[i]); } } } diff --git a/tracker-kinect-face/kinect_face_tracker.h b/tracker-kinect-face/kinect_face_tracker.h index 220a126f..de71f081 100644 --- a/tracker-kinect-face/kinect_face_tracker.h +++ b/tracker-kinect-face/kinect_face_tracker.h @@ -74,6 +74,8 @@ private: void ProcessFaces(); HRESULT UpdateBodyData(IBody** ppBodies); void ExtractFaceRotationInDegrees(const Vector4* pQuaternion, float* pPitch, float* pYaw, float* pRoll); + static IBody* FindClosestBody(IBody** aBodies); + static IBody* FindTrackedBodyById(IBody** aBodies,UINT64 aTrackingId); // Current Kinect IKinectSensor* m_pKinectSensor; @@ -110,4 +112,7 @@ private: // std::unique_ptr iVideoWidget; std::unique_ptr iLayout; + + // Id of the body currently being tracked + UINT64 iTrackingId = 0; }; -- cgit v1.2.3 From ecaf8b6403f57e5813190f4ae1b89780c45339d8 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sun, 3 Feb 2019 13:45:03 +0100 Subject: Kinect: Fixing tracking validity check. --- tracker-kinect-face/kinect_face_tracker.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/kinect_face_tracker.cpp b/tracker-kinect-face/kinect_face_tracker.cpp index b7ccfecf..51990693 100644 --- a/tracker-kinect-face/kinect_face_tracker.cpp +++ b/tracker-kinect-face/kinect_face_tracker.cpp @@ -167,7 +167,9 @@ bool KinectFaceTracker::center() return true; } - +// +// +// void KinectFaceTracker::data(double *data) { const double dt = t.elapsed_seconds(); @@ -178,7 +180,7 @@ void KinectFaceTracker::data(double *data) ExtractFaceRotationInDegrees(&iFaceRotationQuaternion, &iFaceRotation.X, &iFaceRotation.Y, &iFaceRotation.Z); //Check if data is valid - if (!IsNullPoint(iFacePosition) && !IsNullPoint(iFaceRotation)) + if (IsValidRect(iFaceBox)) { // We have valid tracking retain position and rotation iLastFacePosition = iFacePosition; -- cgit v1.2.3 From 58cc914eb84cb4257904e401a91c0cd5fe3aebf7 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sun, 3 Feb 2019 14:11:00 +0100 Subject: Kinect: CMake adjustments following code review. --- tracker-kinect-face/CMakeLists.txt | 20 +++++++++++--------- tracker-kinect-face/kinect_face_tracker.cpp | 5 +++++ 2 files changed, 16 insertions(+), 9 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/CMakeLists.txt b/tracker-kinect-face/CMakeLists.txt index e12534b2..690f82e7 100644 --- a/tracker-kinect-face/CMakeLists.txt +++ b/tracker-kinect-face/CMakeLists.txt @@ -4,18 +4,18 @@ if (WIN32) if(OpenCV_FOUND) # Setup cache variable to Kinect SDK path if(DEFINED ENV{KINECTSDK20_DIR}) - set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH $ENV{KINECTSDK20_DIR}) + set(SDK_KINECT20 $ENV{KINECTSDK20_DIR} CACHE PATH $ENV{KINECTSDK20_DIR}) else() - set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH "") + set(SDK_KINECT20 $ENV{KINECTSDK20_DIR} CACHE PATH "") endif() # If we have a valid SDK path, try build that tracker - if(EXISTS ${KINECTSDK20_DIR}) + if(EXISTS ${SDK_KINECT20}) # Register our module otr_module(tracker-kinect-face) # Add include path to Kinect SDK - target_include_directories(opentrack-tracker-kinect-face SYSTEM PUBLIC ${KINECTSDK20_DIR}/inc) + target_include_directories(opentrack-tracker-kinect-face SYSTEM PUBLIC "${SDK_KINECT20}/inc") # Check processor architecture if(CMAKE_SIZEOF_VOID_P EQUAL 4) @@ -23,18 +23,20 @@ if (WIN32) set (kinect-arch-dir "x86") elseif(CMAKE_SIZEOF_VOID_P EQUAL 8) # 64 bits - set (kinect-arch-dir "x64") + set (kinect-arch-dir "x64") + else() + message(FATAL_ERROR "Kinect: architecture not supported!") endif() # Link against Kinect SDK libraries - target_link_libraries(opentrack-tracker-kinect-face ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.lib ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.Face.lib) + target_link_libraries(opentrack-tracker-kinect-face "${SDK_KINECT20}/lib/${kinect-arch-dir}/Kinect20.lib" "${SDK_KINECT20}/lib/${kinect-arch-dir}/Kinect20.Face.lib") # Link against OpenCV stuff, needed for video preview target_link_libraries(opentrack-tracker-kinect-face opencv_imgproc opentrack-cv opencv_core) # Install Kinect Face DLL - install(FILES ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/Kinect20.Face.dll DESTINATION ./modules/ PERMISSIONS ${opentrack-perms-exec}) + install(FILES "${SDK_KINECT20}/Redist/Face/${kinect-arch-dir}/Kinect20.Face.dll" DESTINATION "./modules/" PERMISSIONS ${opentrack-perms-exec}) # Install Kinect Face Database - install(DIRECTORY ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/NuiDatabase DESTINATION ./modules/) - endif(EXISTS ${KINECTSDK20_DIR}) + install(DIRECTORY "${SDK_KINECT20}/Redist/Face/${kinect-arch-dir}/NuiDatabase" DESTINATION "./modules/") + endif(EXISTS ${SDK_KINECT20}) endif(OpenCV_FOUND) endif(WIN32) \ No newline at end of file diff --git a/tracker-kinect-face/kinect_face_tracker.cpp b/tracker-kinect-face/kinect_face_tracker.cpp index 51990693..0b7775bc 100644 --- a/tracker-kinect-face/kinect_face_tracker.cpp +++ b/tracker-kinect-face/kinect_face_tracker.cpp @@ -186,6 +186,11 @@ void KinectFaceTracker::data(double *data) iLastFacePosition = iFacePosition; iLastFaceRotation = iFaceRotation; } + else + { + //TODO: after like 5s without tracking reset position to zero + //TODO: Instead of hardcoding that delay add it to our settings + } // Feed our framework our last valid position and rotation data[0] = (iLastFacePosition.X - iFacePositionCenter.X) * 100; // Convert to centimer to be in a range that suites OpenTrack. -- cgit v1.2.3 From 7397242cb156ae0d1e942fba9d209b8627a2072e Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sun, 3 Feb 2019 14:17:38 +0100 Subject: Kinect: Removing dead code. --- tracker-kinect-face/kinect_face_tracker.cpp | 3 --- 1 file changed, 3 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/kinect_face_tracker.cpp b/tracker-kinect-face/kinect_face_tracker.cpp index 0b7775bc..40571a28 100644 --- a/tracker-kinect-face/kinect_face_tracker.cpp +++ b/tracker-kinect-face/kinect_face_tracker.cpp @@ -155,9 +155,6 @@ module_status KinectFaceTracker::start_tracker(QFrame* aFrame) return error("Kinect init failed!"); } -#ifdef EMIT_NAN -# include -#endif bool KinectFaceTracker::center() { -- cgit v1.2.3 From 4508be24ef4adc4dac0c40ef316b07da75765b77 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sun, 3 Feb 2019 15:03:57 +0100 Subject: Kinect: Limit actual frame rate to 30Hz. Fixing settings dialog. --- tracker-kinect-face/kinect_face_settings.cpp | 16 +++++++------ tracker-kinect-face/kinect_face_settings.h | 10 ++++---- tracker-kinect-face/kinect_face_settings.ui | 10 ++++---- tracker-kinect-face/kinect_face_tracker.cpp | 35 +++++++++++++++++----------- tracker-kinect-face/kinect_face_tracker.h | 4 +++- tracker-kinect-face/lang/nl_NL.ts | 8 +++---- tracker-kinect-face/lang/ru_RU.ts | 8 +++---- tracker-kinect-face/lang/stub.ts | 8 +++---- tracker-kinect-face/lang/zh_CN.ts | 8 +++---- 9 files changed, 59 insertions(+), 48 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/kinect_face_settings.cpp b/tracker-kinect-face/kinect_face_settings.cpp index 53aada06..238a3da6 100644 --- a/tracker-kinect-face/kinect_face_settings.cpp +++ b/tracker-kinect-face/kinect_face_settings.cpp @@ -17,29 +17,31 @@ #include -test_dialog::test_dialog() +KinectFaceSettings::KinectFaceSettings() { ui.setupUi(this); connect(ui.buttonBox, &QDialogButtonBox::clicked, [this](QAbstractButton* btn) { - if (btn == ui.buttonBox->button(QDialogButtonBox::Abort)) - // NOLINTNEXTLINE - *(volatile int*)nullptr = 0; + if (btn == ui.buttonBox->button(QDialogButtonBox::Abort)) + { + // NOLINTNEXTLINE + //*(volatile int*)nullptr = 0; + } }); connect(ui.buttonBox, SIGNAL(accepted()), this, SLOT(doOK())); connect(ui.buttonBox, SIGNAL(rejected()), this, SLOT(doCancel())); } -void test_dialog::doOK() +void KinectFaceSettings::doOK() { //s.b->save(); close(); } -void test_dialog::doCancel() +void KinectFaceSettings::doCancel() { close(); } -OPENTRACK_DECLARE_TRACKER(KinectFaceTracker, test_dialog, test_metadata) +OPENTRACK_DECLARE_TRACKER(KinectFaceTracker, KinectFaceSettings, KinectFaceMetadata) diff --git a/tracker-kinect-face/kinect_face_settings.h b/tracker-kinect-face/kinect_face_settings.h index 0f8c170b..647aa754 100644 --- a/tracker-kinect-face/kinect_face_settings.h +++ b/tracker-kinect-face/kinect_face_settings.h @@ -1,16 +1,16 @@ #pragma once -#include "ui_test.h" +#include "ui_kinect_face_settings.h" #include "compat/macros.hpp" #include "api/plugin-api.hpp" -class test_dialog : public ITrackerDialog +class KinectFaceSettings : public ITrackerDialog { Q_OBJECT - Ui::test_ui ui; + Ui::KinectFaceUi ui; public: - test_dialog(); + KinectFaceSettings(); void register_tracker(ITracker *) override {} void unregister_tracker() override {} private slots: @@ -18,7 +18,7 @@ private slots: void doCancel(); }; -class test_metadata : public Metadata +class KinectFaceMetadata : public Metadata { Q_OBJECT diff --git a/tracker-kinect-face/kinect_face_settings.ui b/tracker-kinect-face/kinect_face_settings.ui index 509eb819..a6192d9b 100644 --- a/tracker-kinect-face/kinect_face_settings.ui +++ b/tracker-kinect-face/kinect_face_settings.ui @@ -1,7 +1,7 @@ - test_ui - + KinectFaceUi + Qt::NonModal @@ -14,11 +14,11 @@ - Sine wave + Kinect Face Tracker - ../gui/images/opentrack.png../gui/images/opentrack.png + :/images/kinect.png:/images/kinect.png Qt::LeftToRight @@ -36,7 +36,7 @@ - Pressing "Abort" will immediately crash the application. + Start OpenTrack to check if Kinect Face Tracker is working. diff --git a/tracker-kinect-face/kinect_face_tracker.cpp b/tracker-kinect-face/kinect_face_tracker.cpp index 40571a28..ab568f1e 100644 --- a/tracker-kinect-face/kinect_face_tracker.cpp +++ b/tracker-kinect-face/kinect_face_tracker.cpp @@ -170,25 +170,32 @@ bool KinectFaceTracker::center() void KinectFaceTracker::data(double *data) { const double dt = t.elapsed_seconds(); - t.start(); - - Update(); - - ExtractFaceRotationInDegrees(&iFaceRotationQuaternion, &iFaceRotation.X, &iFaceRotation.Y, &iFaceRotation.Z); - //Check if data is valid - if (IsValidRect(iFaceBox)) + const double KMinDelayInSeconds = 1.0 / 30.0; // Pointless running faster than Kinect hardware itself + if (dt > KMinDelayInSeconds) { - // We have valid tracking retain position and rotation - iLastFacePosition = iFacePosition; - iLastFaceRotation = iFaceRotation; + t.start(); // Reset our timer + //OutputDebugStringA("Updating frame!\n"); + Update(); + ExtractFaceRotationInDegrees(&iFaceRotationQuaternion, &iFaceRotation.X, &iFaceRotation.Y, &iFaceRotation.Z); + //Check if data is valid + if (IsValidRect(iFaceBox)) + { + // We have valid tracking retain position and rotation + iLastFacePosition = iFacePosition; + iLastFaceRotation = iFaceRotation; + } + else + { + //TODO: after like 5s without tracking reset position to zero + //TODO: Instead of hardcoding that delay add it to our settings + } } else { - //TODO: after like 5s without tracking reset position to zero - //TODO: Instead of hardcoding that delay add it to our settings + //OutputDebugStringA("Skipping frame!\n"); } - + // Feed our framework our last valid position and rotation data[0] = (iLastFacePosition.X - iFacePositionCenter.X) * 100; // Convert to centimer to be in a range that suites OpenTrack. data[1] = (iLastFacePosition.Y - iFacePositionCenter.Y) * 100; @@ -344,7 +351,7 @@ void KinectFaceTracker::Update() if (SUCCEEDED(hr)) { hr = pColorFrame->get_RawColorImageFormat(&imageFormat); - } + } if (SUCCEEDED(hr)) { diff --git a/tracker-kinect-face/kinect_face_tracker.h b/tracker-kinect-face/kinect_face_tracker.h index de71f081..9f75507d 100644 --- a/tracker-kinect-face/kinect_face_tracker.h +++ b/tracker-kinect-face/kinect_face_tracker.h @@ -51,7 +51,9 @@ private: }; - +// +// +// class KinectFaceTracker : public ITracker { public: diff --git a/tracker-kinect-face/lang/nl_NL.ts b/tracker-kinect-face/lang/nl_NL.ts index f4fe07d8..a0a737ee 100644 --- a/tracker-kinect-face/lang/nl_NL.ts +++ b/tracker-kinect-face/lang/nl_NL.ts @@ -2,20 +2,20 @@ - test_metadata + KinectFaceMetadata Kinect Face 0.1 - test_ui + KinectFaceUi - Sine wave + Kinect Face Tracker - Pressing "Abort" will immediately crash the application. + Start OpenTrack to check if Kinect Face Tracker is working. diff --git a/tracker-kinect-face/lang/ru_RU.ts b/tracker-kinect-face/lang/ru_RU.ts index d4108ed3..72f91367 100644 --- a/tracker-kinect-face/lang/ru_RU.ts +++ b/tracker-kinect-face/lang/ru_RU.ts @@ -2,20 +2,20 @@ - test_metadata + KinectFaceMetadata Kinect Face 0.1 - test_ui + KinectFaceUi - Sine wave + Kinect Face Tracker - Pressing "Abort" will immediately crash the application. + Start OpenTrack to check if Kinect Face Tracker is working. diff --git a/tracker-kinect-face/lang/stub.ts b/tracker-kinect-face/lang/stub.ts index f82903a3..055ec4e1 100644 --- a/tracker-kinect-face/lang/stub.ts +++ b/tracker-kinect-face/lang/stub.ts @@ -2,20 +2,20 @@ - test_metadata + KinectFaceMetadata Kinect Face 0.1 - test_ui + KinectFaceUi - Sine wave + Kinect Face Tracker - Pressing "Abort" will immediately crash the application. + Start OpenTrack to check if Kinect Face Tracker is working. diff --git a/tracker-kinect-face/lang/zh_CN.ts b/tracker-kinect-face/lang/zh_CN.ts index f82903a3..055ec4e1 100644 --- a/tracker-kinect-face/lang/zh_CN.ts +++ b/tracker-kinect-face/lang/zh_CN.ts @@ -2,20 +2,20 @@ - test_metadata + KinectFaceMetadata Kinect Face 0.1 - test_ui + KinectFaceUi - Sine wave + Kinect Face Tracker - Pressing "Abort" will immediately crash the application. + Start OpenTrack to check if Kinect Face Tracker is working. -- cgit v1.2.3 From 1f82111bbee522c4d31372f9f64b903a08ccc11e Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sun, 3 Feb 2019 15:20:36 +0100 Subject: Kinect: Don't do video preview when not needed. --- tracker-kinect-face/kinect_face_tracker.cpp | 76 ++++++++++++++++------------- 1 file changed, 43 insertions(+), 33 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/kinect_face_tracker.cpp b/tracker-kinect-face/kinect_face_tracker.cpp index ab568f1e..5663de0f 100644 --- a/tracker-kinect-face/kinect_face_tracker.cpp +++ b/tracker-kinect-face/kinect_face_tracker.cpp @@ -5,6 +5,7 @@ #include #include +#include "compat/check-visible.hpp" /// @@ -308,6 +309,8 @@ HRESULT KinectFaceTracker::InitializeDefaultSensor() return hr; } + + /// /// Main processing function /// @@ -353,50 +356,57 @@ void KinectFaceTracker::Update() hr = pColorFrame->get_RawColorImageFormat(&imageFormat); } - if (SUCCEEDED(hr)) - { - // Fetch color buffer - if (imageFormat == ColorImageFormat_Rgba) - { - hr = pColorFrame->AccessRawUnderlyingBuffer(&nBufferSize, reinterpret_cast(&pBuffer)); - } - else if (m_pColorRGBX) - { - pBuffer = m_pColorRGBX; - nBufferSize = cColorWidth * cColorHeight * sizeof(RGBQUAD); - hr = pColorFrame->CopyConvertedFrameDataToArray(nBufferSize, reinterpret_cast(pBuffer), ColorImageFormat_Rgba); - } - else - { - hr = E_FAIL; - } - - } - if (SUCCEEDED(hr)) { //DrawStreams(nTime, pBuffer, nWidth, nHeight); ProcessFaces(); } - if (SUCCEEDED(hr)) - { - // Setup our image - QImage image((const unsigned char*)pBuffer, cColorWidth, cColorHeight, sizeof(RGBQUAD)*cColorWidth, QImage::Format_RGBA8888); - if (IsValidRect(iFaceBox)) + if (check_is_visible()) + { + //OutputDebugStringA("Widget visible!\n"); + // If our widget is visible we feed it our frame + if (SUCCEEDED(hr)) { - // Draw our face bounding box - QPainter painter(&image); - painter.setBrush(Qt::NoBrush); - painter.setPen(QPen(Qt::red, 8)); - painter.drawRect(iFaceBox.Left, iFaceBox.Top, iFaceBox.Right - iFaceBox.Left, iFaceBox.Bottom - iFaceBox.Top); - bool bEnd = painter.end(); + // Fetch color buffer + if (imageFormat == ColorImageFormat_Rgba) + { + hr = pColorFrame->AccessRawUnderlyingBuffer(&nBufferSize, reinterpret_cast(&pBuffer)); + } + else if (m_pColorRGBX) + { + pBuffer = m_pColorRGBX; + nBufferSize = cColorWidth * cColorHeight * sizeof(RGBQUAD); + hr = pColorFrame->CopyConvertedFrameDataToArray(nBufferSize, reinterpret_cast(pBuffer), ColorImageFormat_Rgba); + } + else + { + hr = E_FAIL; + } + + } + + if (SUCCEEDED(hr)) + { + // Setup our image + QImage image((const unsigned char*)pBuffer, cColorWidth, cColorHeight, sizeof(RGBQUAD)*cColorWidth, QImage::Format_RGBA8888); + if (IsValidRect(iFaceBox)) + { + // Draw our face bounding box + QPainter painter(&image); + painter.setBrush(Qt::NoBrush); + painter.setPen(QPen(Qt::red, 8)); + painter.drawRect(iFaceBox.Left, iFaceBox.Top, iFaceBox.Right - iFaceBox.Left, iFaceBox.Bottom - iFaceBox.Top); + bool bEnd = painter.end(); + } + + // Update our video preview + iVideoWidget->update_image(image); } - // Update our video preview - iVideoWidget->update_image(image); } + SafeRelease(pFrameDescription); } -- cgit v1.2.3 From 0011f928da90b6b7116f3c633a0a0cb855d75024 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Sun, 3 Feb 2019 15:35:16 +0100 Subject: Kinect: Tracker code cleaning. --- tracker-kinect-face/kinect_face_tracker.cpp | 86 +++++++++++++---------------- tracker-kinect-face/kinect_face_tracker.h | 38 ++++++------- 2 files changed, 56 insertions(+), 68 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/kinect_face_tracker.cpp b/tracker-kinect-face/kinect_face_tracker.cpp index 5663de0f..9f4f133b 100644 --- a/tracker-kinect-face/kinect_face_tracker.cpp +++ b/tracker-kinect-face/kinect_face_tracker.cpp @@ -7,6 +7,8 @@ #include "compat/check-visible.hpp" +static const int KColorWidth = 1920; +static const int KColorHeight = 1080; /// bool IsValidRect(const RectI& aRect) @@ -82,62 +84,48 @@ bool IsNullPoint(const CameraSpacePoint& aPoint) } -KinectFaceTracker::KinectFaceTracker(): - m_pKinectSensor(nullptr), - m_pCoordinateMapper(nullptr), - m_pColorFrameReader(nullptr), - m_pColorRGBX(nullptr), - m_pBodyFrameReader(nullptr), - iLastFacePosition{0,0,0}, - iFacePositionCenter{ 0,0,0 }, - iFacePosition{ 0,0,0 }, - iLastFaceRotation{ 0,0,0 }, - iFaceRotationCenter{ 0,0,0 }, - iFaceRotation{ 0,0,0 } +KinectFaceTracker::KinectFaceTracker() { - m_pFaceFrameSource = nullptr; - m_pFaceFrameReader = nullptr; - // create heap storage for color pixel data in RGBX format - m_pColorRGBX = new RGBQUAD[cColorWidth * cColorHeight]; + iColorRGBX = new RGBQUAD[KColorWidth * KColorHeight]; } KinectFaceTracker::~KinectFaceTracker() { - if (m_pColorRGBX) + if (iColorRGBX) { - delete[] m_pColorRGBX; - m_pColorRGBX = nullptr; + delete[] iColorRGBX; + iColorRGBX = nullptr; } // clean up Direct2D //SafeRelease(m_pD2DFactory); // done with face sources and readers - SafeRelease(m_pFaceFrameSource); - SafeRelease(m_pFaceFrameReader); + SafeRelease(iFaceFrameSource); + SafeRelease(iFaceFrameReader); // done with body frame reader - SafeRelease(m_pBodyFrameReader); + SafeRelease(iBodyFrameReader); // done with color frame reader - SafeRelease(m_pColorFrameReader); + SafeRelease(iColorFrameReader); // done with coordinate mapper - SafeRelease(m_pCoordinateMapper); + SafeRelease(iCoordinateMapper); // close the Kinect Sensor - if (m_pKinectSensor) + if (iKinectSensor) { - m_pKinectSensor->Close(); + iKinectSensor->Close(); } - SafeRelease(m_pKinectSensor); + SafeRelease(iKinectSensor); } module_status KinectFaceTracker::start_tracker(QFrame* aFrame) { - t.start(); + iTimer.start(); if (SUCCEEDED(InitializeDefaultSensor())) { @@ -170,12 +158,12 @@ bool KinectFaceTracker::center() // void KinectFaceTracker::data(double *data) { - const double dt = t.elapsed_seconds(); + const double dt = iTimer.elapsed_seconds(); const double KMinDelayInSeconds = 1.0 / 30.0; // Pointless running faster than Kinect hardware itself if (dt > KMinDelayInSeconds) { - t.start(); // Reset our timer + iTimer.start(); // Reset our timer //OutputDebugStringA("Updating frame!\n"); Update(); ExtractFaceRotationInDegrees(&iFaceRotationQuaternion, &iFaceRotation.X, &iFaceRotation.Y, &iFaceRotation.Z); @@ -254,28 +242,28 @@ HRESULT KinectFaceTracker::InitializeDefaultSensor() HRESULT hr; // Get and open Kinect sensor - hr = GetDefaultKinectSensor(&m_pKinectSensor); + hr = GetDefaultKinectSensor(&iKinectSensor); if (SUCCEEDED(hr)) { - hr = m_pKinectSensor->Open(); + hr = iKinectSensor->Open(); } // TODO: check if we still need that guy if (SUCCEEDED(hr)) { - hr = m_pKinectSensor->get_CoordinateMapper(&m_pCoordinateMapper); + hr = iKinectSensor->get_CoordinateMapper(&iCoordinateMapper); } // Create color frame reader if (SUCCEEDED(hr)) { UniqueInterface colorFrameSource; - hr = m_pKinectSensor->get_ColorFrameSource(colorFrameSource.PtrPtr()); + hr = iKinectSensor->get_ColorFrameSource(colorFrameSource.PtrPtr()); colorFrameSource.Reset(); if (SUCCEEDED(hr)) { - hr = colorFrameSource->OpenReader(&m_pColorFrameReader); + hr = colorFrameSource->OpenReader(&iColorFrameReader); } } @@ -283,12 +271,12 @@ HRESULT KinectFaceTracker::InitializeDefaultSensor() if (SUCCEEDED(hr)) { UniqueInterface bodyFrameSource; - hr = m_pKinectSensor->get_BodyFrameSource(bodyFrameSource.PtrPtr()); + hr = iKinectSensor->get_BodyFrameSource(bodyFrameSource.PtrPtr()); bodyFrameSource.Reset(); if (SUCCEEDED(hr)) { - hr = bodyFrameSource->OpenReader(&m_pBodyFrameReader); + hr = bodyFrameSource->OpenReader(&iBodyFrameReader); } } @@ -296,14 +284,14 @@ HRESULT KinectFaceTracker::InitializeDefaultSensor() if (SUCCEEDED(hr)) { // create the face frame source by specifying the required face frame features - hr = CreateHighDefinitionFaceFrameSource(m_pKinectSensor, &m_pFaceFrameSource); + hr = CreateHighDefinitionFaceFrameSource(iKinectSensor, &iFaceFrameSource); } // Create HD face frame reader if (SUCCEEDED(hr)) { // open the corresponding reader - hr = m_pFaceFrameSource->OpenReader(&m_pFaceFrameReader); + hr = iFaceFrameSource->OpenReader(&iFaceFrameReader); } return hr; @@ -316,13 +304,13 @@ HRESULT KinectFaceTracker::InitializeDefaultSensor() /// void KinectFaceTracker::Update() { - if (!m_pColorFrameReader || !m_pBodyFrameReader) + if (!iColorFrameReader || !iBodyFrameReader) { return; } IColorFrame* pColorFrame = nullptr; - HRESULT hr = m_pColorFrameReader->AcquireLatestFrame(&pColorFrame); + HRESULT hr = iColorFrameReader->AcquireLatestFrame(&pColorFrame); if (SUCCEEDED(hr)) { @@ -373,10 +361,10 @@ void KinectFaceTracker::Update() { hr = pColorFrame->AccessRawUnderlyingBuffer(&nBufferSize, reinterpret_cast(&pBuffer)); } - else if (m_pColorRGBX) + else if (iColorRGBX) { - pBuffer = m_pColorRGBX; - nBufferSize = cColorWidth * cColorHeight * sizeof(RGBQUAD); + pBuffer = iColorRGBX; + nBufferSize = KColorWidth * KColorHeight * sizeof(RGBQUAD); hr = pColorFrame->CopyConvertedFrameDataToArray(nBufferSize, reinterpret_cast(pBuffer), ColorImageFormat_Rgba); } else @@ -389,7 +377,7 @@ void KinectFaceTracker::Update() if (SUCCEEDED(hr)) { // Setup our image - QImage image((const unsigned char*)pBuffer, cColorWidth, cColorHeight, sizeof(RGBQUAD)*cColorWidth, QImage::Format_RGBA8888); + QImage image((const unsigned char*)pBuffer, KColorWidth, KColorHeight, sizeof(RGBQUAD)*KColorWidth, QImage::Format_RGBA8888); if (IsValidRect(iFaceBox)) { // Draw our face bounding box @@ -423,10 +411,10 @@ HRESULT KinectFaceTracker::UpdateBodyData(IBody** ppBodies) { HRESULT hr = E_FAIL; - if (m_pBodyFrameReader != nullptr) + if (iBodyFrameReader != nullptr) { IBodyFrame* pBodyFrame = nullptr; - hr = m_pBodyFrameReader->AcquireLatestFrame(&pBodyFrame); + hr = iBodyFrameReader->AcquireLatestFrame(&pBodyFrame); if (SUCCEEDED(hr)) { hr = pBodyFrame->GetAndRefreshBodyData(BODY_COUNT, ppBodies); @@ -539,7 +527,7 @@ void KinectFaceTracker::ProcessFaces() if (SUCCEEDED(hr)) { // Tell our face source to use the given body id - hr = m_pFaceFrameSource->put_TrackingId(iTrackingId); + hr = iFaceFrameSource->put_TrackingId(iTrackingId); //OutputDebugStringA("Tracking new body!\n"); } } @@ -549,7 +537,7 @@ void KinectFaceTracker::ProcessFaces() IHighDefinitionFaceFrame* pFaceFrame = nullptr; if (SUCCEEDED(hr)) { - hr = m_pFaceFrameReader->AcquireLatestFrame(&pFaceFrame); + hr = iFaceFrameReader->AcquireLatestFrame(&pFaceFrame); } BOOLEAN bFaceTracked = false; diff --git a/tracker-kinect-face/kinect_face_tracker.h b/tracker-kinect-face/kinect_face_tracker.h index 9f75507d..a6d4e89b 100644 --- a/tracker-kinect-face/kinect_face_tracker.h +++ b/tracker-kinect-face/kinect_face_tracker.h @@ -64,13 +64,9 @@ public: bool center() override; private: - Timer t; + // Kinect stuff - static const int cColorWidth = 1920; - static const int cColorHeight = 1080; - - void Update(); HRESULT InitializeDefaultSensor(); void ProcessFaces(); @@ -78,39 +74,43 @@ private: void ExtractFaceRotationInDegrees(const Vector4* pQuaternion, float* pPitch, float* pYaw, float* pRoll); static IBody* FindClosestBody(IBody** aBodies); static IBody* FindTrackedBodyById(IBody** aBodies,UINT64 aTrackingId); + + // + Timer iTimer; // Current Kinect - IKinectSensor* m_pKinectSensor; + IKinectSensor* iKinectSensor = nullptr; // Coordinate mapper - ICoordinateMapper* m_pCoordinateMapper; + ICoordinateMapper* iCoordinateMapper = nullptr; // Color reader - IColorFrameReader* m_pColorFrameReader; + IColorFrameReader* iColorFrameReader = nullptr; // Body reader - IBodyFrameReader* m_pBodyFrameReader; + IBodyFrameReader* iBodyFrameReader = nullptr; // Face sources - IHighDefinitionFaceFrameSource* m_pFaceFrameSource; + IHighDefinitionFaceFrameSource* iFaceFrameSource = nullptr; // Face readers - IHighDefinitionFaceFrameReader* m_pFaceFrameReader; + IHighDefinitionFaceFrameReader* iFaceFrameReader = nullptr; // - RGBQUAD* m_pColorRGBX; + RGBQUAD* iColorRGBX = nullptr; RectI iFaceBox = { 0 }; - CameraSpacePoint iLastFacePosition; - CameraSpacePoint iFacePosition; - CameraSpacePoint iFacePositionCenter; + // Face position + CameraSpacePoint iLastFacePosition = { 0 }; + CameraSpacePoint iFacePosition = { 0 }; + CameraSpacePoint iFacePositionCenter = { 0 }; - Vector4 iFaceRotationQuaternion; + Vector4 iFaceRotationQuaternion = { 0 }; // As Yaw, Pitch, Roll - CameraSpacePoint iLastFaceRotation; - CameraSpacePoint iFaceRotation; - CameraSpacePoint iFaceRotationCenter; + CameraSpacePoint iLastFaceRotation = { 0 }; + CameraSpacePoint iFaceRotation = { 0 }; + CameraSpacePoint iFaceRotationCenter = { 0 }; // std::unique_ptr iVideoWidget; std::unique_ptr iLayout; -- cgit v1.2.3 From 26ad3d2b154f9161cf577060f9b161e3312b5e76 Mon Sep 17 00:00:00 2001 From: Stéphane Lenclud Date: Thu, 7 Feb 2019 12:41:00 +0100 Subject: Kinect: Removing unused coordinate mapper. --- tracker-kinect-face/kinect_face_tracker.cpp | 9 --------- tracker-kinect-face/kinect_face_tracker.h | 3 --- 2 files changed, 12 deletions(-) (limited to 'tracker-kinect-face') diff --git a/tracker-kinect-face/kinect_face_tracker.cpp b/tracker-kinect-face/kinect_face_tracker.cpp index 9f4f133b..104ecda8 100644 --- a/tracker-kinect-face/kinect_face_tracker.cpp +++ b/tracker-kinect-face/kinect_face_tracker.cpp @@ -111,9 +111,6 @@ KinectFaceTracker::~KinectFaceTracker() // done with color frame reader SafeRelease(iColorFrameReader); - // done with coordinate mapper - SafeRelease(iCoordinateMapper); - // close the Kinect Sensor if (iKinectSensor) { @@ -248,12 +245,6 @@ HRESULT KinectFaceTracker::InitializeDefaultSensor() hr = iKinectSensor->Open(); } - // TODO: check if we still need that guy - if (SUCCEEDED(hr)) - { - hr = iKinectSensor->get_CoordinateMapper(&iCoordinateMapper); - } - // Create color frame reader if (SUCCEEDED(hr)) { diff --git a/tracker-kinect-face/kinect_face_tracker.h b/tracker-kinect-face/kinect_face_tracker.h index a6d4e89b..6273cba1 100644 --- a/tracker-kinect-face/kinect_face_tracker.h +++ b/tracker-kinect-face/kinect_face_tracker.h @@ -81,9 +81,6 @@ private: // Current Kinect IKinectSensor* iKinectSensor = nullptr; - // Coordinate mapper - ICoordinateMapper* iCoordinateMapper = nullptr; - // Color reader IColorFrameReader* iColorFrameReader = nullptr; -- cgit v1.2.3