Kinect: now with basic video preview.

We have a massive memory leak, could have been there before though.

3 files changed, 75 insertions, 38 deletions

diff --git a/tracker-kinect-face/CMakeLists.txt b/tracker-kinect-face/CMakeLists.txt
index cbdc2a97..e12534b2 100644
--- a/tracker-kinect-face/CMakeLists.txt
+++ b/tracker-kinect-face/CMakeLists.txt
@@ -1,37 +1,40 @@
 # Kinect SDK is Windows only
 if (WIN32)
-	# Setup cache variable to Kinect SDK path
-	if(DEFINED ENV{KINECTSDK20_DIR})
-		set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH $ENV{KINECTSDK20_DIR})
-	else()
-		set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH "")
-	endif()
+	find_package(OpenCV QUIET)
+	if(OpenCV_FOUND)
+		# Setup cache variable to Kinect SDK path
+		if(DEFINED ENV{KINECTSDK20_DIR})
+			set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH $ENV{KINECTSDK20_DIR})
+		else()
+			set(KINECTSDK20_DIR $ENV{KINECTSDK20_DIR} CACHE PATH "")
+		endif()
 
-	# If we have a valid SDK path, try build that tracker
-	if(EXISTS ${KINECTSDK20_DIR})
-		# Register our module
-		otr_module(tracker-kinect-face)
+		# If we have a valid SDK path, try build that tracker
+		if(EXISTS ${KINECTSDK20_DIR})
+			# Register our module
+			otr_module(tracker-kinect-face)
 
-		# Add include path to Kinect SDK 
-		target_include_directories(opentrack-tracker-kinect-face SYSTEM PUBLIC ${KINECTSDK20_DIR}/inc)
+			# Add include path to Kinect SDK 
+			target_include_directories(opentrack-tracker-kinect-face SYSTEM PUBLIC ${KINECTSDK20_DIR}/inc)
 	
-		# Check processor architecture 
-		if(CMAKE_SIZEOF_VOID_P EQUAL 4)
-			# 32 bits        
-			set (kinect-arch-dir "x86")
-		elseif(CMAKE_SIZEOF_VOID_P EQUAL 8)
-			# 64 bits
-			set (kinect-arch-dir "x64")		
-		endif()
+			# Check processor architecture 
+			if(CMAKE_SIZEOF_VOID_P EQUAL 4)
+				# 32 bits        
+				set (kinect-arch-dir "x86")
+			elseif(CMAKE_SIZEOF_VOID_P EQUAL 8)
+				# 64 bits
+				set (kinect-arch-dir "x64")		
+			endif()
 	
-		# Link against Kinect SDK libraries
-		target_link_libraries(opentrack-tracker-kinect-face ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.lib ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.Face.lib)
-
-		# Install Kinect Face DLL
-		install(FILES ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/Kinect20.Face.dll DESTINATION ./modules/ PERMISSIONS ${opentrack-perms-exec})
-		# Install Kinect Face Database
-		install(DIRECTORY ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/NuiDatabase DESTINATION ./modules/)
-
-	endif()
+			# Link against Kinect SDK libraries
+			target_link_libraries(opentrack-tracker-kinect-face ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.lib ${KINECTSDK20_DIR}/lib/${kinect-arch-dir}/Kinect20.Face.lib)
+			# Link against OpenCV stuff, needed for video preview
+			target_link_libraries(opentrack-tracker-kinect-face opencv_imgproc opentrack-cv opencv_core)
 
-endif()
\ No newline at end of file
+			# Install Kinect Face DLL
+			install(FILES ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/Kinect20.Face.dll DESTINATION ./modules/ PERMISSIONS ${opentrack-perms-exec})
+			# Install Kinect Face Database
+			install(DIRECTORY ${KINECTSDK20_DIR}/Redist/Face/${kinect-arch-dir}/NuiDatabase DESTINATION ./modules/)
+		endif(EXISTS ${KINECTSDK20_DIR})
+	endif(OpenCV_FOUND)
+endif(WIN32)
\ No newline at end of file
diff --git a/tracker-kinect-face/tracker.cpp b/tracker-kinect-face/tracker.cpp
index c5517f89..073cf0be 100644
--- a/tracker-kinect-face/tracker.cpp
+++ b/tracker-kinect-face/tracker.cpp
@@ -2,6 +2,9 @@
 
 #include "tracker.h"
 
+#include <QLayout>
+#include <QPainter>
+
 
 
 ///
@@ -137,6 +140,15 @@ module_status KinectFaceTracker::start_tracker(QFrame* aFrame)
 
 	if (SUCCEEDED(InitializeDefaultSensor()))
 	{
+		// Setup our video preview widget
+		iVideoWidget = std::make_unique<cv_video_widget>(aFrame);
+		iLayout = std::make_unique<QHBoxLayout>(aFrame);
+		iLayout->setContentsMargins(0, 0, 0, 0);
+		iLayout->addWidget(iVideoWidget.get());
+		aFrame->setLayout(iLayout.get());
+		//video_widget->resize(video_frame->width(), video_frame->height());
+		aFrame->show();
+
 		return status_ok();
 	}
 
@@ -161,12 +173,11 @@ void KinectFaceTracker::data(double *data)
 	const double dt = t.elapsed_seconds();
 	t.start();
 
-
 	Update();
-	//TODO: check if data is valid
 
 	ExtractFaceRotationInDegrees(&iFaceRotationQuaternion, &iFaceRotation.X, &iFaceRotation.Y, &iFaceRotation.Z);
 
+	//Check if data is valid
 	if (!IsNullPoint(iFacePosition) && !IsNullPoint(iFaceRotation))
 	{
 		// We have valid tracking retain position and rotation
@@ -218,8 +229,6 @@ void KinectFaceTracker::ExtractFaceRotationInDegrees(const Vector4* pQuaternion,
 	*pPitch = dPitch;
 	*pYaw = dYaw;
 	*pRoll = dRoll;
-
-
 }
 
 
@@ -348,7 +357,8 @@ void KinectFaceTracker::Update()
 
 		if (SUCCEEDED(hr))
 		{
-			if (imageFormat == ColorImageFormat_Bgra)
+			// Fetch color buffer
+			if (imageFormat == ColorImageFormat_Rgba)
 			{
 				hr = pColorFrame->AccessRawUnderlyingBuffer(&nBufferSize, reinterpret_cast<BYTE**>(&pBuffer));
 			}
@@ -356,12 +366,13 @@ void KinectFaceTracker::Update()
 			{
 				pBuffer = m_pColorRGBX;
 				nBufferSize = cColorWidth * cColorHeight * sizeof(RGBQUAD);
-				hr = pColorFrame->CopyConvertedFrameDataToArray(nBufferSize, reinterpret_cast<BYTE*>(pBuffer), ColorImageFormat_Bgra);
+				hr = pColorFrame->CopyConvertedFrameDataToArray(nBufferSize, reinterpret_cast<BYTE*>(pBuffer), ColorImageFormat_Rgba);				
 			}
 			else
 			{
 				hr = E_FAIL;
 			}
+
 		}
 
 		if (SUCCEEDED(hr))
@@ -370,6 +381,24 @@ void KinectFaceTracker::Update()
 			ProcessFaces();
 		}
 
+		if (SUCCEEDED(hr))
+		{			
+			// Setup our image 
+			QImage image((const unsigned char*)pBuffer, cColorWidth, cColorHeight, sizeof(RGBQUAD)*cColorWidth, QImage::Format_RGBA8888);
+			if (IsValidRect(iFaceBox))
+			{
+				// Draw our face bounding box
+				QPainter painter(&image);
+				painter.setBrush(Qt::NoBrush);
+				painter.setPen(QPen(Qt::red, 8));
+				painter.drawRect(iFaceBox.Left, iFaceBox.Top, iFaceBox.Right - iFaceBox.Left, iFaceBox.Bottom - iFaceBox.Top);
+				bool bEnd = painter.end();
+			}
+
+			// Update our video preview
+			iVideoWidget->update_image(image);
+		}
+
 		SafeRelease(pFrameDescription);
 	}
 
@@ -465,7 +494,6 @@ void KinectFaceTracker::ProcessFaces()
 				//IFaceFrameResult* pFaceFrameResult = nullptr;
 				IFaceAlignment* pFaceAlignment = nullptr;
 				CreateFaceAlignment(&pFaceAlignment); // TODO: check return?
-				RectI faceBox = { 0 };
 				//D2D1_POINT_2F faceTextLayout;				
 
 				//hr = pFaceFrame->get_FaceFrameResult(&pFaceFrameResult);
@@ -475,7 +503,7 @@ void KinectFaceTracker::ProcessFaces()
 				// need to verify if pFaceFrameResult contains data before trying to access it
 				if (SUCCEEDED(hr) && pFaceAlignment != nullptr)
 				{
-					hr = pFaceAlignment->get_FaceBoundingBox(&faceBox);
+					hr = pFaceAlignment->get_FaceBoundingBox(&iFaceBox);
 					//pFaceFrameResult->get_FaceBoundingBoxInColorSpace();
 
 					if (SUCCEEDED(hr))
diff --git a/tracker-kinect-face/tracker.h b/tracker-kinect-face/tracker.h
index 1f986e22..5068f185 100644
--- a/tracker-kinect-face/tracker.h
+++ b/tracker-kinect-face/tracker.h
@@ -6,6 +6,7 @@
 #include "api/plugin-api.hpp"
 #include "compat/timer.hpp"
 #include "compat/macros.hpp"
+#include "cv/video-widget.hpp"
 
 // Kinect Header files
 #include <Kinect.h>
@@ -69,6 +70,8 @@ private:
 	//
 	RGBQUAD*               m_pColorRGBX;
 
+	RectI iFaceBox = { 0 };
+
 	CameraSpacePoint iLastFacePosition;
 	CameraSpacePoint iFacePosition;
 	CameraSpacePoint iFacePositionCenter;
@@ -78,4 +81,7 @@ private:
 	CameraSpacePoint iLastFaceRotation;
 	CameraSpacePoint iFaceRotation;
 	CameraSpacePoint iFaceRotationCenter;
+	//
+	std::unique_ptr<cv_video_widget> iVideoWidget;
+	std::unique_ptr<QLayout> iLayout;
 };