Kinect: IR camera now provides raw 16 bits buffer.

Easy Tracker: Support for 16 bits and RGB frame buffers.

5 files changed, 79 insertions, 12 deletions

diff --git a/tracker-easy/cv-point-extractor.cpp b/tracker-easy/cv-point-extractor.cpp
index 368f5587..c7a3c958 100644
--- a/tracker-easy/cv-point-extractor.cpp
+++ b/tracker-easy/cv-point-extractor.cpp
@@ -32,19 +32,62 @@ namespace EasyTracker
     }
 
 
-    void CvPointExtractor::extract_points(const cv::Mat& frame, cv::Mat* aPreview, std::vector<vec2>& aPoints)
+    void CvPointExtractor::extract_points(const cv::Mat& aFrame, cv::Mat* aPreview, std::vector<vec2>& aPoints)
     {
+        //TODO: Assert if channel size is neither one nor two
+        // Make sure our frame channel is 8 bit
+        size_t channelSize = aFrame.elemSize1();
+        if (channelSize == 2)
+        {
+            // We have a 16 bits single channel. Typically coming from Kinect V2 IR sensor
+            // Resample to 8-bits
+            double min = std::numeric_limits<uint16_t>::min();
+            double max = std::numeric_limits<uint16_t>::max();
+            //cv::minMaxLoc(raw, &min, &max); // Should we use 16bit min and max instead?
+            // For scalling to have more precission in the range we are interrested in
+            min = max - 255;
+            // See: https://stackoverflow.com/questions/14539498/change-type-of-mat-object-from-cv-32f-to-cv-8u/14539652
+            aFrame.convertTo(iFrameChannelSizeOne, CV_8U, 255.0 / (max - min), -255.0*min / (max - min));
+        }
+        else
+        {
+            iFrameChannelSizeOne = aFrame;
+        }
+
+
+        // Make sure our frame has a single channel
+        // Make an extra copy if needed
+        const int channelCount = iFrameChannelSizeOne.channels();
+        if (channelCount == 3)
+        {
+            // Convert to grayscale
+            // TODO: What's our input format, BRG or RGB?
+            // That won't make our point extraction work but at least it won't crash
+            cv::cvtColor(iFrameChannelSizeOne, iFrameGray, cv::COLOR_BGR2GRAY);
+            // TODO: Instead convert to HSV and use a key color together with cv::inRange to sport the color we want.
+            // Key color should be defined in settings.
+        }
+        else if (channelCount == 1)
+        {
+            // No further convertion needed
+            iFrameGray = iFrameChannelSizeOne;
+        }
+        else
+        {
+            eval_once(qDebug() << "tracker/easy: camera frame depth not supported" << aFrame.channels());
+            return;
+        }
 
         // Contours detection
         std::vector<std::vector<cv::Point> > contours;
-        cv::findContours(frame, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);
+        cv::findContours(iFrameGray, contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_NONE);
     
         // Workout which countours are valid points
         for (size_t i = 0; i < contours.size(); i++)
         {
             if (aPreview)
             {
-                cv::drawContours(*aPreview, contours, i, CV_RGB(255, 0, 0), 2);
+                cv::drawContours(*aPreview, contours, (int)i, CV_RGB(255, 0, 0), 2);
             }
         
 
@@ -87,7 +130,7 @@ namespace EasyTracker
         while (aPoints.size() > 3) // Until we have no more than three points
         {
             int maxY = 0;
-            int index = -1;
+            size_t index = -1;
 
             // Search for the point with highest Y coordinate
             for (size_t i = 0; i < aPoints.size(); i++)
diff --git a/tracker-easy/cv-point-extractor.h b/tracker-easy/cv-point-extractor.h
index 18229d4a..677be292 100644
--- a/tracker-easy/cv-point-extractor.h
+++ b/tracker-easy/cv-point-extractor.h
@@ -27,10 +27,14 @@ namespace EasyTracker
     public:
         // extracts points from frame and draws some processing info into frame, if draw_output is set
         // dt: time since last call in seconds
-        void extract_points(const cv::Mat& frame, cv::Mat* aPreview, std::vector<vec2>& aPoints) override;
+        void extract_points(const cv::Mat& aFrame, cv::Mat* aPreview, std::vector<vec2>& aPoints) override;
         CvPointExtractor();
-
+        // Settings
         pt_settings s;
+        // Our frame with a channel size of 8 bits
+        cv::Mat iFrameChannelSizeOne;
+        // Our frame with a single 8 bits channel
+        cv::Mat iFrameGray;
     };
 
 }
diff --git a/tracker-easy/preview.cpp b/tracker-easy/preview.cpp
index 404ad299..7f245ae4 100644
--- a/tracker-easy/preview.cpp
+++ b/tracker-easy/preview.cpp
@@ -19,18 +19,37 @@ namespace EasyTracker
 
     Preview& Preview::operator=(const cv::Mat& aFrame)
     {
+        //TODO: Assert if channel size is neither one nor two
+
+        // Make sure our frame channel is 8 bit
+        size_t channelSize = aFrame.elemSize1();
+        if (channelSize == 2)
+        {
+            // First resample to 8-bits            
+            double min = std::numeric_limits<uint16_t>::min();
+            double max = std::numeric_limits<uint16_t>::max();
+            //cv::minMaxLoc(raw, &min, &max); // Should we use 16bit min and max instead?
+            // For scalling to have more precission in the range we are interrested in
+            //min = max - 255;
+            // See: https://stackoverflow.com/questions/14539498/change-type-of-mat-object-from-cv-32f-to-cv-8u/14539652
+            aFrame.convertTo(iFrameChannelSizeOne, CV_8U, 255.0 / (max - min), -255.0*min / (max - min));
+        }
+        else
+        {
+            iFrameChannelSizeOne = aFrame;
+        }
 
         // Make sure our frame is RGB
         // Make an extra copy if needed
-        int channelCount = aFrame.channels();
+        int channelCount = iFrameChannelSizeOne.channels();
         if (channelCount == 1)
         {
             // Convert to RGB
-            cv::cvtColor(aFrame, iFrameRgb, cv::COLOR_GRAY2BGR);
+            cv::cvtColor(iFrameChannelSizeOne, iFrameRgb, cv::COLOR_GRAY2BGR);
         }
         else if (channelCount == 3)
         {
-            iFrameRgb = aFrame;
+            iFrameRgb = iFrameChannelSizeOne;
         }
         else
         {
diff --git a/tracker-easy/preview.h b/tracker-easy/preview.h
index c3ed9f6b..03de6684 100644
--- a/tracker-easy/preview.h
+++ b/tracker-easy/preview.h
@@ -34,6 +34,7 @@ namespace EasyTracker
     public:
         cv::Mat iFrameResized, frame_out;
         cv::Mat iFrameRgb;
+        cv::Mat iFrameChannelSizeOne;
     };
 
 }
diff --git a/tracker-easy/tracker-easy.cpp b/tracker-easy/tracker-easy.cpp
index 42666677..083bd951 100644
--- a/tracker-easy/tracker-easy.cpp
+++ b/tracker-easy/tracker-easy.cpp
@@ -126,9 +126,9 @@ namespace EasyTracker
 
             if (new_frame)
             {
-                //TODO: We should not assume channel size of 1 byte
-                // Though in practice since cv::findContours needs CV_8U we would still need to convert our frame from 16 bits to 8 bits.
-                iMatFrame = cv::Mat(iFrame.height, iFrame.width, CV_MAKETYPE(CV_8U, iFrame.channels), iFrame.data, iFrame.stride);
+                // Create OpenCV matrix from our frame
+                // TODO: Assert channel size is one or two
+                iMatFrame = cv::Mat(iFrame.height, iFrame.width, CV_MAKETYPE((iFrame.channelSize == 2 ? CV_16U : CV_8U), iFrame.channels), iFrame.data, iFrame.stride);
 
 
                 const bool preview_visible = check_is_visible();