When selecting chroma keying, include unsaturated colours for overexposed pixels

10 files changed, 199 insertions, 135 deletions

diff --git a/tracker-pt/FTNoIR_PT_Controls.ui b/tracker-pt/FTNoIR_PT_Controls.ui
index 4cf4344c..fc1e0392 100644
--- a/tracker-pt/FTNoIR_PT_Controls.ui
+++ b/tracker-pt/FTNoIR_PT_Controls.ui
@@ -10,7 +10,7 @@
     <x>0</x>
     <y>0</y>
     <width>413</width>
-    <height>575</height>
+    <height>604</height>
    </rect>
   </property>
   <property name="sizePolicy">
@@ -79,16 +79,48 @@
           <string>Camera settings</string>
          </property>
          <layout class="QGridLayout" name="gridLayout_2">
-          <item row="8" column="1">
-           <widget class="QPushButton" name="camera_settings">
+          <item row="4" column="1">
+           <widget class="QCheckBox" name="use_mjpeg">
+            <property name="text">
+             <string/>
+            </property>
+           </widget>
+          </item>
+          <item row="3" column="0">
+           <widget class="QLabel" name="label_37">
             <property name="sizePolicy">
-             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+             <sizepolicy hsizetype="Minimum" vsizetype="Maximum">
               <horstretch>0</horstretch>
               <verstretch>0</verstretch>
              </sizepolicy>
             </property>
             <property name="text">
-             <string>Open</string>
+             <string>FPS</string>
+            </property>
+            <property name="buddy">
+             <cstring>fps_spin</cstring>
+            </property>
+           </widget>
+          </item>
+          <item row="1" column="1">
+           <widget class="QSpinBox" name="res_x_spin">
+            <property name="sizePolicy">
+             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+              <horstretch>0</horstretch>
+              <verstretch>0</verstretch>
+             </sizepolicy>
+            </property>
+            <property name="toolTip">
+             <string>Desired capture width</string>
+            </property>
+            <property name="suffix">
+             <string> px</string>
+            </property>
+            <property name="maximum">
+             <number>2000</number>
+            </property>
+            <property name="singleStep">
+             <number>10</number>
             </property>
            </widget>
           </item>
@@ -111,37 +143,53 @@
             </property>
            </widget>
           </item>
-          <item row="8" column="0">
-           <widget class="QLabel" name="label_9">
+          <item row="3" column="1">
+           <widget class="QSpinBox" name="fps_spin">
             <property name="sizePolicy">
-             <sizepolicy hsizetype="Minimum" vsizetype="Maximum">
+             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
               <horstretch>0</horstretch>
               <verstretch>0</verstretch>
              </sizepolicy>
             </property>
-            <property name="text">
-             <string>Camera settings (when available)</string>
+            <property name="toolTip">
+             <string>Desired capture framerate</string>
+            </property>
+            <property name="suffix">
+             <string> Hz</string>
+            </property>
+            <property name="maximum">
+             <number>2000</number>
             </property>
            </widget>
           </item>
-          <item row="3" column="0">
-           <widget class="QLabel" name="label_37">
+          <item row="6" column="1">
+           <widget class="QCheckBox" name="dynamic_pose">
             <property name="sizePolicy">
-             <sizepolicy hsizetype="Minimum" vsizetype="Maximum">
+             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
               <horstretch>0</horstretch>
               <verstretch>0</verstretch>
              </sizepolicy>
             </property>
             <property name="text">
-             <string>FPS</string>
+             <string/>
             </property>
-            <property name="buddy">
-             <cstring>fps_spin</cstring>
+           </widget>
+          </item>
+          <item row="8" column="1">
+           <widget class="QPushButton" name="camera_settings">
+            <property name="sizePolicy">
+             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+              <horstretch>0</horstretch>
+              <verstretch>0</verstretch>
+             </sizepolicy>
+            </property>
+            <property name="text">
+             <string>Open</string>
             </property>
            </widget>
           </item>
-          <item row="1" column="0">
-           <widget class="QLabel" name="label_36">
+          <item row="2" column="0">
+           <widget class="QLabel" name="label_41">
             <property name="sizePolicy">
              <sizepolicy hsizetype="Minimum" vsizetype="Maximum">
               <horstretch>0</horstretch>
@@ -149,7 +197,61 @@
              </sizepolicy>
             </property>
             <property name="text">
-             <string>Width</string>
+             <string>Height</string>
+            </property>
+           </widget>
+          </item>
+          <item row="0" column="1">
+           <widget class="QComboBox" name="camdevice_combo">
+            <property name="sizePolicy">
+             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+              <horstretch>0</horstretch>
+              <verstretch>0</verstretch>
+             </sizepolicy>
+            </property>
+            <property name="minimumContentsLength">
+             <number>10</number>
+            </property>
+           </widget>
+          </item>
+          <item row="7" column="1">
+           <widget class="QSpinBox" name="init_phase_timeout">
+            <property name="sizePolicy">
+             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+              <horstretch>0</horstretch>
+              <verstretch>0</verstretch>
+             </sizepolicy>
+            </property>
+            <property name="suffix">
+             <string> ms</string>
+            </property>
+            <property name="minimum">
+             <number>50</number>
+            </property>
+            <property name="maximum">
+             <number>5000</number>
+            </property>
+           </widget>
+          </item>
+          <item row="5" column="1">
+           <widget class="QSpinBox" name="fov">
+            <property name="sizePolicy">
+             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+              <horstretch>0</horstretch>
+              <verstretch>0</verstretch>
+             </sizepolicy>
+            </property>
+            <property name="suffix">
+             <string>°</string>
+            </property>
+            <property name="prefix">
+             <string/>
+            </property>
+            <property name="minimum">
+             <number>10</number>
+            </property>
+            <property name="maximum">
+             <number>90</number>
             </property>
            </widget>
           </item>
@@ -218,8 +320,8 @@
             </item>
            </widget>
           </item>
-          <item row="2" column="0">
-           <widget class="QLabel" name="label_41">
+          <item row="1" column="0">
+           <widget class="QLabel" name="label_36">
             <property name="sizePolicy">
              <sizepolicy hsizetype="Minimum" vsizetype="Maximum">
               <horstretch>0</horstretch>
@@ -227,12 +329,12 @@
              </sizepolicy>
             </property>
             <property name="text">
-             <string>Height</string>
+             <string>Width</string>
             </property>
            </widget>
           </item>
-          <item row="6" column="0">
-           <widget class="QLabel" name="label_5">
+          <item row="8" column="0">
+           <widget class="QLabel" name="label_9">
             <property name="sizePolicy">
              <sizepolicy hsizetype="Minimum" vsizetype="Maximum">
               <horstretch>0</horstretch>
@@ -240,20 +342,7 @@
              </sizepolicy>
             </property>
             <property name="text">
-             <string>Dynamic pose (for caps only, never clips)</string>
-            </property>
-           </widget>
-          </item>
-          <item row="6" column="1">
-           <widget class="QCheckBox" name="dynamic_pose">
-            <property name="sizePolicy">
-             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
-              <horstretch>0</horstretch>
-              <verstretch>0</verstretch>
-             </sizepolicy>
-            </property>
-            <property name="text">
-             <string/>
+             <string>Camera settings (when available)</string>
             </property>
            </widget>
           </item>
@@ -298,98 +387,16 @@
             </property>
            </widget>
           </item>
-          <item row="0" column="1">
-           <widget class="QComboBox" name="camdevice_combo">
-            <property name="sizePolicy">
-             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
-              <horstretch>0</horstretch>
-              <verstretch>0</verstretch>
-             </sizepolicy>
-            </property>
-            <property name="minimumContentsLength">
-             <number>10</number>
-            </property>
-           </widget>
-          </item>
-          <item row="1" column="1">
-           <widget class="QSpinBox" name="res_x_spin">
-            <property name="sizePolicy">
-             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
-              <horstretch>0</horstretch>
-              <verstretch>0</verstretch>
-             </sizepolicy>
-            </property>
-            <property name="toolTip">
-             <string>Desired capture width</string>
-            </property>
-            <property name="suffix">
-             <string> px</string>
-            </property>
-            <property name="maximum">
-             <number>2000</number>
-            </property>
-            <property name="singleStep">
-             <number>10</number>
-            </property>
-           </widget>
-          </item>
-          <item row="3" column="1">
-           <widget class="QSpinBox" name="fps_spin">
-            <property name="sizePolicy">
-             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
-              <horstretch>0</horstretch>
-              <verstretch>0</verstretch>
-             </sizepolicy>
-            </property>
-            <property name="toolTip">
-             <string>Desired capture framerate</string>
-            </property>
-            <property name="suffix">
-             <string> Hz</string>
-            </property>
-            <property name="maximum">
-             <number>2000</number>
-            </property>
-           </widget>
-          </item>
-          <item row="7" column="1">
-           <widget class="QSpinBox" name="init_phase_timeout">
-            <property name="sizePolicy">
-             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
-              <horstretch>0</horstretch>
-              <verstretch>0</verstretch>
-             </sizepolicy>
-            </property>
-            <property name="suffix">
-             <string> ms</string>
-            </property>
-            <property name="minimum">
-             <number>50</number>
-            </property>
-            <property name="maximum">
-             <number>5000</number>
-            </property>
-           </widget>
-          </item>
-          <item row="5" column="1">
-           <widget class="QSpinBox" name="fov">
+          <item row="6" column="0">
+           <widget class="QLabel" name="label_5">
             <property name="sizePolicy">
-             <sizepolicy hsizetype="Preferred" vsizetype="Maximum">
+             <sizepolicy hsizetype="Minimum" vsizetype="Maximum">
               <horstretch>0</horstretch>
               <verstretch>0</verstretch>
              </sizepolicy>
             </property>
-            <property name="suffix">
-             <string>°</string>
-            </property>
-            <property name="prefix">
-             <string/>
-            </property>
-            <property name="minimum">
-             <number>10</number>
-            </property>
-            <property name="maximum">
-             <number>90</number>
+            <property name="text">
+             <string>Dynamic pose (for caps only, never clips)</string>
             </property>
            </widget>
           </item>
@@ -441,8 +448,15 @@
             </property>
            </widget>
           </item>
-          <item row="4" column="1">
-           <widget class="QCheckBox" name="use_mjpeg">
+          <item row="10" column="0">
+           <widget class="QLabel" name="label_16">
+            <property name="text">
+             <string>Chroma key includes overexposed pixels</string>
+            </property>
+           </widget>
+          </item>
+          <item row="10" column="1">
+           <widget class="QCheckBox" name="chroma_key_overexposed">
             <property name="text">
              <string/>
             </property>
diff --git a/tracker-pt/ftnoir_tracker_pt_dialog.cpp b/tracker-pt/ftnoir_tracker_pt_dialog.cpp
index 160eb831..f3025a0c 100644
--- a/tracker-pt/ftnoir_tracker_pt_dialog.cpp
+++ b/tracker-pt/ftnoir_tracker_pt_dialog.cpp
@@ -110,6 +110,10 @@ TrackerDialog_PT::TrackerDialog_PT(const QString& module_name) :
 
     tie_setting(s.blob_color, ui.blob_color);
 
+    tie_setting(s.chroma_key_overexposed, ui.chroma_key_overexposed);
+    connect(ui.blob_color, &QComboBox::currentTextChanged, this, &TrackerDialog_PT::chroma_key_overexp_enable);
+    chroma_key_overexp_enable("");
+
     tie_setting(s.threshold_slider, ui.threshold_value_display, [this](const slider_value& val) {
         return threshold_display_text(int(val));
     });
@@ -248,6 +252,18 @@ void TrackerDialog_PT::show_camera_settings()
         (void)video::show_dialog(s.camera_name);
 }
 
+void TrackerDialog_PT::chroma_key_overexp_enable(const QString&)
+{
+    QVariant data = ui.blob_color->currentData();
+    if (!data.isValid())
+        ui.chroma_key_overexposed->setEnabled(false);
+    else
+    {
+        pt_color_type blob_color = pt_color_type(data.toInt());
+        ui.chroma_key_overexposed->setEnabled(blob_color >= pt_color_red_chromakey && blob_color <= pt_color_magenta_chromakey);
+    }
+}
+
 void TrackerDialog_PT::trans_calib_step()
 {
     QMutexLocker l(&calibrator_mutex);
diff --git a/tracker-pt/ftnoir_tracker_pt_dialog.h b/tracker-pt/ftnoir_tracker_pt_dialog.h
index f4b0ff8c..ea631949 100644
--- a/tracker-pt/ftnoir_tracker_pt_dialog.h
+++ b/tracker-pt/ftnoir_tracker_pt_dialog.h
@@ -39,6 +39,8 @@ public slots:
     void poll_tracker_info_impl();
     void set_camera_settings_available(const QString& camera_name);
     void show_camera_settings();
+    void chroma_key_overexp_enable(const QString&);
+
 protected:
     QString threshold_display_text(int threshold_value);
 
diff --git a/tracker-pt/lang/nl_NL.ts b/tracker-pt/lang/nl_NL.ts
index 90383b97..33542d79 100644
--- a/tracker-pt/lang/nl_NL.ts
+++ b/tracker-pt/lang/nl_NL.ts
@@ -304,6 +304,10 @@ Don&apos;t roll or change position.</source>
         <source>Filter</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Chroma key includes overexposed pixels</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>pt_impl::TrackerDialog_PT</name>
diff --git a/tracker-pt/lang/ru_RU.ts b/tracker-pt/lang/ru_RU.ts
index b7146449..7714689d 100644
--- a/tracker-pt/lang/ru_RU.ts
+++ b/tracker-pt/lang/ru_RU.ts
@@ -309,6 +309,10 @@ ROLL или X/Y-смещения.</translation>
         <source>Filter</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Chroma key includes overexposed pixels</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>pt_impl::TrackerDialog_PT</name>
diff --git a/tracker-pt/lang/stub.ts b/tracker-pt/lang/stub.ts
index 3d5e9ae7..3c90c586 100644
--- a/tracker-pt/lang/stub.ts
+++ b/tracker-pt/lang/stub.ts
@@ -304,6 +304,10 @@ Don&apos;t roll or change position.</source>
         <source>Filter</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Chroma key includes overexposed pixels</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>pt_impl::TrackerDialog_PT</name>
diff --git a/tracker-pt/lang/zh_CN.ts b/tracker-pt/lang/zh_CN.ts
index d8aff4b5..43164b09 100644
--- a/tracker-pt/lang/zh_CN.ts
+++ b/tracker-pt/lang/zh_CN.ts
@@ -304,6 +304,10 @@ Don&apos;t roll or change position.</source>
         <source>Filter</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <source>Chroma key includes overexposed pixels</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>pt_impl::TrackerDialog_PT</name>
diff --git a/tracker-pt/module/point_extractor.cpp b/tracker-pt/module/point_extractor.cpp
index 17819d78..4a4ac964 100644
--- a/tracker-pt/module/point_extractor.cpp
+++ b/tracker-pt/module/point_extractor.cpp
@@ -111,11 +111,25 @@ void PointExtractor::extract_single_channel(const cv::Mat& orig_frame, int idx,
     cv::mixChannels(&orig_frame, 1, &dest, 1, from_to, 1);
 }
 
-void PointExtractor::filter_single_channel(const cv::Mat& orig_frame, float r, float g, float b, cv::Mat1b& dest)
+void PointExtractor::filter_single_channel(const cv::Mat& orig_frame, float r, float g, float b, bool overexp, cv::Mat1b& dest)
 {
     ensure_channel_buffers(orig_frame);
 
     cv::transform(orig_frame, dest, cv::Mat(cv::Matx13f(b, g, r)));
+
+    // decrease color saturation logic in proportion to key color intensity?
+    if (overexp)
+    {
+        // get the intensity of the key color (i.e. +ve coefficients)
+        cv::Mat1b temp1(orig_frame.rows, orig_frame.cols);
+        cv::transform(orig_frame, temp1, cv::Mat(cv::Matx13f(std::max(b, 0.0f), std::max(g, 0.0f), std::max(r, 0.0f))));
+        // get the intensity of the non-key color (i.e. -ve coefficients)
+        cv::Mat1b temp2(orig_frame.rows, orig_frame.cols);
+        cv::transform(orig_frame, temp2, cv::Mat(cv::Matx13f(std::max(-b, 0.0f), std::max(-g, 0.0f), std::max(-r, 0.0f))));
+        // add non-key color back into the result in proportion to key color intensity
+        cv::multiply(temp1, temp2, temp1, 1.0 / 255);
+        cv::add(dest, temp1, dest);
+    }
 }
 
 void PointExtractor::color_to_grayscale(const cv::Mat& frame, cv::Mat1b& output)
@@ -146,32 +160,33 @@ void PointExtractor::color_to_grayscale(const cv::Mat& frame, cv::Mat1b& output)
     }
     case pt_color_red_chromakey:
     {
-        filter_single_channel(frame, 1, -0.5, -0.5, output);
+        filter_single_channel(frame, 1, -0.5, -0.5, s.chroma_key_overexposed, output);
         break;
     }
     case pt_color_green_chromakey:
     {
-        filter_single_channel(frame, -0.5, 1, -0.5, output);
+        filter_single_channel(frame, -0.5, 1, -0.5, s.chroma_key_overexposed, output);
+
         break;
     }
     case pt_color_blue_chromakey:
     {
-        filter_single_channel(frame, -0.5, -0.5, 1, output);
+        filter_single_channel(frame, -0.5, -0.5, 1, s.chroma_key_overexposed, output);
         break;
     }
     case pt_color_cyan_chromakey:
     {
-        filter_single_channel(frame, -1, 0.5, 0.5, output);
+        filter_single_channel(frame, -1, 0.5, 0.5, s.chroma_key_overexposed, output);
         break;
     }
     case pt_color_yellow_chromakey:
     {
-        filter_single_channel(frame, 0.5, 0.5, -1, output);
+        filter_single_channel(frame, 0.5, 0.5, -1, s.chroma_key_overexposed, output);
         break;
     }
     case pt_color_magenta_chromakey:
     {
-        filter_single_channel(frame, 0.5, -1, 0.5, output);
+        filter_single_channel(frame, 0.5, -1, 0.5, s.chroma_key_overexposed, output);
         break;
     }
     case pt_color_hardware:
diff --git a/tracker-pt/module/point_extractor.h b/tracker-pt/module/point_extractor.h
index 3f7fb4ee..fbfdbb0b 100644
--- a/tracker-pt/module/point_extractor.h
+++ b/tracker-pt/module/point_extractor.h
@@ -51,7 +51,7 @@ private:
     void ensure_buffers(const cv::Mat& frame);
 
     void extract_single_channel(const cv::Mat& orig_frame, int idx, cv::Mat1b& dest);
-    void filter_single_channel(const cv::Mat& orig_frame, float r, float g, float b, cv::Mat1b& dest);
+    void filter_single_channel(const cv::Mat& orig_frame, float r, float g, float b, bool overexp, cv::Mat1b& dest);
 
     void color_to_grayscale(const cv::Mat& frame, cv::Mat1b& output);
     void threshold_image(const cv::Mat& frame_gray, cv::Mat1b& output);
diff --git a/tracker-pt/pt-settings.hpp b/tracker-pt/pt-settings.hpp
index 6f7a18d5..29bb348f 100644
--- a/tracker-pt/pt-settings.hpp
+++ b/tracker-pt/pt-settings.hpp
@@ -65,6 +65,7 @@ struct pt_settings final : options::opts
     value<bool> auto_threshold { b, "automatic-threshold", true };
     value<pt_color_type> blob_color { b, "blob-color", pt_color_bt709 };
     value<bool> use_mjpeg { b, "use-mjpeg", false };
+    value<bool> chroma_key_overexposed{ b, "chroma-key-overexposed", false };
 
     value<slider_value> threshold_slider { b, "threshold-slider", { 128, 0, 255 } };