Merge pull request #3 from Techainer/fix_#1

Fix #1: Add support for Landmark

Author: lamhoangtung

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
         id: cache-opencv-linux
         with:
           path: /home/runner/work/sort-node/sort-node/opencv/build
-          key: cache-opencv-linux
+          key: cache-opencv-linux-2
 
       - name: Install OpenCV for Linux
         if: matrix.os == 'ubuntu-latest'
@@ -95,7 +95,7 @@ jobs:
         run: yarn build-prebuild
 
       - name: Build
-        run: yarn install
+        run: yarn install --build-from-source --verbose
 
       - name: Run test suite
         run: yarn test

.github/workflows/publish-prebuild.yml

@@ -33,7 +33,7 @@ jobs:
         id: cache-opencv-linux
         with:
           path: /home/runner/work/sort-node/sort-node/opencv/build
-          key: cache-opencv-linux
+          key: cache-opencv-linux-2
 
       - name: Install OpenCV for Linux
         if: matrix.os == 'ubuntu-latest'

.vscode/c_cpp_properties.json

@@ -4,15 +4,18 @@
             "name": "Mac",
             "includePath": [
                 "${workspaceFolder}/**",
-                "${workspaceFolder}/include/"
+                "${workspaceFolder}/include/",
+                "/usr/local/Cellar/opencv/4.5.2/include/opencv4",
+                "${workspaceFolder}/node_modules/node-addon-api",
+                "~/anaconda3/envs/sort-cpp/include/node"
             ],
             "defines": [],
             "macFrameworkPath": [
                 "/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/System/Library/Frameworks"
             ],
             "compilerPath": "/usr/bin/clang",
             "cStandard": "c11",
-            "cppStandard": "c++98",
+            "cppStandard": "c++11",
             "intelliSenseMode": "macos-clang-x64"
         }
     ],

.vscode/settings.json

@@ -4,6 +4,8 @@
         "utility": "cpp",
         "chrono": "cpp",
         "locale": "cpp",
-        "string": "cpp"
+        "string": "cpp",
+        "ios": "cpp",
+        "iostream": "cpp"
     }
 }

README.md

@@ -9,7 +9,7 @@ This package is maintained by [Techainer](https://techainer.com)
 
 ## Install
 To install this package, make sure you have the following dependencies installed:
-- NodeJS 14.x.x
+- NodeJS 12+
 - cmake 3.9+
 - OpenCV 3.x.x (Build from source recommened)
 - Eigen 3.x.x (`sudo apt-get install -y libeigen3-dev`)
@@ -19,6 +19,8 @@ Noted that we have provide a `Dockerfile` contain all 3rd dependencies. To use i
 docker build -t sort .
 ./docker_run.sh
 ```
+Or you can reference our github actions [CI](.github/workflows/ci.yml) flow to install dependencies for your own OS.
+
 Then you can install the package from npm:
 
 ```bash
@@ -35,9 +37,16 @@ The `SortNode` object can be initialize with 2 arguments:
 
 With each frame, you will need to call `update` method.
 
-This method except a single arguments that had a the format `List[List[float]]`, which means a list of detected object in that frame. Each object will have the format: `[x_top, y_top, width, height, confidence]`.
+This method except a single arguments that had a the format `List[List[float]]`, which means a list of detected object in that frame. Each object will have the format: `[x_top, y_top, width, height, confidence]` or `[x_top, y_top, width, height, confidence, landmark_x1, landmark_y1, ...]` for addtional landmark associated with each bounding box
 
-The `update` method will return a list of tracked object in the format `[List[List[int]]`, each object will have the format: `[x_top, y_top, width, height, track_id]`
+The `update` method will return a list of tracked object in the format `[List[Object]]`, each object will have the following structure:
+```js
+{
+    bbox: List[(int) x_top, y_top, width, height],
+    track_id: int,
+    landmarks: List[(float) x1, y1, x2, y2, ..., x_n, y_n],
+}
+```
 
 Please noted that the number of returned object might not be the same as the number of inputed object.
 

include/track.h

@@ -11,13 +11,14 @@ class Track {
     // Destructor
     ~Track() = default;
 
-    void Init(const cv::Rect& bbox);
+    void Init(const std::pair<cv::Rect, std::vector<float>> &bbox);
     void Predict();
-    void Update(const cv::Rect& bbox);
+    void Update(const std::pair<cv::Rect, std::vector<float>> &bbox);
     cv::Rect GetStateAsBbox() const;
     float GetNIS() const;
 
     int coast_cycles_ = 0, hit_streak_ = 0;
+    std::vector<float> landmarks;
 
 private:
     Eigen::VectorXd ConvertBboxToObservation(const cv::Rect& bbox) const;

include/tracker.h

@@ -27,13 +27,13 @@ class Tracker {
  * @param unmatched_det
  * @param iou_threshold
  */
-    static void AssociateDetectionsToTrackers(const std::vector<cv::Rect>& detection,
-                                       std::map<int, Track>& tracks,
-                                       std::map<int, cv::Rect>& matched,
-                                       std::vector<cv::Rect>& unmatched_det,
-                                       float iou_threshold = 0.3);
+    static void AssociateDetectionsToTrackers(const std::vector<std::pair<cv::Rect, std::vector<float>>> &detection,
+                                              std::map<int, Track> &tracks,
+                                              std::map<int, std::pair<cv::Rect, std::vector<float>>> &matched,
+                                              std::vector<std::pair<cv::Rect, std::vector<float>>> &unmatched_det,
+                                              float iou_threshold = 0.3);
 
-    void Run(const std::vector<cv::Rect>& detections);
+    void Run(const std::vector<std::pair<cv::Rect, std::vector<float>>> &detections);
 
     std::map<int, Track> GetTracks();
 

package.json

@@ -18,7 +18,7 @@
   },
   "name": "@techainer1t/sort-node",
   "description": "Node binding of SORT: Simple, online, and real-time tracking of multiple objects in a video sequence.",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "directories": {
     "doc": "docs"
   },

src/sort.cpp

@@ -69,7 +69,7 @@ namespace sortnode
 
         if (!info[0].IsArray())
         {
-            Napi::TypeError::New(env, "dets must have format: List[List[x1, y1, w, h, conf]]")
+            Napi::TypeError::New(env, "dets must have format: List[List[x1, y1, w, h, conf]] or List[List[x1, y1, w, h, conf, landmark_x1, landmark_y1, ...]]")
                 .ThrowAsJavaScriptException();
             return env.Null();
         }
@@ -82,26 +82,32 @@ namespace sortnode
             Napi::Value _each_bbox = jsList[i];
             if (!_each_bbox.IsArray())
             {
-                Napi::TypeError::New(env, "dets must have format: List[List[x1, y1, w, h, conf]]")
+                Napi::TypeError::New(env, "dets must have format: List[List[x1, y1, w, h, conf]] or List[List[x1, y1, w, h, conf, landmark_x1, landmark_y1, ...]]")
                     .ThrowAsJavaScriptException();
                 return env.Null();
             }
             auto each_bbox = _each_bbox.As<Napi::Array>();
             auto each_bbox_len = each_bbox.Length();
-            if (each_bbox_len != 5)
+            if (each_bbox_len < 5)
             {
                 Napi::TypeError::New(env, "each bbox must have format [x1, y1, w, h, conf]")
                     .ThrowAsJavaScriptException();
                 return env.Null();
             }
 
+            if (each_bbox_len > 5 && (each_bbox_len - 5) % 2 != 0){
+                Napi::TypeError::New(env, "each bbox with landmark must have format [x1, y1, w, h, conf, landmark_x1, landmark_y1]")
+                    .ThrowAsJavaScriptException();
+                return env.Null();
+            }
+
             std::vector<float> current_bbox;
-            for (uint32_t j = 0; j < 5; j++)
+            for (uint32_t j = 0; j < each_bbox_len; j++)
             {
                 Napi::Value _val = each_bbox[j];
                 if (!_val.IsNumber())
                 {
-                    Napi::TypeError::New(env, "each value x1, y1, w, h, conf must be a number")
+                    Napi::TypeError::New(env, "each value x1, y1, w, h, conf, landmark_xn, landmark_yn, ... must be a float")
                         .ThrowAsJavaScriptException();
                     return env.Null();
                 }
@@ -115,21 +121,27 @@ namespace sortnode
         this->frame_index++;
 
         // Convert vector of number to cv::Rect to use
-        std::vector<cv::Rect> bbox_per_frame;
+        std::vector<std::pair<cv::Rect, std::vector<float>>> bbox_per_frame;
         for (auto &each_bbox : dets)
         {
             if (each_bbox[4] >= this->kMinConfidence)
             {
-                bbox_per_frame.emplace_back(each_bbox[0], each_bbox[1], each_bbox[2], each_bbox[3]);
+                cv::Rect rect{int(each_bbox[0]), int(each_bbox[1]), int(each_bbox[2]), int(each_bbox[3])};
+                std::vector<float> landmarks;
+                for (auto i = 5; i < each_bbox.size(); i++){
+                    landmarks.push_back(each_bbox[i]);
+                }
+                bbox_per_frame.push_back(std::make_pair(rect, landmarks));
             }
         }
 
         // Run SORT tracker
         this->tracker.Run(bbox_per_frame);
         const auto tracks = this->tracker.GetTracks();
 
-        // Convert results from cv::Rect to normal int vector
+        // Convert results from cv::Rect to normal float vector
         std::vector<std::vector<int> > res;
+        std::vector<std::vector<float>> res_landmarks;
         for (auto &trk : tracks)
         {
             const auto &bbox = trk.second.GetStateAsBbox();
@@ -142,7 +154,9 @@ namespace sortnode
             {
                 std::vector<int> current_object{bbox.tl().x, bbox.tl().y, bbox.width, bbox.height, trk.first};
                 // Last value is track id
+                std::vector<float> landmarks = trk.second.landmarks;
                 res.push_back(current_object);
+                res_landmarks.push_back(landmarks);
             }
         }
 
@@ -152,11 +166,26 @@ namespace sortnode
         {
             auto bbox = res[i];
             auto jsBbox = Napi::Array::New(env);
-            for (uint32_t j = 0; j < bbox.size(); j++)
+            for (uint32_t j = 0; j < bbox.size() - 1; j++)
             {
                 jsBbox[j] = Napi::Number::New(env, bbox[j]);
             }
-            jsOutputList[i] = jsBbox;
+            // std::cout << "We got the bbox set" << std::endl;
+
+            auto landmarks = res_landmarks[i];
+            auto jsLandmarks = Napi::Array::New(env);
+            for (uint32_t j = 0; j < landmarks.size(); j++){
+                jsLandmarks[j] = Napi::Number::New(env, landmarks[j]);
+            }
+            // std::cout << "We got the landmarks set" << std::endl;
+
+            auto jsDict = Napi::Object::New(env);
+            jsDict.Set("bbox", jsBbox);
+            jsDict.Set("track_id", Napi::Number::New(env, bbox[4]));
+            jsDict.Set("landmarks", jsLandmarks);
+
+
+            jsOutputList[i] = jsDict;
         }
         return jsOutputList;
     }