avoid gpu decode stuck

Author: CarlosLeeGit

src/drivers/devices/cuda/flowunit/video_decoder/nvcodec_video_decoder.cc

@@ -23,6 +23,39 @@
 
 #define MIN_ALLOWABLE_DECODE_SURFACE_NUM 1
 
+NvcodecConcurrencyLimiter *NvcodecConcurrencyLimiter::GetInstance() {
+  static NvcodecConcurrencyLimiter limiter;
+  return &limiter;
+}
+
+void NvcodecConcurrencyLimiter::Init(uint32_t limit) {
+  if (limit == 0) {
+    limited_ = false;
+  }
+
+  count_ = limit;
+}
+
+void NvcodecConcurrencyLimiter::Acquire() {
+  if (!limited_) {
+    return;
+  }
+
+  std::unique_lock<std::mutex> lock(count_lock_);
+  count_cv_.wait(lock, [=] { return count_ > 0; });
+  --count_;
+}
+
+void NvcodecConcurrencyLimiter::Release() {
+  if (!limited_) {
+    return;
+  }
+
+  std::unique_lock<std::mutex> lock(count_lock_);
+  ++count_;
+  count_cv_.notify_one();
+}
+
 #define NVDEC_THROW_ERROR(err_str, err_code)                                \
   throw NVDECException::MakeNVDECException(err_str, err_code, __FUNCTION__, \
                                            __FILE__, __LINE__);
@@ -87,7 +120,7 @@ NvcodecVideoDecoder::~NvcodecVideoDecoder() {
 modelbox::Status NvcodecVideoDecoder::Init(const std::string &device_id,
                                            AVCodecID codec_id,
                                            std::string &file_url,
-                                           bool skip_err_frame) {
+                                           bool skip_err_frame, bool no_delay) {
   gpu_id_ = std::stoi(device_id);
   MBLOG_INFO << "Init decode in gpu " << gpu_id_;
   // Use cuda runtime CUContext on same device in whole modelbox process to
@@ -122,8 +155,8 @@ modelbox::Status NvcodecVideoDecoder::Init(const std::string &device_id,
   videoParserParams.CodecType = codec_id_;
   videoParserParams.ulMaxNumDecodeSurfaces = 1;
   videoParserParams.ulMaxDisplayDelay =
-      2;  // setting ulMaxDisplayDelay to 2 achieves max decoding rate, based on
-          // several tests.
+      no_delay ? 0 : 2;  // setting ulMaxDisplayDelay to 2 achieves max decoding
+                         // rate, based on several tests.
   videoParserParams.pUserData = (void *)this;
   videoParserParams.pfnSequenceCallback = HandleVideoSequenceProc;
   videoParserParams.pfnDecodePicture = HandlePictureDecodeProc;
@@ -170,7 +203,14 @@ modelbox::Status NvcodecVideoDecoder::Decode(
     return modelbox::STATUS_FAULT;
   }
 
+  NvcodecConcurrencyLimiter::GetInstance()->Acquire();
+  is_limiter_released_ = false;
   CUDA_API_CALL(cuvidParseVideoData(video_parser_, &packet));
+  if (!is_limiter_released_) {
+    // might release when handle display
+    NvcodecConcurrencyLimiter::GetInstance()->Release();
+    is_limiter_released_ = true;
+  }
 
   for (size_t i = 0; i < frame_count_in_one_decode_; ++i) {
     auto frame = std::make_shared<NvcodecFrame>();
@@ -389,6 +429,9 @@ int32_t NvcodecVideoDecoder::HandlePictureDisplay(
     return 1;
   }
 
+  NvcodecConcurrencyLimiter::GetInstance()->Release();
+  is_limiter_released_ = true;
+
   ++frame_count_in_one_decode_;
   SaveFrame(src_frame_ptr, src_pitch);
   SaveTimestamp(display_info->timestamp);

src/drivers/devices/cuda/flowunit/video_decoder/nvcodec_video_decoder.h

@@ -14,15 +14,15 @@
  * limitations under the License.
  */
 
-
 #ifndef MODELBOX_FLOWUNIT_NVCODEC_VIDEO_DECODER_H_
 #define MODELBOX_FLOWUNIT_NVCODEC_VIDEO_DECODER_H_
 
-#include <modelbox/base/status.h>
 #include <cuda.h>
 #include <libavformat/avformat.h>
+#include <modelbox/base/status.h>
 #include <nvcuvid.h>
 
+#include <condition_variable>
 #include <iostream>
 #include <map>
 #include <memory>
@@ -31,6 +31,25 @@
 #include <utility>
 #include <vector>
 
+class NvcodecConcurrencyLimiter {
+ public:
+  static NvcodecConcurrencyLimiter *GetInstance();
+
+  void Init(uint32_t limit);
+
+  void Acquire();
+
+  void Release();
+
+ private:
+  NvcodecConcurrencyLimiter() = default;
+
+  std::mutex count_lock_;
+  std::condition_variable count_cv_;
+  uint32_t count_{0};
+  bool limited_{false};
+};
+
 class NVDECException : public std::exception {
  public:
   NVDECException(std::string err_str, const CUresult err_code)
@@ -98,7 +117,8 @@ class NvcodecVideoDecoder {
   virtual ~NvcodecVideoDecoder();
 
   modelbox::Status Init(const std::string &device_id, AVCodecID codec_id,
-                      std::string &file_url, bool skip_err_frame);
+                        std::string &file_url, bool skip_err_frame,
+                        bool no_delay);
 
   modelbox::Status Decode(
       const std::shared_ptr<NvcodecPacket> &pkt,
@@ -115,7 +135,7 @@ class NvcodecVideoDecoder {
   modelbox::Status InitCuCtx(const std::string &device_id);
 
   modelbox::Status GetCudaVideoCodec(AVCodecID codec_id,
-                                   cudaVideoCodec &cuda_codec_id);
+                                     cudaVideoCodec &cuda_codec_id);
 
   std::string GetVideoCodecString(cudaVideoCodec cuda_codec_id);
 
@@ -193,6 +213,8 @@ class NvcodecVideoDecoder {
   bool skip_err_frame_{false};
   int64_t latest_pts_{0};
   int32_t gpu_id_{0};
+
+  bool is_limiter_released_{false};
 };
 
-#endif  // MODELBOX_FLOWUNIT_NVCODEC_VIDEO_DECODER_H_
+#endif  // MODELBOX_FLOWUNIT_NVCODEC_VIDEO_DECODER_H_

src/drivers/devices/cuda/flowunit/video_decoder/video_decoder_flowunit.cc

@@ -35,6 +35,8 @@ modelbox::Status VideoDecoderFlowUnit::Open(
   }
 
   skip_err_frame_ = opts->GetBool("skip_error_frame", false);
+  concurrency_limit_ = opts->GetUint32("concurrency_limit", 0);
+  NvcodecConcurrencyLimiter::GetInstance()->Init(concurrency_limit_);
   return modelbox::STATUS_OK;
 }
 
@@ -146,11 +148,11 @@ modelbox::Status VideoDecoderFlowUnit::WriteData(
     std::shared_ptr<modelbox::DataContext> &data_ctx,
     std::vector<std::shared_ptr<NvcodecFrame>> &frame_list, bool eos,
     const std::string &file_url) {
-  auto last_frame =
-      std::static_pointer_cast<modelbox::Buffer>(data_ctx->GetPrivate(LAST_FRAME));
+  auto last_frame = std::static_pointer_cast<modelbox::Buffer>(
+      data_ctx->GetPrivate(LAST_FRAME));
   data_ctx->SetPrivate(LAST_FRAME, nullptr);
-  auto color_cvt =
-      std::static_pointer_cast<NppiColorConverter>(data_ctx->GetPrivate(CVT_CTX));
+  auto color_cvt = std::static_pointer_cast<NppiColorConverter>(
+      data_ctx->GetPrivate(CVT_CTX));
   auto frame_buff_list = data_ctx->Output(FRAME_INFO_OUTPUT);
   if (last_frame != nullptr) {
     frame_buff_list->PushBack(last_frame);  // Send last frame in cache
@@ -263,8 +265,10 @@ modelbox::Status VideoDecoderFlowUnit::DataPre(
   }
 
   auto video_decoder = std::make_shared<NvcodecVideoDecoder>();
+  // when concurrency limit set, no delay must be true to avoid gpu cache
+  auto no_delay = concurrency_limit_ != 0;
   auto ret = video_decoder->Init(GetBindDevice()->GetDeviceID(), *codec_id,
-                                 *source_url, skip_err_frame_);
+                                 *source_url, skip_err_frame_, no_delay);
   if (ret != modelbox::STATUS_SUCCESS) {
     MBLOG_ERROR << "Video decoder init failed";
     return modelbox::STATUS_FAULT;
@@ -311,6 +315,9 @@ MODELBOX_FLOWUNIT(VideoDecoderFlowUnit, desc) {
   desc.AddFlowUnitOption(modelbox::FlowUnitOption(
       "skip_error_frame", "bool", true, "false",
       "whether the video decoder skip the error frame"));
+  desc.AddFlowUnitOption(modelbox::FlowUnitOption(
+      "concurrency_limit", "int", false, "0",
+      "limit gpu decode concurrency to avoid decode stuck"));
 }
 
 MODELBOX_DRIVER_FLOWUNIT(desc) {

src/drivers/devices/cuda/flowunit/video_decoder/video_decoder_flowunit.h

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-
 #ifndef MODELBOX_FLOWUNIT_VIDEO_DECODER_CPU_H_
 #define MODELBOX_FLOWUNIT_VIDEO_DECODER_CPU_H_
 
@@ -58,7 +57,8 @@ constexpr const char *FLOWUNIT_DESC =
     "\t\tField Name: shape,         Type: vector<size_t>\n"
     "\t\tField Name: type,          Type: ModelBoxDataType::MODELBOX_UINT8\n"
     "\t@Constraint: The flowuint 'video_decoder' must be used pair "
-    "with 'video_demuxer. the output buffer meta fields 'pix_fmt' is 'brg_packed' or 'rgb_packed', 'layout' is 'hcw'.";
+    "with 'video_demuxer. the output buffer meta fields 'pix_fmt' is "
+    "'brg_packed' or 'rgb_packed', 'layout' is 'hcw'.";
 constexpr const char *CODEC_META = "codec_meta";
 constexpr const char *DECODER_CTX = "decoder_ctx";
 constexpr const char *CVT_CTX = "converter_ctx";
@@ -113,6 +113,8 @@ class VideoDecoderFlowUnit : public modelbox::FlowUnit {
   std::string out_pix_fmt_str_;
   bool skip_err_frame_{false};
   std::string device_id_;
+  // limit decode concurrency to avoid decoder stuck bug in gpu driver
+  uint32_t concurrency_limit_{0};
 };
 
 #endif  // MODELBOX_FLOWUNIT_VIDEO_DECODER_CPU_H_