Merge pull request #3130 from alibaba/feature/sync

MNN:Sync: Sync Interal 3.0.2

Author: jxt1234

.gitignore

@@ -361,10 +361,3 @@ pymnn_build/
 
 # mnncompress generated
 MNN_compression_pb2.py
-
-# model path
-model/
-
-# datasets
-datasets/*
-!datasets/*.sh

3rd_party/OpenCLHeaders/CL/cl2.hpp

@@ -3810,6 +3810,24 @@ class Buffer : public Memory
         }
     }
 
+    Buffer(
+        const Context& context,
+        cl_mem_flags flags,
+        const cl_import_properties_arm *properties,
+        void *memory,
+        size_type size,
+        cl_int* err = NULL)
+    {
+        cl_int error;
+        object_ = ::clImportMemoryARM(context(), flags, properties, memory, size, &error);
+
+        detail::errHandler(error, __CREATE_BUFFER_ERR);
+        if (err != NULL) {
+            *err = error;
+        }
+    }
+
+
     /*!
      * \brief Construct a Buffer from a host container via iterators using a specified context.
      * IteratorType must be random access.

3rd_party/OpenCLHeaders/CL/cl_ext.h

@@ -430,6 +430,23 @@ typedef struct _cl_mem_android_native_buffer_host_ptr
 } cl_mem_android_native_buffer_host_ptr;
 
 
+/*********************************
+* cl_qcom_ahardwarebuffer_host_ptr extension
+*********************************/
+
+#define CL_MEM_ANDROID_AHARDWAREBUFFER_HOST_PTR_QCOM                0x4119
+
+typedef struct _cl_mem_ahardwarebuffer_host_ptr
+{
+    /* Type of external memory allocation. */
+    /* Must be CL_MEM_ANDROID_AHARDWAREBUFFER_HOST_PTR_QCOM for Android Hardware buffers. */
+    cl_mem_ext_host_ptr  ext_host_ptr;
+
+    /* Virtual pointer to the android hardware buffer */
+    void*                ahb_ptr;
+
+} cl_mem_ahardwarebuffer_host_ptr;
+
 /******************************************
  * cl_img_yuv_image extension *
  ******************************************/
@@ -583,6 +600,11 @@ typedef intptr_t cl_import_properties_arm;
 
 /* Protected DMA BUF memory type value for CL_IMPORT_TYPE_ARM property */
 #define CL_IMPORT_TYPE_PROTECTED_ARM              0x40B5
+#define CL_IMPORT_TYPE_ANDROID_HARDWARE_BUFFER_ARM          0x41E2
+#define CL_IMPORT_DMA_BUF_DATA_CONSISTENCY_WITH_HOST_ARM    0x41E3
+#define CL_IMPORT_MEMORY_WHOLE_ALLOCATION_ARM               SIZE_MAX
+#define CL_IMPORT_ANDROID_HARDWARE_BUFFER_PLANE_INDEX_ARM   0x41EF
+#define CL_IMPORT_ANDROID_HARDWARE_BUFFER_LAYER_INDEX_ARM   0x41F0
 
 /* This extension adds a new function that allows for direct memory import into
  * OpenCL via the clImportMemoryARM function.

CMakeLists.txt

@@ -20,9 +20,7 @@ endif()
 project(MNN VERSION ${MNN_VERSION} LANGUAGES C CXX ASM)
 # complier options
 set(CMAKE_C_STANDARD 99)
-IF (NOT (CMAKE_CXX_STANDARD EQUAL 17))
-  set(CMAKE_CXX_STANDARD 11)
-ENDIF()
+set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_MODULE_PATH
   ${CMAKE_MODULE_PATH}
   "${CMAKE_CURRENT_LIST_DIR}/cmake"
@@ -49,7 +47,7 @@ option(MNN_BUILD_TOOLS "Build tools/cpp or not" ON)
 option(MNN_BUILD_QUANTOOLS "Build Quantized Tools or not" OFF)
 option(MNN_EVALUATION "Build Evaluation Tools or not" OFF)
 option(MNN_BUILD_CONVERTER "Build Converter" OFF)
-option(MNN_SUPPORT_DEPRECATED_OP "Enable MNN's tflite quantized op" ON)
+option(MNN_SUPPORT_DEPRECATED_OP "Enable MNN's tflite quantized op" OFF)
 option(MNN_DEBUG_MEMORY "MNN Debug Memory Access" OFF)
 option(MNN_DEBUG_TENSOR_SIZE "Enable Tensor Size" OFF)
 option(MNN_GPU_TRACE "Enable MNN Gpu Debug" OFF)
@@ -74,6 +72,7 @@ option(MNN_JNI "Build MNN Jni for java to use" OFF)
 option(MNN_SUPPORT_BF16 "Enable MNN's bf16 op" OFF)
 option(MNN_LOW_MEMORY "Build MNN support low memory for weight quant model." OFF)
 option(MNN_CPU_WEIGHT_DEQUANT_GEMM "Build MNN CPU weight dequant related gemm kernels." OFF)
+option(MNN_BUILD_AUDIO "Build audio api in MNN." OFF)
 
 IF (OHOS AND MNN_INTERNAL)
   include($ENV{NODE_PATH}/@ali/tcpkg/tcpkg.cmake)
@@ -192,6 +191,9 @@ endif()
 if(MNN_SUPPORT_TRANSFORMER_FUSE)
     add_definitions(-DMNN_SUPPORT_TRANSFORMER_FUSE)
 endif()
+if(MNN_BUILD_AUDIO)
+    add_definitions(-DMNN_BUILD_AUDIO)
+endif()
 # debug options
 if(MNN_DEBUG_MEMORY)
     add_definitions(-DMNN_DEBUG_MEMORY)
@@ -287,7 +289,7 @@ if(CMAKE_SYSTEM_NAME MATCHES "^Android")
 endif()
 option(MNN_USE_CPP11 "Enable MNN use c++11" ON)
 if (NOT MSVC)
-    if((MNN_CUDA AND MNN_SUPPORT_TRANSFORMER_FUSE) OR (CMAKE_CXX_STANDARD EQUAL 17))
+    if(MNN_CUDA AND MNN_SUPPORT_TRANSFORMER_FUSE)
         set(CMAKE_CXX_STANDARD 17)
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu99")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
@@ -463,6 +465,10 @@ IF(MNN_BUILD_OPENCV)
   list(APPEND MNN_EXTRA_HEADERS ${MNN_CV_HDRS})
   list(APPEND MNN_EXTRA_HEADERS ${MNN_CV_IMGHDRS})
 ENDIF()
+IF(MNN_BUILD_AUDIO)
+  file(GLOB MNN_AUDIO_HDRS ${CMAKE_CURRENT_SOURCE_DIR}/tools/audio/include/audio/*.hpp PARENT_SCOPE)
+  list(APPEND MNN_EXTRA_HEADERS ${MNN_AUDIO_HDRS})
+ENDIF()
 IF(MNN_BUILD_LLM)
   file(GLOB MNN_LLM_HDRS ${CMAKE_CURRENT_SOURCE_DIR}/transformers/llm/engine/include/llm/*)
   list(APPEND MNN_EXTRA_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/transformers/llm/engine/include/llm/llm.hpp)
@@ -775,6 +781,14 @@ IF(MNN_BUILD_OPENCV AND NOT MNN_SEP_BUILD)
   ENDIF()
   target_sources(MNN PRIVATE $<TARGET_OBJECTS:MNNOpenCV>)
 ENDIF()
+add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/tools/audio)
+IF(MNN_BUILD_AUDIO AND NOT MNN_SEP_BUILD)
+  IF(MSVC)
+    target_compile_definitions(MNNAudio PRIVATE "-DBUILDING_MNN_DLL" INTERFACE "-DUSING_MNN_DLL")
+  ENDIF()
+  message(STATUC "### build MNNAudio into MNN")
+  target_sources(MNN PRIVATE $<TARGET_OBJECTS:MNNAudio>)
+ENDIF()
 
 
 if(CMAKE_SYSTEM_NAME MATCHES "^Linux")
@@ -884,6 +898,14 @@ ELSE()
       SET_SOURCE_FILES_PROPERTIES(${HDR} PROPERTIES MACOSX_PACKAGE_LOCATION Headers/cv/imgproc )
     ENDFOREACH()
   ENDIF()
+  IF(MNN_BUILD_AUDIO)
+    if (NOT MNN_AAPL_FMWK)
+      INSTALL(FILES ${MNN_AUDIO_HDRS} DESTINATION include/MNN/audio)
+    endif()
+    FOREACH(HDR ${MNN_AUDIO_HDRS})
+      SET_SOURCE_FILES_PROPERTIES(${HDR} PROPERTIES MACOSX_PACKAGE_LOCATION Headers/audio/ )
+    ENDFOREACH()
+  ENDIF()
   IF(MNN_BUILD_LLM)
     if (NOT MNN_AAPL_FMWK)
         INSTALL(FILES ${MNN_LLM_HDRS} DESTINATION include/MNN/llm)

source/backend/cpu/compute/DeconvolutionWithStride.cpp renamed to backupcode/cpubackend/compute/DeconvolutionWithStride.cpp

@@ -16,7 +16,7 @@ MNN使用CMake构建项目，CMake中的宏定义列表如下：
 | MNN_BUILD_QUANTOOLS  | 是否构建MNN的量化工具，默认为`OFF` |
 | MNN_EVALUATION       | 是否构建MNN的评估工具，默认为`OFF` |
 | MNN_BUILD_CONVERTER  | 是否构建MNN的转换工具，默认为`OFF` |
-| MNN_SUPPORT_DEPRECATED_OP | 是否支持Tflite的量化算子，默认为`ON` |
+| MNN_SUPPORT_DEPRECATED_OP | 是否支持Tflite的量化算子等已经废弃的算子，用于兼容历史模型(1.1.0版本之前)，默认为`OFF` |
 | MNN_DEBUG_MEMORY     | 是否开启MNN内存调试，默认为`OFF` |
 | MNN_DEBUG_TENSOR_SIZE | 是否开启MNN tensor size调试，默认为`OFF` |
 | MNN_GPU_TRACE        | 是否开启MNN GPU调试，默认为`OFF` |
@@ -32,6 +32,7 @@ MNN使用CMake构建项目，CMake中的宏定义列表如下：
 | MNN_ENABLE_COVERAGE  | 是否开启MNN的代码覆盖率，默认为`OFF` |
 | MNN_BUILD_PROTOBUFFER | 是否使用MNN中的`protobuffer`，默认为`ON` |
 | MNN_BUILD_OPENCV     | 是否构建MNN的OpenCV功能，默认为`OFF` |
+| MNN_BUILD_AUDIO      | 是否构建MNN的Audio功能，默认为`OFF` |
 | MNN_INTERNAL         | 是否构建MNN的一些内部功能，如：日志；默认为`OFF` |
 | MNN_JNI              | 是否构建MNN的JNI支持，默认为`OFF` |
 | MNN_METAL            | 是否构建`Metal`后端，默认为`OFF` |
@@ -79,6 +80,7 @@ MNN使用CMake构建项目，CMake中的宏定义列表如下：
 | MNN_CVCORE           | 构建MNN的OpenCV功能是否开启`core`功能，默认为`ON` |
 | MNN_OPENCV_TEST      | 构建MNN的OpenCV功能是否开启单元测试，默认为`OFF` |
 | MNN_OPENCV_BENCH     | 构建MNN的OpenCV功能是否开启性能benchmark，默认为`OFF` |
+| MNN_AUDIO_TEST       | 构建MNN的Audio功能是否开启单元测试，默认为`OFF` |
 | MNN_VULKAN_IMAGE     | 构建MNN的Vulkan后端时采用Image内存模式，以便支持FP16和部分移动端上GPU的加速，默认为`ON` |
 | MNN_LOW_MEMORY       | 是否支持低内存模式，支持低内存模式使用权值量化模型并设置`low_memory`则会使用计算时反量化，默认为`OFF` |
 | MNN_CPU_WEIGHT_DEQUANT_GEMM       | 是否编译CPU权重反量化的矩阵乘Kernel， 如果打开该编译宏并且在CPU推理时设置MNN::BackendConfig::MemoryMode=Memory_Normal，就会使用权重反量化算子进行权重量化模型的推理，默认为`OFF` |

source/backend/cpu/compute/DeconvolutionWithStride.hpp renamed to backupcode/cpubackend/compute/DeconvolutionWithStride.hpp

@@ -133,6 +133,19 @@
   - `libMNNOpenCV.so` MNN OpenCV函数库
   - `opencv_test` MNN OpenCV单元测试
   - `opencv_bench` MNN OpenCV性能测试
+## MNN Audio库
+- 相关编译选项
+  - `MNN_BUILD_AUDIO` 是否编译Audio函数接口
+  - `MNN_AUDIO_TEST` 是否编译Audio单元测试
+- 编译命令
+    ```bash
+    mkdir build && cd build
+    cmake .. -MNN_BUILD_AUDIO=ON -MNN_AUDIO_TEST=ON
+    make -j4
+    ```
+- 编译产物
+  - `libMNNAudio.so` MNN Audio函数库
+  - `audio_test` MNN Audio单元测试
 
 ## 示例工程
 - 相关编译选项