[TensorRT] Clip TensorRT code update (#426)

* add trt clip mode in models * add tensorrt sd compile option * add tensorrt clip test code * add tensorrt tokenizer and vocab * tensorrt clip code implement * add trtsd namespace * add tensorrt clip namespace * update trt code * delete useless code * update clip tensorrt test code * update clip class doesn't need threads * update trt clip code to infer dynamic shape --------- Co-authored-by: wangzijian <2422717601@qq.com>
DefTruth · Aug 8, 2024 · c83b489 · c83b489
1 parent bbeacc8
commit c83b489
Show file tree

Hide file tree

Showing 10 changed files with 525,393 additions and 4 deletions.
diff --git a/cmake/tensorrt.cmake b/cmake/tensorrt.cmake
@@ -44,13 +44,16 @@ file(GLOB TENSORRT_CORE_SRCS ${CMAKE_SOURCE_DIR}/lite/trt/core/*.cpp)
 file(GLOB TENSORRT_CV_SRCS ${CMAKE_SOURCE_DIR}/lite/trt/cv/*.cpp)
 file(GLOB TENSORRT_NLP_SRCS ${CMAKE_SOURCE_DIR}/lite/trt/nlp/*.cpp)
 file(GLOB TENSORRT_ASR_SRCS ${CMAKE_SOURCE_DIR}/lite/trt/asr/*.cpp)
+file(GLOB TENSORRT_SD_SRCS ${CMAKE_SOURCE_DIR}/lite/trt/sd/*.cpp)
+
 # 2. glob headers files
 file(GLOB TENSORRT_CORE_HEAD ${CMAKE_SOURCE_DIR}/lite/trt/core/*.h)
 file(GLOB TENSORRT_CV_HEAD ${CMAKE_SOURCE_DIR}/lite/trt/cv/*.h)
 file(GLOB TENSORRT_NLP_HEAD ${CMAKE_SOURCE_DIR}/lite/trt/nlp/*.h)
 file(GLOB TENSORRT_ASR_HEAD ${CMAKE_SOURCE_DIR}/lite/trt/asr/*.h)
+file(GLOB TENSORRT_SD_HEAD ${CMAKE_SOURCE_DIR}/lite/trt/sd/*.h)
 
-set(TRT_SRCS ${TENSORRT_CV_SRCS} ${TENSORRT_NLP_SRCS} ${TENSORRT_ASR_SRCS} ${TENSORRT_CORE_SRCS})
+set(TRT_SRCS ${TENSORRT_CV_SRCS} ${TENSORRT_NLP_SRCS} ${TENSORRT_ASR_SRCS} ${TENSORRT_CORE_SRCS} ${TENSORRT_SD_SRCS})
 # 3. copy
 message("[Lite.AI.Toolkit][I] Installing Lite.AI.ToolKit Headers for TensorRT Backend ...")
 # "INSTALL" can copy all files from the list to the specified path.
@@ -59,3 +62,5 @@ file(INSTALL ${TENSORRT_CORE_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/l
 file(INSTALL ${TENSORRT_CV_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/trt/cv)
 file(INSTALL ${TENSORRT_ASR_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/trt/asr)
 file(INSTALL ${TENSORRT_NLP_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/trt/nlp)
+file(INSTALL ${TENSORRT_SD_HEAD} DESTINATION ${CMAKE_INSTALL_PREFIX}/include/lite/trt/sd)
+
diff --git a/examples/lite/sd/test_lite_clip.cpp b/examples/lite/sd/test_lite_clip.cpp
@@ -21,10 +21,29 @@ static void test_default()
 }
 
 
+static void test_tensorrt()
+{
+    std::string engine_path = "../../../examples/hub/trt/dynamic_text_model_fp32.engine";
+
+    lite::trt::sd::text_encoder::Clip *clip = new lite::trt::sd::text_encoder::Clip(engine_path);
+
+    std::vector<std::string> input_vector = {"i am not good at cpp","goi ofg go !"};
+
+    std::vector<std::vector<float>> output;
+
+    clip->inference(input_vector,output);
+
+    delete clip;
+
+}
+
+
 
 static void test_lite()
 {
     test_default();
+
+    test_tensorrt();
 }
 
 int main(__unused int argc, __unused char *argv[])

diff --git a/lite/models.h b/lite/models.h
@@ -129,6 +129,7 @@
 #include "lite/trt/cv/trt_yolov8.h"
 #include "lite/trt/cv/trt_yolov6.h"
 #include "lite/trt/cv/trt_yolov5_blazeface.h"
+#include "lite/trt/sd/trt_clip.h"
 #endif
 
 // ENABLE_MNN
@@ -714,6 +715,15 @@ namespace lite{
                 }
             }
         }
+        namespace sd
+        {
+
+            typedef trtsd::TRTClip _TRT_Clip;
+            namespace text_encoder
+            {
+                typedef _TRT_Clip Clip;
+            }
+        }
     }
 #endif
 }

diff --git a/lite/ort/sd/clip.h b/lite/ort/sd/clip.h
@@ -44,11 +44,8 @@ namespace ortsd
     public:
         void encode_text(std::vector<std::string> input_text, std::vector<std::vector<int>> &output);
 
-        void inference(std::vector<int> input,std::vector<float> &output);
-
         void inference(std::vector<std::string> input,std::vector<std::vector<float>> &output);
 
-        Ort::Value transform(const cv::Mat &mat);
     };
 }
 

diff --git a/lite/trt/core/trt_core.h b/lite/trt/core/trt_core.h
@@ -22,6 +22,10 @@ namespace trtcv{
     using trtcore::BasicTRTHandler;
 }
 
+namespace trtsd{
+    class LITE_EXPORTS TRTClip;
+}
+
 
 
 #endif //LITE_AI_TOOLKIT_TRT_CORE_H
diff --git a/lite/trt/core/trt_types.h b/lite/trt/core/trt_types.h
@@ -10,5 +10,10 @@ namespace trtcv{
     namespace types = lite::types;
 }
 
+namespace trtsd{
+    namespace types = lite::types;
+}
+
+
 
 #endif //LITE_AI_TOOLKIT_TRT_TYPES_H