Merge pull request DayBreak-u#41 from nihui/ncnn-lstm-model

ncnn lstm model

ncnn_project/models/crnn_lite_lstm_v2.param

@@ -0,0 +1,40 @@
+7767517
+38 38
+Input            input                    0 1 input
+Convolution      Conv_0                   1 1 input 77 0=24 1=5 11=5 2=1 12=1 3=2 13=2 4=2 14=2 15=2 16=2 5=1 6=600
+ReLU             Relu_1                   1 1 77 78
+ConvolutionDepthWise Conv_2                   1 1 78 79 0=24 1=3 11=3 2=1 12=1 3=1 13=1 4=1 14=1 15=1 16=1 5=1 6=216 7=24
+ReLU             Relu_3                   1 1 79 80
+Convolution      Conv_4                   1 1 80 81 0=128 1=1 11=1 2=1 12=1 3=1 13=1 4=0 14=0 15=0 16=0 5=1 6=3072
+Pooling          MaxPool_5                1 1 81 82 0=0 1=2 11=2 2=2 12=2 3=0 13=0 14=0 15=0 5=1
+ConvolutionDepthWise Conv_6                   1 1 82 83 0=128 1=3 11=3 2=1 12=1 3=1 13=1 4=1 14=1 15=1 16=1 5=1 6=1152 7=128
+ReLU             Relu_8                   1 1 83 85
+Convolution      Conv_9                   1 1 85 86 0=256 1=1 11=1 2=1 12=1 3=1 13=1 4=0 14=0 15=0 16=0 5=1 6=32768
+ConvolutionDepthWise Conv_11                  1 1 86 88 0=256 1=3 11=3 2=1 12=1 3=1 13=1 4=1 14=1 15=1 16=1 5=1 6=2304 7=256
+ReLU             Relu_12                  1 1 88 89
+Convolution      Conv_13                  1 1 89 90 0=256 1=1 11=1 2=1 12=1 3=1 13=1 4=0 14=0 15=0 16=0 5=1 6=65536
+Pooling          MaxPool_14               1 1 90 91 0=0 1=2 11=2 2=1 12=2 3=1 13=0 14=1 15=0 5=1
+ConvolutionDepthWise Conv_15                  1 1 91 92 0=256 1=3 11=3 2=1 12=1 3=1 13=1 4=1 14=1 15=1 16=1 5=1 6=2304 7=256
+ReLU             Relu_17                  1 1 92 94
+Convolution      Conv_18                  1 1 94 95 0=512 1=1 11=1 2=1 12=1 3=1 13=1 4=0 14=0 15=0 16=0 5=1 6=131072
+ConvolutionDepthWise Conv_20                  1 1 95 97 0=512 1=3 11=3 2=1 12=1 3=1 13=1 4=1 14=1 15=1 16=1 5=1 6=4608 7=512
+ReLU             Relu_21                  1 1 97 98
+Convolution      Conv_22                  1 1 98 99 0=512 1=1 11=1 2=1 12=1 3=1 13=1 4=0 14=0 15=0 16=0 5=1 6=262144
+Pooling          MaxPool_23               1 1 99 100 0=0 1=2 11=2 2=1 12=2 3=1 13=0 14=1 15=0 5=1
+ConvolutionDepthWise Conv_24                  1 1 100 101 0=512 1=2 11=2 2=1 12=1 3=1 13=1 4=0 14=0 15=0 16=0 5=1 6=2048 7=512
+ReLU             Relu_26                  1 1 101 103
+Convolution      Conv_27                  1 1 103 104 0=512 1=1 11=1 2=1 12=1 3=1 13=1 4=0 14=0 15=0 16=0 5=1 6=262144
+Reshape          Squeeze_29               1 1 104 106 0=-1 1=512 2=-233
+Permute          Transpose_30             1 1 106 107 0=1
+LSTM             LSTM_37                  1 1 107 234 0=128 1=524288 2=2
+Permute          Transpose_38             1 1 234 237 0=2
+Reshape          Reshape_40               1 1 237 239 0=-1 1=0
+Reshape          Reshape_54               1 1 239 253 0=256
+InnerProduct     Gemm_55                  1 1 253 254 0=256 1=1 2=65536
+Reshape          Reshape_59               1 1 254 260 0=-1 1=1
+LSTM             LSTM_66                  1 1 260 387 0=64 1=131072 2=2
+Permute          Transpose_67             1 1 387 390 0=2
+Reshape          Reshape_69               1 1 390 392 0=-1 1=0
+Reshape          Reshape_83               1 1 392 406 0=128
+InnerProduct     Gemm_84                  1 1 406 407 0=5530 1=1 2=707840
+Reshape          Reshape_88               1 1 407 out 0=-1 1=1

ncnn_project/ocr/src/ocr.cpp

@@ -1,14 +1,19 @@
 #include "ocr.h"
 
+#define CRNN_LSTM 0
 
 OCR::OCR()
 {
     psenet.load_param("../../models/psenet_lite_mbv2.param");
     psenet.load_model("../../models/psenet_lite_mbv2.bin");
 
+#if CRNN_LSTM
+    crnn_net.load_param("../../models/crnn_lite_lstm_v2.param");
+    crnn_net.load_model("../../models/crnn_lite_lstm_v2.bin");
+#else
     crnn_net.load_param("../../models/crnn_lite_dw_dense.param");
     crnn_net.load_model("../../models/crnn_lite_dw_dense.bin");
-
+#endif
 //    crnn_net.load_param("../../models/crnn_lite_dw_dense_1x5.param");
 //    crnn_net.load_model("../../models/crnn_lite_dw_dense_1x5.bin");
 
@@ -388,7 +393,50 @@ void  OCR::detect(cv::Mat im_bgr,int long_size)
         if (angle_index ==0 || angle_index ==1 ){
             ncnn::Extractor crnn_ex = crnn_net.create_extractor();
             crnn_ex.input("input", crnn_in);
+#if CRNN_LSTM
+            // lstm
+            ncnn::Mat blob162;
+            crnn_ex.extract("234", blob162);
+
+            // batch fc
+            ncnn::Mat blob182(256, blob162.h);
+            for (int i=0; i<blob162.h; i++)
+            {
+                ncnn::Extractor crnn_ex_1 = crnn_net.create_extractor();
+
+                ncnn::Mat blob162_i = blob162.row_range(i, 1);
+                crnn_ex_1.input("253", blob162_i);
+
+                ncnn::Mat blob182_i;
+                crnn_ex_1.extract("254", blob182_i);
+
+                memcpy(blob182.row(i), blob182_i, 256 * sizeof(float));
+            }
+
+            // lstm
+            ncnn::Mat blob243;
+            crnn_ex.input("260", blob182);
+            crnn_ex.extract("387", blob243);
+
+            // batch fc
+            ncnn::Mat blob263(5530, blob243.h);
+            for (int i=0; i<blob243.h; i++)
+            {
+                ncnn::Extractor crnn_ex_2 = crnn_net.create_extractor();
+
+                ncnn::Mat blob243_i = blob243.row_range(i, 1);
+                crnn_ex_2.input("406", blob243_i);
+
+                ncnn::Mat blob263_i;
+                crnn_ex_2.extract("407", blob263_i);
+
+                memcpy(blob263.row(i), blob263_i, 5530 * sizeof(float));
+            }
+
+            crnn_preds = blob263;
+#else // CRNN_LSTM
             crnn_ex.extract("out", crnn_preds);
+#endif // CRNN_LSTM
         }
         else{
             ncnn::Extractor crnn_ex = crnn_vertical_net.create_extractor();