多进程版yolov3tiny

没试过多线程，post process都是cpu在跑，直接用了多进程搞，FPS大概20.

1. import logging as log
   
2. import time
   
3. import numpy as np
   
4. import cv2
   
5. from rknn.api import RKNN
   
6. from multiprocessing import Process, Queue
   
7. 
   
8. GRID0 = 13
   
9. GRID1 = 26
   
10. LISTSIZE = 85
    
11. SPAN = 3
    
12. NUM_CLS = 80
    
13. MAX_BOXES = 500
    
14. OBJ_THRESH = 0.2
    
15. NMS_THRESH = 0.2
    
16. 
    
17. CLASSES = ("person", "bicycle", "car","motorbike ","aeroplane ","bus ","train","truck ","boat","traffic light",
    
18.            "fire hydrant","stop sign ","parking meter","bench","bird","cat","dog ","horse ","sheep","cow","elephant",
    
19.            "bear","zebra ","giraffe","backpack","umbrella","handbag","tie","suitcase","frisbee","skis","snowboard","sports ball","kite",
    
20.            "baseball bat","baseball glove","skateboard","surfboard","tennis racket","bottle","wine glass","cup","fork","knife ",
    
21.            "spoon","bowl","banana","apple","sandwich","orange","broccoli","carrot","hot dog","pizza ","donut","cake","chair","sofa",
    
22.            "pottedplant","bed","diningtable","toilet ","tvmonitor","laptop        ","mouse        ","remote ","keyboard ","cell phone","microwave ",
    
23.            "oven ","toaster","sink","refrigerator ","book","clock","vase","scissors ","teddy bear ","hair drier", "toothbrush ")
    
24. 
    
25. def sigmoid(x):
    
26.     return 1 / (1 + np.exp(-x))
    
27. 
    
28. 
    
29. def process(input, mask, anchors):
    
30. 
    
31.     anchors = [anchors[i] for i in mask]
    
32.     grid_h, grid_w = map(int, input.shape[0:2])
    
33. 
    
34.     box_confidence = sigmoid(input[..., 4])
    
35.     box_confidence = np.expand_dims(box_confidence, axis=-1)
    
36. 
    
37.     box_class_probs = sigmoid(input[..., 5:])
    
38. 
    
39.     box_xy = sigmoid(input[..., :2])
    
40.     box_wh = np.exp(input[..., 2:4])
    
41.     box_wh = box_wh * anchors
    
42. 
    
43.     col = np.tile(np.arange(0, grid_w), grid_w).reshape(-1, grid_w)
    
44.     row = np.tile(np.arange(0, grid_h).reshape(-1, 1), grid_h)
    
45. 
    
46.     col = col.reshape(grid_h, grid_w, 1, 1).repeat(3, axis=-2)
    
47.     row = row.reshape(grid_h, grid_w, 1, 1).repeat(3, axis=-2)
    
48.     grid = np.concatenate((col, row), axis=-1)
    
49. 
    
50.     box_xy += grid
    
51.     box_xy /= (grid_w, grid_h)
    
52.     box_wh /= (416, 416)
    
53.     box_xy -= (box_wh / 2.)
    
54.     box = np.concatenate((box_xy, box_wh), axis=-1)
    
55. 
    
56.     return box, box_confidence, box_class_probs
    
57. 
    
58. def filter_boxes(boxes, box_confidences, box_class_probs):
    
59.     """Filter boxes with object threshold.
    
60. 
    
61.     # Arguments
    
62.         boxes: ndarray, boxes of objects.
    
63.         box_confidences: ndarray, confidences of objects.
    
64.         box_class_probs: ndarray, class_probs of objects.
    
65. 
    
66.     # Returns
    
67.         boxes: ndarray, filtered boxes.
    
68.         classes: ndarray, classes for boxes.
    
69.         scores: ndarray, scores for boxes.
    
70.     """
    
71.     box_scores = box_confidences * box_class_probs
    
72.     box_classes = np.argmax(box_scores, axis=-1)
    
73.     box_class_scores = np.max(box_scores, axis=-1)
    
74.     pos = np.where(box_class_scores >= OBJ_THRESH)
    
75. 
    
76.     boxes = boxes[pos]
    
77.     classes = box_classes[pos]
    
78.     scores = box_class_scores[pos]
    
79. 
    
80.     return boxes, classes, scores
    
81. 
    
82. def nms_boxes(boxes, scores):
    
83.     """Suppress non-maximal boxes.
    
84. 
    
85.     # Arguments
    
86.         boxes: ndarray, boxes of objects.
    
87.         scores: ndarray, scores of objects.
    
88. 
    
89.     # Returns
    
90.         keep: ndarray, index of effective boxes.
    
91.     """
    
92.     x = boxes[:, 0]
    
93.     y = boxes[:, 1]
    
94.     w = boxes[:, 2]
    
95.     h = boxes[:, 3]
    
96. 
    
97.     areas = w * h
    
98.     order = scores.argsort()[::-1]
    
99. 
    
100.     keep = []
     
101.     while order.size > 0:
     
102.         i = order[0]
     
103.         keep.append(i)
     
104. 
     
105.         xx1 = np.maximum(x[i], x[order[1:]])
     
106.         yy1 = np.maximum(y[i], y[order[1:]])
     
107.         xx2 = np.minimum(x[i] + w[i], x[order[1:]] + w[order[1:]])
     
108.         yy2 = np.minimum(y[i] + h[i], y[order[1:]] + h[order[1:]])
     
109. 
     
110.         w1 = np.maximum(0.0, xx2 - xx1 + 0.00001)
     
111.         h1 = np.maximum(0.0, yy2 - yy1 + 0.00001)
     
112.         inter = w1 * h1
     
113. 
     
114.         ovr = inter / (areas[i] + areas[order[1:]] - inter)
     
115.         inds = np.where(ovr <= NMS_THRESH)[0]
     
116.         order = order[inds + 1]
     
117.     keep = np.array(keep)
     
118.     return keep
     
119. 
     
120. 
     
121. def yolov3_post_process(input_data):
     
122.     # # yolov3
     
123.     # masks = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
     
124.     # anchors = [[10, 13], [16, 30], [33, 23], [30, 61], [62, 45],
     
125.     #            [59, 119], [116, 90], [156, 198], [373, 326]]
     
126.     # yolov3-tiny
     
127.     masks = [[3, 4, 5], [0, 1, 2]]
     
128.     anchors = [[10, 14], [23, 27], [37, 58], [81, 82], [135, 169], [344, 319]]
     
129. 
     
130.     boxes, classes, scores = [], [], []
     
131.     for input,mask in zip(input_data, masks):
     
132.         b, c, s = process(input, mask, anchors)
     
133.         b, c, s = filter_boxes(b, c, s)
     
134.         boxes.append(b)
     
135.         classes.append(c)
     
136.         scores.append(s)
     
137. 
     
138.     boxes = np.concatenate(boxes)
     
139.     classes = np.concatenate(classes)
     
140.     scores = np.concatenate(scores)
     
141. 
     
142.     # # Scale boxes back to original image shape.
     
143.     # width, height = 416, 416 #shape[1], shape[0]
     
144.     # image_dims = [width, height, width, height]
     
145.     # boxes = boxes * image_dims
     
146. 
     
147.     nboxes, nclasses, nscores = [], [], []
     
148.     for c in set(classes):
     
149.         inds = np.where(classes == c)
     
150.         b = boxes[inds]
     
151.         c = classes[inds]
     
152.         s = scores[inds]
     
153. 
     
154.         keep = nms_boxes(b, s)
     
155. 
     
156.         nboxes.append(b[keep])
     
157.         nclasses.append(c[keep])
     
158.         nscores.append(s[keep])
     
159. 
     
160.     if not nclasses and not nscores:
     
161.         return None, None, None
     
162. 
     
163.     boxes = np.concatenate(nboxes)
     
164.     classes = np.concatenate(nclasses)
     
165.     scores = np.concatenate(nscores)
     
166. 
     
167.     return boxes, classes, scores
     
168. 
     
169. def draw(image, boxes, scores, classes):
     
170.     """Draw the boxes on the image.
     
171. 
     
172.     # Argument:
     
173.         image: original image.
     
174.         boxes: ndarray, boxes of objects.
     
175.         classes: ndarray, classes of objects.
     
176.         scores: ndarray, scores of objects.
     
177.         all_classes: all classes name.
     
178.     """
     
179.     for box, score, cl in zip(boxes, scores, classes):
     
180.         x, y, w, h = box
     
181.         # print('class: {}, score: {}'.format(CLASSES[cl], score))
     
182.         # print('box coordinate left,top,right,down: [{}, {}, {}, {}]'.format(x, y, x+w, y+h))
     
183.         x *= image.shape[1]
     
184.         y *= image.shape[0]
     
185.         w *= image.shape[1]
     
186.         h *= image.shape[0]
     
187.         top = max(0, np.floor(x + 0.5).astype(int))
     
188.         left = max(0, np.floor(y + 0.5).astype(int))
     
189.         right = min(image.shape[1], np.floor(x + w + 0.5).astype(int))
     
190.         bottom = min(image.shape[0], np.floor(y + h + 0.5).astype(int))
     
191. 
     
192.         # print('class: {}, score: {}'.format(CLASSES[cl], score))
     
193.         # print('box coordinate left,top,right,down: [{}, {}, {}, {}]'.format(top, left, right, bottom))
     
194. 
     
195.         cv2.rectangle(image, (top, left), (right, bottom), (255, 0, 0), 2)
     
196.         cv2.putText(image, '{0} {1:.2f}'.format(CLASSES[cl], score),
     
197.                     (top, left - 6),
     
198.                     cv2.FONT_HERSHEY_SIMPLEX,
     
199.                     0.6, (0, 0, 255), 2)
     
200. 
     
201.         # print('class: {0}, score: {1:.2f}'.format(CLASSES[cl], score))
     
202.         # print('box coordinate x,y,w,h: {0}'.format(box))
     
203. 
     
204. def load_model():
     
205.         rknn = RKNN()
     
206.         print('-->loading model')
     
207.         rknn.load_rknn('./model/yolov3_tiny.rknn')
     
208.         #rknn.load_rknn('./yolov3.rknn')
     
209.         print('loading model done')
     
210. 
     
211.         print('--> Init runtime environment')
     
212.         ret = rknn.init_runtime(target='rk3399pro')
     
213.         if ret != 0:
     
214.                 print('Init runtime environment failed')
     
215.                 exit(ret)
     
216.         print('done')
     
217.         return rknn
     
218. 
     
219. 
     
220. def video_capture(src, q_frame:Queue):
     
221.     #video = VideoCapture.Video(src)
     
222.     video = cv2.VideoCapture(int(src))
     
223.     while True:
     
224.         s = time.time()
     
225.         #frame = video.get_last_frame()
     
226.         ret, frame = video.read()
     
227.         assert ret, 'read video frame failed.'
     
228.         if q_frame.empty():
     
229.             s = time.time()
     
230.             q_frame.put(frame)
     
231.             print("get frame and put to queue used {} ms".format((time.time()-s)*1000))
     
232.         else:
     
233.             print("get frame but skiped, used {} ms".format((time.time()-s)*1000))
     
234. 
     
235. def infer_rknn(q_image:Queue, q_infer:Queue):
     
236.     rknn = load_model()
     
237.     rknn.get_sdk_version()
     
238. 
     
239.     while True:
     
240.         s = time.time()
     
241.         image = q_image.get()
     
242.         print('Infer recv frame, used time {} ms. '.format((time.time() - s) * 1000))
     
243.         s = time.time()
     
244.         out_boxes, out_boxes2 = rknn.inference(inputs=[image])
     
245.         out_boxes = out_boxes.reshape(SPAN, LISTSIZE, GRID0, GRID0)
     
246.         out_boxes2 = out_boxes2.reshape(SPAN, LISTSIZE, GRID1, GRID1)
     
247.         input_data = []
     
248.         input_data.append(np.transpose(out_boxes, (2, 3, 0, 1)))
     
249.         input_data.append(np.transpose(out_boxes2, (2, 3, 0, 1)))
     
250. 
     
251.         q_infer.put(input_data)
     
252.         print('Infer done, used time {} ms. '.format((time.time()-s)*1000))
     
253. 
     
254. def post_process(q_infer, q_objs):
     
255.     while True:
     
256.         s = time.time()
     
257.         input_data = q_infer.get()
     
258.         log.info('Post process recv input, used time {} ms. '.format((time.time() - s) * 1000))
     
259.         s = time.time()
     
260.         boxes, classes, scores = yolov3_post_process(input_data)
     
261. 
     
262.         q_objs.put((boxes, classes, scores))
     
263.         log.info('Post process done, used time {} ms. '.format((time.time()-s)*1000))
     
264. 
     
265. if __name__ == '__main__':
     
266.     log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.DEBUG)
     
267. 
     
268.     q_frame = Queue()
     
269.     q_image = Queue()
     
270.     q_infer = Queue()
     
271.     q_objs = Queue()
     
272. 
     
273.     p_cap = Process(target=video_capture, args=('0',q_frame))
     
274.     p_infer = Process(target=infer_rknn, args=(q_image, q_infer))
     
275.     p_post = Process(target=post_process, args=(q_infer, q_objs))
     
276. 
     
277. 
     
278.     p_infer.start()
     
279.     p_cap.start()
     
280.     p_post.start()
     
281.     #p_show.start()
     
282. 
     
283.     l_frame = []
     
284.     fps = 0
     
285.     l_used_time = []
     
286. 
     
287.     while True:
     
288.         s = time.time()
     
289.         frame = q_frame.get()
     
290. 
     
291.         image = cv2.resize(frame, (416, 416))
     
292.         image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
     
293.         q_image.put(image)
     
294. 
     
295.         l_frame.append(frame)
     
296. 
     
297.         if q_objs.empty() and (len(l_frame) < 3):
     
298.             continue
     
299.         else:
     
300.             objects = q_objs.get()
     
301. 
     
302.         frame = l_frame.pop(0)
     
303.         boxes, classes, scores = objects
     
304.         if boxes is not None:
     
305.             draw(frame, boxes, scores, classes)
     
306.             cv2.putText(frame, text='FPS: {}'.format(fps), org=(3, 15), fontFace=cv2.FONT_HERSHEY_SIMPLEX,
     
307.                                      fontScale=0.50, color=(255, 0, 0), thickness=2)
     
308.             cv2.imshow("results", frame)
     
309. 
     
310.             c = cv2.waitKey(5) & 0xff
     
311.             if c == 27:
     
312.                 cv2.destroyAllWindows()
     
313.                 break
     
314. 
     
315.         used_time = time.time() - s
     
316.         l_used_time.append(used_time)
     
317.         if len(l_used_time) > 20:
     
318.             l_used_time.pop(0)
     
319.         fps = int(1/np.mean(l_used_time))
     
320. 
     
321.         print('All done, used time {} ms. '.format(used_time*1000))
     
322. 
     
323.     p_cap.terminate()
     
324.     p_infer.terminate()
     
325.     p_post.terminate()
     
326.     exit()
     
327. 
     

复制代码

20FPS的瓶颈在哪里？post process吗？

zhangzj 发表于 2019-4-11 18:34
20FPS的瓶颈在哪里？post process吗？

推理和后处理并行，大概2，30ms, 画图和显示大概10ms(wait了5ms), 队列搞进搞出的大概10ms.
显示也可以丢到进程里面去的，有空我搞搞。

为什么我运行了代码后一直显示get frame but skiped,也没有弹出摄像头的界面。

MinskyLee 发表于 2019-4-12 15:09
为什么我运行了代码后一直显示get frame but skiped,也没有弹出摄像头的界面。

估计是推理进程没跑起来，看看启动阶段报了什么错。注意，rknn.init_runtime(target='rk3399pro')。

hisping 发表于 2019-4-18 09:01
感谢你提供的多进程版本，我在你的基础上做了优化，可以达到30帧，已上传，帖子里可以下载下来 ...

好的，这个30帧是tiny才能达到的吧？完整的yolov3在推理这块就用了近0.5s，实在太慢了，有办法解决吗，还是我弄的有问题

感谢分享！

你好，请问 c++版 demo 与 python 版 demo，二者推理速度差多少，请问您试过吗？

这速度有点对不起这价格啊