OpenVino (三） tensorflow yolo3 同步/异步 实战

环境：

1. python3.7 / anacoda3
2. tensorflow 1.14
   networkx>=1.11,<2.4
   numpy 1.16.
   defusedxml>=0.5.0
3. openvino (python) 2019 - R3
4. opencv4.0

一、下载yolo3.weight与coco.names

以coco数据集训练的权重文件为例。

下载coco.names。

下载yolov3.weights或者yolov3-tiny.weights。（非tiny选输入为416*416)

ps:如果是转换自己训练的数据集，则将coco.names和yolov3.weights替换成自己相应的文件就可以了。

二、转换下载源码

git clone https://github.com/mystic123/tensorflow-yolo-v3.git
cd tensorflow-yolo-v3

三、从Darknet模型转换成Tensorflow模型（cmd管理员执行噢!!）

YOLOv3：

python convert_weights_pb.py --class_names coco.names --data_format NHWC --weights_file yolov3.weights

YOLOv3-tiny：

python convert_weights_pb.py --class_names coco.names --data_format NHWC --weights_file yolov3-tiny.weights --tiny

ps:(convert_weights_pb.py 会用到的包 pip install pillow)

转换成ckpt文件格式(暂时未用到)

YOLOv3：

python convert_weights.py --class_names coco.names --data_format NHWC --weights_file yolov3.weights

YOLOv3-tiny：

python convert_weights.py --class_names coco.names --data_format NHWC --weights_file yolov3-tiny.weights --tiny

四、Tensorflow模型转换成OpenVINO模型（cmd管理员执行噢!!）

1.MO工具：mo.py

所在目录：

cd C:\IntelSWTools\openvino\deployment_tools\model_optimizer

2.yolov3 操作文件：yolo_v3.json

所在目录：

C:\IntelSWTools\openvino\deployment_tools\model_optimizer\extensions\front\tf

3.把上面（三）得到的：frozen_darknet_yolov3_model.pb 与 yolo_v3.json 放到 mo.py 同级目录上

 执行（cmd管理员执行噢!!）ps:缺包补充包哦

python mo.py --input_model ./frozen_darknet_yolov3_model.pb -b 1 --tensorflow_use_custom_operations_config ./yolo_v3.json

五、测试OpenVINO模型

测试代码：object_detection_demo_yolov3_async.py

所在目录：

C:\IntelSWTools\openvino\deployment_tools\open_model_zoo\demos\python_demos\object_detection_demo_yolov3_async

这里我稍微修改了一下执行运行：（某些地方按实际再修改）

python object_detection_demo_yolov3_async.py 

代码：

# -*- coding:utf-8 -*-
import logging
import os
import sys
from argparse import ArgumentParser, SUPPRESS
from math import exp as exp
from time import time

import cv2
from openvino.inference_engine import IENetwork, IECore

logging.basicConfig(format="[ %(levelname)s ] %(message)s", level=logging.INFO, stream=sys.stdout)
log = logging.getLogger()


def build_argparser():
    parser = ArgumentParser(add_help=False)
    args = parser.add_argument_group('Options')
    args.add_argument('-h', '--help', action='help', default=SUPPRESS, help='显示此帮助消息和退出')
    args.add_argument("-m", "--model", help="具有经过训练的模型的.xml文件的路径",
                      default='frozen_darknet_yolov3_model.xml', type=str)
    args.add_argument("-i", "--input", help="图像/视频文件的路径。（指定要使用的“cam”->摄像机）", default='cam', type=str)
    args.add_argument("-l", "--cpu_extension",
                      help="可选。CPU自定义层需要。与内核实现共享库的绝对路径。", type=str,
                      default='C:/Users/catt/Documents/Intel/OpenVINO/inference_engine_samples_build/intel64/Release/cpu_extension.dll')
    args.add_argument("-d", "--device",
                      help="可选。指定要推断的目标设备；可接受CPU、GPU、FPGA、HDDL或MYRIAD。示例将为指定的设备寻找合适的插件。默认值为CPU",
                      default="GPU", type=str)
    args.add_argument("--labels", help="可选。标签映射文件", default='coco.names', type=str)
    args.add_argument("-t", "--prob_threshold", help="可选。检测滤波的概率阈值",
                      default=0.5, type=float)
    args.add_argument("-iout", "--iou_threshold", help="可选。重叠检测滤波的联合阈值交叉", default=0.4, type=float)
    args.add_argument("-ni", "--number_iter", help="可选。推理迭代次数", default=1, type=int)
    args.add_argument("-pc", "--perf_counts", help="可选。演示性能计数器", default=False,
                      action="store_true")
    args.add_argument("-r", "--raw_output_message", help="可选。输出推断结果原始值显示",
                      default=False, action="store_true")
    return parser


class YoloParams:
    # ------------------------------------------- 提取图层参数 ------------------------------------------
    # Magic numbers 从yolo样本中复制出来的
    def __init__(self, param, side):
        self.num = 3 if 'num' not in param else int(param['num'])
        self.coords = 4 if 'coords' not in param else int(param['coords'])
        self.classes = 80 if 'classes' not in param else int(param['classes'])
        self.anchors = [10.0, 13.0, 16.0, 30.0, 33.0, 23.0, 30.0, 61.0, 62.0, 45.0, 59.0, 119.0, 116.0, 90.0, 156.0,
                        198.0,
                        373.0, 326.0] if 'anchors' not in param else [float(a) for a in param['anchors'].split(',')]

        if 'mask' in param:
            mask = [int(idx) for idx in param['mask'].split(',')]
            self.num = len(mask)

            maskedAnchors = []
            for idx in mask:
                maskedAnchors += [self.anchors[idx * 2], self.anchors[idx * 2 + 1]]
            self.anchors = maskedAnchors

        self.side = side
        self.isYoloV3 = 'mask' in param  # Weak way to determine but the only one.


    def log_params(self):
        params_to_print = {'classes': self.classes, 'num': self.num, 'coords': self.coords, 'anchors': self.anchors}
        [log.info("         {:8}: {}".format(param_name, param)) for param_name, param in params_to_print.items()]


def entry_index(side, coord, classes, location, entry):
    side_power_2 = side ** 2
    n = location // side_power_2
    loc = location % side_power_2
    return int(side_power_2 * (n * (coord + classes + 1) + entry) + loc)


def scale_bbox(x, y, h, w, class_id, confidence, h_scale, w_scale):
    xmin = int((x - w / 2) * w_scale)
    ymin = int((y - h / 2) * h_scale)
    xmax = int(xmin + w * w_scale)
    ymax = int(ymin + h * h_scale)
    return dict(xmin=xmin, xmax=xmax, ymin=ymin, ymax=ymax, class_id=class_id, confidence=confidence)


def parse_yolo_region(blob, resized_image_shape, original_im_shape, params, threshold):
    # ------------------------------------------ 验证输出参数 ------------------------------------------
    _, _, out_blob_h, out_blob_w = blob.shape
    assert out_blob_w == out_blob_h, "Invalid size of output blob. It sould be in NCHW layout and height should " \
                                     "be equal to width. Current height = {}, current width = {}" \
                                     "".format(out_blob_h, out_blob_w)

    # ------------------------------------------ 提取图层参数 -------------------------------------------
    orig_im_h, orig_im_w = original_im_shape
    resized_image_h, resized_image_w = resized_image_shape
    objects = list()
    predictions = blob.flatten()
    side_square = params.side * params.side

    # ------------------------------------------- 分析YOLO区域输出 -------------------------------------------
    for i in range(side_square):
        row = i // params.side
        col = i % params.side
        for n in range(params.num):
            obj_index = entry_index(params.side, params.coords, params.classes, n * side_square + i, params.coords)
            scale = predictions[obj_index]
            if scale < threshold:
                continue
            box_index = entry_index(params.side, params.coords, params.classes, n * side_square + i, 0)
            # Network produces location predictions in absolute coordinates of feature maps.
            # Scale it to relative coordinates.
            x = (col + predictions[box_index + 0 * side_square]) / params.side
            y = (row + predictions[box_index + 1 * side_square]) / params.side
            # Value for exp is very big number in some cases so following construction is using here
            try:
                w_exp = exp(predictions[box_index + 2 * side_square])
                h_exp = exp(predictions[box_index + 3 * side_square])
            except OverflowError:
                continue
            # Depends on topology we need to normalize sizes by feature maps (up to YOLOv3) or by input shape (YOLOv3)
            w = w_exp * params.anchors[2 * n] / (resized_image_w if params.isYoloV3 else params.side)
            h = h_exp * params.anchors[2 * n + 1] / (resized_image_h if params.isYoloV3 else params.side)
            for j in range(params.classes):
                class_index = entry_index(params.side, params.coords, params.classes, n * side_square + i,
                                          params.coords + 1 + j)
                confidence = scale * predictions[class_index]
                if confidence < threshold:
                    continue
                objects.append(scale_bbox(x=x, y=y, h=h, w=w, class_id=j, confidence=confidence,
                                          h_scale=orig_im_h, w_scale=orig_im_w))
    return objects


def intersection_over_union(box_1, box_2):
    width_of_overlap_area = min(box_1['xmax'], box_2['xmax']) - max(box_1['xmin'], box_2['xmin'])
    height_of_overlap_area = min(box_1['ymax'], box_2['ymax']) - max(box_1['ymin'], box_2['ymin'])
    if width_of_overlap_area < 0 or height_of_overlap_area < 0:
        area_of_overlap = 0
    else:
        area_of_overlap = width_of_overlap_area * height_of_overlap_area
    box_1_area = (box_1['ymax'] - box_1['ymin']) * (box_1['xmax'] - box_1['xmin'])
    box_2_area = (box_2['ymax'] - box_2['ymin']) * (box_2['xmax'] - box_2['xmin'])
    area_of_union = box_1_area + box_2_area - area_of_overlap
    if area_of_union == 0:
        return 0
    return area_of_overlap / area_of_union


def main():
    args = build_argparser().parse_args()

    model_xml = args.model
    model_bin = os.path.splitext(model_xml)[0] + ".bin"

    # ------------- 1. 指定设备和加载扩展库的插件初始化(如果指定)-------------
    log.info("创建推理引擎...")
    ie = IECore()
    if args.cpu_extension and 'CPU' in args.device:
        ie.add_extension(args.cpu_extension, "CPU")

    # -------------------- 2. 读取模型优化器生成的IR（.xml和.bin文件） --------------------
    log.info("加载网络文件:\n\t{}\n\t{}".format(model_xml, model_bin))
    net = IENetwork(model=model_xml, weights=model_bin)

    # ---------------------------------- 3. 为支持特定层加载CPU扩展 ------------------------------
    if "CPU" in args.device:
        supported_layers = ie.query_network(net, "CPU")
        not_supported_layers = [l for l in net.layers.keys() if l not in supported_layers]
        if len(not_supported_layers) != 0:
            log.error("指定设备的插件不支持以下层 {}:\n {}".
                      format(args.device, ', '.join(not_supported_layers)))
            log.error("请尝试使用 -l 在示例的命令行参数中指定cpu扩展库路径 "
                      "或 --cpu_extension 命令行参数")
            sys.exit(1)

    assert len(net.inputs.keys()) == 1, "示例仅支持基于YOLO V3的单输入拓扑"

    # ---------------------------------------------- 4. 准备输入 -----------------------------------------------
    log.info("准备输入")
    input_blob = next(iter(net.inputs))

    #  默认批量是1
    net.batch_size = 1

    # 读取并预处理输入图像
    n, c, h, w = net.inputs[input_blob].shape



    if args.labels:
        with open(args.labels, 'r') as f:
            labels_map = [x.strip() for x in f]
    else:
        labels_map = None

    input_stream = 0 if args.input == "cam" else args.input

    is_async_mode = True #异步
    cap = cv2.VideoCapture(input_stream)
    number_input_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) #获得总视频的总帧数,摄像机 是-1，视频是 大于0,图片是 1

    number_input_frames = 1 if number_input_frames != -1 and number_input_frames < 0 else number_input_frames

    wait_key_code = 1

    # 图片中的帧数为1，将按周期读取。同步模式是这种情况下的默认值
    if number_input_frames != 1:
        ret, frame = cap.read()
    else:
        is_async_mode = False #如果是图片
        wait_key_code = 0

    # ----------------------------------------- 5. 将模型加载到插件 -----------------------------------------
    log.info("将模型加载到插件")
    exec_net = ie.load_network(network=net, num_requests=2, device_name=args.device)

    cur_request_id = 0
    next_request_id = 1
    render_time = 0
    parsing_time = 0


    # ----------------------------------------------- 6. Doing inference -----------------------------------------------
    log.info("开始推断...")
    print("要关闭应用程序，请按此处的“CTRL+C”或切换到输出窗口并按ESC键")
    print("要在同步/异步模式之间切换，请在“输出”窗口中按TAB键")
    while cap.isOpened():
        # Here is the first asynchronous point: in the Async mode, we capture frame to populate the NEXT infer request
        # in the regular mode, we capture frame to the CURRENT infer request
        if is_async_mode:
            ret, next_frame = cap.read()
        else:
            ret, frame = cap.read()

        if not ret:
            break

        if is_async_mode:
            request_id = next_request_id
            in_frame = cv2.resize(next_frame, (w, h))
        else:
            request_id = cur_request_id
            in_frame = cv2.resize(frame, (w, h))

        # 将输入框调整为网络大小
        in_frame = in_frame.transpose((2, 0, 1))  # Change data layout from HWC to CHW
        in_frame = in_frame.reshape((n, c, h, w))

        # 开始推断
        start_time = time()
        exec_net.start_async(request_id=request_id, inputs={input_blob: in_frame})
        det_time = time() - start_time #推理时间

        # 收集目标检测结果
        objects = list()
        if exec_net.requests[cur_request_id].wait(-1) == 0:
            output = exec_net.requests[cur_request_id].outputs

            start_time = time()
            for layer_name, out_blob in output.items():
                out_blob = out_blob.reshape(net.layers[net.layers[layer_name].parents[0]].shape)
                layer_params = YoloParams(net.layers[layer_name].params, out_blob.shape[2])
                log.info("Layer {} parameters: ".format(layer_name))
                layer_params.log_params()
                objects += parse_yolo_region(out_blob, in_frame.shape[2:],
                                             frame.shape[:-1], layer_params,
                                             args.prob_threshold)
            parsing_time = time() - start_time

        # 根据--iou_threshold CLI参数筛选重叠框
        objects = sorted(objects, key=lambda obj : obj['confidence'], reverse=True)
        for i in range(len(objects)):
            if objects[i]['confidence'] == 0:
                continue
            for j in range(i + 1, len(objects)):
                if intersection_over_union(objects[i], objects[j]) > args.iou_threshold:
                    objects[j]['confidence'] = 0

        # 根据--prob_threshold CLI参数绘制对象
        objects = [obj for obj in objects if obj['confidence'] >= args.prob_threshold]

        if len(objects) and args.raw_output_message:
            log.info("\nDetected boxes for batch {}:".format(1))
            log.info(" Class ID | Confidence | XMIN | YMIN | XMAX | YMAX | COLOR ")

        origin_im_size = frame.shape[:-1]
        for obj in objects:
            # 检测对象的验证框
            if obj['xmax'] > origin_im_size[1] or obj['ymax'] > origin_im_size[0] or obj['xmin'] < 0 or obj['ymin'] < 0:
                continue
            color = (int(min(obj['class_id'] * 12.5, 255)),
                     min(obj['class_id'] * 7, 255), min(obj['class_id'] * 5, 255))
            det_label = labels_map[obj['class_id']] if labels_map and len(labels_map) >= obj['class_id'] else \
                str(obj['class_id'])

            if args.raw_output_message:
                log.info(
                    "{:^9} | {:10f} | {:4} | {:4} | {:4} | {:4} | {} ".format(det_label, obj['confidence'], obj['xmin'],
                                                                              obj['ymin'], obj['xmax'], obj['ymax'],
                                                                              color))

            cv2.rectangle(frame, (obj['xmin'], obj['ymin']), (obj['xmax'], obj['ymax']), color, 2)
            cv2.putText(frame,
                        "#" + det_label + ' ' + str(round(obj['confidence'] * 100, 1)) + ' %',
                        (obj['xmin'], obj['ymin'] - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1)

        # 在帧上绘制性能统计信息
        inf_time_message = "Inference time: N\A for async mode" if is_async_mode else \
            "Inference time: {:.3f} ms".format(det_time * 1e3)
        render_time_message = "OpenCV rendering time: {:.3f} ms".format(render_time * 1e3)
        async_mode_message = "Async mode is on. Processing request {}".format(cur_request_id) if is_async_mode else \
            "Async mode is off. Processing request {}".format(cur_request_id)
        parsing_message = "YOLO parsing time is {:.3f}".format(parsing_time * 1e3)

        cv2.putText(frame, inf_time_message, (15, 15), cv2.FONT_HERSHEY_COMPLEX, 0.5, (200, 10, 10), 1)
        cv2.putText(frame, render_time_message, (15, 45), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)
        cv2.putText(frame, async_mode_message, (10, int(origin_im_size[0] - 20)), cv2.FONT_HERSHEY_COMPLEX, 0.5,
                    (10, 10, 200), 1)
        cv2.putText(frame, parsing_message, (15, 30), cv2.FONT_HERSHEY_COMPLEX, 0.5, (10, 10, 200), 1)

        start_time = time()
        cv2.imshow("DetectionResults", frame)
        render_time = time() - start_time

        if is_async_mode:
            cur_request_id, next_request_id = next_request_id, cur_request_id
            frame = next_frame

        key = cv2.waitKey(wait_key_code)

        # ESC key
        if key == 27:
            break
        # Tab key
        if key == 9:
            exec_net.requests[cur_request_id].wait()
            is_async_mode = not is_async_mode
            log.info("Switched to {} mode".format("async" if is_async_mode else "sync"))

    cv2.destroyAllWindows()
    del net
    del exec_net
    del ie


if __name__ == '__main__':
    sys.exit(main() or 0)