YOLO11 推理速度大對決：OpenVINO vs. PyTorch vs. ONNX Runtime

pip install ultralytics

from ultralytics import YOLO

model = YOLO("yolo11n.pt")
image_path = "bus.jpg"

# average the inference time
count = 100
inference_time_list = []
for _ in range(count):
    results = model(image_path, imgsz=(640, 640), device="CPU", rect=False)
    inference_time_list.append(results[0].speed["inference"])

print(f"Total: {sum(inference_time_list) / 1000} seconds")
print(f"Average: {sum(inference_time_list) / 1000 / count} seconds")
print(f"{1 / sum(inference_time_list) * count * 1000} FPS")

yolo export model=yolo11n.pt format=openvino dynamic=False nms=True

from PIL import Image, ImageDraw, ImageOps
import numpy as np

def letterbox(
    img: Image,
    new_shape: tuple[int, int] | int = (640, 640),
    color: tuple[int, int, int] = (114, 114, 114),
):
    shape = img.size  # W, H
    if isinstance(new_shape, int):
        new_shape = (new_shape, new_shape)  # W, H

    # scale ratio (new / old)
    r = min(new_shape[0] / shape[0], new_shape[1] / shape[1])
    # only scale down, do not scale up (for better test mAP)
    r = min(r, 1.0)

    # compute padding
    new_unpad = int(round(shape[0] * r)), int(round(shape[1] * r))
    dw, dh = (
        new_shape[0] - new_unpad[0],
        new_shape[1] - new_unpad[1],
    )
    left = int(round(dw / 2))
    right = dw - left
    top = int(round(dh / 2))
    bottom = dh - top

    # resize
    img = img.resize(new_unpad)

    # border
    image_with_border = ImageOps.expand(
        img, border=(left, top, right, bottom), fill=color
    )
    return image_with_border

def preprocess_image(img: Image):
    # Normalization
    img = np.array(img).astype(np.float32)
    img = img / 255

    # Convert HWC to CHW
    img = img.transpose(2, 0, 1)
    img = np.ascontiguousarray(img)

    # add batch dim
    img = np.expand_dims(img, axis=0)
    return img

from PIL import Image
import openvino as ov
import time

def openvino(img_input: np.ndarray, device_name: str):
    model_path = "model/yolo11n_openvino_model/yolo11n.xml"

    core = ov.Core()
    model = core.read_model(model_path)
    compiled_model = core.compile_model(model, device_name=device_name)

    # average the inference time
    count = 100
    start = time.time()
    for _ in range(count):
        outputs = compiled_model(img_input)
    end = time.time()

    print(f"Device Name: {device_name}")
    print(f"Total: {end - start} seconds")
    print(f"Average: {(end - start) / count} seconds")
    print(f"{1 / (end - start) * count} FPS")
    result = outputs[compiled_model.output(0)][0]

    return result

if __name__ == "__main__":

    image_path = "bus.jpg"

    img = Image.open(image_path)
    img_border = letterbox(img)
    img_input = preprocess_image(img_border)

    device_name_list = ["CPU", "GPU", "HETERO:GPU,CPU"]
    for device_name in device_name_list:
        result = openvino(img_input, device_name)
        draw_result(img_border, result)

pip install onnxruntime

yolo export model=yolo11n.pt format=onnx dynamic=False nms=True

from PIL import Image
import onnxruntime as ort
import time

def onnxruntime(img_input: np.ndarray):
    model_path = "model/onnx/yolo11n.onnx"

    session = ort.InferenceSession(
        model_path,
        providers=["CPUExecutionProvider"],
        provider_options=[{}],
    )

    count = 100
    start = time.time()
    for _ in range(count):
        outputs = session.run(None, {"images": img_input})
    end = time.time()

    print(f"Total: {end - start} seconds")
    print(f"Average: {(end - start) / count} seconds")
    print(f"{1 / (end - start) * count} FPS")
    result = outputs[0][0]

    return result

if __name__ == "__main__":

    image_path = "bus.jpg"

    img = Image.open(image_path)
    img_border = letterbox(img)
    img_input = preprocess_image(img_border)

    result = onnxruntime(img_input)
    draw_result(img_border, result)

pip install onnxruntime-openvino

import onnxruntime as ort

session = ort.InferenceSession(
    model_path,
    providers=["OpenVINOExecutionProvider"],
    provider_options=[{"device_type": "CPU"}],
)

from PIL import Image
import onnxruntime as ort
import time

def onnxruntime(img_input: np.ndarray, device_name: str):
    model_path = "model/onnx/yolo11n.onnx"

    session = ort.InferenceSession(
        model_path,
        providers=["OpenVINOExecutionProvider"],
        provider_options=[{"device_type": device_name}],
    )

    count = 100
    start = time.time()
    for _ in range(count):
        outputs = session.run(None, {"images": img_input})
    end = time.time()

    print(f"Total: {end - start} seconds")
    print(f"Average: {(end - start) / count} seconds")
    print(f"{1 / (end - start) * count} FPS")
    result = outputs[0][0]

    return result

if __name__ == "__main__":

    image_path = "bus.jpg"

    img = Image.open(image_path)
    img_border = letterbox(img)
    img_input = preprocess_image(img_border)

    device_name_list = ["CPU", "GPU", "HETERO:GPU,CPU"]
    for device_name in device_name_list:
        result = onnxruntime(img_input, device_name)
        draw_result(img_border, result)

2025-05-31 15:54:23.4379916 [W:onnxruntime:, session_state.cc:1263 onnxruntime::VerifyEachNodeIsAssignedToAnEp] Some nodes were not assigned to the preferred execution providers which may or may not have an negative impact on performance. e.g. ORT explicitly assigns shape related ops to CPU to improve perf.
2025-05-31 15:54:23.4436529 [W:onnxruntime:, session_state.cc:1265 onnxruntime::VerifyEachNodeIsAssignedToAnEp] Rerunning with verbose output on a non-minimal build will show node assignments.