I am doing a template matching project using rotation angle and scale ratio, I have an issue with draw bounding box of the object with rotation angle, the scale is no problem, but I couldn’t find the solution for drawing rotated object bounding box, in my project I resize the template image for scaled but I rotated the search ROI because I don’t want to lose the information of the corner so I rotated the search ROI but I cannot draw bounding box for object founded, I use Pyqt5, opencv and python, all my code is down below and when you click Test, the code will run manual_test function, thanks for all your help !
import sys
import cv2 as cv
import os
import json
import numpy as np
from PyQt5 import QtGui, QtWidgets, QtCore
from PyQt5.QtWidgets import QApplication
from Tools.Base import ImageProcessingTool, OutputResult
from UI.TemplateMatchingUI import Ui_Form
from pathlib import Path
class RunApp(QtWidgets.QWidget):
def __init__(self, tool: str, tool_name: str, image: np.ndarray = None):
self.tool = tool
self.tool_name = tool_name
self.temp_image = image
super().__init__()
self.ui = Ui_Form()
self.ui.setupUi(self)
# Initialize Object
self.template_matching = TemplateMatchingTool(self.ui)
# Init variable
self.base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
self.x_start = None
self.y_start = None
self.x_start_real = None
self.y_start_real = None
self.x_end = None
self.y_end = None
self.x_end_real = None
self.y_end_real = None
self.is_draw = False
self.mask_template_crop = None
self.temp_save_roi = [0, 0, 0, 0]
self.template_image = [0, 0, 0, 0]
# Set up attribute and implement function
self.set_ui_attribute()
self.implement_function()
# Autoload param from param json file
self.load_param(tool, tool_name)
# Setup UI attribute here
def set_ui_attribute(self):
# Automatically adjust the window size to fit the content
self.adjustSize()
self.setFixedSize(self.size())
# Setup HEADER
self.setWindowTitle(f"{self.tool_name}")
color = QtGui.QColor(70, 136, 224)
color.setAlpha(130)
effect_1 = QtWidgets.QGraphicsDropShadowEffect(offset=QtCore.QPoint(3, 3), blurRadius=4, color=color)
effect_3 = QtWidgets.QGraphicsDropShadowEffect(offset=QtCore.QPoint(3, 3), blurRadius=4, color=color)
self.ui.btn_save.setGraphicsEffect(effect_1)
self.ui.btn_save_template.setGraphicsEffect(effect_3)
color = QtGui.QColor(196, 64, 51)
color.setAlpha(130)
effect_2 = QtWidgets.QGraphicsDropShadowEffect(offset=QtCore.QPoint(3, 3), blurRadius=4, color=color)
self.ui.btn_exit.setGraphicsEffect(effect_2)
color = QtGui.QColor(196, 201, 52)
color.setAlpha(130)
effect_4 = QtWidgets.QGraphicsDropShadowEffect(offset=QtCore.QPoint(3, 3), blurRadius=4, color=color)
self.ui.btn_test.setGraphicsEffect(effect_4)
color = QtGui.QColor(167, 243, 208)
color.setAlpha(130)
effect_5 = QtWidgets.QGraphicsDropShadowEffect(offset=QtCore.QPoint(3, 3), blurRadius=4, color=color)
self.ui.btn_save_roi.setGraphicsEffect(effect_5)
widgets1 = [
self.ui.dp_view,
self.ui.dp_view_crop
]
for widget in widgets1:
effect = QtWidgets.QGraphicsDropShadowEffect(offset=QtCore.QPoint(3, 3), color=QtGui.QColor(217, 217, 217))
widget.setGraphicsEffect(effect)
self.ui.cbb_operator.addItems([
">",
">=",
"=",
"<",
"<=",
])
self.ui.spb_spec_value.setMinimum(0)
self.ui.spb_spec_value.setMaximum(100)
self.ui.spb_angle_from.setMinimum(-360)
self.ui.spb_angle_from.setMaximum(360)
self.ui.spb_angle_to.setMinimum(-360)
self.ui.spb_angle_to.setMaximum(360)
self.ui.spb_angle_from.setSingleStep(1)
self.ui.spb_angle_to.setSingleStep(1)
self.ui.spb_scale_from.setMinimum(0)
self.ui.spb_scale_from.setMaximum(3)
self.ui.spb_scale_to.setMinimum(0)
self.ui.spb_scale_to.setMaximum(3)
self.ui.spb_scale_from.setSingleStep(float(0.1))
self.ui.spb_scale_to.setSingleStep(float(0.1))
self.ui.spb_scale_from.setValue(float(1))
self.ui.spb_scale_to.setValue(float(1))
# Auto display image when open tool setting window
self.auto_display_image()
# Implement function for UI element here
def implement_function(self):
self.ui.btn_exit.clicked.connect(self.close)
self.ui.btn_save.clicked.connect(self.save_config)
self.ui.btn_save_roi.clicked.connect(self.save_roi)
self.ui.btn_save_template.clicked.connect(self.save_template)
self.ui.btn_test.clicked.connect(self.manual_test)
self.ui.dp_view.mousePressEvent = self.mouse_press_event
self.ui.dp_view.mouseMoveEvent = self.mouse_move_event
self.ui.dp_view.mouseReleaseEvent = self.mouse_release_event
def auto_display_image(self):
frame = self.temp_image
ret = False
if frame is not None or np.any(frame != [0, 0, 0]):
ret = True
else:
QtWidgets.QMessageBox.critical(self, "Error !!!", "Cannot read frame from camera !")
if ret:
self.template_matching.add_image_to_qlabel(self.ui.dp_view, frame)
# Load param from JSON file
def load_param(self, tool: str, tool_name: str):
try:
# Get run tool param path
param_file_path = os.path.join(self.base_path, "Configs", tool, tool_name)
param_file_path = f"{param_file_path}.json"
param = json.load(open(param_file_path, "r", encoding="utf-8"))
# self.temp_save_roi = param["ROI_XYXY"]
self.template_image = param["Template_XYXY"]
operators = [self.ui.cbb_operator.itemText(i) for i in range(self.ui.cbb_operator.count())]
for i in range(len(operators)):
if operators[i] == param["Operator"]:
self.ui.cbb_operator.setCurrentIndex(i)
self.ui.spb_spec_value.setValue(int(param["Spec_Score"]))
except:
self.ui.cbb_operator.setCurrentIndex(0)
self.ui.spb_spec_value.setValue(75)
pass
def mouse_press_event(self, event=QtGui.QMouseEvent):
try:
dp_view = self.ui.dp_view
crop_x_ratio = self.temp_image.shape[1] / dp_view.size().width()
crop_y_ratio = self.temp_image.shape[0] / dp_view.size().height()
if event.buttons() == QtCore.Qt.MouseButton.LeftButton:
self.x_start = event.x()
self.y_start = event.y()
self.x_start_real = round(event.x() * crop_x_ratio)
self.y_start_real = round(event.y() * crop_y_ratio)
self.is_draw = True
except:
pass
def mouse_move_event(self, event=QtGui.QMouseEvent):
try:
if self.is_draw:
image = self.temp_image
image = cv.resize(image, (self.ui.dp_view.size().width(), self.ui.dp_view.size().height()))
cv.rectangle(image, (self.x_start, self.y_start), (event.x(), event.y()), (0, 0, 255),
thickness=1)
self.template_matching.add_image_to_qlabel(self.ui.dp_view, image)
except:
pass
def mouse_release_event(self, event=QtGui.QMouseEvent):
try:
crop_x_ratio = self.temp_image.shape[1] / self.ui.dp_view.size().width()
crop_y_ratio = self.temp_image.shape[0] / self.ui.dp_view.size().height()
self.is_draw = False
self.x_end = event.x()
self.y_end = event.y()
self.x_end_real = round(event.x() * crop_x_ratio)
self.y_end_real = round(event.y() * crop_y_ratio)
# Recalculate x_start, x_end
if self.x_start > self.x_end:
temp = self.x_start
self.x_start = self.x_end
self.x_end = temp
del temp
# Recalculate x_start, x_end
if self.y_start > self.y_end:
temp = self.y_start
self.y_start = self.y_end
self.y_end = temp
del temp
# Recalculate x_start_real, x_end_real
if self.x_start_real > self.x_end_real:
temp = self.x_start_real
self.x_start_real = self.x_end_real
self.x_end_real = temp
del temp
# Recalculate x_start_real, x_end_real
if self.y_start_real > self.y_end_real:
temp = self.y_start_real
self.y_start_real = self.y_end_real
self.y_end_real = temp
del temp
self.mask_template_crop = self.temp_image[
round(self.y_start_real):round(self.y_end_real),
round(self.x_start_real):round(self.x_end_real)
]
self.template_matching.add_image_to_qlabel(self.ui.dp_view_crop, self.mask_template_crop, is_cropped=True)
except:
pass
def save_template(self):
try:
self.template_image = [self.x_start_real, self.y_start_real, self.x_end_real, self.y_end_real]
QtWidgets.QMessageBox.information(self, "Success !", "Save template successfully !")
except:
QtWidgets.QMessageBox.critical(self, "Error !!!", "Some error occur while segmenting, please try again !")
pass
def manual_test(self):
try:
lower_rot_ang = int(self.ui.spb_angle_from.value())
upper_rot_ang = int(self.ui.spb_angle_to.value())
if lower_rot_ang > upper_rot_ang:
QtWidgets.QMessageBox.warning(self, "Warning !!!",
"Please set upper rotation angle greater then lower rotation angle")
return
lower_rot_scale = int(self.ui.spb_scale_from.value() * 10)
upper_rot_scale = int(self.ui.spb_scale_to.value() * 10)
if lower_rot_scale > upper_rot_scale:
QtWidgets.QMessageBox.warning(self, "Warning !!!",
"Please set upper scale greater then lower scale")
return
image = np.copy(self.temp_image)
draw_image = np.copy(image)
search_roi = image[round(self.y_start_real): round(self.y_end_real),
round(self.x_start_real):round(self.x_end_real)]
template = np.copy(self.temp_image)[self.template_image[1]:self.template_image[3],
self.template_image[0]:self.template_image[2]]
match_results = []
for scale in range(lower_rot_scale, upper_rot_scale + 1):
# Rescale template
scaled_templ = self.rescale_image(template, float(scale / 10))
for angle in range(lower_rot_ang, upper_rot_ang + 1):
# Rotate search ROI
rotated_search_roi, rotated_image_width, rotated_image_height = self.rotate_image(search_roi, angle)
# Do template matching
result = cv.matchTemplate(rotated_search_roi, scaled_templ, cv.TM_CCOEFF_NORMED)
min_val, max_val, min_loc, max_loc = cv.minMaxLoc(result)
match_results.append((max_val, max_loc, angle, scale))
best_match = max(match_results, key=lambda x: x[0])
max_val, max_loc, best_angle, best_scale = best_match
w, h = template.shape[1::-1]
top_left = (int(self.x_start_real + max_loc[0]), int(self.y_start_real + max_loc[1]))
bottom_right = (int(top_left[0] + (w * best_scale / 10)), int(top_left[1] + (h * best_scale / 10)))
result_val = round(max_val * 100)
spec_val = self.ui.spb_spec_value.value()
operator = self.ui.cbb_operator.currentText()
self.ui.lbl_result_value.setText(str(result_val))
rot_point = (int(self.x_start_real + template.shape[1] / 2), int(self.y_start_real + template.shape[0] / 2))
if result_val >= spec_val:
self.draw_rotate_rectangle(
draw_image,
top_left,
bottom_right,
rot_point,
-best_angle,
(0, 255, 0),
3
)
cv.rectangle(draw_image, (self.x_start_real, self.y_start_real), (self.x_end_real, self.y_end_real), (0, 255, 255), 3)
cv.putText(draw_image, f"Score: {round(max_val * 100)}/100", (self.x_start_real, self.y_start_real - 4),
cv.FONT_ITALIC, 1.0, (0, 255, 0), 2)
else:
cv.rectangle(draw_image, (self.x_start_real, self.y_start_real),
(self.x_end_real, self.y_end_real), (0, 0, 255), 3)
if operator == "=" and result_val == spec_val:
result = True
elif operator == ">" and result_val > spec_val:
result = True
elif operator == ">=" and result_val >= spec_val:
result = True
elif operator == "<" and result_val < spec_val:
result = True
elif operator == "<=" and result_val <= spec_val:
result = True
else:
result = False
if result:
css = "background-color: green; color: white"
result_txt = "PASS"
else:
css = "background-color: red; color: white"
result_txt = "FAILED"
self.ui.lbl_test_result.setStyleSheet(css)
self.ui.lbl_test_result.setText(result_txt)
self.template_matching.add_image_to_qlabel(self.ui.dp_view, draw_image)
except Exception as ex:
QtWidgets.QMessageBox.critical(self, "Error !!!", str(ex))
pass
@staticmethod
def rotate_image(image: np.ndarray, angle: int):
if angle == 0:
return image, image.shape[1], image.shape[0]
# Get image height and width
height, width = image.shape[:2]
# Rotation point is center point
image_center = (width / 2, height / 2)
# Get rotation matrix
rotation_mat = cv.getRotationMatrix2D(image_center, angle, 1.)
# Get absolute cos and sin
abs_cos = abs(rotation_mat[0, 0])
abs_sin = abs(rotation_mat[0, 1])
# print(abs_cos * abs_cos + abs_sin * abs_sin)
# Calculate new bound width and height
bound_w = int(height * abs_sin + width * abs_cos)
bound_h = int(height * abs_cos + width * abs_sin)
# Subtract old image center (bringing image back to origo) and adding the new image center coordinates
rotation_mat[0, 2] += bound_w / 2 - image_center[0]
rotation_mat[1, 2] += bound_h / 2 - image_center[1]
# rotate image with the new bounds and translated rotation matrix
rotated_image = cv.warpAffine(image, rotation_mat, (bound_w, bound_h))
return rotated_image, bound_w, bound_h
@staticmethod
def rescale_image(img: np.ndarray, scale: float = 0.75):
if scale == 1:
return img
return cv.resize(img, (0, 0), fx=scale, fy=scale)
@staticmethod
def draw_rotate_rectangle(
image: np.ndarray,
top_left: tuple,
bottom_right: tuple,
rot_point: tuple,
angle: int,
color: tuple,
thickness: int
):
pt1 = top_left
pt2 = bottom_right
center_x, center_y = rot_point
# Get rotation matrix
rotation_matrix = cv.getRotationMatrix2D((center_x, center_y), angle, 1.0)
# Calculate rectangle point
rect_points = np.array([
[pt1[0], pt1[1]],
[pt2[0], pt1[1]],
[pt2[0], pt2[1]],
[pt1[0], pt2[1]]
], dtype=np.float32)
# Recalculate rotation matrix
ones = np.ones(shape=(len(rect_points), 1))
points_ones = np.hstack([rect_points, ones])
# Rotate points
rotated_points = rotation_matrix.dot(points_ones.T).T
rotated_points = np.int0(rotated_points)
# Draw rectangle
cv.drawContours(image, [rotated_points], 0, color, thickness)
return image
@staticmethod
def get_point_by_rot_angle(point: tuple, center: tuple, angle_degrees: int):
"""
Rotate a point around another point.
Parameters:
point (tuple): The point to rotate, in the form (x, y).
center (tuple): The center point to rotate around, in the form (cx, cy).
angle_degrees (float): The angle to rotate in degrees.
Returns:
tuple: The rotated point, in the form (x', y').
"""
angle_radians = np.radians(angle_degrees) # Chuyển đổi góc từ độ sang radian
# Dịch chuyển điểm gốc (center) về gốc tọa độ
translated_x = point[0] - center[0]
translated_y = point[1] - center[1]
# Xoay điểm quanh gốc tọa độ
rotated_x = translated_x * np.cos(angle_radians) - translated_y * np.sin(angle_radians)
rotated_y = translated_x * np.sin(angle_radians) + translated_y * np.cos(angle_radians)
# Dịch chuyển điểm trở lại vị trí ban đầu
final_x = int(rotated_x + center[0])
final_y = int(rotated_y + center[1])
return final_x, final_y
def closeEvent(self, e):
return_value = QtWidgets.QMessageBox.warning(self, "Are you sure you want to quit ?",
"If you do not save your config,"
" all your setting will be lost,"
" are you sure you want to quit ?",
QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No,
QtWidgets.QMessageBox.No)
if return_value == QtWidgets.QMessageBox.Yes:
e.accept()
else:
e.ignore()
def save_roi(self):
self.temp_save_roi = [self.x_start_real, self.y_start_real, self.x_end_real, self.y_end_real]
QtWidgets.QMessageBox.information(self, "Success !", "Save ROI successfully !")
def save_config(self):
data = {}
if self.temp_save_roi == [0, 0, 0, 0]:
region_xyxy = [self.x_start_real, self.y_start_real, self.x_end_real, self.y_end_real]
else:
region_xyxy = self.temp_save_roi
template_xyxy = self.template_image
operator = self.ui.cbb_operator.currentText()
spec_score = self.ui.spb_spec_value.value()
data = {
"ROI_XYXY": region_xyxy,
"Template_XYXY": template_xyxy,
"Operator": operator,
"Spec_Score": spec_score
}
# Create directory if it does not
config_dir = os.path.join(self.base_path, "Configs", self.tool)
Path(str(config_dir)).mkdir(parents=True, exist_ok=True)
# os.makedirs(config_dir, exist_ok=True)
# File path for the JSON configuration file
file_path = os.path.join(str(config_dir), f"{self.tool_name}.json")
# Write data to the JSON file
with open(file_path, 'w') as file:
json.dump(data, file, indent=4)
QtWidgets.QMessageBox.information(self, "Success !", "Save configuration successfully !")
def keyPressEvent(self, e):
if e.key() == 0x01000000:
self.close()
The solution that I can think is that rotating template instead of rotating search ROI, that is much easier, but I will lose information in the template image corner, so I rotate the search ROI, that is higher accuracy and reliability.