can't detect equal sign on image - opencv

Question

I'm working on a computer vision project that understands mathematical signs. It detects all correctly except the "Equal" mark. But the "equal" sign is perceived as two separate "minus" signs.
image = cv2.imread('./deneme.png')
grey = cv2.cvtColor(image.copy(), cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(grey.copy(), 0, 255, cv2.THRESH_BINARY_INV)
contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
preprocessed_digits = []
for c in contours:
x,y,w,h = cv2.boundingRect(c)
cv2.rectangle(image, (x,y), (x+w, y+h), color=(0, 255, 0), thickness=2)
digit = thresh[y:y+h, x:x+w]
digit = make_square(digit)
preprocessed_digits.append(digit)
plt.imshow(image, cmap="gray")
plt.show()
I have no idea how to fix this problem. How can I detect the equal sign?

Answer 1

Is that the result you're trying to achieve?
Since it seems that your math expressions are written horizontally, as #Micka points out, you have very strong priors on the relationship between the two different components of an equal sign, so there is a straight-forward (but hacky) way to detect when to minus signs are actually an equal:
import cv2
import matplotlib.pyplot as plt
import numpy as np
class Rect:
def __init__(self,
a,
b,
c,
d):
self.a=a
self.b=b
self.c=c
self.d=d
self.center=np.array([(a+c)/2, (b+d)/2])
def merge_rectangles(r_1, r_2):
a_m= min(r_1.a, r_2.a)
b_m= min(r_1.b, r_2.b)
c_M= max(r_1.c, r_2.c)
d_M= max(r_1.d, r_2.d)
return Rect(a_m, b_m, c_M, d_M)
def they_form_an_equal_sign(rect_1,
rect_2,
tol_w=10,#in pixels
tol_h=10):
#check if the bounding boxes approximately align
b0 = (np.abs(rect_1.a - rect_2.a) < tol_w ) and (np.abs(rect_1.c - rect_2.c) < tol_w)
#check if the bounding boxes have approximately the same height
h1 = np.abs(rect_1.d - rect_1.b)
h2 = np.abs(rect_2.d - rect_2.b)
b1 = np.abs(h1-h2) < tol_h
return b0 and b1
image = cv2.imread('/tmp/m.png')
grey = cv2.cvtColor(image.copy(), cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(grey.copy(), 0, 255, cv2.THRESH_BINARY_INV)
contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
candidate_rectangles=[]
for c in contours:
x,y,w,h = cv2.boundingRect(c)
candidate_rectangles.append(Rect(x,y,x+w,y+h))
kept=np.ones(len(candidate_rectangles))
new_rectangles=[]
for i in range(len(candidate_rectangles)):
for j in range(i+1,len(candidate_rectangles)):
b=they_form_an_equal_sign(candidate_rectangles[i], candidate_rectangles[j])
if b:
new_rect=merge_rectangles(candidate_rectangles[i], candidate_rectangles[j])
new_rectangles.append(new_rect)
kept[i]=0
kept[j]=0
for i in range(len(kept)):
if kept[i]:
rect=candidate_rectangles[i]
cv2.rectangle(image, (rect.a, rect.b), (rect.c, rect.d), color=(0, 255, 0), thickness=2)
for rect in new_rectangles:
cv2.rectangle(image, (rect.a, rect.b), (rect.c, rect.d), color=(0, 255, 0), thickness=2)
plt.imshow(image, cmap="gray")
plt.show()
Basically, this takes the brute-force approach of comparing every two bounding boxes that have been detected in your code, and merging them into a larger one if they meet the requirements from your prior: i.e. if they are horizontally algined (as the top and bottom parts of a minus sign should) and if their height is approximately the same.
However, this is obviously not robust: you need to adjust thresholds and the entire thing will fall apart if your expressions aren't horizontal and clearly separated. If you want a more robust/useful system, basic ML approaches for character recognition are a better place to start.