OpenCV crossword grid parser - opencv

Question

I am experimenting with OpenCV.js and attempting to get it to parse a Crossword Grid.
I have a couple of samples of Crossword grids that I am working with:
I would like to be able to parse the grid into an array that reads something like this:
[[-1, 1, -1, 2, 0, ...],[-1, 1, -1, 2, 0, ...]]
where:
-1 = "black square"
0 = "white square"
n > 0 = numbered square (where n is the number itself).
I am using opencv.js but cannot quite get the results I am looking for.
Here is a list of functions I have written to parse the crosswords:
function convertImageToRBGA(input, width, height, save = true) {
let output = new cv.Mat();
cv.cvtColor(input, output, cv.COLOR_RGB2RGBA, 0);
if (save) saveImage(width, height, output, "output-grey.jpg");
return output;
}
function convertImageToGrayscale(input, width, height, save = true) {
let output = new cv.Mat();
cv.cvtColor(input, output, cv.COLOR_RGB2GRAY, 0);
if (save) saveImage(width, height, output, "output-grey.jpg");
return output;
}
function adaptiveThresholdImage(input, width, height, save = true) {
let output = new cv.Mat();
//cv.threshold(input, output, 128, 255, cv.THRESH_BINARY | cv.THRESH_OTSU);
cv.adaptiveThreshold(input, output, 255, cv.ADAPTIVE_THRESH_GAUSSIAN_C, cv.THRESH_BINARY, 5, 2);
if (save) saveImage(width, height, output, "output-adaptive-threshold.jpg");
return output;
}
function denoiseImage(input, width, height, save = true) {
let output = new cv.Mat();
cv.fastNlMeansDenoising(input, output, 7);
if (save) saveImage(width, height, output, "output-denoised.jpg");
return output;
}
function thresholdImage(input, width, height, type, save = true) {
let output = new cv.Mat();
cv.threshold(input, output, 128, 255, cv.THRESH_BINARY_INV);
if (save) saveImage(width, height, output, "output-threshold.jpg");
return output;
}
function morphOpenImage(input, width, height, save = true) {
let output = new cv.Mat();
let M = cv.Mat.ones(5, 5, cv.CV_8U);
let anchor = new cv.Point(-1, -1);
// You can try more different parameters
cv.morphologyEx(input, output, cv.MORPH_OPEN, M, anchor, 1, cv.BORDER_CONSTANT, cv.morphologyDefaultBorderValue());
if (save) saveImage(width, height, output, "output-morph-open.jpg");
return output;
}
function medianBlurImage(input, width, height, save = true) {
let output = new cv.Mat();
cv.medianBlur(input, output, 3);
if (save) saveImage(width, height, output, "output-median-blur.jpg");
return output;
}
function blurImage(input, width, height, kunit = 3, name = "output-blur", save = true) {
let output = new cv.Mat();
// let M = cv.Mat.eye(3, 3, cv.CV_32FC1);
// let anchor = new cv.Point(-1, -1);
// cv.filter2D(input, output, cv.CV_8U, M, anchor, 0, cv.BORDER_DEFAULT);
let anchor = new cv.Point(-1, -1);
let ksize = new cv.Size(kunit, kunit);
cv.blur(input, output, ksize, anchor, cv.BORDER_DEFAULT);
if (save) saveImage(width, height, output, `${name}.jpg`);
return output;
}
function gaussianBlur(input, width, height, name = "output-gaussian-blur", save = true) {
let output = new cv.Mat();
let ksize = new cv.Size(5, 5);
cv.GaussianBlur(input, output, ksize, 0, 0, cv.BORDER_DEFAULT);
if (save) saveImage(width, height, output, `${name}.jpg`);
return output;
}
function cannyImage(input, width, height, save = true) {
let output = new cv.Mat();
cv.Canny(input, output, 50, 150, 3, false);
if (save) saveImage(width, height, output, "output-canny.jpg");
return output;
}
function sharpenImage(input, width, height, name = "output-sharpen", save = true) {
// change kernal
let kernel = cv.matFromArray(3, 3, cv.CV_32FC1, [0, -1, 0, -1, 5, -1, 0, -1, 0]);
let anchor = new cv.Point(-1, -1);
let output = new cv.Mat();
cv.filter2D(input, output, cv.CV_8U, kernel, anchor, 0, cv.BORDER_CONSTANT);
saveImage(width, height, output, `${name}.jpg`);
return output;
}
function detectHoughLinesInImage(input, width, height, save = true) {
let output = cv.Mat.zeros(input.rows, input.cols, cv.CV_8UC3);
let lines = new cv.Mat();
cv.HoughLines(input, lines, 1, Math.PI / 180, 30, 0, 0, 0, Math.PI);
// draw lines
for (let i = 0; i < lines.rows; ++i) {
let rho = lines.data32F[i * 2];
console.log(rho);
let theta = lines.data32F[i * 2 + 1];
let a = Math.cos(theta);
let b = Math.sin(theta);
let x0 = a * rho;
let y0 = b * rho;
let startPoint = { x: x0 - 1000 * b, y: y0 + 1000 * a };
let endPoint = { x: x0 + 1000 * b, y: y0 - 1000 * a };
cv.line(output, startPoint, endPoint, [255, 0, 0, 255]);
}
lines.delete();
if (save) saveImage(width, height, output, "output-lines.jpg");
return output;
}
function detectHoughLinesPInImage(input, width, height, save = true) {
let output = cv.Mat.zeros(input.rows, input.cols, cv.CV_8UC3);
let lines = new cv.Mat();
let color = new cv.Scalar(255, 0, 0);
cv.HoughLinesP(input, lines, 1, Math.PI / 180, 2, 0, 0);
for (let i = 0; i < lines.rows; ++i) {
let startPoint = new cv.Point(lines.data32S[i * 4], lines.data32S[i * 4 + 1]);
let endPoint = new cv.Point(lines.data32S[i * 4 + 2], lines.data32S[i * 4 + 3]);
cv.line(output, startPoint, endPoint, color);
}
lines.delete();
if (save) saveImage(width, height, output, "output-lines.jpg");
return output;
}
function dilateImage(input, width, height, save = true) {
let M = cv.Mat.ones(5, 5, cv.CV_8U);
let anchor = new cv.Point(-1, -1);
let output = new cv.Mat();
cv.dilate(input, output, M, anchor, 1, cv.BORDER_CONSTANT, cv.morphologyDefaultBorderValue());
if (save) saveImage(width, height, output, "output-dilated.jpg");
return output;
}
function erodeImage(input, width, height, save = true) {
let M = cv.Mat.eye(5, 5, cv.CV_8U);
let anchor = new cv.Point(-1, -1);
let output = new cv.Mat();
cv.erode(input, output, M, anchor, 1, cv.BORDER_CONSTANT, cv.morphologyDefaultBorderValue());
if (save) saveImage(width, height, output, "output-eroded.jpg");
return output;
}
The above functions are not exclusive as there are other functions that can drawContours, etc. I haven't included them here for brevity but can do if necessary. Also, the above functions are a product of experimenting but, so far, the best results I have had is by using only a subset of them (shown below).
Apart from there being certain parameters that can be adjusted, the general issue appears to be the order that the functions.
The simplest order I can come up with is this:
mat = convertImageToGrayscale(src, width, height);
mat = gaussianBlur(mat, width, height);
mat = sharpenImage(mat, width, height);
mat = cannyImage(mat, width, height);
cv.findContours(mat, contours, hierarchy, cv.RETR_CCOMP, cv.CHAIN_APPROX_SIMPLE);
Which produces this output (in order):
The canny image looks good but when I run the cv.findContours function, this is what I get:
I have added a couple of functions to determine the colour of the ROI and add a letter value to indicate that.
I just can't get the findContours function to read all of the white squares (I believe if I can read all of the white squares, I can use some basic cartesian Math to work out the rest of the grid).
The numbers within the squares seem to cause the canny function some issues when it comes to reading the squares.
Not sure at this point if I need to start complicating things with erosion/dilation functions or even get into thresholding.
But I feel that with the way the Canny function is reading the grid, it feels like I am halfway there but can't quite get the contours I need.
Any help here would be apprciated.

Answer 1

I agree, the Canny output does look fine, I wouldn't think you would need to do anything more with the image to enable findContours to get all the squares. Have you tried using the opencv function 'drawContours'? If used correctly, that can help you see what findContours is doing.
In addition, you are using 'RETR_CCOMP', which, according to the docs:
"retrieves all of the contours and organizes them into a two-level hierarchy. At the top level, there are external boundaries of the components. At the second level, there are boundaries of the holes"
Are you using the hierarchy system from findContours already? And if so, perhaps some of those squares are being incorrectly classified in the hierarchy by findContours.