Use opencv to crop boxes in car documents - opencv

Question

I'm new to opencv and would like to use it to crop portions of an image and then use tesseract to read them. I'm not sure what's the best way to crop all the necessary boxes that i need.
Here is an easy example of the document i need to transform:
Any advice on what would be the best?
I tried with ORB and the following image as template:
But without success.
On the template, some lines are selected as keypoints but on the image i want to process it's mainly the text and not the lines. Is it a bad template? Do i need to process the image first?
and my code:
Feature2D f2d = ORB.create(5000); // SIFT.create(1000);
MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
Mat descriptors1 = new Mat();
Mat mask1 = new Mat();
f2d.detectAndCompute(img1, mask1, keypoints1, descriptors1);
MatOfKeyPoint keypoints2 = new MatOfKeyPoint();
Mat descriptors2 = new Mat();
Mat mask2 = new Mat();
f2d.detectAndCompute(img2, mask2, keypoints2, descriptors2);
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_HAMMING);
MatOfDMatch matches = new MatOfDMatch();
matcher.match(descriptors1, descriptors2, matches);
Mat outputImg = new Mat();
MatOfByte drawnMatches = new MatOfByte();
Features2d.drawMatches(img1, keypoints1, img2, keypoints2, matches, outputImg, new Scalar(0, 255, 0), new Scalar(255, 0, 0), drawnMatches, Features2d.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS);

Answer 1

I could get good results by using a template that contains all the text that never change in the form. Furthermore, creating 2 templates (1 per page) and using SIFT instead of ORB helped a lot too.
Here is my solution:
public static Mat matchTEmplateSIFT(Mat img1, Mat template, boolean showKeypoints, boolean drawMatchs) {
Feature2D f2d = SIFT.create(15000);
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE_SL2); // or FLANNBASED for better performance
return matchTEmplate(img1, template, f2d, matcher);
}
public static Mat matchTEmplate(Mat baseImage, Mat template, Feature2D f2d, DescriptorMatcher matcher) {
int dilateSize = 5;
Mat scene = dilateBitwise(dilateSize, baseImage.clone());
template = dilateBitwise(dilateSize, template.clone());
MatOfKeyPoint keypoints1 = new MatOfKeyPoint();
Mat descriptors1 = new Mat();
f2d.detectAndCompute(scene, new Mat(), keypoints1, descriptors1);
MatOfKeyPoint keypoints2 = new MatOfKeyPoint();
Mat descriptors2 = new Mat();
f2d.detectAndCompute(template, new Mat(), keypoints2, descriptors2);
List<MatOfDMatch> matches = new ArrayList<>();
matcher.knnMatch(descriptors1, descriptors2, matches, 2);
MatOfDMatch goodMatches = getBestMatches(matches);
Mat result = transformAndWarp(baseImage, template, keypoints1, keypoints2, goodMatches);
return result;
}
private static Mat transformAndWarp(Mat baseImage, Mat template, MatOfKeyPoint keypoints1, MatOfKeyPoint keypoints2, MatOfDMatch goodMatches) {
Mat H = findHomographyMatrix(keypoints1, keypoints2, goodMatches);
perspectiveTransform(template, H);
Mat result = new Mat();
Imgproc.warpPerspective(baseImage, result, H, new Size(template.cols(), template.rows()));
return result;
}
private static void perspectiveTransform(Mat template, Mat H) {
Mat obj_corners = new Mat(4, 1, CvType.CV_32FC2);
obj_corners.put(0, 0, new double[]{0, 0});
obj_corners.put(0, 0, new double[]{template.cols(), 0});
obj_corners.put(0, 0, new double[]{template.cols(), template.rows()});
obj_corners.put(0, 0, new double[]{0, template.rows()});
Mat scene_corners = new Mat(4, 1, CvType.CV_32FC2);
Core.perspectiveTransform(obj_corners, scene_corners, H);
}
private static Mat findHomographyMatrix(MatOfKeyPoint keypoints1, MatOfKeyPoint keypoints2, MatOfDMatch goodMatches) {
LinkedList<Point> templateList = new LinkedList<>();
LinkedList<Point> sceneList = new LinkedList<>();
List<KeyPoint> templateKeyPoints = keypoints1.toList();
List<KeyPoint> sceneKeypoints = keypoints2.toList();
for (int i = 0; i < goodMatches.toList().size(); i++) {
templateList.addLast(templateKeyPoints.get(goodMatches.toList().get(i).queryIdx).pt);
sceneList.addLast(sceneKeypoints.get(goodMatches.toList().get(i).trainIdx).pt);
}
MatOfPoint2f templateMat = new MatOfPoint2f();
templateMat.fromList(templateList);
MatOfPoint2f sceneMat = new MatOfPoint2f();
sceneMat.fromList(sceneList);
return Calib3d.findHomography(templateMat, sceneMat, Calib3d.RANSAC);
}
// https://docs.opencv.org/3.4/d5/d6f/tutorial_feature_flann_matcher.html
private static MatOfDMatch getBestMatches(List<MatOfDMatch> knnMatches) {
//-- Filter matches using the Lowe's ratio test
float ratioThresh = 0.5f;
List<DMatch> listOfGoodMatches = new ArrayList<>();
for (int i = 0; i < knnMatches.size(); i++) {
if (knnMatches.get(i).rows() > 1) {
DMatch[] matches = knnMatches.get(i).toArray();
if (matches[0].distance < ratioThresh * matches[1].distance) {
listOfGoodMatches.add(matches[0]);
}
}
}
MatOfDMatch matOfDMatch = new MatOfDMatch();
matOfDMatch.fromList(listOfGoodMatches);
return matOfDMatch;
}