Related
I am working in a robot application, in which I have a camera fixed to a robot gripper. In order to calculate the matrix transformation between the camera and the gripper Hcg I am using the calibrateHandEye new function provided in the OpenCV version 4.1.0
I had taken 10 pictures of the chessboard from the camera mounted in the gripper and at the same time I recorded the robot position.
The code I am working on:
// My_handeye.cpp : This file contains the 'main' function. Program execution begins and ends there.
//
#include <iostream>
#include <sstream>
#include <string>
#include <ctime>
#include <cstdio>
#include "pch.h"
#include <opencv2/opencv.hpp>
#include <opencv2/core.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
using namespace cv;
using namespace std;
Mat eulerAnglesToRotationMatrix(Vec3f &theta);
Vec3f rotationMatrixToEulerAngles(Mat &R);
float rad2deg(float radian);
float deg2rad(float degree);
int main()
{
// Camera calibration information
std::vector<double> distortionCoefficients(5); // camera distortion
distortionCoefficients[0] = 2.4472856611074989e-01;
distortionCoefficients[1] = -8.1042032574246325e-01;
distortionCoefficients[2] = 0;
distortionCoefficients[3] = 0;
distortionCoefficients[4] = 7.8769462320821060e-01;
double f_x = 1.3624172121852105e+03; // Focal length in x axis
double f_y = 1.3624172121852105e+03; // Focal length in y axis (usually the same?)
double c_x = 960; // Camera primary point x
double c_y = 540; // Camera primary point y
cv::Mat cameraMatrix(3, 3, CV_32FC1);
cameraMatrix.at<float>(0, 0) = f_x;
cameraMatrix.at<float>(0, 1) = 0.0;
cameraMatrix.at<float>(0, 2) = c_x;
cameraMatrix.at<float>(1, 0) = 0.0;
cameraMatrix.at<float>(1, 1) = f_y;
cameraMatrix.at<float>(1, 2) = c_y;
cameraMatrix.at<float>(2, 0) = 0.0;
cameraMatrix.at<float>(2, 1) = 0.0;
cameraMatrix.at<float>(2, 2) = 1.0;
Mat rvec(3, 1, CV_32F), tvec(3, 1, CV_32F);
//
std::vector<Mat> R_gripper2base;
std::vector<Mat> t_gripper2base;
std::vector<Mat> R_target2cam;
std::vector<Mat> t_target2cam;
Mat R_cam2gripper = (Mat_<float>(3, 3));
Mat t_cam2gripper = (Mat_<float>(3, 1));
vector<String> fn;
glob("images/*.bmp", fn, false);
vector<Mat> images;
size_t num_images = fn.size(); //number of bmp files in images folder
Size patternsize(6, 8); //number of centers
vector<Point2f> centers; //this will be filled by the detected centers
float cell_size = 30;
vector<Point3f> obj_points;
R_gripper2base.reserve(num_images);
t_gripper2base.reserve(num_images);
R_target2cam.reserve(num_images);
t_target2cam.reserve(num_images);
for (int i = 0; i < patternsize.height; ++i)
for (int j = 0; j < patternsize.width; ++j)
obj_points.push_back(Point3f(float(j*cell_size),
float(i*cell_size), 0.f));
for (size_t i = 0; i < num_images; i++)
images.push_back(imread(fn[i]));
Mat frame;
for (size_t i = 0; i < num_images; i++)
{
frame = imread(fn[i]); //source image
bool patternfound = findChessboardCorners(frame, patternsize, centers);
if (patternfound)
{
drawChessboardCorners(frame, patternsize, Mat(centers), patternfound);
//imshow("window", frame);
//int key = cv::waitKey(0) & 0xff;
solvePnP(Mat(obj_points), Mat(centers), cameraMatrix, distortionCoefficients, rvec, tvec);
Mat R;
Rodrigues(rvec, R); // R is 3x3
R_target2cam.push_back(R);
t_target2cam.push_back(tvec);
Mat T = Mat::eye(4, 4, R.type()); // T is 4x4
T(Range(0, 3), Range(0, 3)) = R * 1; // copies R into T
T(Range(0, 3), Range(3, 4)) = tvec * 1; // copies tvec into T
cout << "T = " << endl << " " << T << endl << endl;
}
cout << patternfound << endl;
}
Vec3f theta_01{ deg2rad(-153.61), deg2rad(8.3), deg2rad(-91.91) };
Vec3f theta_02{ deg2rad(-166.71), deg2rad(3.04), deg2rad(-93.31) };
Vec3f theta_03{ deg2rad(-170.04), deg2rad(24.92), deg2rad(-88.29) };
Vec3f theta_04{ deg2rad(-165.71), deg2rad(24.68), deg2rad(-84.85) };
Vec3f theta_05{ deg2rad(-160.18), deg2rad(-15.94),deg2rad(-56.24) };
Vec3f theta_06{ deg2rad(175.68), deg2rad(10.95), deg2rad(180) };
Vec3f theta_07{ deg2rad(175.73), deg2rad(45.78), deg2rad(-179.92) };
Vec3f theta_08{ deg2rad(-165.34), deg2rad(47.37), deg2rad(-166.25) };
Vec3f theta_09{ deg2rad(-165.62), deg2rad(17.95), deg2rad(-166.17) };
Vec3f theta_10{ deg2rad(-151.99), deg2rad(-14.59),deg2rad(-94.19) };
Mat robot_rot_01 = eulerAnglesToRotationMatrix(theta_01);
Mat robot_rot_02 = eulerAnglesToRotationMatrix(theta_02);
Mat robot_rot_03 = eulerAnglesToRotationMatrix(theta_03);
Mat robot_rot_04 = eulerAnglesToRotationMatrix(theta_04);
Mat robot_rot_05 = eulerAnglesToRotationMatrix(theta_05);
Mat robot_rot_06 = eulerAnglesToRotationMatrix(theta_06);
Mat robot_rot_07 = eulerAnglesToRotationMatrix(theta_07);
Mat robot_rot_08 = eulerAnglesToRotationMatrix(theta_08);
Mat robot_rot_09 = eulerAnglesToRotationMatrix(theta_09);
Mat robot_rot_10 = eulerAnglesToRotationMatrix(theta_10);
const Mat robot_tr_01 = (Mat_<float>(3, 1) << 781.2, 338.59, 903.48);
const Mat robot_tr_02 = (Mat_<float>(3, 1) << 867.65, 382.52, 884.42);
const Mat robot_tr_03 = (Mat_<float>(3, 1) << 856.91, 172.99, 964.61);
const Mat robot_tr_04 = (Mat_<float>(3, 1) << 748.81, 146.75, 1043.29);
const Mat robot_tr_05 = (Mat_<float>(3, 1) << 627.66, 554.08, 920.85);
const Mat robot_tr_06 = (Mat_<float>(3, 1) << 715.06, 195.96, 889.38);
const Mat robot_tr_07 = (Mat_<float>(3, 1) << 790.9, 196.29, 1117.38);
const Mat robot_tr_08 = (Mat_<float>(3, 1) << 743.5, 283.93, 1131.92);
const Mat robot_tr_09 = (Mat_<float>(3, 1) << 748.9, 288.19, 910.58);
const Mat robot_tr_10 = (Mat_<float>(3, 1) << 813.18, 400.44, 917.16);
R_gripper2base.push_back(robot_rot_01);
R_gripper2base.push_back(robot_rot_02);
R_gripper2base.push_back(robot_rot_03);
R_gripper2base.push_back(robot_rot_04);
R_gripper2base.push_back(robot_rot_05);
R_gripper2base.push_back(robot_rot_06);
R_gripper2base.push_back(robot_rot_07);
R_gripper2base.push_back(robot_rot_08);
R_gripper2base.push_back(robot_rot_09);
R_gripper2base.push_back(robot_rot_10);
t_gripper2base.push_back(robot_tr_01);
t_gripper2base.push_back(robot_tr_02);
t_gripper2base.push_back(robot_tr_03);
t_gripper2base.push_back(robot_tr_04);
t_gripper2base.push_back(robot_tr_05);
t_gripper2base.push_back(robot_tr_06);
t_gripper2base.push_back(robot_tr_07);
t_gripper2base.push_back(robot_tr_08);
t_gripper2base.push_back(robot_tr_09);
t_gripper2base.push_back(robot_tr_10);
calibrateHandEye(R_gripper2base, t_gripper2base, R_target2cam, t_target2cam, R_cam2gripper, t_cam2gripper, CALIB_HAND_EYE_TSAI);
Vec3f R_cam2gripper_r = rotationMatrixToEulerAngles(R_cam2gripper);
cout << "R_cam2gripper = " << endl << " " << R_cam2gripper << endl << endl;
cout << "R_cam2gripper_r = " << endl << " " << R_cam2gripper_r << endl << endl;
cout << "t_cam2gripper = " << endl << " " << t_cam2gripper << endl << endl;
}
Mat eulerAnglesToRotationMatrix(Vec3f &theta)
{
// Calculate rotation about x axis
Mat R_x = (Mat_<double>(3, 3) <<
1, 0, 0,
0, cos(theta[0]), -sin(theta[0]),
0, sin(theta[0]), cos(theta[0])
);
// Calculate rotation about y axis
Mat R_y = (Mat_<double>(3, 3) <<
cos(theta[1]), 0, sin(theta[1]),
0, 1, 0,
-sin(theta[1]), 0, cos(theta[1])
);
// Calculate rotation about z axis
Mat R_z = (Mat_<double>(3, 3) <<
cos(theta[2]), -sin(theta[2]), 0,
sin(theta[2]), cos(theta[2]), 0,
0, 0, 1);
// Combined rotation matrix
Mat R = R_z * R_y * R_x;
return R;
}
float rad2deg(float radian) {
double pi = 3.14159;
return(radian * (180 / pi));
}
float deg2rad(float degree) {
double pi = 3.14159;
return(degree * (pi / 180));
}
// Checks if a matrix is a valid rotation matrix.
bool isRotationMatrix(Mat &R)
{
Mat Rt;
transpose(R, Rt);
Mat shouldBeIdentity = Rt * R;
Mat I = Mat::eye(3, 3, shouldBeIdentity.type());
return norm(I, shouldBeIdentity) < 1e-6;
}
// Calculates rotation matrix to euler angles
// The result is the same as MATLAB except the order
// of the euler angles ( x and z are swapped ).
Vec3f rotationMatrixToEulerAngles(Mat &R)
{
assert(isRotationMatrix(R));
float sy = sqrt(R.at<double>(0, 0) * R.at<double>(0, 0) + R.at<double>(1, 0) * R.at<double>(1, 0));
bool singular = sy < 1e-6; // If
float x, y, z;
if (!singular)
{
x = atan2(R.at<double>(2, 1), R.at<double>(2, 2));
y = atan2(-R.at<double>(2, 0), sy);
z = atan2(R.at<double>(1, 0), R.at<double>(0, 0));
}
else
{
x = atan2(-R.at<double>(1, 2), R.at<double>(1, 1));
y = atan2(-R.at<double>(2, 0), sy);
z = 0;
}
return Vec3f(x, y, z);
}
The result the function is giving me is the next one:
R_cam2gripper =
[0.3099803593003124, -0.8923086952824562, -0.3281727733547833;
0.7129271761196039, 0.4465219155360299, -0.5406967916458927;
0.6290047840821058, -0.0663579028402444, 0.7745641421680119]
R_cam2gripper_r =
[-0.0854626, -0.680272, 1.16065]
t_cam2gripper =
[-35.02063730299775;
-74.80633768251272;
-307.6725851251873]
I am getting 'good' results provided by other software. With them, the robot got to the exact points I am pointing in the camera (I have a 3D camera, from which I am getting the x, y, z from the camera world) so they are certainly correct, but I am having troubles to repeat the same result with the OpenCV function.
Sorry for the long introduction to my problem. Any understanding of why the solutions are not what is supposed to be? My guess is, that I have a problem understanding the angles or converting them but I couldn't find any way to solve this. Any hint will be well welcome!
I actually managed to solve this problem. The general idea was correct, but:
I was not understanding correctly the vector rotation notation the robot was giving. It was necessary to multiply the actual values by a factor.
I created a new program that extracts directly from the robot and the pictures the matrixes that the algorithm requires and writes these values to a YML file.
The CALIB_HAND_EYE_TSAI method wasn't giving me correct values. But with the four others, the values seem to converge to the actual values
Anyway, thank you for your help. I am stuck to get more precision in the algorithm, but that's for another question.
At the moment I am working on a project, which searches a sample in a video stream and constantly check it.
But after a while (sometimes some seconds, sometimes some minutes) it just stops working. No detecting, no video stream.
I checked my processor, and also tried different resolutions. But it's still not working correctly.
#include "pch.h"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include <iostream>
#include <array>
#include <stdio.h>
#include "opencv2/core/mat.hpp"
using namespace std;
using namespace cv;
/// Create a flag, used for nearly everything
int flag = 0;
int i = 0;
/// BlackPixel counter
float blackPixel;
/// Localizing the best match with minMaxLoc
double minVal, maxVal;
Point minLoc, maxLoc, matchLoc;
/// Punkte für drawMatch
Point middle_x, middle_y;
/// Creating picture Matrix
Mat image_input, image_template, image_display, image_object;
/// Create the result matrix
int result_cols, result_rows;
/// Test Inputs
Mat pictureTest;
Mat objectTest;
string intToString(int number) {
std::stringstream ss;
ss << number;
return ss.str();
}
void drawMatch(Mat object, Mat scene, vector<Point> match_centers)
{
for (size_t i = 0; i < match_centers.size(); i++)
{
/// middle of template at X
middle_x = Point((object.cols / 2) + match_centers[i].x, match_centers[i].y);
/// middle of template at Y
middle_y = Point(match_centers[i].x, match_centers[i].y + (object.rows / 2));
/// Zeichnet Rechteck um Match
rectangle(scene, Point(match_centers[i].x, match_centers[i].y), Point(match_centers[i].x + object.cols, match_centers[i].y + object.rows), Scalar(0, 255, 0), 2);
/// Zeigt das Ergebnis des Tracking
namedWindow("Track", WINDOW_NORMAL);
imshow("Track", scene);
/// Gibt die Koordinaten des Matches im Bild an
//putText(scene, "(" + intToString(match_centers[i].x) + "," + intToString(match_centers[i].y) + ")", Point(match_centers[i].x - 40, match_centers[i].y - 15), 1, 1, Scalar(255, 0, 0));
}
}
vector<Point> imageComparetion(Mat object, Mat scene, int match_method, float peek_percent) {
scene.copyTo(image_display);
object.copyTo(image_object);
result_cols = scene.cols - object.cols + 1;
result_rows = scene.rows - object.rows + 1;
Mat result(result_cols, result_rows, CV_32FC1);
/// match scene with template
matchTemplate(scene, object, result, match_method);
///normalize(result, result, 0, 1, NORM_MINMAX, -1, Mat());
normalize(result, result, 0, 1, NORM_MINMAX, -1, Mat());
/// For SQDIFF and SQDIFF_NORMED, the best matches are lower values. For all the other methods, the higher the better
if (match_method == TM_SQDIFF || match_method == TM_SQDIFF_NORMED)
{
matchLoc = minLoc;
threshold(result, result, 0.1, 1, THRESH_BINARY_INV);
}
else
{
matchLoc = maxLoc;
threshold(result, result, 0.9, 1, THRESH_TOZERO);
}
vector<Point> res;
maxVal = 1.f;
Mat input_matrix = image_display; //webcam image into matrix
Mat match = Mat(input_matrix.size(), input_matrix.type(), Scalar::all(0));
while (maxVal > peek_percent) {
minMaxLoc(result, &minVal, &maxVal, &minLoc, &maxLoc, Mat());
if (maxVal > peek_percent) {
Rect r1(Point(maxLoc.x - object.cols / 2, maxLoc.y + object.rows / 2), Point(maxLoc.x + object.cols / 2, maxLoc.y - object.rows / 2));
rectangle(result, Point(maxLoc.x - object.cols / 2, maxLoc.y - object.rows / 2), Point(maxLoc.x + object.cols / 2, maxLoc.y + object.rows / 2), Scalar::all(0), -1);
res.push_back(maxLoc);
}
}
return res;
}
int main(int argc, char** argv) {
///open Cam
VideoCapture cap(1);
cap.set(CAP_PROP_FRAME_WIDTH, 1920);
cap.set(CAP_PROP_FRAME_HEIGHT, 1080);
///Check if cam is open
if (!cap.isOpened()) {
cout << "\n Bitte Verbindung zur Kamera ueberpruefen! \n";
return 0;
}
///Read the matches template
image_template = imread("C:/Users/Ceraxes/Pictures/glaspol.jpg");
Mat current = imread("C:/Users/Ceraxes/Pictures/glaspol.jpg");
cap >> image_input;
///find location of template in stream
vector<Point> match_centers = imageComparetion(image_template, image_input, TM_CCOEFF_NORMED, 0.3); //template, picture, matching method, treshold(peak percent)
/// shows the found template
drawMatch(image_template, image_input, match_centers);
waitKey(1);
///build window
namedWindow("IDS", WINDOW_NORMAL);
Mat detect = image_input; //img_display
Mat draw = Mat(detect.size(), detect.type(), Scalar::all(0));
while (true) {
cap >> detect;
///Show image from camera
imshow("IDS", detect);
/// check every found match
while (i < match_centers.size()) {
///Extract the found matches from the picture
Rect r3(Point(match_centers[i].x, match_centers[i].y), Point(match_centers[i].x + image_template.cols, match_centers[i].y + image_template.rows));
///compares the camera imgage with the template
absdiff(detect(r3), current, pictureTest);
///counts the differences
blackPixel = countNonZero(pictureTest == 150);
cout << blackPixel << "\n";
i++;
}
i = 0;
waitKey(1);
}
return 0;
}
I'm using a IDS ueye camera, openCV 4 und Visual Studio 2017.
Does anyone have an idea, why the algorithm slows down?
I'm new to opencv. I want to detect very small lines in an image and store them as an array of lines. There will not be more than 10 lines (alignment markers) in my image. I've used a opencv sample tutorial to detect lines using canny and Hough.
For testing, my input image has a set of lines that vary in size: 15, 30, 50, 75 and 100 pixles
The app does not detect the 15 and 30 pixel lines.
It detects 2 of the 4 50 pixel lines.
The 75 and 100 pixel lines are detected fine.
What can I do to detect the 15 and 30 pixel lines?
This is my
Input image
This is my
output image
This is my source
/*
* https://github.com/opencv/opencv/blob/master/samples/cpp/tutorial_code/ImgTrans/HoughLines_Demo.cpp
*/
/**
* #file HoughLines_Demo.cpp
* #brief Demo code for Hough Transform
* #author OpenCV team
*/
#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include <iostream>
using namespace cv;
using namespace std;
/// Global variables
/** General variables */
Mat src, edges;
Mat src_gray;
Mat standard_hough, probabilistic_hough;
int min_threshold = 50;
int max_trackbar = 150;
const char* standard_name = "Standard Hough Lines Demo";
const char* probabilistic_name = "Probabilistic Hough Lines Demo";
int s_trackbar = max_trackbar;
int p_trackbar = max_trackbar;
/// Function Headers
void help();
void Standard_Hough( int, void* );
void Probabilistic_Hough( int, void* );
/**
* #function main
*/
int main( int argc, char** argv )
{
// Read the image
String imageName("../data/building.jpg"); // by default
if (argc > 1)
{
imageName = argv[1];
}
src = imread( imageName, IMREAD_COLOR );
if( src.empty() )
{ help();
return -1;
}
/// Pass the image to gray
cvtColor( src, src_gray, COLOR_RGB2GRAY );
/// Apply Canny edge detector
Canny( src_gray, edges, 50, 200, 3 );
/// Create Trackbars for Thresholds
char thresh_label[50];
sprintf( thresh_label, "Thres: %d + input", min_threshold );
namedWindow( standard_name, WINDOW_AUTOSIZE );
createTrackbar( thresh_label, standard_name, &s_trackbar, max_trackbar, Standard_Hough);
namedWindow( probabilistic_name, WINDOW_AUTOSIZE );
createTrackbar( thresh_label, probabilistic_name, &p_trackbar, max_trackbar, Probabilistic_Hough);
/// Initialize
Standard_Hough(0, 0);
Probabilistic_Hough(0, 0);
waitKey(0);
return 0;
}
/**
* #function help
* #brief Indications of how to run this program and why is it for
*/
void help()
{
printf("\t Hough Transform to detect lines \n ");
printf("\t---------------------------------\n ");
printf(" Usage: ./HoughLines_Demo <image_name> \n");
}
/**
* #function Standard_Hough
*/
void Standard_Hough( int, void* )
{
vector<Vec2f> s_lines;
cvtColor( edges, standard_hough, COLOR_GRAY2BGR );
/// 1. Use Standard Hough Transform
HoughLines( edges, s_lines, 1, CV_PI/180, min_threshold + s_trackbar, 0, 0 );
/// Show the result
for( size_t i = 0; i < s_lines.size(); i++ )
{
float r = s_lines[i][0], t = s_lines[i][1];
double cos_t = cos(t), sin_t = sin(t);
double x0 = r*cos_t, y0 = r*sin_t;
double alpha = 1000;
Point pt1( cvRound(x0 + alpha*(-sin_t)), cvRound(y0 + alpha*cos_t) );
Point pt2( cvRound(x0 - alpha*(-sin_t)), cvRound(y0 - alpha*cos_t) );
line( standard_hough, pt1, pt2, Scalar(255,0,0), 3, LINE_AA);
}
imshow( standard_name, standard_hough );
}
/**
* #function Probabilistic_Hough
*/
void Probabilistic_Hough( int, void* )
{
vector<Vec4i> p_lines;
cvtColor( edges, probabilistic_hough, COLOR_GRAY2BGR );
/// 2. Use Probabilistic Hough Transform
HoughLinesP( edges, p_lines, 1, CV_PI/180, min_threshold + p_trackbar, 30, 10 );
/// Show the result
for( size_t i = 0; i < p_lines.size(); i++ )
{
Vec4i l = p_lines[i];
line( probabilistic_hough, Point(l[0], l[1]), Point(l[2], l[3]), Scalar(255,0,0), 3, LINE_AA);
}
imshow( probabilistic_name, probabilistic_hough );
}
TIA
The variable min_threshold in your code has been set to 50, so it will not consider any line segment candidates smaller than that. With a bit of noise, it is easy for the 50 pixel line segments to get skipped.
You can set this variable to a value less than 15, so that all line segments can be considered.
I fitted an ellipse based on edges of extracted red ball. But it's not accurate.
I extracted this red ball based on HSV Color Space, but it always ignores the contour of this ball. (Perhaps because color of contour is much darker).
Any good ideas to let me fit a better ellipse for this ball? I want to find an ellipse which can embrace the red ball as accurate as possible.
It will be better if I can use existing functions of OpenCV.
I have fixed this problem. It is still unstable, but at most of time it works.
source image. All of those images can be detected: https://www.dropbox.com/sh/daerty94kv5k2n7/AABu9Axewe6mL0NdEX2nG5MIa?dl=0
Fit ellipse based on color
Re-fit ellipse based on color and edges
The Video link: https://www.youtube.com/watch?v=q0TQYREm9uA
Here is source code:
#include <iostream>
#include "opencv2/opencv.hpp"
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
using namespace cv;
using namespace std;
int main(int argc, char** argv)
{
cv::Mat capturedImage = imread(argv[1]);
if( capturedImage.empty() )
{
cout << "Couldn't open image " << argv[1] << "\nUsage: fitellipse <image_name>\n";
return 0;
}
/*============================= Phase 1: Translate Color Space from RGB to HSV =====================================================*/
cv::Mat imgHSV;
cv::cvtColor(capturedImage, imgHSV, cv::COLOR_BGR2HSV); //Convert the captured frame from BGR to HSV
cv::Mat imgGray;
cv::cvtColor(capturedImage, imgGray, CV_RGB2GRAY);
cv::Mat imgThresholded;
cv::inRange(imgHSV, cv::Scalar(160, 80, 70), cv::Scalar(179, 255, 255), imgThresholded); //Threshold the image
//morphological opening
cv::erode(imgThresholded, imgThresholded, cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(7, 7)) );
cv::dilate( imgThresholded, imgThresholded, cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(7, 7)) );
//morphological closing (removes small holes from the foreground)
cv::dilate( imgThresholded, imgThresholded, cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(7, 7)) );
cv::erode(imgThresholded, imgThresholded, cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(7, 7)) );
namedWindow("imgThresholded", WINDOW_NORMAL);
imshow("imgThresholded", imgThresholded);
/*============================= Phase 2: Fit a coarse ellipse based on red color ======================================================*/
vector<vector<cv::Point> > contours;
cv::findContours(imgThresholded, contours, CV_RETR_LIST, CV_CHAIN_APPROX_NONE, cv::Point(0,0));
size_t index = 0;
size_t largestSize = 0;
for(size_t i = 0; i < contours.size(); i++)
{
if (contours[i].size() > largestSize)
{
largestSize = contours[i].size();
index = i;
}
}
if (contours[index].size() < 6)
{
cout << "Do not have enough points" << endl;
return -1;
}
cv::Mat imgContour;
cv::Mat(contours[index]).convertTo(imgContour, CV_32F);
cv::RotatedRect coarseEllipse = cv::fitEllipse(imgContour);
cv::Mat capturedImageClone = capturedImage.clone();
ellipse(capturedImageClone, coarseEllipse.center, coarseEllipse.size*0.5f, coarseEllipse.angle, 0.0, 360.0, cv::Scalar(0,255,255), 3, CV_AA);
namedWindow("capturedImageClone", CV_WINDOW_NORMAL);
imshow("capturedImageClone", capturedImageClone);
/*============================= Phase 3: Re-fit a final ellipse based on combination of color and edge ===============================*/
double cxc = coarseEllipse.center.x;
double cyc = coarseEllipse.center.y;
double ca = coarseEllipse.size.height/2;
double cb = coarseEllipse.size.width/2;
cv::Mat mask(capturedImage.rows, capturedImage.cols, CV_8UC3, cv::Scalar(0,0,0));
cv::circle(mask, cv::Point(coarseEllipse.center.x, coarseEllipse.center.y), coarseEllipse.size.height/2 + 100, cv::Scalar(255,255,255), -1);
cv::Mat imgMask;
cv::Mat edges;
cv::bitwise_and(capturedImage, mask, imgMask);
namedWindow("imgMask", CV_WINDOW_NORMAL);
imshow("imgMask", imgMask);
cv::GaussianBlur(imgMask, edges, cv::Size(5,5), 0);
cv::Canny(edges, edges, 50, 100);
namedWindow("edges", CV_WINDOW_NORMAL);
imshow("edges", edges);
cv::findContours(edges, contours, CV_RETR_LIST, CV_CHAIN_APPROX_NONE, cv::Point(0,0));
index = -1;
double centerDistance = (numeric_limits<double>::max)();
double abRatio = (numeric_limits<double>::max)();
cv::RotatedRect finalEllipse;
for (size_t i = 0; i < contours.size(); i++)
{
if (contours[i].size() < 500 || i == contours.size() - 1 || i == contours.size() - 2)
continue;
cv::Mat(contours[i]).convertTo(imgContour, CV_32F);
cv::RotatedRect tmpEllipse = cv::fitEllipse(imgContour);
double txc = tmpEllipse.center.x;
double tyc = tmpEllipse.center.y;
double ta = tmpEllipse.size.height/2;
double tb = tmpEllipse.size.width/2;
double tmpDis = (cxc - txc) * (cxc - txc) + (cyc - tyc) * (cyc - tyc);
if (tmpDis < centerDistance && fabs(tb/ta - 1) < abRatio && ta > ca && tb > cb)
{
centerDistance = tmpDis;
abRatio = fabs(tb/ta - 1);
index = i;
finalEllipse = tmpEllipse;
}
}
if (index == -1)
finalEllipse = coarseEllipse;
ellipse(capturedImage, finalEllipse.center, finalEllipse.size*0.5f, finalEllipse.angle, 0.0, 360.0, cv::Scalar(0,255,255), 3, CV_AA);
double xc = finalEllipse.center.x; // center x
double yc = finalEllipse.center.y; // center y
double theta = finalEllipse.angle; // rotation angle theta
double a = finalEllipse.size.height / 2; // semi-major axis: a
double b = finalEllipse.size.width / 2; // semi-minor axis: b
double A = a * a * sin(theta) * sin(theta) + b * b * cos(theta) * cos(theta);
double B = 2 * (b * b - a * a) * sin(theta) * cos(theta);
double C = a * a * cos(theta) * cos(theta) + b * b * sin(theta) * sin(theta);
double D = -2 * A * xc - B * yc;
double E = -B * xc - 2 * C * yc;
double F = A * xc * xc + B * xc * yc + C * yc * yc - a * a * b * b;
A = A/F;
B = B/F;
C = C/F;
D = D/F;
E = E/F;
F = F/F;
double ellipseArray[3][3] = {{A, B/2, D/2},
{B/2, C, E/2},
{D/2, E/2, F}};
cv::Mat ellipseMatrix(3,3,CV_64FC1, ellipseArray);
cout << ellipseMatrix << endl;
namedWindow("capturedImage", CV_WINDOW_NORMAL);
imshow("capturedImage", capturedImage);
imwrite(argv[2],capturedImage);
imwrite(argv[3],edges);
imwrite(argv[4],capturedImageClone);
imwrite(argv[5],imgMask);
waitKey(0);
return 0;
}
As an input I have a photo of a simple symbol, e.g.: https://www.dropbox.com/s/nrmsvfd0le0bkke/symbol.jpg
I would like to detect the straight lines in it, like points of start and ends of the lines. In this case, assuming the top left of the symbol is (0,0), the lines would be defined like this:
start end (coordinates of beginning and end of a line)
1. (0,0); (0,10) (vertical line)
2. (0,10); (15, 15)
3. (15,15); (0, 20)
4. (0,20); (0,30)
How can I do it (pereferably using OpenCV)? I though about Hough lines, but they seem to be good for perfect thin straight lines, which is not the case in a drawing. I'll probably work on binarized image, too.
Give a try on this,
Apply thinning algorithm on threshold image.
Find contours.
approxPolyDP for the found contour.
See some reference:
approxpolydp-for-edge-maps
Creating Bounding boxes and circles for contours
maybe you can work on this one.
assume a perfect binarization:
run HoughLinesP
(not implemented) try to group those detected lines
I used this code:
int main()
{
cv::Mat image = cv::imread("HoughLinesP_perfect.png");
cv::Mat gray;
cv::cvtColor(image,gray,CV_BGR2GRAY);
cv::Mat output; image.copyTo(output);
cv::Mat g_thres = gray == 0;
std::vector<cv::Vec4i> lines;
//cv::HoughLinesP( binary, lines, 1, 2*CV_PI/180, 100, 100, 50 );
// cv::HoughLinesP( h_thres, lines, 1, CV_PI/180, 100, image.cols/2, 10 );
cv::HoughLinesP( g_thres, lines, 1, CV_PI/(4*180.0), 50, image.cols/20, 10 );
for( size_t i = 0; i < lines.size(); i++ )
{
cv::line( output, cv::Point(lines[i][0], lines[i][3]),
cv::Point(lines[i][4], lines[i][3]), cv::Scalar(155,255,155), 1, 8 );
}
cv::imshow("g thres", g_thres);
cv::imwrite("HoughLinesP_out.png", output);
cv::resize(output, output, cv::Size(), 0.5,0.5);
cv::namedWindow("output"); cv::imshow("output", output);
cv::waitKey(-1);
std::cout << "finished" << std::endl;
return 0;
}
EDIT:
updated code with simple line clustering (`minimum_distance function taken from SO):
giving this result:
float minimum_distance(cv::Point2f v, cv::Point2f w, cv::Point2f p) {
// Return minimum distance between line segment vw and point p
const float l2 = cv::norm(w-v) * cv::norm(w-v); // i.e. |w-v|^2 - avoid a sqrt
if (l2 == 0.0) return cv::norm(p-v); // v == w case
// Consider the line extending the segment, parameterized as v + t (w - v).
// We find projection of point p onto the line.
// It falls where t = [(p-v) . (w-v)] / |w-v|^2
//const float t = dot(p - v, w - v) / l2;
float t = ((p-v).x * (w-v).x + (p-v).y * (w-v).y)/l2;
if (t < 0.0) return cv::norm(p-v); // Beyond the 'v' end of the segment
else if (t > 1.0) return cv::norm(p-w); // Beyond the 'w' end of the segment
const cv::Point2f projection = v + t * (w - v); // Projection falls on the segment
return cv::norm(p - projection);
}
int main()
{
cv::Mat image = cv::imread("HoughLinesP_perfect.png");
cv::Mat gray;
cv::cvtColor(image,gray,CV_BGR2GRAY);
cv::Mat output; image.copyTo(output);
cv::Mat g_thres = gray == 0;
std::vector<cv::Vec4i> lines;
cv::HoughLinesP( g_thres, lines, 1, CV_PI/(4*180.0), 50, image.cols/20, 10 );
float minDist = 100;
std::vector<cv::Vec4i> lines_filtered;
for( size_t i = 0; i < lines.size(); i++ )
{
bool keep = true;
int overwrite = -1;
cv::Point2f a(lines[i][0], lines[i][6]);
cv::Point2f b(lines[i][7], lines[i][3]);
float lengthAB = cv::norm(a-b);
for( size_t j = 0; j < lines_filtered.size(); j++ )
{
cv::Point2f c(lines_filtered[j][0], lines_filtered[j][8]);
cv::Point2f d(lines_filtered[j][9], lines_filtered[j][3]);
float distCDA = minimum_distance(c,d,a);
float distCDB = minimum_distance(c,d,b);
float lengthCD = cv::norm(c-d);
if((distCDA < minDist) && (distCDB < minDist))
{
if(lengthCD >= lengthAB)
{
keep = false;
}
else
{
overwrite = j;
}
}
}
if(keep)
{
if(overwrite >= 0)
{
lines_filtered[overwrite] = lines[i];
}
else
{
lines_filtered.push_back(lines[i]);
}
}
}
for( size_t i = 0; i < lines_filtered.size(); i++ )
{
cv::line( output, cv::Point(lines_filtered[i][0], lines_filtered[i][10]),
cv::Point(lines_filtered[i][11], lines_filtered[i][3]), cv::Scalar(155,255,155), 2, 8 );
}
cv::imshow("g thres", g_thres);
cv::imwrite("HoughLinesP_out.png", output);
cv::resize(output, output, cv::Size(), 0.5,0.5);
cv::namedWindow("output"); cv::imshow("output", output);
cv::waitKey(-1);
std::cout << "finished" << std::endl;
return 0;
}
You should try the Hough Line Transform. And here is an example from this website
#include "opencv2/highgui/highgui.hpp"
#include "opencv2/imgproc/imgproc.hpp"
#include <iostream>
using namespace cv;
using namespace std;
int main()
{
Mat src = imread("building.jpg", 0);
Mat dst, cdst;
Canny(src, dst, 50, 200, 3);
cvtColor(dst, cdst, CV_GRAY2BGR);
vector<Vec2f> lines;
// detect lines
HoughLines(dst, lines, 1, CV_PI/180, 150, 0, 0 );
// draw lines
for( size_t i = 0; i < lines.size(); i++ )
{
float rho = lines[i][0], theta = lines[i][1];
Point pt1, pt2;
double a = cos(theta), b = sin(theta);
double x0 = a*rho, y0 = b*rho;
pt1.x = cvRound(x0 + 1000*(-b));
pt1.y = cvRound(y0 + 1000*(a));
pt2.x = cvRound(x0 - 1000*(-b));
pt2.y = cvRound(y0 - 1000*(a));
line( cdst, pt1, pt2, Scalar(0,0,255), 3, CV_AA);
}
imshow("source", src);
imshow("detected lines", cdst);
waitKey();
return 0;
}
With this you should be able to tweak and get the proprieties you are looking for (vertices).