Image Rectification Using camera intrinsic and extrinsic parameters - opencv

I want to rectify stereo images using intrinsic and extrinsic camera parameters that obtained from photomodeler software.
I wrote the code (modifying this link https://gist.github.com/anonymous/6586653) and I determined the relative rotation and translation parameters , but when I input the images the obtained results is not as expected although I tried to find the error but I couldn't.
your help is really appreciated:
the input images are:
I couldn't load all the images therefore I have put this link regarding the images and the results:
https://www.dropbox.com/s/5tmj9rk91tkrot4/RECTIFICATION_TEST_DATA.docx?dl=0
the code is
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <iostream>
#include <iomanip>
#include<opencv2/opencv.hpp>
#include <iostream>
#include <opencv/cv.h>
#include <opencv/highgui.h>
#include <fstream>
using namespace std;
using namespace cv;
int main(int argc, char** argv)
{
// Mat img1 = imread("E:\\12_0628.tif", 1);
// Mat img2 = imread("E:\\12_0629.tif", 1);
Mat img1 = imread("E:\\DSC_0483.JPG");
Mat img2 = imread("E:\\DSC_0484.JPG");
//EXTERIOR OREINTATION FOR THE 1ST IMAGE
double omega1 = -172.672440, phi1 = -80.168311, kappa1 = 163.005082, tx1 = -35.100000, ty1 = -56.700000, tz1 = -59.300000;
//EXTERIOR OREINTATION FOR THE 2ND IMAGE
double omega2 = 27.576999, phi2 = -67.089920, kappa2 = 2.826051, tx2 = -37.600000, ty2 = -18.600000, tz2 = -41.700000;
//Rotation matrices of the 1st image
omega1 = omega1 * CV_PI / 180.;
phi1 = phi1 * CV_PI / 180.;
kappa1 = kappa1 * CV_PI / 180.;
omega2 = omega2 * CV_PI / 180.;
phi2 = phi2 * CV_PI / 180.;
kappa2 = kappa2 * CV_PI / 180.;
Mat RX1 = (Mat_<double>(3, 3) <<
1, 0, 0,
0, cos(omega1), sin(omega1),
0, -sin(omega1), cos(omega1));
Mat RY1 = (Mat_<double>(3, 3) <<
cos(phi1), 0, -sin(phi1),
0, 1, 0,
sin(phi1), 0, cos(phi1));
Mat RZ1 = (Mat_<double>(3, 3) <<
cos(kappa1), sin(kappa1), 0,
-sin(kappa1), cos(kappa1), 0,
0, 0, 1);
// Composed rotation matrix with (RX, RY, RZ)
Mat R1 = RX1 * RY1 * RZ1;
Mat T1 = (Mat_<double>(3, 1) <<
tx1,
ty1,
tz1);
/////////////////////Rotation matrices of the 2nd image//////////////////////////////////////////
//Rotation matrices of the 1st image
Mat RX2 = (Mat_<double>(3, 3) <<
1, 0, 0,
0, cos(omega2), sin(omega2),
0, -sin(omega2), cos(omega2));
Mat RY2 = (Mat_<double>(3, 3) <<
cos(phi2), 0, -sin(phi2),
0, 1, 0,
sin(phi2), 0, cos(phi2));
Mat RZ2 = (Mat_<double>(3, 3) <<
cos(kappa2), sin(kappa2), 0,
-sin(kappa2), cos(kappa2), 0,
0, 0, 1);
// Composed rotation matrix with (RX, RY, RZ)
Mat R2 = RX2 * RY2 * RZ2;
Mat T2 = (Mat_<double>(3, 1) <<
tx2,
ty2,
tz2);
/////////////////////////////////////////////////////////////
double f = 2284.;// the focal length of the camera nikon D40, this is equivalant to 18mm
double w = (double)img1.cols;
double h = (double)img1.rows;
Mat M = (Mat_<double>(3, 3) <<//camera matrix
f , 0. , w/2,
0. , f , h/2,
0. , 0. , 1.);
Mat D = (Mat_<double>(5, 1) <<// distortion coefficicents
0,
0,
0,
0,
0.
);
Mat R1inv = R1.inv();
Mat Rrel = R2 * R1inv;
Mat Trel = (-1 * Rrel) * T1+ T2;
Mat T = (Mat_<double>(3, 1) <<//translation matrix
-2376.6,
-740.0,
229.0);
cout << img1.size() << endl;
cout << img2.size() << endl;
//Mat R1, R2,
Mat P1, P2, Q;
stereoRectify(M, D, M, D, img1.size(), Rrel, Trel, //the input data
R1, R2, P1, P2, Q);//the output data
Mat map1x, map1y, map2x, map2y;
Mat imgdst1, imgdst2;
// Size (flaot)imageSize;
// imageSize = img1.size();
initUndistortRectifyMap(M, D, R1, P1, img1.size(), CV_32FC1, map1x, map1y);
initUndistortRectifyMap(M, D, R2, P2, img1.size(), CV_32FC1, map2x, map2y);
remap(img1, imgdst1, map1x, map1y, INTER_LINEAR, BORDER_CONSTANT, Scalar());
remap(img2, imgdst2, map2x, map2y, INTER_LINEAR, BORDER_CONSTANT, Scalar());
namedWindow("image1");
namedWindow("image2");
imshow("image1", imgdst1);
imshow("image2", imgdst2);
// imwrite("DSC_0906_rect.jpg", imgdst1);
// imwrite("DSC_0913_rect.jpg", imgdst2);
imwrite("E:\\Researches\\2016-2017_res\\2_8_epipolar_geometry\\temp_image\\output1.bmp", imgdst1);
imwrite("E:\\Researches\\2016-2017_res\\2_8_epipolar_geometry\\temp_image\\output2.bmp", imgdst2);
waitKey();
return 0;
}

Related

OpenCV: Wrong result in calibrateHandEye function

I am working in a robot application, in which I have a camera fixed to a robot gripper. In order to calculate the matrix transformation between the camera and the gripper Hcg I am using the calibrateHandEye new function provided in the OpenCV version 4.1.0
I had taken 10 pictures of the chessboard from the camera mounted in the gripper and at the same time I recorded the robot position.
The code I am working on:
// My_handeye.cpp : This file contains the 'main' function. Program execution begins and ends there.
//
#include <iostream>
#include <sstream>
#include <string>
#include <ctime>
#include <cstdio>
#include "pch.h"
#include <opencv2/opencv.hpp>
#include <opencv2/core.hpp>
#include <opencv2/core/utility.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/imgcodecs.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
using namespace cv;
using namespace std;
Mat eulerAnglesToRotationMatrix(Vec3f &theta);
Vec3f rotationMatrixToEulerAngles(Mat &R);
float rad2deg(float radian);
float deg2rad(float degree);
int main()
{
// Camera calibration information
std::vector<double> distortionCoefficients(5); // camera distortion
distortionCoefficients[0] = 2.4472856611074989e-01;
distortionCoefficients[1] = -8.1042032574246325e-01;
distortionCoefficients[2] = 0;
distortionCoefficients[3] = 0;
distortionCoefficients[4] = 7.8769462320821060e-01;
double f_x = 1.3624172121852105e+03; // Focal length in x axis
double f_y = 1.3624172121852105e+03; // Focal length in y axis (usually the same?)
double c_x = 960; // Camera primary point x
double c_y = 540; // Camera primary point y
cv::Mat cameraMatrix(3, 3, CV_32FC1);
cameraMatrix.at<float>(0, 0) = f_x;
cameraMatrix.at<float>(0, 1) = 0.0;
cameraMatrix.at<float>(0, 2) = c_x;
cameraMatrix.at<float>(1, 0) = 0.0;
cameraMatrix.at<float>(1, 1) = f_y;
cameraMatrix.at<float>(1, 2) = c_y;
cameraMatrix.at<float>(2, 0) = 0.0;
cameraMatrix.at<float>(2, 1) = 0.0;
cameraMatrix.at<float>(2, 2) = 1.0;
Mat rvec(3, 1, CV_32F), tvec(3, 1, CV_32F);
//
std::vector<Mat> R_gripper2base;
std::vector<Mat> t_gripper2base;
std::vector<Mat> R_target2cam;
std::vector<Mat> t_target2cam;
Mat R_cam2gripper = (Mat_<float>(3, 3));
Mat t_cam2gripper = (Mat_<float>(3, 1));
vector<String> fn;
glob("images/*.bmp", fn, false);
vector<Mat> images;
size_t num_images = fn.size(); //number of bmp files in images folder
Size patternsize(6, 8); //number of centers
vector<Point2f> centers; //this will be filled by the detected centers
float cell_size = 30;
vector<Point3f> obj_points;
R_gripper2base.reserve(num_images);
t_gripper2base.reserve(num_images);
R_target2cam.reserve(num_images);
t_target2cam.reserve(num_images);
for (int i = 0; i < patternsize.height; ++i)
for (int j = 0; j < patternsize.width; ++j)
obj_points.push_back(Point3f(float(j*cell_size),
float(i*cell_size), 0.f));
for (size_t i = 0; i < num_images; i++)
images.push_back(imread(fn[i]));
Mat frame;
for (size_t i = 0; i < num_images; i++)
{
frame = imread(fn[i]); //source image
bool patternfound = findChessboardCorners(frame, patternsize, centers);
if (patternfound)
{
drawChessboardCorners(frame, patternsize, Mat(centers), patternfound);
//imshow("window", frame);
//int key = cv::waitKey(0) & 0xff;
solvePnP(Mat(obj_points), Mat(centers), cameraMatrix, distortionCoefficients, rvec, tvec);
Mat R;
Rodrigues(rvec, R); // R is 3x3
R_target2cam.push_back(R);
t_target2cam.push_back(tvec);
Mat T = Mat::eye(4, 4, R.type()); // T is 4x4
T(Range(0, 3), Range(0, 3)) = R * 1; // copies R into T
T(Range(0, 3), Range(3, 4)) = tvec * 1; // copies tvec into T
cout << "T = " << endl << " " << T << endl << endl;
}
cout << patternfound << endl;
}
Vec3f theta_01{ deg2rad(-153.61), deg2rad(8.3), deg2rad(-91.91) };
Vec3f theta_02{ deg2rad(-166.71), deg2rad(3.04), deg2rad(-93.31) };
Vec3f theta_03{ deg2rad(-170.04), deg2rad(24.92), deg2rad(-88.29) };
Vec3f theta_04{ deg2rad(-165.71), deg2rad(24.68), deg2rad(-84.85) };
Vec3f theta_05{ deg2rad(-160.18), deg2rad(-15.94),deg2rad(-56.24) };
Vec3f theta_06{ deg2rad(175.68), deg2rad(10.95), deg2rad(180) };
Vec3f theta_07{ deg2rad(175.73), deg2rad(45.78), deg2rad(-179.92) };
Vec3f theta_08{ deg2rad(-165.34), deg2rad(47.37), deg2rad(-166.25) };
Vec3f theta_09{ deg2rad(-165.62), deg2rad(17.95), deg2rad(-166.17) };
Vec3f theta_10{ deg2rad(-151.99), deg2rad(-14.59),deg2rad(-94.19) };
Mat robot_rot_01 = eulerAnglesToRotationMatrix(theta_01);
Mat robot_rot_02 = eulerAnglesToRotationMatrix(theta_02);
Mat robot_rot_03 = eulerAnglesToRotationMatrix(theta_03);
Mat robot_rot_04 = eulerAnglesToRotationMatrix(theta_04);
Mat robot_rot_05 = eulerAnglesToRotationMatrix(theta_05);
Mat robot_rot_06 = eulerAnglesToRotationMatrix(theta_06);
Mat robot_rot_07 = eulerAnglesToRotationMatrix(theta_07);
Mat robot_rot_08 = eulerAnglesToRotationMatrix(theta_08);
Mat robot_rot_09 = eulerAnglesToRotationMatrix(theta_09);
Mat robot_rot_10 = eulerAnglesToRotationMatrix(theta_10);
const Mat robot_tr_01 = (Mat_<float>(3, 1) << 781.2, 338.59, 903.48);
const Mat robot_tr_02 = (Mat_<float>(3, 1) << 867.65, 382.52, 884.42);
const Mat robot_tr_03 = (Mat_<float>(3, 1) << 856.91, 172.99, 964.61);
const Mat robot_tr_04 = (Mat_<float>(3, 1) << 748.81, 146.75, 1043.29);
const Mat robot_tr_05 = (Mat_<float>(3, 1) << 627.66, 554.08, 920.85);
const Mat robot_tr_06 = (Mat_<float>(3, 1) << 715.06, 195.96, 889.38);
const Mat robot_tr_07 = (Mat_<float>(3, 1) << 790.9, 196.29, 1117.38);
const Mat robot_tr_08 = (Mat_<float>(3, 1) << 743.5, 283.93, 1131.92);
const Mat robot_tr_09 = (Mat_<float>(3, 1) << 748.9, 288.19, 910.58);
const Mat robot_tr_10 = (Mat_<float>(3, 1) << 813.18, 400.44, 917.16);
R_gripper2base.push_back(robot_rot_01);
R_gripper2base.push_back(robot_rot_02);
R_gripper2base.push_back(robot_rot_03);
R_gripper2base.push_back(robot_rot_04);
R_gripper2base.push_back(robot_rot_05);
R_gripper2base.push_back(robot_rot_06);
R_gripper2base.push_back(robot_rot_07);
R_gripper2base.push_back(robot_rot_08);
R_gripper2base.push_back(robot_rot_09);
R_gripper2base.push_back(robot_rot_10);
t_gripper2base.push_back(robot_tr_01);
t_gripper2base.push_back(robot_tr_02);
t_gripper2base.push_back(robot_tr_03);
t_gripper2base.push_back(robot_tr_04);
t_gripper2base.push_back(robot_tr_05);
t_gripper2base.push_back(robot_tr_06);
t_gripper2base.push_back(robot_tr_07);
t_gripper2base.push_back(robot_tr_08);
t_gripper2base.push_back(robot_tr_09);
t_gripper2base.push_back(robot_tr_10);
calibrateHandEye(R_gripper2base, t_gripper2base, R_target2cam, t_target2cam, R_cam2gripper, t_cam2gripper, CALIB_HAND_EYE_TSAI);
Vec3f R_cam2gripper_r = rotationMatrixToEulerAngles(R_cam2gripper);
cout << "R_cam2gripper = " << endl << " " << R_cam2gripper << endl << endl;
cout << "R_cam2gripper_r = " << endl << " " << R_cam2gripper_r << endl << endl;
cout << "t_cam2gripper = " << endl << " " << t_cam2gripper << endl << endl;
}
Mat eulerAnglesToRotationMatrix(Vec3f &theta)
{
// Calculate rotation about x axis
Mat R_x = (Mat_<double>(3, 3) <<
1, 0, 0,
0, cos(theta[0]), -sin(theta[0]),
0, sin(theta[0]), cos(theta[0])
);
// Calculate rotation about y axis
Mat R_y = (Mat_<double>(3, 3) <<
cos(theta[1]), 0, sin(theta[1]),
0, 1, 0,
-sin(theta[1]), 0, cos(theta[1])
);
// Calculate rotation about z axis
Mat R_z = (Mat_<double>(3, 3) <<
cos(theta[2]), -sin(theta[2]), 0,
sin(theta[2]), cos(theta[2]), 0,
0, 0, 1);
// Combined rotation matrix
Mat R = R_z * R_y * R_x;
return R;
}
float rad2deg(float radian) {
double pi = 3.14159;
return(radian * (180 / pi));
}
float deg2rad(float degree) {
double pi = 3.14159;
return(degree * (pi / 180));
}
// Checks if a matrix is a valid rotation matrix.
bool isRotationMatrix(Mat &R)
{
Mat Rt;
transpose(R, Rt);
Mat shouldBeIdentity = Rt * R;
Mat I = Mat::eye(3, 3, shouldBeIdentity.type());
return norm(I, shouldBeIdentity) < 1e-6;
}
// Calculates rotation matrix to euler angles
// The result is the same as MATLAB except the order
// of the euler angles ( x and z are swapped ).
Vec3f rotationMatrixToEulerAngles(Mat &R)
{
assert(isRotationMatrix(R));
float sy = sqrt(R.at<double>(0, 0) * R.at<double>(0, 0) + R.at<double>(1, 0) * R.at<double>(1, 0));
bool singular = sy < 1e-6; // If
float x, y, z;
if (!singular)
{
x = atan2(R.at<double>(2, 1), R.at<double>(2, 2));
y = atan2(-R.at<double>(2, 0), sy);
z = atan2(R.at<double>(1, 0), R.at<double>(0, 0));
}
else
{
x = atan2(-R.at<double>(1, 2), R.at<double>(1, 1));
y = atan2(-R.at<double>(2, 0), sy);
z = 0;
}
return Vec3f(x, y, z);
}
The result the function is giving me is the next one:
R_cam2gripper =
[0.3099803593003124, -0.8923086952824562, -0.3281727733547833;
0.7129271761196039, 0.4465219155360299, -0.5406967916458927;
0.6290047840821058, -0.0663579028402444, 0.7745641421680119]
R_cam2gripper_r =
[-0.0854626, -0.680272, 1.16065]
t_cam2gripper =
[-35.02063730299775;
-74.80633768251272;
-307.6725851251873]
I am getting 'good' results provided by other software. With them, the robot got to the exact points I am pointing in the camera (I have a 3D camera, from which I am getting the x, y, z from the camera world) so they are certainly correct, but I am having troubles to repeat the same result with the OpenCV function.
Sorry for the long introduction to my problem. Any understanding of why the solutions are not what is supposed to be? My guess is, that I have a problem understanding the angles or converting them but I couldn't find any way to solve this. Any hint will be well welcome!
I actually managed to solve this problem. The general idea was correct, but:
I was not understanding correctly the vector rotation notation the robot was giving. It was necessary to multiply the actual values by a factor.
I created a new program that extracts directly from the robot and the pictures the matrixes that the algorithm requires and writes these values to a YML file.
The CALIB_HAND_EYE_TSAI method wasn't giving me correct values. But with the four others, the values seem to converge to the actual values
Anyway, thank you for your help. I am stuck to get more precision in the algorithm, but that's for another question.

using getoptiamlnewcameramatrix to recover all the original image

enter image description here
this is my program output
and this is my distorted imageenter image description here
to get the original image`s whole pixel ,I used the getoptimalnewcameramatrix,but the output is terrible ,my intrix and distcoffes is totally correct.
here is my program
Mat src = imread("E:\\40_office\\distorted_bot\\0.jpg");
Size newsize(2280,3072);
cout << src.rows;
namedWindow("", WINDOW_AUTOSIZE);
imshow("", src);
waitKey();
Mat mapx,mapy;//size(x,y)
Mat cameraMatrix = (Mat_<double>(3, 3) << 1224.1, 0, 761.7497, 0, 1209.8, 1043.6, 0, 0, 1);
//Mat cameraMatrix = (Mat_<double>(3, 3) << 1224.1, 0, 1141.7497, 0, 1209.8, 1442.6, 0, 0, 1);
//Mat cameraMatrix = (Mat_<double>(3, 3) << 1209.8, 0, 1043.6, 0, 1224.1, 761.7497, 0, 0, 1);
Mat distCoeffs = (Mat_<double>(5, 1) << -0.3649, 0.1451, -0.0273, -0.000035214,0.0012);
double alpha = 1;
Mat newcameramatrix = getOptimalNewCameraMatrix(cameraMatrix, distCoeffs, src.size(), alpha, newsize,0);
cout << newcameramatrix<<"src.size"<<src.size();
fisheye::initUndistortRectifyMap(cameraMatrix, distCoeffs,Mat(), newcameramatrix ,newsize, CV_16SC2, mapx,mapy);
Mat newimage=Mat(newsize,CV_8UC3);
remap(src, newimage, mapx, mapy, INTER_LINEAR);
imwrite("C:\\Users\\wk\\Desktop\\1_cali100.jpg", newimage);
return;

Opencv : project points manually

I'm trying to reproduce the behavior of the method projectPoints() from OpenCV.
In the two images below, red/green/blue axis are obtained with OpenCV's method, whereas magenta/yellow/cyan axis are obtained with my own method :
image1
image2
With my method, axis seem to have a good orientation but translations are incorrect.
Here is my code :
void drawVector(float x, float y, float z, float r, float g, float b, cv::Mat &pose, cv::Mat &cameraMatrix, cv::Mat &dst) {
//Origin = (0, 0, 0, 1)
cv::Mat origin(4, 1, CV_64FC1, double(0));
origin.at<double>(3, 0) = 1;
//End = (x, y, z, 1)
cv::Mat end(4, 1, CV_64FC1, double(1));
end.at<double>(0, 0) = x; end.at<double>(1, 0) = y; end.at<double>(2, 0) = z;
//multiplies transformation matrix by camera matrix
cv::Mat mat = cameraMatrix * pose.colRange(0, 4).rowRange(0, 3);
//projects points
origin = mat * origin;
end = mat * end;
//draws corresponding line
cv::line(dst, cv::Point(origin.at<double>(0, 0), origin.at<double>(1, 0)),
cv::Point(end.at<double>(0, 0), end.at<double>(1, 0)),
CV_RGB(255 * r, 255 * g, 255 * b));
}
void drawVector_withProjectPointsMethod(float x, float y, float z, float r, float g, float b, cv::Mat &pose, cv::Mat &cameraMatrix, cv::Mat &dst) {
std::vector<cv::Point3f> points;
std::vector<cv::Point2f> projectedPoints;
//fills input array with 2 points
points.push_back(cv::Point3f(0, 0, 0));
points.push_back(cv::Point3f(x, y, z));
//Gets rotation vector thanks to cv::Rodrigues() method.
cv::Mat rvec;
cv::Rodrigues(pose.colRange(0, 3).rowRange(0, 3), rvec);
//projects points using cv::projectPoints method
cv::projectPoints(points, rvec, pose.colRange(3, 4).rowRange(0, 3), cameraMatrix, std::vector<double>(), projectedPoints);
//draws corresponding line
cv::line(dst, projectedPoints[0], projectedPoints[1],
CV_RGB(255 * r, 255 * g, 255 * b));
}
void drawAxis(cv::Mat &pose, cv::Mat &cameraMatrix, cv::Mat &dst) {
drawVector(0.1, 0, 0, 1, 1, 0, pose, cameraMatrix, dst);
drawVector(0, 0.1, 0, 0, 1, 1, pose, cameraMatrix, dst);
drawVector(0, 0, 0.1, 1, 0, 1, pose, cameraMatrix, dst);
drawVector_withProjectPointsMethod(0.1, 0, 0, 1, 0, 0, pose, cameraMatrix, dst);
drawVector_withProjectPointsMethod(0, 0.1, 0, 0, 1, 0, pose, cameraMatrix, dst);
drawVector_withProjectPointsMethod(0, 0, 0.1, 0, 0, 1, pose, cameraMatrix, dst);
}
What am I doing wrong ?
I just forgot to divide the resulting points by their last component after projection :
Given the matrix of the camera wich serve to take an image, and for any point (x, y, z, 1) in 3d space, its projection on that image is computed like following :
//point3D has 4 component (x, y, z, w), point2D has 3 (x, y, z).
point2D = cameraMatrix * point3D;
//then we have to divide the 2 first component of point2D by the third.
point2D /= point2D.z;

cv::solvePnPRansac returns identity matrix

I want to find the object pose given 3D-to-2D correspondences using cv::solvePnPRansac. My correspondences might have outliers, so I don't want to use cv::solvePnP. I have used 3D-to-2D correspondences of a snapshot taken from a camera in opengl. So after backprojecting, I find that the rotation matrix returned after using solvePnP is correct. But, when I use solvePnPRansac, I get a identity matrix, and zero translation vector. Please help! I have used the following code. Thanks in advance !
// Parameters to solvePnP
Mat camera_mat(3, 3, CV_64FC1);
Mat distCoeffs(4, 1, CV_64FC1);
Mat rvec(3, 1, CV_64FC1);
Mat tvec(3, 1, CV_64FC1);
Mat d(3, 3, CV_64FC1);
vector<Point3f> list_points3d_model_match;
vector<Point2f> list_points2d_scene_match;
while(map >> u >> v >> x >> y >> z)
{
Point2f ip = Point2f(u, v);
Point3f sp = Point3f(x, y, z);
// cout << x << " " << y << " " << z << endl;
list_points3d_model_match.push_back(sp);
list_points2d_scene_match.push_back(ip);
}
camera_mat.at<double>(0, 0) = 600;
camera_mat.at<double>(1, 1) = 600;
camera_mat.at<double>(0, 2) = WIDTH / 2;
camera_mat.at<double>(1, 0) = HEIGHT / 2;
camera_mat.at<double>(3, 3) = 1;
// solvePnP( list_points3d_model_match, list_points2d_scene_match, camera_mat, distCoeffs, rvec, tvec,
// useExtrinsicGuess, CV_ITERATIVE);
tvec.at<float>(0) = 0;
tvec.at<float>(1) = 0;
tvec.at<float>(2) = 2;
d.at<float>(0, 0) = 1;
d.at<float>(1, 2) = -1;
d.at<float>(2, 1) = 1;
Rodrigues(d, rvec);
solvePnPRansac(list_points3d_model_match, list_points2d_scene_match, camera_mat, distCoeffs, rvec, tvec, false, CV_ITERATIVE);
Rodrigues(rvec, d);
double* _r = d.ptr<double>();
printf("rotation mat: \n %.3f %.3f %.3f\n%.3f %.3f %.3f\n%.3f %.3f %.3f\n",
_r[0],_r[1],_r[2],_r[3],_r[4],_r[5],_r[6],_r[7],_r[8]);
}
cout << "translation vec:\n" << tvec << endl;
I get the following output after running the code.
rotation mat:
1.000 0.000 0.000
0.000 1.000 0.000
0.000 0.000 1.000
translation vec:
[0; 0; 0]
Your camera matrix is wrong.
Your code should read:
camera_mat.at<double>(0, 0) = 600;
camera_mat.at<double>(1, 1) = 600;
camera_mat.at<double>(0, 2) = WIDTH / 2;
camera_mat.at<double>(1, 2) = HEIGHT / 2;
camera_mat.at<double>(3, 3) = 1;
I had the same problem before, and it's not about the wrong typing of the camera matrix.
Change the declaration of
Mat camera_mat(3, 3, CV_64FC1);
Mat distCoeffs(4, 1, CV_64FC1);
to CV_32FC1
and don't give the initial size to Mat tvec and Mat d.

Disparity map colors are backwards in opencv

My problem is that the colors in my disparity map are backwards. As in the farther away things are lighter than the things closer to the camera.
I have tried many things (i.e. convertTo, convertScaleAbs, and various combinations of values in them, etc.) and cannot seem to get the colors in the disparity map to reverse (i.e. be normal - where things closer are lighter than things farther away).
I need some help in doing that.
Also, out of curiosity, how can i change the color space of the disparity map to be like the colorful ones in MATLAB that I see online?
Here's my code and also on pastebin. http://pastebin.com/E3vVN6UU
#include <opencv2/core/core.hpp>
#include <opencv2/imgproc/imgproc.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/calib3d/calib3d.hpp>
#include <iostream>
#include <string>
using namespace cv;
using namespace std;
void show(const char* windowname, Mat image)
{
namedWindow(windowname, CV_WINDOW_AUTOSIZE);
imshow(windowname, image);
}
int main()
{
Mat image1, image2;
Mat camMat1 = (Mat_<double>(3,3) << 793.1338, 0, 337.2309, 0, 792.0555, 256.9991, 0, 0, 1);
Mat camMat2 = (Mat_<double>(3,3) << 799.1271, 0, 319.8581, 0, 797.2460, 243.4638, 0, 0, 1);
Mat dispCoeffs1 = (Mat_<double>(1,5) << 0.0033, -0.1320, -0.0019, 0.0026, 0);
Mat dispCoeffs2 = (Mat_<double>(1,5) << -0.0109, -0.0188, -0.0014, -0.0055, 0);
Mat RotMat = (Mat_<double>(3,3) << 0.9998, -0.0023, 0.0221, 0.0022, 1, 0.0031, -0.0221, -0.0031, 0.9998);
Mat TransMat = (Mat_<double>(3,1) << 374.2306, -1.8319, 5.5745);
//Rectify
Mat R1, R2, P1, P2, Q;
stereoRectify(camMat1, dispCoeffs1, camMat2, dispCoeffs2, Size(640,480), RotMat, TransMat, R1, R2, P1, P2, Q, CV_CALIB_ZERO_DISPARITY, 1, Size(640,480));
//Define the mapping to the done
Mat rx1, ry1;
Mat rx2, ry2;
initUndistortRectifyMap(camMat1, dispCoeffs1, R1, P1, Size(640,480), CV_16SC2, rx1, ry1);
initUndistortRectifyMap(camMat2, dispCoeffs2, R2, P2, Size(640,480), CV_16SC2, rx2, ry2);
//SET THE BM STATE VARIABLES BEGIN - DONE GLOBALLY
StereoBM bm;
bm.state->preFilterSize = 31;
bm.state->preFilterCap = 63;
bm.state->SADWindowSize = 9;
bm.state->minDisparity = -128;
//bm.state->disp12MaxDiff = 2;
bm.state->numberOfDisparities = 128;
bm.state->textureThreshold = 50;
bm.state->uniquenessRatio = 15;
bm.state->speckleWindowSize = 100;
bm.state->speckleRange = 16;
//SET THE BM STATE VARIABLES END
VideoCapture cap3 = VideoCapture(0);
VideoCapture cap4 = VideoCapture(1);
//cap3.set(CV_CAP_PROP_FRAME_WIDTH, 320);
//cap3.set(CV_CAP_PROP_FRAME_HEIGHT, 240);
//cap4.set(CV_CAP_PROP_FRAME_WIDTH, 320);
//cap4.set(CV_CAP_PROP_FRAME_HEIGHT, 240);
cap3 >> image1;
cap4 >> image2;
Size imageSize = image1.size();
Mat gray_image1;
Mat gray_image2;
Mat frame1r;
//frame1r.create(image1.size(), CV_8U);
Mat frame2r;
//frame2r.create(image2.size(), CV_8U);
Mat frame1rf;
Mat frame2rf;
//Mat disp(image1.size(), CV_16S);
//Mat vdisp(image1.size(), CV_8U);
Mat disp, vdisp;
//Mat image3d(image1.size(), CV_32FC3);
Mat image3d;
Mat rectified_pair;
rectified_pair.create(imageSize.height, (imageSize.width)*2, CV_8UC3);
//Actually do the mapping -- based on the mapping definition
while(1)
{
bm.state->preFilterSize = 31;
bm.state->preFilterCap = 63;
bm.state->SADWindowSize = 21;
bm.state->minDisparity = -128;
//bm.state->disp12MaxDiff = 2;
bm.state->numberOfDisparities = 64;
bm.state->textureThreshold = 20;
bm.state->uniquenessRatio = 10;
bm.state->speckleWindowSize = 100;
bm.state->speckleRange = 32;
cvtColor(image1, gray_image1, CV_BGR2GRAY);
cvtColor(image2, gray_image2, CV_BGR2GRAY);
remap(gray_image1, frame1r, rx1, ry1, CV_INTER_LINEAR);
remap(gray_image2, frame2r, rx2, ry2, CV_INTER_LINEAR);
bm(frame1r, frame2r, disp);
normalize(disp, vdisp, 0, 255, NORM_MINMAX, CV_8U);
//convertScaleAbs(vdisp, vdisp, 1, 0);
disp.convertTo(vdisp, CV_8U, 255/(64*16.));
show("disparity", vdisp);
//reprojectImageTo3D(disp, image3d, Q, true);
//show("depth map", image3d);
//display image side by side for rectified window
//copy frame1r to the left side
cvtColor(frame1r, frame1rf, CV_GRAY2BGR);
frame1rf.copyTo(rectified_pair(Rect(0,0,imageSize.width, imageSize.height)));
//copy frame2r to the right side
cvtColor(frame2r, frame2rf, CV_GRAY2BGR);
frame2rf.copyTo(rectified_pair(Rect(imageSize.width,0,imageSize.width, imageSize.height)));
for(int i=0; i<imageSize.height; i+=32)
line(rectified_pair, Point(0,i), Point((imageSize.width)*2, i), CV_RGB(0,255,0));
show("rectified", rectified_pair);
cap3 >> image1;
cap4 >> image2;
if(waitKey(15) == 27)
break;
}
return 0;
}
I'm not using stereo pairs but get the same result using Kinect - far = light, near = dark
To change this I have used the below :
double min, max;
minMaxLoc(depthImage, &min, &max);
depthImage.convertTo(rImage, CV_8U, -255.0/max, 255);
I was facing the same problem then I tried swapping right and left images,and it worked!
Now I am getting correct image.

Resources