Incorrect image borders taken in cv::stereoRectify (calibration.cpp) #6336
Description
Please state the information for your system
- OpenCV version: 3.1
- Host OS: Linux (Ubuntu 14.04) / Windows 7
In which part of the OpenCV library you got the issue?
- calib3d
Introduction
The math that stands behind pinhole camera model, radial distortion and stereo rectification can be easily applied to warping images of different scales. When one performes all the calibration routines on the sourse image in some applications it can be useful to undistort/rectify image of, say 0.25*actual size.
This can be achieved by generating separate remap maps via InitUndistorRectifyMap. The only thing to change is the camera matrices passed to cv::stereoRectify functiuon. Their first 2 lines should be scaled respectively; Distortion coefficient operate to scale-independent coordinates, Rotation and translation describe real world and thus are independent from the output image size too.
bool resizeCameraMatrix(const cv::Mat &in_cm, cv::Mat &dst_cm, double scale)
{
if (in_cm.empty() || in_cm.cols != 3 || in_cm.rows != 3 || in_cm.type() != CV_64FC1)
return false;
dst_cm = in_cm * scale;
dst_cm.at<double>(2, 2) = 1.0;
return true;
}source image
Expected behaviour
stereoRectify of resized image with resized camera matrix would equal to the resize of rectified source image
remap(rectify)->resize result
Actual behaviour
It is not under certain conditions.
resize->remap(rectify) result
Additional description
This trick was working just fine until we have obtained unlucky rectification values (or, from the other side, we were indeed very lucky). The principal point of the resized rectified image moved to the bottom right part of the image and all the stereo reconstructions failed.
The problem appers because of incorrect image border copmutation in cv::stereoRectify on the undistorted image. When dealing with double-valued image coordinates the width of pixel becomes an issue. The top-left corner of a pixel which is used to compute new principal point on current master version has different double coordinates on different image scales. In case of 0.25 scale it moves 3 pixels left and top, respectively. If the actual corners of the image are taken the problem disappears since scaling does not change actual double relative points coordinates, which are passed to cv::undistortPoints and the rectification process is similar for all the scales.
The possible solution is to pass (width+1, height+1) image size to cv::stereoRectify function, however it can be bad for focal length computation that is performed in the beginning of the function.
The more elegant way to solve the problem is to replace image corners to actual corner coordinates: (nx-1) and (ny-1) by nx and ny, respectively
pull request to solve the issue #6337.
Code example to reproduce the issue / Steps to reproduce the issue
The test data and results are attached.
#include <opencv2/core.hpp>
#include <opencv2/imgproc.hpp>
#include <opencv2/calib3d.hpp>
#include <opencv2/highgui.hpp>
#include <iostream>
using namespace std;
using namespace cv;
bool resizeCameraMatrix(const Mat &in_cm, Mat &dst_cm, double scale)
{
if (in_cm.empty() || in_cm.cols != 3 || in_cm.rows != 3 || in_cm.type() != CV_64FC1)
return false;
dst_cm = in_cm * scale;
dst_cm.at<double>(2, 2) = 1.0;
return true;
}
static const Matx33d M1(906.7857732303256, 0.0, 1026.456125870669,
0.0, 906.7857732303256, 540.0531577669913,
0.0, 0.0, 1.0);
static const Matx33d M2(906.782205162265, 0.0, 1014.619997352785,
0.0, 906.782205162265, 561.9990018887295,
0.0, 0.0, 1.0);
static const Matx<double, 5, 1> D1(0.0064836857220181504, 0.033880363848984636, 0.0, 0.0, -0.042996356352306114);
static const Matx<double, 5, 1> D2(0.023754068600491646, -0.02364619610835259, 0.0, 0.0, 0.0015014971456262652);
static const Size imageSize(2048, 1088);
static const double scale = 0.25;
static const Matx33d Rot(0.999788461750194, -0.015696495349844446, -0.013291041528534329,
0.015233019205877604, 0.999296086451901, -0.034282455101525826,
0.01381980018141639, 0.03407274036010432, 0.9993238021218641);
static const Matx31d T(-1.552005597952028, 0.0019508251875105093, -0.023335501616116062);
int main()
{
// generate camera matrices for resized image rectification.
Mat srcM1(M1), srcM2(M2);
Mat rszM1, rszM2;
resizeCameraMatrix(srcM1, rszM1, scale);
resizeCameraMatrix(srcM2, rszM2, scale);
Size rszImageSize(cvRound(scale * imageSize.width), cvRound(scale * imageSize.height));
Size srcImageSize = imageSize;
// apply stereoRectify
Mat srcR[2], srcP[2], srcQ;
Mat rszR[2], rszP[2], rszQ;
stereoRectify(srcM1, D1, srcM2, D2, srcImageSize, Rot, T,
srcR[0], srcR[1], srcP[0], srcP[1], srcQ,
CALIB_ZERO_DISPARITY, 0);
stereoRectify(rszM1, D1, rszM2, D2, rszImageSize, Rot, T,
rszR[0], rszR[1], rszP[0], rszP[1], rszQ,
CALIB_ZERO_DISPARITY, 0);
//generate remap maps
Mat srcRmap[2], rszRmap[2];
initUndistortRectifyMap(srcM1, D1, srcR[0], srcP[0], srcImageSize, CV_32FC2, srcRmap[0], srcRmap[1]);
initUndistortRectifyMap(rszM1, D1, rszR[0], rszP[0], rszImageSize, CV_32FC2, rszRmap[0], rszRmap[1]);
// read source image
Mat image = imread("./image.png");
if (image.empty())
{
cerr << "Can not read input image;" << endl;
return 1;
}
// perform remap-resize
Mat src_result;
remap(image, src_result, srcRmap[0], srcRmap[1], CV_INTER_LINEAR);
resize(src_result, src_result, Size(), scale, scale, CV_INTER_LINEAR);
// perform resize-remap
Mat rsz_result;
resize(image, rsz_result, Size(), scale, scale, CV_INTER_LINEAR);
remap(rsz_result, rsz_result, rszRmap[0], rszRmap[1], CV_INTER_LINEAR);
//output mats;
imwrite("./src_result.png", src_result);
imwrite("./rsz_result.png", rsz_result);
return 0;
}