图像拼接融合 #C++ #OpenCV

imagefusion.cpp

#include "test.h"
#include <iostream>
#include <opencv2/opencv.hpp>
#include "opencv2/stitching.hpp"
#include <opencv2/xfeatures2d.hpp>   
#include "opencv2/xfeatures2d/cuda.hpp"
#include <opencv2/cudaimgproc.hpp>   
#include "opencv2/features2d.hpp"
#include "opencv2/cudafeatures2d.hpp"

using namespace std;
using namespace cv;

inline void ImShow(string name, cv::Mat img) {
	int n = 4;
	cv::namedWindow(name, WINDOW_NORMAL);
	cv::resizeWindow(name, cv::Size(img.cols / n, img.rows / n));
	cv::imshow(name, img);
}

//stitcher->stitch---------------------------------------------------------------------------
int stitch() {
	vector<Mat> imgs;
	Mat img1 = imread("0.jpg");
	Mat img2 = imread("1.jpg");
	Mat img3 = imread("2.jpg");

	ImShow("p1", img1);
	ImShow("p2", img2);
	ImShow("p3", img3);

	imgs.push_back(img1);
	imgs.push_back(img2);
	imgs.push_back(img3);

	cv::Ptr<Stitcher> stitcher = Stitcher::create(Stitcher::Mode::PANORAMA);
	// 使用stitch函数进行拼接
	Mat dst1;

	Stitcher::Status status = stitcher->stitch(imgs, dst1);
	if (status != Stitcher::OK)
	{
		cout << "Can't stitch images, error code = " << int(status) << endl;
		return -1;
	}
	//imgs.erase(imgs.begin());
	//Mat dst2;
	//imgs.push_back(img3);
	//status = stitcher->stitch(imgs, dst2);

	//if (status != Stitcher::OK)
	//{
	//	cout << "Can't stitch images, error code = " << int(status) << endl;
	//	return -1;
	//}
	imwrite("dst1.jpg", dst1);
	//imwrite("dst2.jpg", dst2);

	ImShow("dst1", dst1);
	//ImShow("dst2", dst2);
	waitKey(0);

}

//Speeded Up Robust Features加速鲁棒特征------------------------------------------------------------

struct four_corners_t
{
	Point2f left_top;
	Point2f left_bottom;
	Point2f right_top;
	Point2f right_bottom;
};

four_corners_t corners; //配准图的四个顶点坐标

//计算配准图的四个顶点坐标
void CalcCorners(const Mat& H, const Mat& src)
{
	//左上角(0,0,1)
	double v2[] = { 0, 0, 1 };
	double v1[3];//变换后的坐标值
	Mat V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
	Mat V1 = Mat(3, 1, CV_64FC1, v1);  //列向量
	V1 = H * V2;
	cout << "V2: " << V2 << endl;
	cout << "V1: " << V1 << endl;
	corners.left_top.x = v1[0] / v1[2];
	corners.left_top.y = v1[1] / v1[2];

	//左下角(0,src.rows,1)
	v2[0] = 0;
	v2[1] = src.rows;
	v2[2] = 1;
	V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
	V1 = Mat(3, 1, CV_64FC1, v1);  //列向量
	V1 = H * V2;
	corners.left_bottom.x = v1[0] / v1[2];
	corners.left_bottom.y = v1[1] / v1[2];

	//右上角(src.cols,0,1)
	v2[0] = src.cols;
	v2[1] = 0;
	v2[2] = 1;
	V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
	V1 = Mat(3, 1, CV_64FC1, v1);  //列向量
	V1 = H * V2;
	corners.right_top.x = v1[0] / v1[2];
	corners.right_top.y = v1[1] / v1[2];

	//右下角(src.cols,src.rows,1)
	v2[0] = src.cols;
	v2[1] = src.rows;
	v2[2] = 1;
	V2 = Mat(3, 1, CV_64FC1, v2);  //列向量
	V1 = Mat(3, 1, CV_64FC1, v1);  //列向量
	V1 = H * V2;
	corners.right_bottom.x = v1[0] / v1[2];
	corners.right_bottom.y = v1[1] / v1[2];
}

//优化两图的连接处，使得拼接自然
void OptimizeSeam(Mat& imageL, Mat& imageTransformR, Mat& dst)
{
	int left_border_col = std::min(corners.left_top.x, corners.left_bottom.x);//开始位置，即重叠区域的左边界  

	double processWidth = imageL.cols - left_border_col;//重叠区域的宽度  
	double alpha = 1;//imageL中像素的权重  

	CV_Assert(dst.channels() == 1 || 3);
	if (dst.channels() == 1)
	{
		for (int i = 0; i < imageL.rows; i++)//从上到下从左到右扫描
		{
			uchar* pImgL = imageL.ptr<uchar>(i);  //获取第i行的首地址
			uchar* pTran = imageTransformR.ptr<uchar>(i);
			uchar* pDst = dst.ptr<uchar>(i);
			for (int j = left_border_col; j < imageL.cols; j++)
			{
				//如果遇到imageTransformR中无像素的黑点，则完全拷贝imageL中的数据
				if (pTran[j] == 0)
				{
					alpha = 1;
				}
				else
				{
					//imageL中像素的权重，与当前处理点距重叠区域左边界的距离成正比(距离越远alpha越小) 
					alpha = (processWidth - (j - left_border_col)) / processWidth;
				}
				pDst[j] = pImgL[j] * alpha + pTran[j] * (1 - alpha);
			}
		}
	}
	else if (dst.channels() == 3)
	{
		for (int i = 0; i < imageL.rows; i++)
		{
			uchar* pImgL = imageL.ptr<uchar>(i);  //获取第i行的首地址
			uchar* pTran = imageTransformR.ptr<uchar>(i);
			uchar* pDst = dst.ptr<uchar>(i);
			for (int j = left_border_col; j < imageL.cols; j++)
			{
				//如果遇到imageTransformR中无像素的黑点，则完全拷贝imageL中的数据
				if (pTran[j * 3] == 0 && pTran[j * 3 + 1] == 0 && pTran[j * 3 + 2] == 0)
				{
					alpha = 1;
				}
				else
				{
					//imageL中像素的权重，与当前处理点距重叠区域左边界的距离成正比(距离越远alpha越小) 
					alpha = (processWidth - (j - left_border_col)) / processWidth;
				}

				pDst[j * 3] = pImgL[j * 3] * alpha + pTran[j * 3] * (1 - alpha);
				pDst[j * 3 + 1] = pImgL[j * 3 + 1] * alpha + pTran[j * 3 + 1] * (1 - alpha);
				pDst[j * 3 + 2] = pImgL[j * 3 + 2] * alpha + pTran[j * 3 + 2] * (1 - alpha);
			}
		}
	}
}

int SURF()
{
	Mat imageL = imread("0.jpg");    //左图
	Mat imageR = imread("1.jpg");    //右图

	if (imageL.empty() || imageR.empty())
	{
		cout << "imread error!\n" << endl;
		return -1;
	}

	ImShow("左图", imageL);
	ImShow("右图", imageR);

	//转为灰度图  
	Mat grayL, grayR;
	cvtColor(imageL, grayL, COLOR_RGB2GRAY);
	cvtColor(imageR, grayR, COLOR_RGB2GRAY);

	//提取特征点,
	Ptr<xfeatures2d::SurfFeatureDetector> Detector;

	double hessianThreshold = 3000;//阀值越大，得到的特征点的鲁棒性越好。在处理场景简单的图像时，其阀值可以适当的调低。在复杂的图像中，图像经旋转或者模糊后特征点变化的数量较大，测试需要适当提高阀值。    
	int nOctaves = 4;
	int nOctaveLayers = 4;
	bool isExtended = false;
	bool isUpright = false;
	Ptr<xfeatures2d::SURF> m_surf_det = Detector->create(hessianThreshold, nOctaves, nOctaveLayers, isExtended, isUpright);
	vector<KeyPoint> keyPointR, keyPointL;
	Mat imageDescR, imageDescL;
	m_surf_det->detectAndCompute(grayL, Mat(), keyPointL, imageDescL);
	m_surf_det->detectAndCompute(grayR, Mat(), keyPointR, imageDescR);

	FlannBasedMatcher matcher;//基于Flann的描述符匹配器
	vector<vector<DMatch>> matchePoints;//DMatch用于保存匹配结果
	vector<DMatch> GoodMatchePoints;

	vector<Mat> train_desc(1, imageDescR);
	matcher.add(train_desc); //添加描述符以训练CPU或GPU描述符集合
	matcher.train();//训练描述符匹配器
	matcher.knnMatch(imageDescL, matchePoints, 2);//K最近邻匹配-对每个匹配返回两个最佳匹配
	cout << "total match points: " << matchePoints.size() << endl;

	// Lowe's algorithm,获取优秀匹配点
	for (int i = 0; i < matchePoints.size(); i++)
	{
		if (matchePoints[i][0].distance < 0.4 * matchePoints[i][1].distance)
		{
			GoodMatchePoints.push_back(matchePoints[i][0]);
		}
	}

	//画出匹配图
	Mat first_match;
	cv::drawMatches(imageL, keyPointL, imageR, keyPointR, GoodMatchePoints, first_match,
		Scalar::all(-1), Scalar::all(-1), std::vector<char>(), DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);
	ImShow("first_match ", first_match);

	//优秀匹配点
	vector<Point2f> imagePointsL, imagePointsR;
	for (int i = 0; i < GoodMatchePoints.size(); i++)
	{
		imagePointsR.push_back(keyPointL[GoodMatchePoints[i].queryIdx].pt);
		imagePointsL.push_back(keyPointR[GoodMatchePoints[i].trainIdx].pt);
	}
	//计算单应矩阵H，并细化匹配结果
	//射影变换:获取图像1到图像2的投影映射矩阵 尺寸为3*3  
	Mat H = cv::findHomography(imagePointsL, imagePointsR, cv::RHO);
	if (H.empty())
	{
		cout << "findHomography error!\n" << endl;
		return -1;
	}
	cout << "变换矩阵为：\n" << H << endl; //输出映射矩阵      

   //计算配准图的四个顶点坐标
	CalcCorners(H, imageR);
	cout << "left_top:" << corners.left_top << endl;
	cout << "left_bottom:" << corners.left_bottom << endl;
	cout << "right_top:" << corners.right_top << endl;
	cout << "right_bottom:" << corners.right_bottom << endl;

	//将透视变换应用于图像
	Mat imageTransformR;
	cv::warpPerspective(imageR, imageTransformR, H,
		Size(MAX(corners.right_top.x, corners.right_bottom.x), imageL.rows),
		INTER_LINEAR, BORDER_REPLICATE);
	ImShow("R直接经过透视矩阵变换", imageTransformR);

	//图像拼接
	int dst_width = imageTransformR.cols;  //取最右点的长度为拼接图的长度
	int dst_height = imageL.rows;
	Mat dst(dst_height, dst_width, CV_8UC3);
	imageTransformR.copyTo(dst(Rect(0, 0, imageTransformR.cols, imageTransformR.rows)));
	imageL.copyTo(dst(Rect(0, 0, imageL.cols, imageL.rows)));
	ImShow("b_dst", dst);

	OptimizeSeam(imageL, imageTransformR, dst);	//优化拼接处
	ImShow("dst", dst);
	imwrite("dst.jpg", dst);
	waitKey();
	return 0;
}

int CUDA_SURF()
{
	cv::cuda::printShortCudaDeviceInfo(cv::cuda::getDevice());

	cuda::Stream streamL, streamR;

	//读取原图
	Mat imageMatL, imageMatR;
	imageMatL = imread("0.jpg");    //左图
	imageMatR = imread("1.jpg");    //右图
	ImShow("imageMatL", imageMatL);
	ImShow("imageMatR", imageMatR);

	//直接读取灰度图
	Mat grayMatL, grayMatR;
	cuda::GpuMat grayGpuMatL, grayGpuMatR;
	grayMatL = imread("0.jpg", IMREAD_GRAYSCALE);//左灰度图
	grayMatR = imread("1.jpg", IMREAD_GRAYSCALE);//右灰度图
	CV_Assert(!grayMatL.empty());
	CV_Assert(!grayMatR.empty());

	grayGpuMatL.upload(grayMatL, streamL);
	grayGpuMatR.upload(grayMatR, streamR);

	double hessianThreshold = 3000;//阀值越大，得到的特征点的鲁棒性越好。在处理场景简单的图像时，其阀值可以适当的调低。在复杂的图像中，图像经旋转或者模糊后特征点变化的数量较大，测试需要适当提高阀值。    
	int nOctaves = 4;
	int nOctaveLayers = 4;
	bool isExtended = false;
	float keypointsRatio = 0.01f;
	bool isUpright = false;
	cuda::SURF_CUDA surf_cuda(hessianThreshold, nOctaves, nOctaveLayers, isExtended, keypointsRatio, isUpright);

	// 检测关键点 & 计算描述符
	cuda::GpuMat keypointsGpuMatL, keypointsGpuMatR;
	cuda::GpuMat descriptorsGpuMatL, descriptorsGpuMatR;
	surf_cuda(grayGpuMatL, cuda::GpuMat(), keypointsGpuMatL, descriptorsGpuMatL);
	surf_cuda(grayGpuMatR, cuda::GpuMat(), keypointsGpuMatR, descriptorsGpuMatR);

	cout << "FOUND " << keypointsGpuMatL.cols << " keypoints on first image" << endl;
	cout << "FOUND " << keypointsGpuMatR.cols << " keypoints on second image" << endl;

	// 下载结果
	vector<KeyPoint> keypointsL, keypointsR;
	vector<float> descriptorsL, descriptorsR;
	surf_cuda.downloadKeypoints(keypointsGpuMatL, keypointsL);
	surf_cuda.downloadKeypoints(keypointsGpuMatR, keypointsR);
	surf_cuda.downloadDescriptors(descriptorsGpuMatL, descriptorsL);
	surf_cuda.downloadDescriptors(descriptorsGpuMatR, descriptorsR);

	// 匹配描述符
	Ptr<cv::cuda::DescriptorMatcher> matcher = cv::cuda::DescriptorMatcher::createBFMatcher(surf_cuda.defaultNorm());//暴力描述符匹配器
	vector<vector<DMatch>> matchePoints;
	matcher->knnMatch(descriptorsGpuMatL, descriptorsGpuMatR, matchePoints, 2, cuda::GpuMat(), false);

	// Lowe's algorithm,获取优秀匹配点
	vector<DMatch> GoodMatchePoints;
	for (int i = 0; i < matchePoints.size(); i++)
	{
		if (matchePoints[i][0].distance < 0.4 * matchePoints[i][1].distance)
		{
			GoodMatchePoints.push_back(matchePoints[i][0]);
		}
	}

	// 画出匹配结果
	Mat img_matches;
	drawMatches(imageMatL, keypointsL, imageMatR, keypointsR, GoodMatchePoints, img_matches,
		Scalar::all(-1), Scalar::all(-1), std::vector<char>(), DrawMatchesFlags::NOT_DRAW_SINGLE_POINTS);
	ImShow("img_matches", img_matches);

	//优秀匹配点
	vector<Point2f> imagePointsL, imagePointsR;
	for (int i = 0; i < GoodMatchePoints.size(); i++)
	{
		imagePointsL.push_back(keypointsR[GoodMatchePoints[i].trainIdx].pt);
		imagePointsR.push_back(keypointsL[GoodMatchePoints[i].queryIdx].pt);
	}

	//计算单应矩阵H，并细化匹配结果
	//射影变换:获取图像1到图像2的投影映射矩阵 尺寸为3*3  
	Mat H = cv::findHomography(imagePointsL, imagePointsR, cv::RHO);
	if (H.empty())
	{
		cout << "findHomography error!\n" << endl;
		return -1;
	}
	cout << "变换矩阵为：\n" << H << endl; //输出映射矩阵      

   //计算配准图的四个顶点坐标
	CalcCorners(H, grayMatR);
	cout << "left_top:" << corners.left_top << endl;
	cout << "left_bottom:" << corners.left_bottom << endl;
	cout << "right_top:" << corners.right_top << endl;
	cout << "right_bottom:" << corners.right_bottom << endl;

	//将透视变换应用于图像
	Mat imageTransformR;
	cv::warpPerspective(imageMatR, imageTransformR, H,
		Size(MAX(corners.right_top.x, corners.right_bottom.x), imageMatL.rows),
		INTER_LINEAR, BORDER_REPLICATE);
	ImShow("R直接经过透视矩阵变换", imageTransformR);

	//图像拼接
	int dst_width = imageTransformR.cols;  //取最右点的长度为拼接图的长度
	int dst_height = grayMatL.rows;
	Mat dst(dst_height, dst_width, imageTransformR.channels()==3?CV_8UC3: CV_8UC1);
	imageTransformR.copyTo(dst(Rect(0, 0, imageTransformR.cols, imageTransformR.rows)));
	imageMatL.copyTo(dst(Rect(0, 0, imageMatL.cols, imageMatL.rows)));
	ImShow("b_dst", dst);

	OptimizeSeam(imageMatL, imageTransformR, dst);	//优化拼接处
	ImShow("dst", dst);
	imwrite("dst.jpg", dst);
	waitKey(0);
}

int main(int argc, char * argv[])
{
	//stitch();
  
	//int result=SURF();
	//if (result!=-1)
	//{
	//	cout << "error" << endl;
	//}

	int result = CUDA_SURF();
	if (result != -1)
	{
		cout << "error" << endl;
	}
}