I'm learning cuda texture memory. Now, I got a opencv Iplimage, and I get its imagedata. Then I bind a texture to this uchar array, like below:
Iplimage *image = cvCreateImage(cvSize(width, height), IPL_DEPTH_8U, 3);
unsigned char* imageDataArray = (unsigned char*)image->imagedata;
texture<unsigned char,2,cudaReadModeElementType> tex;
cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(8, 8, 8, 0,
cudaChannelFormatKindUnsigned);
cudaArray *cuArray = NULL;
CudaSafeCall(cudaMallocArray(&cuArray,&channelDesc,width,height));
cudaMemcpy2DToArray(cuArray,0,0,imageDataArray,image->widthstep,
width * sizeof(unsigned char), height, cudaMemcpyHostToDevice);
cudaBindTextureToArray(texC1_cf,cuArray_currentFrame, channelDesc);
Now I lanch my kernel, and I want to access each pixel, every channel of that image. This is where I get confused.
I use this code to get the pixel coordinate (X,Y):
int X = (blockIdx.x*blockDim.x+threadIdx.x);
int Y = (blockIdx.y*blockDim.y+threadIdx.y);
And how can I access each channel of this (X,Y)? what's the code below return?
tex2D(tex, X, Y);
Besides this, Can you tell me how texture memory using texture to access an array, and how this transform looks like?
To bind a 3 channel OpenCV image to cudaArray texture, you have to create a cudaArray of width equal to image->width * image->nChannels, because the channels are stored interleaved by OpenCV.
cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc<unsigned char>();
cudaArray *cuArray = NULL;
CudaSafeCall(cudaMallocArray(&cuArray,&channelDesc,width * image->nChannels,height));
cudaMemcpy2DToArray(cuArray,0,0,imageDataArray,image->widthstep, width * image->nChannels * sizeof(unsigned char), height, cudaMemcpyHostToDevice);
cudaBindTextureToArray(texC1_cf,cuArray_currentFrame, channelDesc);
Now, to access each channel separately in the kernel, you just have to multiply the x index with number of channels and add the offset of desired channel like this:
unsigned char blue = tex2D(tex, (3 * X) , Y);
unsigned char green = tex2D(tex, (3 * X) + 1, Y);
unsigned char red = tex2D(tex, (3 * X) + 2, Y);
First one is blue because OpenCV stores images with channel sequence BGR.
As for the error you get when you try to access texture<uchar3,..> using tex2D; CUDA only supports creating 2D textures of 1,2 and 4 element vector types. Unfortunately, ONLY 3 is not supported which is very good for binding RGB images and is a really desirable feature.
Related
I would like to draw a red rectangle using the following function on a YUV420P frame. Following code alters the frame and I can see two black line(top and bottom) remaining black dots scattered. Any suggestions?
void draw_rectangle(uint8_t *frame, int x, int y,
int width, int height,
int img_width, int img_height)
{
cv::Mat frame_yuv;
int size[2];
Point pt1, pt2;
cv::Scalar color = Scalar(255, 0, 0);
size[0] = img_width + img_width/2;
size[1] = img_height;
frame_yuv = cv::Mat(2, size, CV_8UC1, frame);
pt1.x = x;
pt1.y = y;
pt2.x = x + width;
pt2.y = y + height;
rectangle(frame_yuv, pt1, pt2, Scalar(0, 0, 255));
}
Finally, I got my code working. Steps are given below for reference.
frame_yuv = cv::Mat(2, size, CV_8UC3, frame);
cv::Mat C(2,2, CV_8UC3, color);
cv::Mat C_yuv;
cvtColor(C, C_yuv, cv::COLOR_BGR2YUV_I420);
// Set the R, G, B values to C_yuv
// Extract the Y, U, V components to separate Mat's
// Apply rectange first on Y component
// Devide each points pt1, pt2 by 2
// Apply the rectange on U, V
No extra copy of the frame is done.
As you haven't provided any sample data, please use the file kindly provided by #zindarod with dimensions 144x176.
Here is how the YUV data look in memory:
Notice in the stream along the bottom... all the Y pixels come first. Then all the U pixels but downsampled by a factor of 4. Then all the V pixels, also downsampled by a factor of 4.
I haven't got time to write the code in OpenCV, but I can show you how to make a regular Mat out of it.
Step 1 - Extract Y channel
Take the first 144x176 bytes and put them into an 144x176 8UC1 Mat called Y.
Step 2 - Extract U channel
Skip the first 144x176 bytes and then take the next 72x88 bytes and put them into another 72x88 8UC1 Mat called U. Resize this Mat to double the width and double the height, i.e. 144x176.
Step 3 - Extract the V channel
Skip the first (144x176) + (88x72) bytes and then take the next 72x88 bytes and put them into another 72x88 8UC1 Mat called V. Resize this Mat to double the width and double the height, i.e. 144x176.
Step 4 - Merge
Take the Y, U, and V Mats and merge them into an 8UC3 Mat:
// Now merge the 3 individual channels into 3-band bad boy
auto channels = std::vector<cv::Mat>{Y, U, V};
cv::Mat ThreeBandBoy;
cv::merge(channels, ThreeBandBoy);
There is some code here that does more or less exactly what is needed for Steps 1-3.
I am reading this YUV image from file, which is YUV_I420.
fstream file;
file.open("yuv_i420.yuv", ios::in | ios::binary);
// size of image in RGB
size_t rows = 144, cols = 176;
if (!file.is_open())
stderr<<"Error opening file"<<endl;
else {
// get total size of file
auto size = file.tellg();
file.seekg(0,ios::end);
size = file.tellg() - size;
file.seekg(0,ios::beg);
char *buffer = new char[size];
// read image from file
if (file.read(buffer, size)) {
// create YUV Mat
Mat yuv_I420(rows + rows / 2, cols, CV_8UC1, buffer);
// draw a rectangle on YUV image, keep in mind that the YUV image is a
// single channel grayscale image, size is different than the BGR image
rectangle(yuv_I420, Point(10, 10), Point(50, 50), Scalar(255));
// convert to BGR to check validity
Mat bgr;
cvtColor(yuv_I420, bgr, cv::COLOR_YUV2BGR_I420);
cv::imshow("image", bgr);
cv::waitKey(0);
}
file.close();
delete[] buffer;
}
I would like to perform screenshots (or screen captures) the fastest possible.
Googling this question brings many answeers but my concern is more specific :
I am not interested in the image itself, I would like to grab in near real time the screen brightness, not the hardware one, but the image one, given that, for example, the firefox white google page gives a brighter image than a dark xterm (when both are maximzed).
To make me as clear as possible, here is one way I already managed to implement with X11 and CImg library :
Here is the header :
#include <CImg.h>
using namespace cimg_library;
#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include <X11/Xos.h>
and the core part which extract an X11 image and make a loop on very pixel :
Display *display = XOpenDisplay(NULL);
Window root = DefaultRootWindow(display);
Screen* screen = DefaultScreenOfDisplay(display);
const int W = WidthOfScreen(screen);
const int H = HeightOfScreen(screen);
XImage *image = XGetImage(display, root, 0, 0, W, H, AllPlanes, ZPixmap);
unsigned long red_count(0), green_count(0), blue_count(0), count(0);
const unsigned long red_mask = image->red_mask;
const unsigned long green_mask = image->green_mask;
const unsigned long blue_mask = image->blue_mask;
CImg<unsigned char> screenshot(W, H, 1, 3, 0);
for (int x = 0; x < W; x += pixel_stride)
for (int y = 0; y < H; y += pixel_stride)
{
unsigned long pixel = XGetPixel(image, x, y);
screenshot(x, y, 0) = (pixel & red_mask) >> 16;
screenshot(x, y, 1) = (pixel & green_mask) >> 8;
screenshot(x, y, 2) = pixel & blue_mask;
red_count += (int) screenshot(x, y, 0);
green_count += (int) screenshot(x, y, 1);
blue_count += (int) screenshot(x, y, 2);
count++;
}
As I said, I do not keep the image itself, I just try to compute an average luminance value with respective values of red, green and blue pixels.
XFree(image);
const double luminance_relative = (red_luminance * double(red_count) +
green_luminance * double(green_count) +
blue_luminance * double(blue_count))
/ (double(255) * double(count));
The underlying idea is to adjust the hardware screen brightness depending on the image luminance. In short, the whiter is the screenshot, the more the brightness can be reduced and conversely.
I want to do that because I have sensitive eyes, it usually hurts my eyes when I switch from xterm to firefox.
To do so, the hardware brightness must be adjusted in a very short time, the screenshot, that is to say, the loop on pixels must be as fast as possible.
I began to implement it with X11 methods, but I wonder if there could be faster access methods ? Which comes to the question : what is the fastest way/library to get a screenshot ?
Thanks in advance for your help.
Regards
Android:
import android.graphics.Bitmap;
public void getPixels (int[] pixels, int offset, int stride, int x, int y, int width, int height);
Bitmap bmap = source.renderCroppedGreyscaleBitmap();
int w=bmap.getWidth(),h=bmap.getHeight();
int[] pix = new int[w * h];
bmap.getPixels(pix, 0, w, 0, 0, w, h);
Returns in pixels[] a copy of the data in the bitmap.
Each value is a packed int representing a Color.
The stride parameter allows the caller to allow for gaps in the returned pixels array between rows.
For normal packed results, just pass width for the stride value.
The returned colors are non-premultiplied ARGB values.
iOS:
#implementation UIImage (Pixels)
-(unsigned char*) rgbaPixels
{
// The amount of bits per pixel, in this case we are doing RGBA so 4 byte = 32 bits
#define BITS_PER_PIXEL 32
// The amount of bits per component, in this it is the same as the bitsPerPixel divided by 4 because each component (such as Red) is only 8 bits
#define BITS_PER_COMPONENT (BITS_PER_PIXEL/4)
// The amount of bytes per pixel, in this case a pixel is made up of Red, Green, Blue and Alpha so it will be 4
#define BYTES_PER_PIXEL (BITS_PER_PIXEL/BITS_PER_COMPONENT)
// Define the colour space (in this case it's gray)
CGColorSpaceRef colourSpace = CGColorSpaceCreateDeviceRGB();
// Find out the number of bytes per row (it's just the width times the number of bytes per pixel)
size_t bytesPerRow = self.size.width * BYTES_PER_PIXEL;
// Allocate the appropriate amount of memory to hold the bitmap context
unsigned char* bitmapData = (unsigned char*) malloc(bytesPerRow*self.size.height);
// Create the bitmap context, we set the alpha to none here to tell the bitmap we don't care about alpha values
CGContextRef context = CGBitmapContextCreate(bitmapData,self.size.width,self.size.height,BITS_PER_COMPONENT,bytesPerRow,colourSpace,kCGImageAlphaFirst);//It returns null
/* We are done with the colour space now so no point in keeping it around*/
CGColorSpaceRelease(colourSpace);
// Create a CGRect to define the amount of pixels we want
CGRect rect = CGRectMake(0.0,0.0,self.size.width,self.size.height);
// Draw the bitmap context using the rectangle we just created as a bounds and the Core Graphics Image as the image source
CGContextDrawImage(context,rect,self.CGImage);
// Obtain the pixel data from the bitmap context
unsigned char* pixelData = (unsigned char*)CGBitmapContextGetData(context);
// Release the bitmap context because we are done using it
CGContextRelease(context);
//CGColorSpaceRelease(colourSpace);
return pixelData;
#undef BITS_PER_PIXEL
#undef BITS_PER_COMPONENT
}
But it can't work.
CGBitmapContextCreate(bitmapData,self.size.width,self.size.height,BITS_PER_COMPONENT,bytesPerRow,colourSpace,kCGImageAlphaFirst);
It returns NULL.
I need the same array as pix[ ] above,how can I make it?
I use OpenCV to undestort set of points after camera calibration.
The code follows.
const int npoints = 2; // number of point specified
// Points initialization.
// Only 2 ponts in this example, in real code they are read from file.
float input_points[npoints][2] = {{0,0}, {2560, 1920}};
CvMat * src = cvCreateMat(1, npoints, CV_32FC2);
CvMat * dst = cvCreateMat(1, npoints, CV_32FC2);
// fill src matrix
float * src_ptr = (float*)src->data.ptr;
for (int pi = 0; pi < npoints; ++pi) {
for (int ci = 0; ci < 2; ++ci) {
*(src_ptr + pi * 2 + ci) = input_points[pi][ci];
}
}
cvUndistortPoints(src, dst, &camera1, &distCoeffs1);
After the code above dst contains following numbers:
-8.82689655e-001 -7.05507338e-001 4.16228324e-001 3.04863811e-001
which are too small in comparison with numbers in src.
At the same time if I undistort image via the call:
cvUndistort2( srcImage, dstImage, &camera1, &dist_coeffs1 );
I receive good undistorted image which means that pixel coordinates are not modified so drastically in comparison with separate points.
How to obtain the same undistortion for specific points as for images?
Thanks.
The points should be "unnormalized" using camera matrix.
More specifically, after call of cvUndistortPoints following transformation should be also added:
double fx = CV_MAT_ELEM(camera1, double, 0, 0);
double fy = CV_MAT_ELEM(camera1, double, 1, 1);
double cx = CV_MAT_ELEM(camera1, double, 0, 2);
double cy = CV_MAT_ELEM(camera1, double, 1, 2);
float * dst_ptr = (float*)dst->data.ptr;
for (int pi = 0; pi < npoints; ++pi) {
float& px = *(dst_ptr + pi * 2);
float& py = *(dst_ptr + pi * 2 + 1);
// perform transformation.
// In fact this is equivalent to multiplication to camera matrix
px = px * fx + cx;
py = py * fy + cy;
}
More info on camera matrix at OpenCV 'Camera Calibration and 3D Reconstruction'
UPDATE:
Following C++ function call should work as well:
std::vector<cv::Point2f> inputDistortedPoints = ...
std::vector<cv::Point2f> outputUndistortedPoints;
cv::Mat cameraMatrix = ...
cv::Mat distCoeffs = ...
cv::undistortPoints(inputDistortedPoints, outputUndistortedPoints, cameraMatrix, distCoeffs, cv::noArray(), cameraMatrix);
It may be your matrix size :)
OpenCV expects a vector of points - a column or a row matrix with two channels. But because your input matrix is only 2 pts, and the number of channels is also 1, it cannot figure out what's the input, row or colum.
So, fill a longer input mat with bogus values, and keep only the first:
const int npoints = 4; // number of point specified
// Points initialization.
// Only 2 ponts in this example, in real code they are read from file.
float input_points[npoints][4] = {{0,0}, {2560, 1920}}; // the rest will be set to 0
CvMat * src = cvCreateMat(1, npoints, CV_32FC2);
CvMat * dst = cvCreateMat(1, npoints, CV_32FC2);
// fill src matrix
float * src_ptr = (float*)src->data.ptr;
for (int pi = 0; pi < npoints; ++pi) {
for (int ci = 0; ci < 2; ++ci) {
*(src_ptr + pi * 2 + ci) = input_points[pi][ci];
}
}
cvUndistortPoints(src, dst, &camera1, &distCoeffs1);
EDIT
While OpenCV specifies undistortPoints accept only 2-channel input, actually, it accepts
1-column, 2-channel, multi-row mat or (and this case is not documented)
2 column, multi-row, 1-channel mat or
multi-column, 1 row, 2-channel mat
(as seen in undistort.cpp, line 390)
But a bug inside (or lack of available info), makes it wrongly mix the second one with the third one, when the number of columns is 2. So, your data is considered a 2-column, 2-row, 1-channel.
I also reach this problems, and I take some time to research an finally understand.
Formula
You see the formula above, in the open system, distort operation is before camera matrix, so the process order is:
image_distorted ->camera_matrix -> un-distort function->camera_matrix->back to image_undistorted.
So you need a small fix to and camera1 again.
Mat eye3 = Mat::eye(3, 3, CV_64F);
cvUndistortPoints(src, dst, &camera1, &distCoeffs1, &eye3,&camera1);
Otherwise, if the last two parameters is empty, It would be project to a Normalized image coordinate.
See codes: opencv-3.4.0-src\modules\imgproc\src\undistort.cpp :297
cvUndistortPointsInternal()
I need to exchange data between FreeImage (FIBITMAP) and OpenCV format (IplImage and/or Mat). I'm fine with getting data from a FIBITMAP into an IplImage or Mat since FreeImage gives you a function FreeImage_GetScanLine which you can set the OPenCV imageData ptr equal to.
However, I'm stuck on how to do the reverse, i.e. once I have an OpenCV image, how do I get its data into a FreeImage image?
Here is a more detailed code for the conversion. There are many image data types in either library, I tried supporting most common ones. This assumes you are passing a cv::Mat as the source. FreeImage has the view perspective of lower left!
/* These are openCV types
#define CV_8U 0
#define CV_8S 1
#define CV_16U 2
#define CV_16S 3
#define CV_32S 4
#define CV_32F 5
#define CV_64F 6
*/
/* these are FI types
FIT_UNKNOWN = 0, // unknown type
FIT_BITMAP = 1, // standard image : 1-, 4-, 8-, 16-, 24-, 32-bit
FIT_UINT16 = 2, // array of unsigned short : unsigned 16-bit
FIT_INT16 = 3, // array of short : signed 16-bit
FIT_UINT32 = 4, // array of unsigned long : unsigned 32-bit
FIT_INT32 = 5, // array of long : signed 32-bit
FIT_FLOAT = 6, // array of float : 32-bit IEEE floating point
FIT_DOUBLE = 7, // array of double : 64-bit IEEE floating point
FIT_COMPLEX = 8, // array of FICOMPLEX : 2 x 64-bit IEEE floating point
FIT_RGB16 = 9, // 48-bit RGB image : 3 x 16-bit
FIT_RGBA16 = 10, // 64-bit RGBA image : 4 x 16-bit
FIT_RGBF = 11, // 96-bit RGB float image : 3 x 32-bit IEEE floating point
FIT_RGBAF = 12 // 128-bit RGBA float image : 4 x 32-bit IEEE floating point
*/
if(_dib) // get rid of the current dib.
FreeImage_Unload(_dib);
int width = src.size().width;
int height = src.size().height;
switch(src.type())
{
case CV_8U :{_dib = FreeImage_AllocateT(FIT_BITMAP,width, height, 8) ;}break; // 8 bit grayscale
case CV_8UC3:{_dib = FreeImage_AllocateT(FIT_BITMAP,width, height, 24);}break; // 24 bit RGB
case CV_16U :{_dib = FreeImage_AllocateT(FIT_UINT16,width, height, 16);}break; // 16 bit grayscale
case CV_16S :{_dib = FreeImage_AllocateT(FIT_INT16 ,width, height, 16);}break;
case CV_32S :{_dib = FreeImage_AllocateT(FIT_INT32 ,width, height, 32);}break;
case CV_32F :{_dib = FreeImage_AllocateT(FIT_FLOAT ,width, height, 32);}break;
case CV_64F :{_dib = FreeImage_AllocateT(FIT_DOUBLE,width, height, 32);}break;
default:ASSERT(FALSE);
}
if(_dib==NULL)
return FALSE;
int srcRowBytes = width * src.elemSize();
for (int ih=0;ih<height;ih++)
{
BYTE* ptr2Line = FreeImage_GetScanLine(_dib,(height-1)-ih);
memcpy(ptr2Line,src.ptr(ih),srcRowBytes);
}
_bHasChanged = TRUE;
return TRUE;
You have to be a little careful just copying the data pointer, a lot of image formats have padding to start each new line on eg a 4byte boundary.
If your image has a GetScanLine() function then it's probably safer to make an empty plImage*/cv::Mat and memcopy each row with the pointer returned from GetScanLine() and the .ptr() member of cv::MAt
cv::Mat &src
int srcRowBytes = width * src.elemSize();
for (int ih=0;ih<height;ih++) {
memcpy(dest.pointer_to_row(ih),src.ptr(ih),srcRowBytes);
}
Well, if you don't mind the copy, you can just create an IplImage*/cv::Mat header for the FIBITMAP and then copy (using the opencv function), like this:
cv::Mat src; // your source image
FIBITMAP whatever; // allocate space for your FIBITMAP here
cv::Mat wrapper(height, width, CV_8UC3, ptr_from_FIBITMAP, step);
src.copyTo(wrapper);