yv12 to rgb using glsl in iOS ,result image attached - ios

Question

referred to this question
i convert the yv12 frame data to rgb data using glsl shader,the raw image below:
but the result image is not same with the former,attached below:
following is my code for uploading the three planar data to textures:
- (GLuint) textureY: (Byte*)imageData
widthType: (int) width
heightType: (int) height
{
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
}
- (GLuint) textureU: (Byte*)imageData
widthType: (int) width
heightType: (int) height
{
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
}
- (GLuint) textureV: (Byte*)imageData
widthType: (int) width
heightType: (int) height
{
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
}
- (void) readYUVFile
{
NSString *file = [[NSBundle mainBundle] pathForResource:#"video" ofType:#"yv12"];
NSLog(#"%#",file);
NSData* fileData = [NSData dataWithContentsOfFile:file];
//NSLog(#"%#",[fileData description]);
NSInteger width = 352;
NSInteger height = 288;
NSInteger uv_width = width / 2;
NSInteger uv_height = height / 2;
NSInteger dataSize = [fileData length];
NSLog(#"%i\n",dataSize);
GLint nYsize = width * height;
GLint nUVsize = uv_width * uv_height;
GLint nCbOffSet = nYsize;
GLint nCrOffSet = nCbOffSet + nUVsize;
Byte *spriteData = (Byte *)malloc(dataSize);
[fileData getBytes:spriteData length:dataSize];
Byte* uData = spriteData + nCbOffSet;
//NSLog(#"%#\n",[[NSData dataWithBytes:uData length:nUVsize] description]);
Byte* vData = spriteData + nCrOffSet;
//NSLog(#"%#\n",[[NSData dataWithBytes:vData length:nUVsize] description]);
/**
Byte *YPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<nYsize; i++) {
YPlanarData[i]= spriteData[i];
}
Byte *UPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<height; i++) {
for (int j=0; j<width; j++) {
int numInUVsize = (i/2)*uv_width+j/2;
UPlanarData[i*width+j]=uData[numInUVsize];
}
}
Byte *VPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<height; i++) {
for (int j=0; j<width; j++) {
int numInUVsize = (i/2)*uv_width+j/2;
VPlanarData[i*width+j]=vData[numInUVsize];
}
}
**/
_YPlanarTexture = [self textureY:spriteData widthType:width heightType:height];
_UPlanarTexture = [self textureU:uData widthType:uv_width heightType:uv_height];
_VPlanarTexture = [self textureV:vData widthType:uv_width heightType:uv_height];
free(spriteData);
}
and my fragment shaders code:
precision highp float;
uniform sampler2D SamplerY;
uniform sampler2D SamplerU;
uniform sampler2D SamplerV;
varying highp vec2 coordinate;
void main()
{
highp vec3 yuv,yuv1;
highp vec3 rgb;
yuv.x = texture2D(SamplerY, coordinate).r;
yuv.y = texture2D(SamplerU, coordinate).r-0.5;
yuv.z = texture2D(SamplerV, coordinate).r-0.5 ;
rgb = mat3( 1, 1, 1,
0, -.34414, 1.772,
1.402, -.71414, 0) * yuv;
gl_FragColor = vec4(rgb, 1);
}
my confusion is the conversion formula
while i using this formula directly converting the yv12 data to rgb24,and draw a image with the
CGImageCreate(iwidth,
iheight,
8,
24,
iwidth*3,
colorSpace,
bitmapInfo,
provider,
NULL,
NO,
kCGRenderingIntentDefault);
the result image is correct.
but using the shader (for the direct transform approach running on iOS device is dump) turns to this problem ,i've tried some tricks(expand the UV planers to (2*uv_width)*2(uv_height) rectangle and then upload the texture),but failed in the same more red image.
How to resolve this issue?
attached with my whole glView.m code:
#import "OpenGLView.h"
typedef struct {
float Position[3];
float TexCoord[2];
} Vertex;
const Vertex Vertices[] = {
{{1, -1, 0},{1,1}},
{{1, 1, 0},{1,0}},
{{-1, 1, 0},{0,0}},
{{-1, -1, 0},{0,1}}
};
const GLubyte Indices[] = {
0, 1, 2,
2, 3, 0
};
#interface OpenGLView ()
- (void)setupLayer;
- (void)setupContext;
- (void)setupRenderBuffer;
- (void)setupFrameBuffer;
- (void)render;
- (GLuint)compileShader:(NSString*)shaderName withType:(GLenum)shaderType;
- (void)setupVBOs;
- (void)compileShaders;
- (void) readYUVFile;
#end
#implementation OpenGLView
- (void)setupVBOs {
GLuint vertexBuffer;
glGenBuffers(1, &vertexBuffer);
glBindBuffer(GL_ARRAY_BUFFER, vertexBuffer);
glBufferData(GL_ARRAY_BUFFER, sizeof(Vertices), Vertices, GL_STATIC_DRAW);
GLuint indexBuffer;
glGenBuffers(1, &indexBuffer);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, indexBuffer);
glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(Indices), Indices, GL_STATIC_DRAW);
}
- (id)initWithFrame:(CGRect)frame
{
self = [super initWithFrame:frame];
if (self) {
// Initialization code[]
self.backgroundColor = [UIColor redColor];
[self setupLayer];
[self setupContext];
[self setupRenderBuffer];
[self setupFrameBuffer];
[self setupVBOs];
[self compileShaders];
[self readYUVFile];
[self render];
}
return self;
}
+ (Class)layerClass{
return [CAEAGLLayer class];
}
-(void)setupLayer{
_eaglLayer = (CAEAGLLayer *)self.layer;
_eaglLayer.opaque = YES;
}
- (void)setupContext{
EAGLRenderingAPI api = kEAGLRenderingAPIOpenGLES2;
_context = [[[EAGLContext alloc] initWithAPI:api] autorelease];
if (!_context) {
NSLog(#"Failed to initialize OpenGLES 2.0 context");
exit(1);
}
if (![EAGLContext setCurrentContext:_context]) {
NSLog(#"Failed to set current OpenGL context");
exit(1);
}
}
- (void)setupRenderBuffer {
glGenRenderbuffers(1, &_colorRenderBuffer);
glBindRenderbuffer(GL_RENDERBUFFER, _colorRenderBuffer);
[_context renderbufferStorage:GL_RENDERBUFFER fromDrawable:_eaglLayer];
}
- (void)setupFrameBuffer {
GLuint framebuffer;
glGenFramebuffers(1, &framebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_RENDERBUFFER, _colorRenderBuffer);
}
- (GLuint) textureY: (Byte*)imageData
widthType: (int) width
heightType: (int) height
{
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
}
- (GLuint) textureU: (Byte*)imageData
widthType: (int) width
heightType: (int) height
{
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_RED_EXT, width, height, 0, GL_RED_EXT, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
}
- (GLuint) textureV: (Byte*)imageData
widthType: (int) width
heightType: (int) height
{
GLuint texName;
glGenTextures( 1, &texName );
glBindTexture(GL_TEXTURE_2D, texName);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri( GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D( GL_TEXTURE_2D, 0, GL_RED_EXT, width, height, 0, GL_RED_EXT, GL_UNSIGNED_BYTE, imageData );
//free(imageData);
return texName;
}
- (void) readYUVFile
{
NSString *file = [[NSBundle mainBundle] pathForResource:#"video" ofType:#"yv12"];
NSLog(#"%#",file);
NSData* fileData = [NSData dataWithContentsOfFile:file];
//NSLog(#"%#",[fileData description]);
NSInteger width = 352;
NSInteger height = 288;
NSInteger uv_width = width / 2;
NSInteger uv_height = height / 2;
NSInteger dataSize = [fileData length];
NSLog(#"%i\n",dataSize);
GLint nYsize = width * height;
GLint nUVsize = uv_width * uv_height;
GLint nCbOffSet = nYsize;
GLint nCrOffSet = nCbOffSet + nUVsize;
Byte *spriteData = (Byte *)malloc(dataSize);
[fileData getBytes:spriteData length:dataSize];
Byte* uData = spriteData + nCbOffSet;
//NSLog(#"%#\n",[[NSData dataWithBytes:uData length:nUVsize] description]);
Byte* vData = spriteData + nCrOffSet;
//NSLog(#"%#\n",[[NSData dataWithBytes:vData length:nUVsize] description]);
Byte *YPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<nYsize; i++) {
YPlanarData[i]= spriteData[i];
}
Byte *UPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<height; i++) {
for (int j=0; j<width; j++) {
int numInUVsize = (i/2)*uv_width+j/2;
UPlanarData[i*width+j]=uData[numInUVsize];
}
}
Byte *VPlanarData = (Byte *)malloc(nYsize);
for (int i=0; i<height; i++) {
for (int j=0; j<width; j++) {
int numInUVsize = (i/2)*uv_width+j/2;
VPlanarData[i*width+j]=vData[numInUVsize];
}
}
_YPlanarTexture = [self textureY:YPlanarData widthType:width heightType:height];
_UPlanarTexture = [self textureU:UPlanarData widthType:width heightType:height];
_VPlanarTexture = [self textureV:VPlanarData widthType:width heightType:height];
free(spriteData);
}
- (void)render {
glClearColor(0,0,0 , 1.0);
glClear(GL_COLOR_BUFFER_BIT);
// 1
glViewport(0, 200, self.frame.size.width, 558);
// 2
glVertexAttribPointer(_positionSlot, 3, GL_FLOAT, GL_FALSE,
sizeof(Vertex), 0);
glVertexAttribPointer(_texCoordSlot, 2, GL_FLOAT, GL_FALSE,
sizeof(Vertex), (GLvoid*) (sizeof(float) *3));
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, _YPlanarTexture);
glUniform1i(_textureUniformY, 0);
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, _UPlanarTexture);
glUniform1i(_textureUniformU, 1);
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, _VPlanarTexture);
glUniform1i(_textureUniformV, 2);
// 3
glDrawElements(GL_TRIANGLES, sizeof(Indices)/sizeof(Indices[0]),
GL_UNSIGNED_BYTE, 0);
[_context presentRenderbuffer:GL_RENDERBUFFER];
}
/*
// Only override drawRect: if you perform custom drawing.
// An empty implementation adversely affects performance during animation.
- (void)drawRect:(CGRect)rect
{
// Drawing code
}
*/
- (GLuint)compileShader:(NSString*)shaderName withType:(GLenum)shaderType {
// 1
NSString* shaderPath = [[NSBundle mainBundle] pathForResource:shaderName
ofType:#"glsl"];
NSError* error;
NSString* shaderString = [NSString stringWithContentsOfFile:shaderPath
encoding:NSUTF8StringEncoding error:&error];
if (!shaderString) {
NSLog(#"Error loading shader: %#", error.localizedDescription);
exit(1);
}
// 2
GLuint shaderHandle = glCreateShader(shaderType);
// 3
const char* shaderStringUTF8 = [shaderString UTF8String];
int shaderStringLength = [shaderString length];
glShaderSource(shaderHandle, 1, &shaderStringUTF8, &shaderStringLength);
// 4
glCompileShader(shaderHandle);
// 5
GLint compileSuccess;
glGetShaderiv(shaderHandle, GL_COMPILE_STATUS, &compileSuccess);
if (compileSuccess == GL_FALSE) {
GLchar messages[256];
glGetShaderInfoLog(shaderHandle, sizeof(messages), 0, &messages[0]);
NSString *messageString = [NSString stringWithUTF8String:messages];
NSLog(#"%#", messageString);
exit(1);
}
return shaderHandle;
}
- (void)compileShaders {
// 1
GLuint vertexShader = [self compileShader:#"SimpleVertex"
withType:GL_VERTEX_SHADER];
GLuint fragmentShader = [self compileShader:#"SimpleFragment"
withType:GL_FRAGMENT_SHADER];
// 2
GLuint programHandle = glCreateProgram();
glAttachShader(programHandle, vertexShader);
glAttachShader(programHandle, fragmentShader);
glLinkProgram(programHandle);
// 3
GLint linkSuccess;
glGetProgramiv(programHandle, GL_LINK_STATUS, &linkSuccess);
if (linkSuccess == GL_FALSE) {
GLchar messages[256];
glGetProgramInfoLog(programHandle, sizeof(messages), 0, &messages[0]);
NSString *messageString = [NSString stringWithUTF8String:messages];
NSLog(#"%#", messageString);
exit(1);
}
// 4
glUseProgram(programHandle);
// 5
_positionSlot = glGetAttribLocation(programHandle, "position");
glEnableVertexAttribArray(_positionSlot);
_texCoordSlot = glGetAttribLocation(programHandle, "textureCoordinate");
glEnableVertexAttribArray(_texCoordSlot);
_YPlanarTexture = glGetUniformLocation(programHandle, "SamplerY");
_UPlanarTexture = glGetUniformLocation(programHandle, "SamplerU");
_VPlanarTexture = glGetUniformLocation(programHandle, "SamplerV");
}
#end

Answer 1

my foolish mistake,the three uploading-texture process and fragment shader are correct ,but the following code are not compatible:
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, _YPlanarTexture);
glUniform1i(_textureUniformY, 0);
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, _UPlanarTexture);
glUniform1i(_textureUniformU, 1);
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, _VPlanarTexture);
glUniform1i(_textureUniformV, 2);
and below:
_YPlanarTexture = glGetUniformLocation(programHandle, "SamplerY");
_UPlanarTexture = glGetUniformLocation(programHandle, "SamplerU");
_VPlanarTexture = glGetUniformLocation(programHandle, "SamplerV");
so replace with this:
_textureUniformY = glGetUniformLocation(programHandle, "SamplerY");
_textureUniformU = glGetUniformLocation(programHandle, "SamplerU");
_textureUniformV = glGetUniformLocation(programHandle, "SamplerV");
then it will do right thing .