Could anyone please help me to port this DX9 code to DX10 (not 11!). It is a simple drawing to texture.
// create the texture
m_pDevice->CreateTexture(m_width, m_height, 1, 0, D3DFMT_X8R8G8B8, D3DPOOL_DEFAULT, &m_pTexture, NULL);
// get its surface (target surface)
m_pTexture->GetSurfaceLevel(0, &m_pSurface);
// create the second surface (source surface)
m_pDevice->CreateOffscreenPlainSurface(m_width, m_height, D3DFMT_X8R8G8B8, D3DPOOL_SYSTEMMEM, &m_pSurfaceMem, NULL);
// drawing to Direct3D, this is called in loop
...
m_pSurfaceMem->GetDC(&hdc);
HRGN hrgn = CreateRectRgnIndirect(lprc);
SelectClipRgn(hdc, hrgn);
RECTL clipRect = { 0, 0, m_width, m_height };
// This draws to the source surface
m_pViewObject->Draw(DVASPECT_TRANSPARENT, 1, NULL, NULL, NULL, hdc, &clipRect, &clipRect, NULL, 0);
m_pSurfaceMem->ReleaseDC(hdc);
DeleteObject(hrgn);
POINT pt = {lprc->left, lprc->top};
// Now update the target surface
m_pDevice->UpdateSurface(m_pSurfaceMem, lprc, m_pSurface, &pt);
Error handling and the likes can be omitted for simplicity.
Related
I want to render a cube on a picture like in this tutorial. Problem is that it renders only the picture and the cube doesn't render. Can you help me ? Thankyou
m_spriteBatch->Begin();
m_spriteBatch->Draw(m_background.Get(), m_fullscreenRect);
//
// Clear the back buffer
//
g_pImmediateContext->ClearRenderTargetView( g_pRenderTargetView, Colors::MidnightBlue );
g_pImmediateContext->ClearDepthStencilView(g_pDepthStencilView, D3D11_CLEAR_DEPTH, 1.0f, 0);
g_pImmediateContext->OMSetRenderTargets(1, &g_pRenderTargetView, g_pDepthStencilView);
//
// Update variables
//
ConstantBuffer cb;
cb.mWorld = XMMatrixTranspose( g_World );
cb.mView = XMMatrixTranspose( g_View );
cb.mProjection = XMMatrixTranspose( g_Projection );
g_pImmediateContext->UpdateSubresource( g_pConstantBuffer, 0, nullptr, &cb, 0, 0 );
//
// Renders a triangle
//
g_pImmediateContext->VSSetShader( g_pVertexShader, nullptr, 0 );
g_pImmediateContext->VSSetConstantBuffers( 0, 1, &g_pConstantBuffer );
g_pImmediateContext->PSSetShader( g_pPixelShader, nullptr, 0 );
g_pImmediateContext->DrawIndexed( 36, 0, 0 ); // 36 vertices needed for 12 triangles in a triangle list
//
// Present our back buffer to our front buffer
//
m_spriteBatch->End();
g_pSwapChain->Present( 0, 0 );
SpriteBatch batches up draws for performance, so it's likely being drawn after the cube draw. If you want to make sure the sprite background draws first, then you need to call End before you submit your cube. You also need to call Begin after you set up the render target:
// Clear
g_pImmediateContext->ClearDepthStencilView(g_pDepthStencilView, D3D11_CLEAR_DEPTH, 1.0f, 0);
g_pImmediateContext->OMSetRenderTargets(1, &g_pRenderTargetView, g_pDepthStencilView);
// Draw background image
m_spriteBatch->Begin();
m_spriteBatch->Draw(m_background.Get(), m_fullscreenRect);
m_spriteBatch->End();
// Draw objects
context->OMSetBlendState(…);
context->OMSetDepthStencilState(…);
context->IASetInputLayout(…);
context->IASetVertexBuffers(…);
context->IASetIndexBuffer(…);
context->IASetPrimitiveTopology(…);
You can omit the ClearRenderTargetView if the m_background texture covers the whole screen.
For more on how SpriteBatch draw order and batching works, see the wiki.
Based on this answer by #ChuckWalbourn I fixed the problem.
g_pImmediateContext->ClearRenderTargetView( g_pRenderTargetView, Colors::MidnightBlue );
g_pImmediateContext->ClearDepthStencilView(g_pDepthStencilView, D3D11_CLEAR_DEPTH |
D3D11_CLEAR_STENCIL, 1.0f, 0);
m_spriteBatch->Begin();
m_spriteBatch->Draw(m_background.Get(), m_fullscreenRect);
m_spriteBatch->End();
states = std::make_unique<CommonStates>(g_pd3dDevice);
g_pImmediateContext->OMSetBlendState(states->Opaque(), Colors::Black, 0xFFFFFFFF);
g_pImmediateContext->OMSetDepthStencilState(states->DepthDefault(), 0);
// Set the input layout
g_pImmediateContext->IASetInputLayout(g_pVertexLayout);
UINT stride = sizeof(SimpleVertex);
UINT offset = 0;
g_pImmediateContext->IASetVertexBuffers(0, 1, &g_pVertexBuffer, &stride, &offset);
// Set index buffer
g_pImmediateContext->IASetIndexBuffer(g_pIndexBuffer, DXGI_FORMAT_R16_UINT, 0);
// Set primitive topology
g_pImmediateContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
// Draw objects
Do a good deed and help get someone (me) out of their misery, since it's New Year's Eve soon. I'm working on an iOS app, a coloring book for kids and I haven't stumbled upon OpenGL before (more precisely OpenGLES 2.0) so there's a big chance there's stuff I don't actually get in my code.
One of the tasks is to not let the brush spill out of the contour in which the user started drawing.
After reading and understanding some OpenGL basics, I found that using the stencil buffer is the right solution. This is my stencil buffer setup:
glClearStencil(0);
//clear the stencil
glClear(GL_STENCIL_BUFFER_BIT);
//disable writing to color buffer
glColorMask( GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE );
//disable depth buffer
glDisable(GL_DEPTH_TEST);
//enable writing to stencil buffer
glEnable(GL_STENCIL_TEST);
glStencilFunc(GL_NEVER, 1, 0xFF);
glStencilOp(GL_REPLACE, GL_REPLACE, GL_REPLACE);
[self drawStencil];
//re-enable color buffer
glColorMask( GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE );
//only draw where there is a 1
glStencilFunc(GL_EQUAL, 1, 1);
//keep the pixels in the stencil buffer
glStencilOp( GL_KEEP, GL_KEEP, GL_KEEP );
Right now, I'm just trying to draw a square in the stencil buffer and see if I can limit my drawing only to that square. This is the method drawing the square:
- (void)drawStencil
{
// Create a renderbuffer
GLuint renderbuffer;
glGenRenderbuffers(1, &renderbuffer);
glBindRenderbuffer(GL_RENDERBUFFER, renderbuffer);
[context renderbufferStorage:GL_RENDERBUFFER fromDrawable:(CAEAGLLayer*)self.layer];
// Create a framebuffer
GLuint framebuffer;
glGenFramebuffers(1, &framebuffer);
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer);
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_RENDERBUFFER, renderbuffer);
// Clear
glClearColor(1, 1, 1, 1);
glClear(GL_COLOR_BUFFER_BIT);
// Read vertex shader source
NSString *vertexShaderSource = [NSString stringWithContentsOfFile:[[NSBundle mainBundle] pathForResource:#"VertexShader" ofType:#"vsh"] encoding:NSUTF8StringEncoding error:nil];
const char *vertexShaderSourceCString = [vertexShaderSource cStringUsingEncoding:NSUTF8StringEncoding];
// Create and compile vertex shader
GLuint _vertexShader = glCreateShader(GL_VERTEX_SHADER);
glShaderSource(_vertexShader, 1, &vertexShaderSourceCString, NULL);
glCompileShader(_vertexShader);
// Read fragment shader source
NSString *fragmentShaderSource = [NSString stringWithContentsOfFile:[[NSBundle mainBundle] pathForResource:#"FragmentShader" ofType:#"fsh"] encoding:NSUTF8StringEncoding error:nil];
const char *fragmentShaderSourceCString = [fragmentShaderSource cStringUsingEncoding:NSUTF8StringEncoding];
// Create and compile fragment shader
GLuint _fragmentShader = glCreateShader(GL_FRAGMENT_SHADER);
glShaderSource(_fragmentShader, 1, &fragmentShaderSourceCString, NULL);
glCompileShader(_fragmentShader);
// Create and link program
GLuint program = glCreateProgram();
glAttachShader(program, _vertexShader);
glAttachShader(program, _fragmentShader);
glLinkProgram(program);
// Use program
glUseProgram(program);
// Define geometry
GLfloat square[] = {
-0.5, -0.5,
0.5, -0.5,
-0.5, 0.5,
0.5, 0.5};
//Send geometry to vertex shader
const char *aPositionCString = [#"a_position" cStringUsingEncoding:NSUTF8StringEncoding];
GLuint aPosition = glGetAttribLocation(program, aPositionCString);
glVertexAttribPointer(aPosition, 2, GL_FLOAT, GL_FALSE, 0, square);
glEnableVertexAttribArray(aPosition);
// Draw
glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
// Present renderbuffer
[context presentRenderbuffer:GL_RENDERBUFFER];
}
So much code and nothing happens... I can draw relentlessly wherever I want without a single stencil test stopping me.
What can I do? How do I check if the stencil buffer has something drawn inside it? If there's a missing puzzle for any of you, I will happily share any other parts of the code.
Any help is greatly appreciated! This has been torturing me for a while now. I will be forever in your debt!
UPDATE
I got the contour thing to work but I didn't use the stencil buffer. I created masks for every drawing area and textures for each mask which I loaded in the fragment shader along with the brush texture. When I tap on an area, I iterate through the array of masks and see which one was selected and bind the mask texture. I will make another post on SO with a more appropriate title and explain it there.
The way you allocate the renderbuffer storage looks problematic:
[context renderbufferStorage:GL_RENDERBUFFER fromDrawable:(CAEAGLLayer*)self.layer];
The documentation says about this method:
The width, height, and internal color buffer format are derived from the characteristics of the drawable object.
The way I understand it, since your "drawable object" will normally be a color buffer, this will create a color renderbuffer. But you need a renderbuffer with stencil format in your case. I'm not sure if there's a way to do this with a utility method in the context class (the documentation says something about "overriding the internal color buffer format"), but the easiest way is probably to simply call the corresponding OpenGL function directly:
glRenderbufferStorage(GL_RENDERBUFFER, GL_STENCIL_INDEX8, width, height);
If you want to use your own FBO for this rendering, you will also need to create a color buffer for it, and attach it to the FBO. Otherwise you're not really producing any rendering output.
Instead of creating a new FBO, it might be easier to make sure that the default framebuffer has a stencil buffer, and render to it directly. To do this, you can request a stencil buffer for your GLKView derived view by making this call during setup:
[view setDrawableStencilFormat: GLKViewDrawableStencilFormat8];
I am trying to find an efficient way to pan a 2D image in place using DirectX 9
I've attached a picture below that hopefully explains what I want to do. Basically, I want to scroll the tu and tv coordinates of all the quad's vertices across the texture to produce a "scrolling in place" effect for a 2D texture.
The first image below represents my loaded texture.
The second image is the texture with the tu,tv coordinates of the four vertices in each corner showing the standard rendered image.
The third image illustrates what I want to happen; I want to move the vertices in such a way that the box that is rendered straddles the end of the image and wraps back around in such a way that the texture will be rendered as shown with the two halves of the cloud separated.
The fourth image shows my temporary (wasteful) solution; I simply doubled the image and pan across until I reach the far right edge, at which point I reset the vertices' tu and tv so that the box being rendered is back on the far right.
Is there a legitimate way to do this without breaking everything into two separate quads?
I've added details of my set up and my render code below, if that helps clarify a path to a solution with my current design.
I have a function that sets up DirectX for 2D render as follows. I've added wrap properties to texture stage 0 as recommended:
VOID SetupDirectXFor2DRender()
{
pd3dDevice->SetSamplerState( 0, D3DSAMP_MINFILTER, D3DTEXF_POINT );
pd3dDevice->SetSamplerState( 0, D3DSAMP_MAGFILTER, D3DTEXF_POINT );
pd3dDevice->SetSamplerState( 0, D3DSAMP_MIPFILTER, D3DTEXF_POINT );
// Set for wrapping textures to enable panning sprite render
pd3dDevice->SetSamplerState( 0, D3DSAMP_ADDRESSU, D3DTADDRESS_WRAP );
pd3dDevice->SetSamplerState( 0, D3DSAMP_ADDRESSV, D3DTADDRESS_WRAP );
pd3dDevice->SetRenderState( D3DRS_ALPHAFUNC, D3DCMP_GREATEREQUAL );
pd3dDevice->SetRenderState( D3DRS_ALPHAREF, 0 );
pd3dDevice->SetRenderState( D3DRS_ALPHABLENDENABLE, true );
pd3dDevice->SetRenderState( D3DRS_ALPHATESTENABLE, false );
pd3dDevice->SetRenderState( D3DRS_SRCBLEND , D3DBLEND_SRCALPHA );
pd3dDevice->SetRenderState( D3DRS_DESTBLEND, D3DBLEND_INVSRCALPHA) ;
pd3dDevice->SetTextureStageState( 0, D3DTSS_COLOROP, D3DTOP_MODULATE );
pd3dDevice->SetTextureStageState( 0, D3DTSS_COLORARG1, D3DTA_TEXTURE );
pd3dDevice->SetTextureStageState( 0, D3DTSS_COLORARG2, D3DTA_DIFFUSE );
pd3dDevice->SetTextureStageState( 0, D3DTSS_ALPHAOP, D3DTOP_MODULATE );
pd3dDevice->SetTextureStageState( 0, D3DTSS_ALPHAARG1, D3DTA_TEXTURE );
pd3dDevice->SetTextureStageState( 0, D3DTSS_ALPHAARG2, D3DTA_DIFFUSE );
pd3dDevice->SetTextureStageState( 0, D3DTSS_COLOROP, D3DTOP_SELECTARG1 );
pd3dDevice->SetTextureStageState( 0, D3DTSS_COLORARG1, D3DTA_TEXTURE );
pd3dDevice->SetTextureStageState( 0, D3DTSS_COLORARG2, D3DTA_DIFFUSE );
return;
}
On each frame, I render things as follows:
VOID RenderAllEntities()
{
HRESULT hResult;
// Void pointer for DirectX buffer locking
VOID* pVoid;
hResult = pd3dDevice->Clear( 0,
NULL,
D3DCLEAR_TARGET,
0x0,
1.0f,
0 );
hResult = pd3dDevice->BeginScene();
// Do rendering on the back buffer here
hResult = pd3dDevice->SetFVF( CUSTOMFVF );
hResult = pd3dDevice->SetStreamSource( 0, pVertexBuffer, 0, sizeof(CUSTOM_VERTEX) );
for ( std::vector<RenderContext>::iterator renderContextIndex = queuedContexts.begin(); renderContextIndex != queuedContexts.end(); ++renderContextIndex )
{
// Render each sprite
for ( UINT uiIndex = 0; uiIndex < (*renderContextIndex).uiNumSprites; ++uiIndex )
{
// Lock the vertex buffer into memory
hResult = pVertexBuffer->Lock( 0, 0, &pVoid, 0 );
// Copy our vertex buffer to memory
::memcpy( pVoid, &renderContextIndex->vertexLists[uiIndex], sizeof(vertexList) );
// Unlock buffer
hResult = pVertexBuffer->Unlock();
hResult = pd3dDevice->SetTexture( 0, (*renderContextIndex).textures[uiIndex]->GetTexture() );
hResult = pd3dDevice->DrawPrimitive( D3DPT_TRIANGLELIST, 0, 6 );
}
}
// Complete and present the rendered scene
hResult = pd3dDevice->EndScene();
hResult = pd3dDevice->Present( NULL, NULL, NULL, NULL );
return;
}
To test SetTransform, I tried adding the following (sloppy but temporary) code block inside the render code before the call to DrawPrimitive:
{
static FLOAT di = 0.0f;
static FLOAT dy = 0.0f;
di += 0.03f;
dy += 0.03f;
// Build and set translation matrix
D3DXMATRIX ret;
D3DXMatrixIdentity(&ret);
ret(3, 0) = di;
ret(3, 1) = dy;
//ret(3, 2) = dz;
hResult = pd3dDevice->SetTransform( D3DTS_TEXTURE0, &ret );
}
This does not make any of my rendered sprites pan about.
I've been working through DirectX tutorials and reading the MS documentation to catch up on things but there are definitely holes in my knowledge, so I hope I'm not doing anything too completely brain-dead.
Any help super appreciated.
Thanks!
This should be quiet easy to do with one quad.
Assuming that you're using DX9 with the fixed function pipeline, you can translate your texture with IDirect3DDevice9::SetTransform (doc) with the properly texture as D3DTRANSFORMSTATETYPE(doc) and a 2D-Translation Matrix. You must ensure that your sampler state D3DSAMP_ADDRESSU and D3DSAMP_ADDRESSV (doc) is set to D3DTADDRESS_WRAP (doc). This tiles the texture virtually, so that negativ uv-values or values greater 1 are mapped to an infinit repetition of your texture.
If you're using shaders or another version of directx, you can translate you texture coordinate by yourself in the shader or manipulate the uv-values of your vertices.
I am trying to use DirectX to create and paint into a bitmap, and then later use that bitmap to paint to the screen. I have the following test code which I have pieced together from a number of sources. I seem to be able to create the bitmap without errors, and I try and fill it with red. However, when I use the bitmap to paint to the screen I just get a black rectangle. Can anyone tell me what I might be doing wrong? Note that this is running as a Windows 8 Metro app, if that makes a difference.
Note that the formatter seems to be removing my angle brackets and I don't how stop it from doing that. The 'device' variable is of class ID3D11Device, the 'context' variable is ID3D11DeviceContext, the d2dContext variable is ID2D1DeviceContext, the dgixDevice variable is IDXGIDevice, and the 'd2dDevice' is ID2D1Device.
void PaintTestBitmap(GRectF& rect)
{
HRESULT hr;
UINT creationFlags = D3D11_CREATE_DEVICE_BGRA_SUPPORT;
D3D_FEATURE_LEVEL featureLevels[] =
{
D3D_FEATURE_LEVEL_11_1,
D3D_FEATURE_LEVEL_11_0,
D3D_FEATURE_LEVEL_10_1,
D3D_FEATURE_LEVEL_10_0,
D3D_FEATURE_LEVEL_9_3,
D3D_FEATURE_LEVEL_9_2,
D3D_FEATURE_LEVEL_9_1
};
ComPtr<ID3D11Device> device;
ComPtr<ID3D11DeviceContext> context;
ComPtr<ID2D1DeviceContext> d2dContext;
hr = D3D11CreateDevice( NULL,
D3D_DRIVER_TYPE_HARDWARE,
NULL,
creationFlags,
featureLevels,
ARRAYSIZE(featureLevels),
D3D11_SDK_VERSION,
&device,
NULL,
&context);
if (SUCCEEDED(hr))
{
ComPtr<IDXGIDevice> dxgiDevice;
device.As(&dxgiDevice);
// create D2D device context
ComPtr<ID2D1Device> d2dDevice;
hr = D2D1CreateDevice(dxgiDevice.Get(), NULL, &d2dDevice);
if (SUCCEEDED(hr))
{
hr = d2dDevice->CreateDeviceContext(D2D1_DEVICE_CONTEXT_OPTIONS_NONE, &d2dContext);
if (SUCCEEDED(hr))
{
ID2D1Bitmap1 *pBitmap;
D2D1_SIZE_U size = { rect.m_width, rect.m_height };
D2D1_BITMAP_PROPERTIES1 properties = {{ DXGI_FORMAT_R8G8B8A8_UNORM, D2D1_ALPHA_MODE_PREMULTIPLIED }, 96, 96, D2D1_BITMAP_OPTIONS_TARGET, 0 };
hr = d2dContext->CreateBitmap(size, (const void *) 0, 0, &properties, &pBitmap);
if (SUCCEEDED(hr))
{
d2dContext->SetTarget(pBitmap);
d2dContext->BeginDraw();
d2dContext->Clear(D2D1::ColorF(D2D1::ColorF::Red)); // Clear the bitmap to Red
d2dContext->EndDraw();
// If I now use pBitmap to paint to the screen, I get a black rectange
// rather than the red one I was expecting
}
}
}
}
}
Note that I am fairly new to both DirectX and Metro style apps.
Thanks
what you're probably forgetting is that you dont draw your bitmap on your context.
what you can try is
hr = d2dContext->CreateBitmap(size, (const void *) 0, 0, &properties, &pBitmap);
if (SUCCEEDED(hr))
{
ID2D1Image *pImage = nullptr;
d2dContext->GetTarget(&image);
d2dContext->SetTarget(pBitmap);
d2dContext->BeginDraw();
d2dContext->Clear(D2D1::ColorF(D2D1::ColorF::Red)); // Clear the bitmap to Red
d2dContext->SetTarget(pImage);
d2dContext->DrawBitmap(pBitmap);
d2dContext->EndDraw();
}
what you do is store your rendertarget and then draw your bitmap and at the end you set your context to the right render target and let it draw your bitmap on it
How to use the stencil buffer to create a mask in OGRE?
That is, some objects should be rendered first into the stencil buffer and generate a mask (say 0 is background, 0xFF is foreground); then render the scene itself, using the stencil buffer as a mask so only pixels where it is 0xFF are rendered.
I guess I should use a RenderQueueListener, but I can't make it work. Here's what I'm doing right now:
void StencilOpQueueListener::renderQueueStarted(Ogre::uint8 queueGroupId, const Ogre::String& invocation, bool& skipThisInvocation) {
if (queueGroupId == Ogre::RENDER_QUEUE_1) {
Ogre::RenderSystem *rs = Ogre::Root::getSingleton().getRenderSystem();
rs->clearFrameBuffer(Ogre::FBT_STENCIL, Ogre::ColourValue::Black, 1.0, 0xFF);
rs->setStencilCheckEnabled(true);
rs->_setColourBufferWriteEnabled(false, false, false, false);
rs->setStencilBufferParams(
Ogre::CMPF_ALWAYS_PASS, // compare
0x1, // refvalue
0xFFFFFFFF, // mask
Ogre::SOP_REPLACE, Ogre::SOP_REPLACE, // stencil fail, depth fail
Ogre::SOP_REPLACE, // stencil pass + depth pass
false); // two-sided operation? no
}
}
void StencilOpQueueListener::renderQueueEnded(Ogre::uint8 queueGroupId, const Ogre::String& invocation, bool& repeatThisInvocation) {
if (queueGroupId == Ogre::RENDER_QUEUE_1) {
Ogre::RenderSystem *rs = Ogre::Root::getSingleton().getRenderSystem();
rs->setStencilCheckEnabled(false);
rs->setStencilBufferParams();
}
}
And I set the entities that should render to the stencil buffer are set with:
entity->setRenderQueueGroup(RENDER_QUEUE_1);
What am I doing wrong? Any examples of how this is done in Ogre? Thanks!
For reference, this is how I do it in pure OpenGL:
/* Enable stencil test and leave it enabled throughout */
glClearStencil(0xFF);
glClearColor(1, 1, 1, 1);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
glEnable(GL_STENCIL_TEST);
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
glDepthMask(GL_FALSE);
glStencilFunc(GL_ALWAYS, 0x1, 0xFFFFFFFF);
glStencilOp(GL_REPLACE, GL_REPLACE, GL_REPLACE);
// render into the stencil buffer. This should render only the selector objects
renderStencil();
// restore the rendering
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
glDepthMask(GL_TRUE);
// only render "inside" the silhouette -- we want the overlap
glStencilFunc(GL_EQUAL, 0x1, 0xFFFFFFFF);
glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
render(); // now we render objects against that mask
you should synthesize the second step as well, which you mentioned in
// only render "inside" the silhouette -- we want the overlap
glStencilFunc(GL_EQUAL, 0x1, 0xFFFFFFFF); glStencilOp(GL_KEEP,
GL_KEEP, GL_KEEP);
This means, you should apply the mask you set in RENDER_QUEUE_1 to the rendered scene, like you do in the case of gl calls. The main rendering takes place in Ogre's RENDER_QUEUE_MAIN by default, thus you should add this to your renderQueueStarted method:
if (queueGroupId == Ogre::RENDER_QUEUE_MAIN)
{
Ogre::RenderSystem * rs = Ogre::Root::getSingleton().getRenderSystem();
rs->_setColourBufferWriteEnabled(true, true, true, true);
rs->setStencilCheckEnabled(true);
rs->setStencilBufferParams(Ogre::CMPF_EQUAL,0x1,0xFFFFFFFF,
Ogre::SOP_KEEP,Ogre::SOP_KEEP,Ogre::SOP_KEEP,false);
}
also, switch off the stenciling in renderQueueEnded only after main rendering is finished:
if ( queueGroupId == Ogre::RENDER_QUEUE_MAIN )
{
Ogre::RenderSystem *rs = Ogre::Root::getSingleton().getRenderSystem();
rs->setStencilCheckEnabled(false);
rs->setStencilBufferParams();
}