glFramebufferTexture2D on webgl2 with mipmaps levels - webgl

With webGL2 derived from ES3.0 I thought that we can use mipmap levels as the last parameter of:
void glFramebufferTexture2D(GLenum target,
GLenum attachment,
GLenum textarget,
GLuint texture,
GLint level);
Now from Khronos ES3.0 official documentation states that mipmap levels are supposed to work:
level:
Specifies the mipmap level of texture to attach.
From Khronos ES2.0 instead it says it must be 0
level:
Specifies the mipmap level of the texture image to be attached, which must be 0.
Now, the I cannot find any docs from WebGL2.0 context about glFramebufferTexture2D, but the mozilla docs states that mipmap layer must be 0, as in ES2.0, here:
Mozilla WebGL doc
level:
A GLint specifying the mipmap level of the texture image to be attached. Must be 0.
That page I think refers to WebGL1 context but it has mentions of WebGL2 features in it, and I cannot find glFramebufferTexture2D on WebGL2 docs.
So to wrap it up, is there a way to use mipmap levels on framebuffer targets on WebGL2.0?
(I've looked into layered images but AFAIK layered rendering is not available for WebGL2.0)

is there a way to use mipmap levels on framebuffer targets on WebGL2.0
Yes
I'd close the answer there but I guess I wonder did you actually try something and have it not work? You have to create a WebGL2 context to use mipmap levels as framebuffer attachments but otherwise yes, it works. On WebGL1 it will not work.
function main() {
const gl = document.querySelector('canvas').getContext('webgl2');
if (!gl) {
return alert('need webgl2');
}
const vs = `#version 300 es
void main() {
// just draw an 8x8 pixel point in the center of the target
// this shader needs/uses no attributes
gl_Position = vec4(0, 0, 0, 1);
gl_PointSize = 8.0;
}
`;
const fsColor = `#version 300 es
precision mediump float;
uniform vec4 color;
out vec4 outColor;
void main() {
outColor = color;
}
`;
const fsTexture = `#version 300 es
precision mediump float;
uniform sampler2D tex;
out vec4 outColor;
void main() {
// this shader needs no texcoords since we just
// use gl_PoitnCoord provided by rendering a point with gl.POINTS
// bias lets select the mip level so no need for
// some fancier shader just to show that it's working.
float bias = gl_PointCoord.x * gl_PointCoord.y * 4.0;
outColor = texture(tex, gl_PointCoord.xy, bias);
}
`;
// compile shaders, link into programs, look up attrib/uniform locations
const colorProgramInfo = twgl.createProgramInfo(gl, [vs, fsColor]);
const textureProgramInfo = twgl.createProgramInfo(gl, [vs, fsTexture]);
const tex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, tex);
const levels = 4;
const width = 8;
const height = 8;
gl.texStorage2D(gl.TEXTURE_2D, levels, gl.RGBA8, width, height);
// make a framebuffer for each mip level
const fbs = [];
for (let level = 0; level < levels; ++level) {
const fb = gl.createFramebuffer();
fbs.push(fb);
gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
gl.framebufferTexture2D(
gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0,
gl.TEXTURE_2D, tex, level);
}
// render a different color to each level
const colors = [
[1, 0, 0, 1], // red
[0, 1, 0, 1], // green
[0, 0, 1, 1], // blue
[1, 1, 0, 1], // yellow
];
gl.useProgram(colorProgramInfo.program);
for (let level = 0; level < levels; ++level) {
gl.bindFramebuffer(gl.FRAMEBUFFER, fbs[level]);
const size = width >> level;
gl.viewport(0, 0, size, size);
twgl.setUniforms(colorProgramInfo, { color: colors[level] });
const offset = 0;
const count = 1;
gl.drawArrays(gl.POINTS, offset, count); // draw 1 point
}
// draw the texture's mips to the canvas
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.useProgram(textureProgramInfo.program);
// no need to bind the texture it's already bound
// no need to set the uniform it defaults to 0
gl.drawArrays(gl.POINT, 0, 1); // draw 1 point
}
main();
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<canvas width="8" height="8" style="width: 128px; height: 128px;"></canvas>
You can also render to layers of TEXTURE_2D_ARRAY texture.
function main() {
const gl = document.querySelector('canvas').getContext('webgl2');
if (!gl) {
return alert('need webgl2');
}
const vs = `#version 300 es
void main() {
// just draw an 8x8 pixel point in the center of the target
// this shader needs/uses no attributes
gl_Position = vec4(0, 0, 0, 1);
gl_PointSize = 8.0;
}
`;
const fsColor = `#version 300 es
precision mediump float;
uniform vec4 color;
out vec4 outColor;
void main() {
outColor = color;
}
`;
const fsTexture = `#version 300 es
precision mediump float;
uniform mediump sampler2DArray tex;
out vec4 outColor;
void main() {
// this shader needs no texcoords since we just
// use gl_PoitnCoord provided by rendering a point with gl.POINTS
float layer = gl_PointCoord.x * gl_PointCoord.y * 4.0;
outColor = texture(tex, vec3(gl_PointCoord.xy, layer));
}
`;
// compile shaders, link into programs, look up attrib/uniform locations
const colorProgramInfo = twgl.createProgramInfo(gl, [vs, fsColor]);
const textureProgramInfo = twgl.createProgramInfo(gl, [vs, fsTexture]);
const tex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D_ARRAY, tex);
const levels = 1;
const width = 8;
const height = 8;
const layers = 4;
gl.texStorage3D(gl.TEXTURE_2D_ARRAY, levels, gl.RGBA8, width, height, layers);
// only use level 0 (of course we could render to levels in layers as well)
gl.texParameteri(gl.TEXTURE_2D_ARRAY, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
// make a framebuffer for each layer
const fbs = [];
for (let layer = 0; layer < layers; ++layer) {
const fb = gl.createFramebuffer();
fbs.push(fb);
gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
const level = 0;
gl.framebufferTextureLayer(
gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0,
tex, level, layer);
}
// render a different color to each layer
const colors = [
[1, 0, 0, 1], // red
[0, 1, 0, 1], // green
[0, 0, 1, 1], // blue
[1, 1, 0, 1], // yellow
];
gl.useProgram(colorProgramInfo.program);
for (let layer = 0; layer < layers; ++layer) {
gl.bindFramebuffer(gl.FRAMEBUFFER, fbs[layer]);
gl.viewport(0, 0, width, height);
twgl.setUniforms(colorProgramInfo, { color: colors[layer] });
const offset = 0;
const count = 1;
gl.drawArrays(gl.POINTS, offset, count); // draw 1 point
}
// draw the texture's mips to the canvas
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.useProgram(textureProgramInfo.program);
// no need to bind the texture it's already bound
// no need to set the uniform it defaults to 0
gl.drawArrays(gl.POINT, 0, 1); // draw 1 point
}
main();
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<canvas width="8" height="8" style="width: 128px; height: 128px; image-rendering: pixelated;"></canvas>

Related

Can an layout(location=n) out skip an index for drawBuffers in WebGL?

I'm working on MRT in my graphics engine.
An interesting point i'm at (and aim to fix) has my generated fragment shader spitting out:
layout(location = 0) out vec4 thing1;
layout(location = 2) out vec4 thing2;
The drawBuffers call on the application side calls something like this:
gl.drawBuffers([gl.COLOR_ATTACHMENT0, gl.NONE, gl.COLOR_ATTACHMENT1]);
However, I'm getting an error:
WebGL: INVALID_OPERATION: drawBuffers: COLOR_ATTACHMENTi_EXT or NONE
So obviously, this would appear to not be allowed. From the documentation I've read from a wikipedia article discussing it:
https://www.khronos.org/opengl/wiki/Fragment_Shader
It states along the lines that the layout location specified refers to the array index specified from the drawBuffers call. So, in theory I would have thought this shader to configuration would be valid.
What am I missing from my understanding that makes this not work?
I ask for understanding mostly and not to fix my program, my generator will correct the indices when I'm done to be 'correct' with no location index skipping.
Update: As noted below, you CAN skip layout locations in the shader. My issue was the improper formatting of the drawBuffers call where I had COLOR_ATTACHMENT1 in the index where ONLY COLOR_ATTACHMENT2 is valid.
This is wrong
gl.drawBuffers([gl.COLOR_ATTACHMENT0, gl.NONE, gl.COLOR_ATTACHMENT1]);
the i-th attachment must be gl.NONE or gl.COLOR_ATTACHMENTi
so it has to be this
gl.drawBuffers([gl.COLOR_ATTACHMENT0, gl.NONE, gl.COLOR_ATTACHMENT2]);
function main() {
const gl = document.querySelector('canvas').getContext('webgl2');
const vs = `#version 300 es
void main() {
gl_Position = vec4(0, 0, 0, 1);
gl_PointSize = 100.0;
}
`;
const fs = `#version 300 es
precision highp float;
layout(location = 0) out vec4 thing1;
layout(location = 2) out vec4 thing2;
void main () {
thing1 = vec4(1, 0, 0, 1);
thing2 = vec4(0, 0, 1, 1);
}
`;
const prg = twgl.createProgram(gl, [vs, fs]);
const fb = gl.createFramebuffer();
gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
createTextureAndAttach(gl, gl.COLOR_ATTACHMENT0);
createTextureAndAttach(gl, gl.COLOR_ATTACHMENT2);
gl.drawBuffers([
gl.COLOR_ATTACHMENT0,
gl.NONE,
gl.COLOR_ATTACHMENT2,
]);
const status = gl.checkFramebufferStatus(gl.FRAMEBUFFER);
if (status !== gl.FRAMEBUFFER_COMPLETE) {
console.error("can't render to this framebuffer combo");
return;
}
gl.useProgram(prg);
gl.viewport(0, 0, 1, 1);
gl.drawArrays(gl.POINTS, 0, 1);
checkError();
read(gl.COLOR_ATTACHMENT0);
read(gl.COLOR_ATTACHMENT2);
checkError();
function checkError() {
const err = gl.getError();
if (err) {
console.error(twgl.glEnumToString(gl, err));
}
}
function read(attachmentPoint) {
gl.readBuffer(attachmentPoint);
const pixel = new Uint8Array(4);
gl.readPixels(0, 0, 1, 1, gl.RGBA, gl.UNSIGNED_BYTE, pixel);
console.log(Array.from(pixel).join(','));
}
function createTextureAndAttach(gl, attachmentPoint) {
const tex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, tex);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA8, 1, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);
gl.framebufferTexture2D(gl.FRAMEBUFFER, attachmentPoint, gl.TEXTURE_2D, tex, 0);
}
}
main();
<script src="https://twgljs.org/dist/4.x/twgl.min.js"></script>
<canvas></canvas>
note: Referencing the OpenGL docs for WebGL are often wrong and/or misleading for WebGL. You need to reference the OpenGL 3.0 ES spec for WebGL2

WebGL rendering outside of browser paint time

We are building a WebGL application that has some high render-load objects. Is there a way we can render those object outside of browser-paint time, i.e. in the background? We don't want our FPS going down, and breaking up our rendering process is possible (to split between frames).
Three ideas come to mind.
You can render to a texture via a framebuffer over many frames, when you're done you render that texture to the canvas.
const gl = document.querySelector('canvas').getContext('webgl');
const vs = `
attribute vec4 position;
attribute vec2 texcoord;
varying vec2 v_texcoord;
void main() {
gl_Position = position;
v_texcoord = texcoord;
}
`;
const fs = `
precision highp float;
uniform sampler2D tex;
varying vec2 v_texcoord;
void main() {
gl_FragColor = texture2D(tex, v_texcoord);
}
`;
// compile shader, link program, look up locations
const programInfo = twgl.createProgramInfo(gl, [vs, fs]);
// gl.createBuffer, gl.bindBuffer, gl.bufferData
const bufferInfo = twgl.createBufferInfoFromArrays(gl, {
position: {
numComponents: 2,
data: [
-1, -1,
1, -1,
-1, 1,
-1, 1,
1, -1,
1, 1,
],
},
texcoord: {
numComponents: 2,
data: [
0, 0,
1, 0,
0, 1,
0, 1,
1, 0,
1, 1,
],
},
});
// create a framebuffer with a texture and depth buffer
// same size as canvas
// gl.createTexture, gl.texImage2D, gl.createFramebuffer
// gl.framebufferTexture2D
const framebufferInfo = twgl.createFramebufferInfo(gl);
const infoElem = document.querySelector('#info');
const numDrawSteps = 16;
let drawStep = 0;
let time = 0;
// draw over several frames. Return true when ready
function draw() {
// draw to texture
// gl.bindFrambuffer, gl.viewport
twgl.bindFramebufferInfo(gl, framebufferInfo);
if (drawStep == 0) {
// on the first step clear and record time
gl.disable(gl.SCISSOR_TEST);
gl.clearColor(0, 0, 0, 0);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
time = performance.now() * 0.001;
}
// this represents drawing something.
gl.enable(gl.SCISSOR_TEST);
const halfWidth = framebufferInfo.width / 2;
const halfHeight = framebufferInfo.height / 2;
const a = time * 0.1 + drawStep
const x = Math.cos(a ) * halfWidth + halfWidth;
const y = Math.sin(a * 1.3) * halfHeight + halfHeight;
gl.scissor(x, y, 16, 16);
gl.clearColor(
drawStep / 16,
drawStep / 6 % 1,
drawStep / 3 % 1,
1);
gl.clear(gl.COLOR_BUFFER_BIT);
drawStep = (drawStep + 1) % numDrawSteps;
return drawStep === 0;
}
let frameCount = 0;
function render() {
++frameCount;
infoElem.textContent = frameCount;
if (draw()) {
// draw to canvas
// gl.bindFramebuffer, gl.viewport
twgl.bindFramebufferInfo(gl, null);
gl.disable(gl.DEPTH_TEST);
gl.disable(gl.BLEND);
gl.disable(gl.SCISSOR_TEST);
gl.useProgram(programInfo.program);
// gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);
// gl.uniform...
twgl.setUniformsAndBindTextures(programInfo, {
tex: framebufferInfo.attachments[0],
});
// draw the quad
gl.drawArrays(gl.TRIANGLES, 0, 6);
}
requestAnimationFrame(render);
}
requestAnimationFrame(render);
<canvas></canvas>
<div id="info"></div>
<script src="https://twgljs.org/dist/4.x/twgl.min.js"></script>
You can make 2 canvases. A webgl canvas that is not in the DOM. You render to it over many frames and when you're done you draw it to a 2D canvas with ctx.drawImage(webglCanvas, ...) This is basically the same as #1 except you're letting the browser "render that texture to a canvas" part
const ctx = document.querySelector('canvas').getContext('2d');
const gl = document.createElement('canvas').getContext('webgl');
const vs = `
attribute vec4 position;
attribute vec2 texcoord;
varying vec2 v_texcoord;
void main() {
gl_Position = position;
v_texcoord = texcoord;
}
`;
const fs = `
precision highp float;
uniform sampler2D tex;
varying vec2 v_texcoord;
void main() {
gl_FragColor = texture2D(tex, v_texcoord);
}
`;
// compile shader, link program, look up locations
const programInfo = twgl.createProgramInfo(gl, [vs, fs]);
const infoElem = document.querySelector('#info');
const numDrawSteps = 16;
let drawStep = 0;
let time = 0;
// draw over several frames. Return true when ready
function draw() {
if (drawStep == 0) {
// on the first step clear and record time
gl.disable(gl.SCISSOR_TEST);
gl.clearColor(0, 0, 0, 0);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
time = performance.now() * 0.001;
}
// this represents drawing something.
gl.enable(gl.SCISSOR_TEST);
const halfWidth = gl.canvas.width / 2;
const halfHeight = gl.canvas.height / 2;
const a = time * 0.1 + drawStep
const x = Math.cos(a ) * halfWidth + halfWidth;
const y = Math.sin(a * 1.3) * halfHeight + halfHeight;
gl.scissor(x, y, 16, 16);
gl.clearColor(
drawStep / 16,
drawStep / 6 % 1,
drawStep / 3 % 1,
1);
gl.clear(gl.COLOR_BUFFER_BIT);
drawStep = (drawStep + 1) % numDrawSteps;
return drawStep === 0;
}
let frameCount = 0;
function render() {
++frameCount;
infoElem.textContent = frameCount;
if (draw()) {
// draw to canvas
ctx.clearRect(0, 0, ctx.canvas.width, ctx.canvas.height);
ctx.drawImage(gl.canvas, 0, 0);
}
requestAnimationFrame(render);
}
requestAnimationFrame(render);
<canvas></canvas>
<div id="info"></div>
<script src="https://twgljs.org/dist/4.x/twgl.min.js"></script>
You can use OffscreenCanvas and render in a worker. This has only shipped in Chrome though.
Note that if you DOS the GPU (give the GPU too much work) you can still affect the responsiveness of the main thread because most GPUs do not support pre-emptive multitasking. So, if you have a lot of really heavy work then split it up into smaller tasks.
As an example if you took one of the heaviest shaders from shadertoy.com that runs at say 0.5 fps when rendered at 1920x1080, even offscreen it will force the entire machine to run at 0.5 fps. To fix you'd need to render smaller portions over several frames. If it's running at 0.5 fps that suggests you need to split it up into at least 120 smaller parts, maybe more, to keep the main thread responsive and at 120 smaller parts you'd only see the results every 2 seconds.
In fact trying it out shows some issues. Here's Iq's Happy Jumping Example drawn over 960 frames. It still can't keep 60fps on my late 2018 Macbook Air even though it's rendering only 2160 pixels a frame (2 columns of a 1920x1080 canvas). The issue is likely some parts of the scene have to recurse deeply and there is no way knowing before hand which parts of the scene that will be. One reason why shadertoy style shaders using signed distance fields are more of a toy (hence shaderTOY) and not actually a production style technique.
Anyway, the point of that is if you give the GPU too much work you'll still get an unresponsive machine.

WebGL Droste effect

I am trying to use WebGL achieve Droste effect on a cube's faces. There is a single mesh in the viewport, a cube, and all of its faces share the same texture. To achieve Droste effect, I update the texture on each frame and I actually just take a snapshot of the canvas whose WebGL context I am drawing to, which over time results in the Droste effect as the snapshot increasingly contain more and more nested past frames.
There is a demo of what I have right now in action here:
https://tomashubelbauer.github.io/webgl-op-1/?cubeTextured
The code in question follows:
// Set up fragment and vertex shader and attach them to a program, link the program
// Create a vertex buffer, an index buffer and a texture coordinate buffer
// Tesselate the cube's vertices and fill in the index and texture coordinate buffers
const textureCanvas = document.createElement('canvas');
textureCanvas.width = 256;
textureCanvas.height = 256;
const textureContext = textureCanvas.getContext('2d');
// In every `requestAnimationFrame`:
textureContext.drawImage(context.canvas, 0, 0);
const texture = context.createTexture();
context.bindTexture(context.TEXTURE_2D, texture);
context.texImage2D(context.TEXTURE_2D, 0, context.RGBA, context.RGBA, context.UNSIGNED_BYTE, textureCanvas);
context.generateMipmap(context.TEXTURE_2D);
// Clear the viewport completely (depth and color buffers)
// Set up attribute and uniform values, the projection and model view matrices
context.activeTexture(context.TEXTURE0);
context.bindTexture(context.TEXTURE_2D, texture);
context.uniform1i(fragmentShaderTextureSamplerUniformLocation, 0);
context.drawElements(context.TRIANGLES, 36, context.UNSIGNED_SHORT, 0)
The above is the meat of it all, there is a separate canvas from the WebGL one and it gets the WebGL canvas drawn on it before each WebGL frame and this canvas is then used to create the texture for the given frame and the texture is applied to the cube's faces according to the texture coordinate buffer and the texture sampler uniform provided to the fragment shader which just uses gl_FragColor = texture2D(textureSampler, textureCoordinate) like you would expect.
But this is super slow (30 FPS slow on this simple demo with one cube mesh where all my other demoes some with an order of magnitude more tris still edge the 60 FPS requestAnimationFrame cap).
Also it feels weird to do this "outside" of WebGL by using the external canvas when I feel like it should be achievable using WebGL alone.
I know WebGL keeps two buffers, one for the active frame and the back buffer for the recently drawn frame and these two swap with each frame to achieve immediate screen update. Is it possible to tap to this back buffer and use it as a texture? Can you please provide example code of how that would be done?
From this article
The normal way to do this is to render to a texture by attaching that texture to a framebuffer.
const fb = gl.createFramebuffer();
gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
gl.framebufferTexture2D(
gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, tex, 0 /* level */)
Now to render to the texture
gl.bindFramebuffer(gl.FRAMEBUFFER, fb);
gl.viewport(0, 0, textureWidth, textureHeight);
To render to the canvas
gl.bindFramebuffer(gl.FRAMEBUFFER, null);
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
To do what you want you need 2 textures since you can not read from and write to the same texture at the same time so you draw say
Draw Image to TextureA
Draw Previous Frame (TextureB) to TextureA
Draw Cube with TextureA to TextureB
Draw TextureB to Canvas
"use strict";
function main() {
const m4 = twgl.m4;
const gl = document.querySelector('canvas').getContext('webgl')
const vs = `
attribute vec4 position;
attribute vec2 texcoord;
uniform mat4 u_matrix;
varying vec2 v_texcoord;
void main() {
gl_Position = u_matrix * position;
v_texcoord = texcoord;
}
`;
const fs = `
precision mediump float;
varying vec2 v_texcoord;
uniform sampler2D u_tex;
void main() {
gl_FragColor = texture2D(u_tex, v_texcoord);
}
`;
// compile shaders, link program, look up locations
const programInfo = twgl.createProgramInfo(gl, [vs, fs]);
// gl.createBuffer, gl.bufferData for positions and texcoords of a cube
const cubeBufferInfo = twgl.primitives.createCubeBufferInfo(gl, 1);
// gl.createBuffer, gl.bufferData for positions and texcoords of a quad
const quadBufferInfo = twgl.primitives.createXYQuadBufferInfo(gl, 2);
// all the normal stuff for setting up a texture
const imageTexture = twgl.createTexture(gl, {
src: 'https://i.imgur.com/ZKMnXce.png',
});
function makeFramebufferAndTexture(gl, width, height) {
const framebuffer = gl.createFramebuffer();
gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffer);
const texture = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, texture);
gl.texImage2D(gl.TEXTURE_2D,
0, // level
gl.RGBA, // internal format
width,
height,
0, // border
gl.RGBA, // format
gl.UNSIGNED_BYTE, // type
null, // data (no data needed)
);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.framebufferTexture2D(
gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0,
gl.TEXTURE_2D, texture, 0 /* level */);
// note: depending on what you're rendering you might want to atttach
// a depth renderbuffer or depth texture. See linked article
return {
framebuffer,
texture,
width,
height,
};
}
function bindFramebufferAndSetViewport(gl, fbi) {
gl.bindFramebuffer(gl.FRAMEBUFFER, fbi ? fbi.framebuffer : null);
const {width, height} = fbi || gl.canvas;
gl.viewport(0, 0, width, height);
}
let fbiA = makeFramebufferAndTexture(gl, 512, 512);
let fbiB = makeFramebufferAndTexture(gl, 512, 512);
function drawImageAndPreviousFrameToTextureB() {
bindFramebufferAndSetViewport(gl, fbiB);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
// for each attribute
twgl.setBuffersAndAttributes(gl, programInfo, quadBufferInfo);
// calls gl.activeTexture, gl.bindTexture, gl.uniform
twgl.setUniforms(programInfo, {
u_tex: imageTexture,
u_matrix: m4.identity(),
});
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, quadBufferInfo);
// ---------
// draw previous cube texture into current cube texture
{
twgl.setUniforms(programInfo, {
u_tex: fbiA.texture,
u_matrix: m4.scaling([0.8, 0.8, 1]),
});
twgl.drawBufferInfo(gl, quadBufferInfo);
}
}
function drawTexturedCubeToTextureA(time) {
// ---------
// draw cube to "new" dstFB using srcFB.texture on cube
bindFramebufferAndSetViewport(gl, fbiA);
gl.clear(gl.COLOR_BUFFER_BIT);
twgl.setBuffersAndAttributes(gl, programInfo, cubeBufferInfo);
{
const fov = 60 * Math.PI / 180;
const aspect = fbiA.width / fbiA.height;
const near = 0.1;
const far = 100;
let mat = m4.perspective(fov, aspect, near, far);
mat = m4.translate(mat, [0, 0, -2]);
mat = m4.rotateX(mat, time);
mat = m4.rotateY(mat, time * 0.7);
twgl.setUniforms(programInfo, {
u_tex: fbiB.texture,
u_matrix: mat,
});
}
twgl.drawBufferInfo(gl, cubeBufferInfo);
}
function drawTextureAToCanvas() {
// --------
// draw dstFB.texture to canvas
bindFramebufferAndSetViewport(gl, null);
twgl.setBuffersAndAttributes(gl, programInfo, quadBufferInfo);
{
const aspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
const near = -1;
const far = 1;
let mat = m4.ortho(-aspect, aspect, -1, 1, near, far);
twgl.setUniforms(programInfo, {
u_tex: fbiA.texture,
u_matrix: mat,
});
}
twgl.drawBufferInfo(gl, quadBufferInfo);
}
function render(time) {
time *= 0.001; // convert to seconds;
twgl.resizeCanvasToDisplaySize(gl.canvas);
gl.enable(gl.DEPTH_TEST);
gl.enable(gl.CULL_FACE);
// there's only one shader program so let's set it here
gl.useProgram(programInfo.program);
drawImageAndPreviousFrameToTextureB();
drawTexturedCubeToTextureA(time);
drawTextureAToCanvas();
requestAnimationFrame(render);
}
requestAnimationFrame(render);
}
main();
body { margin: 0; }
canvas { width: 100vw; height: 100vh; display: block; }
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<canvas></canvas>
As for the canvas and its 2 buffers, no it is not possible to directly use them as textures. You can call gl.copyTexImage2D or gl.copyTexSubImage2D top copy a portion of the canvas to a texture though so that is another solution. It's less flexible and I believe slower than the framebuffer method

Whats an efficient way to stop these transparency overlaps in webgl?

I would like to make it so that these blocks are all drawn to one layer than that entire layer is made transparent. Or if there is a way I can use blend functions or alpha blending to do it that would be fine too. Thanks a lot.
What is your definition of efficient? Under what circumstances? What conditions?
Here's a few solutions. It's hard to tell if they fit without more details.
First let's repo the issue
const m4 = twgl.m4;
const gl = document.querySelector('canvas').getContext('webgl');
const vs = `
attribute vec4 position;
uniform mat4 u_matrix;
void main() {
gl_Position = u_matrix * position;
}
`;
const fs = `
precision mediump float;
void main() {
gl_FragColor = vec4(0, .5, 0, .5);
}
`;
// compile shaders, link, look up locations
const programInfo = twgl.createProgramInfo(gl, [vs, fs]);
// create buffers and upload vertex data
const bufferInfo = twgl.primitives.createCubeBufferInfo(gl, 1);
render();
function render() {
gl.clearColor(0, .4, 0, 1);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
gl.enable(gl.BLEND);
gl.enable(gl.CULL_FACE);
gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
gl.useProgram(programInfo.program);
const halfHeight = 1;
const aspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
const halfWidth = halfHeight * aspect;
const projection = m4.ortho(
-halfWidth, halfWidth, -halfHeight, halfWidth, 0.1, 20);
const camera = m4.lookAt(
[5, 2, 5], // eye
[0, -.5, 0], // target
[0, 1, 0], // up
);
const view = m4.inverse(camera);
const viewProjection = m4.multiply(projection, view);
twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);
for (let x = -1; x <= 1; ++x) {
let mat = m4.translate(viewProjection, [x, 0, 0]);
twgl.setUniforms(programInfo, {
u_matrix: mat,
});
// calls drawArrays or drawElements
twgl.drawBufferInfo(gl, bufferInfo);
}
}
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<canvas></canvas>
Note the example above just clears the background to [0, .4, 0, 1] which is dark green. It then draws 3 cubes using [0, .5, 0, .5] which is full green (as in [0, 1, 0, 1]) except premultiplied by 50% alpha. Using premultiplied colors the blending is set to gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA) Face culling is on.
As for solutions off the top of my head looking at your picture you could
Draw front to back with z-test on
const m4 = twgl.m4;
const gl = document.querySelector('canvas').getContext('webgl');
const vs = `
attribute vec4 position;
uniform mat4 u_matrix;
void main() {
gl_Position = u_matrix * position;
}
`;
const fs = `
precision mediump float;
void main() {
gl_FragColor = vec4(0, .5, 0, .5);
}
`;
// compile shaders, link, look up locations
const programInfo = twgl.createProgramInfo(gl, [vs, fs]);
// create buffers and upload vertex data
const bufferInfo = twgl.primitives.createCubeBufferInfo(gl, 1);
render();
function render() {
gl.clearColor(0, .4, 0, 1);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
gl.enable(gl.BLEND);
gl.enable(gl.CULL_FACE);
gl.enable(gl.DEPTH_TEST);
gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
gl.useProgram(programInfo.program);
const halfHeight = 1;
const aspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
const halfWidth = halfHeight * aspect;
const projection = m4.ortho(
-halfWidth, halfWidth, -halfHeight, halfWidth, 0.1, 20);
const camera = m4.lookAt(
[5, 2, 5], // eye
[0, -.5, 0], // target
[0, 1, 0], // up
);
const view = m4.inverse(camera);
const viewProjection = m4.multiply(projection, view);
twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);
for (let x = 1; x >= -1; --x) {
let mat = m4.translate(viewProjection, [x, 0, 0]);
twgl.setUniforms(programInfo, {
u_matrix: mat,
});
// calls drawArrays or drawElements
twgl.drawBufferInfo(gl, bufferInfo);
}
}
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<canvas></canvas>
Note the only changes to the top version are the addition of
gl.enable(gl.DEPTH_TEST);
And drawing in reverse order
for (let x = 1; x >= -1; --x) {
I have no idea how your data is stored. Assuming it's a grid you'd have to write code to iterate over the grid in the correct order from the view of the camera.
Your example only shows a green background so you could just draw opaque and multiply or mix by a color, the same color as your background.
const m4 = twgl.m4;
const gl = document.querySelector('canvas').getContext('webgl');
const vs = `
attribute vec4 position;
uniform mat4 u_matrix;
void main() {
gl_Position = u_matrix * position;
}
`;
const fs = `
precision mediump float;
uniform vec4 u_backgroundColor;
uniform float u_mixAmount;
void main() {
gl_FragColor = mix(vec4(0, 1, 0, 1), u_backgroundColor, u_mixAmount);
}
`;
// compile shaders, link, look up locations
const programInfo = twgl.createProgramInfo(gl, [vs, fs]);
// create buffers and upload vertex data
const bufferInfo = twgl.primitives.createCubeBufferInfo(gl, 1);
render();
function render() {
gl.clearColor(0, .4, 0, 1);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
gl.enable(gl.CULL_FACE);
gl.enable(gl.DEPTH_TEST);
gl.useProgram(programInfo.program);
const halfHeight = 1;
const aspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
const halfWidth = halfHeight * aspect;
const projection = m4.ortho(
-halfWidth, halfWidth, -halfHeight, halfWidth, 0.1, 20);
const camera = m4.lookAt(
[5, 2, 5], // eye
[0, -.5, 0], // target
[0, 1, 0], // up
);
const view = m4.inverse(camera);
const viewProjection = m4.multiply(projection, view);
twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);
for (let x = 1; x >= -1; --x) {
let mat = m4.translate(viewProjection, [x, 0, 0]);
twgl.setUniforms(programInfo, {
u_matrix: mat,
u_backgroundColor: [0, 0.4, 0, 1],
u_mixAmount: 0.5,
});
// calls drawArrays or drawElements
twgl.drawBufferInfo(gl, bufferInfo);
}
}
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<canvas></canvas>
The solution above changes the fragment shader to
uniform vec4 u_backgroundColor;
uniform float u_mixAmount;
void main() {
gl_FragColor = mix(vec4(0, 1, 0, 1), u_backgroundColor, u_mixAmount);
}
Where vec4(0, 1, 0, 1) is the cube's green color. We then set u_backgroundColor to match the background color of 0, .4, 0, 1 and set u_mixAmount to .5 (50%)
This solution might sound dumb but it's common to want to fade to a background color which is basically how fog works. You don't actually make things more transparent in the distance you just draw with the fog color.
draw all the tiles without transparency into another texture, then draw that texture with transparency
const m4 = twgl.m4;
const gl = document.querySelector('canvas').getContext('webgl', {alpha: false});
const vs = `
attribute vec4 position;
uniform mat4 u_matrix;
void main() {
gl_Position = u_matrix * position;
}
`;
const fs = `
precision mediump float;
void main() {
gl_FragColor = vec4(0, 1, 0, 1);
}
`;
const mixVs = `
attribute vec4 position;
attribute vec2 texcoord;
uniform mat4 u_matrix;
varying vec2 v_texcoord;
void main() {
gl_Position = u_matrix * position;
v_texcoord = texcoord;
}
`;
const mixFs = `
precision mediump float;
varying vec2 v_texcoord;
uniform sampler2D u_tex;
uniform float u_alpha;
void main() {
gl_FragColor = texture2D(u_tex, v_texcoord) * u_alpha;
}
`;
// compile shaders, link, look up locations
const programInfo = twgl.createProgramInfo(gl, [vs, fs]);
const mixProgramInfo = twgl.createProgramInfo(gl, [mixVs, mixFs]);
// create buffers and upload vertex data
const bufferInfo = twgl.primitives.createCubeBufferInfo(gl, 1);
const xyQuadBufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);
// create framebuffer with RGBA/UNSIGNED_BYTE texture
// and depth buffer renderbuffer that matches the size
// of the canvas
const fbi = twgl.createFramebufferInfo(gl);
render();
function render() {
renderTiles();
renderScene();
}
function renderScene() {
// bind canvas and set viewport
twgl.bindFramebufferInfo(gl, null);
gl.clearColor(0, 0.4, 0, 1);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
gl.enable(gl.BLEND);
gl.blendFunc(gl.ONE, gl.ONE_MINUS_SRC_ALPHA);
gl.useProgram(mixProgramInfo.program);
twgl.setBuffersAndAttributes(gl, mixProgramInfo, xyQuadBufferInfo);
twgl.setUniforms(mixProgramInfo, {
u_matrix: m4.identity(),
u_tex: fbi.attachments[0], // the texture
u_alpha: .5,
});
// calls drawArrays or drawElements
twgl.drawBufferInfo(gl, xyQuadBufferInfo);
}
function renderTiles() {
// bind framebuffer and set viewport
twgl.bindFramebufferInfo(gl, fbi);
gl.clearColor(0, 0, 0, 0);
gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT);
gl.disable(gl.BLEND);
gl.enable(gl.CULL_FACE);
gl.enable(gl.DEPTH_TEST);
gl.useProgram(programInfo.program);
const halfHeight = 1;
const aspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
const halfWidth = halfHeight * aspect;
const projection = m4.ortho(
-halfWidth, halfWidth, -halfHeight, halfWidth, 0.1, 20);
const camera = m4.lookAt(
[5, 2, 5], // eye
[0, -.5, 0], // target
[0, 1, 0], // up
);
const view = m4.inverse(camera);
const viewProjection = m4.multiply(projection, view);
twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);
for (let x = 1; x >= -1; --x) {
let mat = m4.translate(viewProjection, [x, 0, 0]);
twgl.setUniforms(programInfo, {
u_matrix: mat,
u_backgroundColor: [0, 0.4, 0, 1],
u_mixAmount: 0.5,
});
// calls drawArrays or drawElements
twgl.drawBufferInfo(gl, bufferInfo);
}
}
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<canvas></canvas>
The change above creates an RGBA texture and a depth renderbuffer the same size as the canvas and attaches them to a framebuffer. It then renders the tiles into that texture opaquely. Then it renders the texture over the canvas with 50% alpha. Note that the canvas itself is set to {alpha: false} so that the canvas doesn't blend with the elements behind it.
Generate new geometry that doesn't have the hidden surfaces
The problem is your drawing 3 cubes and the edges between them. A Minecraft like solution would probably generate new geometry that didn't have the inner edges. It would be pretty easy to walk a grid of tiles and decide whether or not to add that edge of the cube based on if there is a neighbor or not.
In Minecraft they only have to generate new geometry when blocks are added or removed and with some creative coding that might involve only modifying a few vertices rather than regenerating the entire mesh. They also probably generate in a gird like very 64x64x64 area.

How do I determine the average scene brightness in WebGL?

I am currently doing straightforward direct-to-screen (no multiple passes or postprocessing) rendering in WebGL. I would like to determine the average brightness/luminance of the entire rendered image (i.e. a single number), in a way which is efficient enough to do every frame.
What I'm looking to accomplish is to implement “exposure” adjustment (as a video camera or the human eye would) in the scene, so as to view both indoor and outdoor scenes with realistic lighting and no transitions — the brightness of the current frame will be negative feedback to the brightness of the next frame.
I am currently calculating a very rough approximation on the CPU side by sending a few rays through my scene data to find the brightness at those points; this works, but has too few samples to be stable (brightness varies noticeably with view angle as the rays cross light sources). I would prefer to offload the work to the GPU if at all possible, as my application is typically CPU-bound.
I just thought of a horrible kludge, namely to render to texture and generateMipmaps on it, then read the smallest level. I hope there's a better way.
What's wrong with that? This way is almost entirely done on the GPU, can be worked nicely into an existing render pipeline, and should give reasonable results. I don't know of any reason to recommend against it.
I know this question is 8 years old but hey....
First off, WebGL1, generateMipmap only works for power of 2 images.
I'd suggest either (1) generating a simple shaders like this
function createShader(texWidth, texHeight) {
return `
precision mediump float;
uniform sampler2D tex;
void main() {
vec2 size = vec2(${texWidth}, ${texHeight});
float totalBrightness = 0.0;
float minBrightness = 1.0;
float maxBrightness = 0.0;
for (int y = 0; y < ${texHeight}; ++y) {
for (int x = 0; x < ${texWidth}; ++x) {
vec4 color = texture2D(tex, (vec2(x, y) + 0.5) / size);
vec3 adjusted = color.rgb * vec3(0.2126, 0.7152, 0.0722);
float brightness = adjusted.r + adjusted.g + adjusted.b;
totalBrightness += brightness;
minBrightness = min(brightness, minBrightness);
maxBrightness = max(brightness, maxBrightness);
}
}
float averageBrightness = totalBrightness / (size.x * size.y);
gl_FragColor = vec4(averageBrightness, minBrightness, maxBrightness, 0);
}
`;
}
const startElem = document.querySelector('button');
startElem.addEventListener('click', main, {once: true});
function createShader(texWidth, texHeight) {
return `
precision mediump float;
uniform sampler2D tex;
void main() {
vec2 size = vec2(${texWidth}, ${texHeight});
float totalBrightness = 0.0;
float minBrightness = 1.0;
float maxBrightness = 0.0;
for (int y = 0; y < ${texHeight}; ++y) {
for (int x = 0; x < ${texWidth}; ++x) {
vec4 color = texture2D(tex, (vec2(x, y) + 0.5) / size);
vec3 adjusted = color.rgb * vec3(0.2126, 0.7152, 0.0722);
float brightness = adjusted.r + adjusted.g + adjusted.b;
totalBrightness += brightness;
minBrightness = min(brightness, minBrightness);
maxBrightness = max(brightness, maxBrightness);
}
}
float averageBrightness = totalBrightness / (size.x * size.y);
gl_FragColor = vec4(averageBrightness, minBrightness, maxBrightness, 0);
}
`;
}
const prgs = {}
function getAverageProgram(gl, width, height) {
const id = `${width}x${height}`;
const prg = prgs[id];
if (prg) {
return prg;
}
const vs = `
attribute vec4 position;
void main() {
gl_Position = position;
}
`;
const fs = createShader(width, height);
// compile shaders, link program, look up uniforms
const newPrg = twgl.createProgramInfo(gl, [vs, fs]);
prgs[id] = newPrg;
return newPrg;
}
function main() {
const gl = document.querySelector('canvas').getContext('webgl');
let updateTexture = false;
const video = document.createElement('video');
video.crossOrigin = 'anonymous';
video.loop = true;
video.src = 'https://webglsamples.org/color-adjust/sample-video.mp4';
if (video.requestVideoFrameCallback) {
function update() {
draw();
video.requestVideoFrameCallback(update);
};
video.requestVideoFrameCallback(update);
} else {
function update() {
if (video.currentTime > 0) {
draw();
}
requestAnimationFrame(update);
}
requestAnimationFrame(update);
}
video.volume = 0;
video.play();
// create a 1x1 pixel RGBA/UNSIGNED_BYTE framebuffer
const fbi = twgl.createFramebufferInfo(gl, [
{ internalForamt: gl.RGBA },
], 1, 1);
const tVS = `
attribute vec4 position;
attribute vec2 texcoord;
varying vec2 v_texcoord;
void main() {
gl_Position = position;
v_texcoord = texcoord;
}
`;
const tFS = `
precision mediump float;
uniform sampler2D tex;
varying vec2 v_texcoord;
void main() {
gl_FragColor = texture2D(tex, v_texcoord);
}
`;
// compile shaders, link program, look up uniforms
const textureProgInfo = twgl.createProgramInfo(gl, [tVS, tFS]);
const avgMinMaxVS = `
attribute float id;
varying float v_id;
uniform sampler2D avgMinMaxTex;
void main() {
vec4 avgMinMax = texture2D(avgMinMaxTex, vec2(0.5));
float v = id < 1.0
? avgMinMax.x
: id < 2.0
? avgMinMax.y
: avgMinMax.z;
gl_Position = vec4(1. - (id + 1.0) / 10., v * 2. - 1., 0, 1);
gl_PointSize = 10.0;
v_id = id;
}
`;
const avgMinMaxFS = `
precision mediump float;
varying float v_id;
void main() {
gl_FragColor = vec4(1., v_id / 2., 1. - v_id / 2., 1);
}
`;
// compile shaders, link program, look up uniforms
const avgMinMaxPrgInfo = twgl.createProgramInfo(gl, [avgMinMaxVS, avgMinMaxFS]);
const planeBufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);
const idBufferInfo = twgl.createBufferInfoFromArrays(gl, {
id: {
data: [0, 1, 2],
numComponents: 1,
},
});
const videoTex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, videoTex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
function draw() {
// copy video to texture
gl.bindTexture(gl.TEXTURE_2D, videoTex);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
// --- [ compute average, min, max to single pixel ] ---
const averagePrgInfo = getAverageProgram(gl, video.videoWidth, video.videoHeight);
gl.useProgram(averagePrgInfo.program);
// calls gl.bindFramebuffer and gl.viewport
twgl.bindFramebufferInfo(gl, fbi);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, averagePrgInfo, planeBufferInfo);
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, planeBufferInfo);
// --- [ draw video to texture ] ---
// calls gl.bindFramebuffer and gl.viewport
twgl.bindFramebufferInfo(gl, null);
gl.useProgram(textureProgInfo.program);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, textureProgInfo, planeBufferInfo);
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, planeBufferInfo);
// -- [ draw 3 points showing avg, min, max] ---
gl.useProgram(avgMinMaxPrgInfo.program);
gl.bindTexture(gl.TEXTURE_2D, fbi.attachments[0]);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, avgMinMaxPrgInfo, idBufferInfo);
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, idBufferInfo, gl.POINTS);
}
}
body {
background: #444;
}
canvas {
border: 1px solid black;
display: block;
}
<canvas></canvas>
<button type="button">start</button>
<span style="color: #FF0">■ max brightness</span>
<span style="color: #F80">■ min brightness, </span>
<span style="color: #F0F">■ average brightness, </span>
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
the only problem with this solution is that it can't be parallelized by the GPU AFAIK so (2) I might test doing something similar to generating mipmaps where I say make a shader that does 16x16 pixels and target it to generate a smaller texture and repeat until I get to 1x1. I'd have to test to see if that's actually faster and what size cell 2x2, 4x4, 16x16 etc is best.
Finally, if possible, like the example above, if I don't actually need the result on the CPU then just pass that 1x1 texture as input to some other shader. The example just draws 3 points but of course you could feed those values into the shader that's drawing the video to do some image processing like crank up the exposure if the brightness is low try to auto level the image based on the min and max brightness etc...
Note that in WebGL2 you wouldn't have to generate a different shader per size as WebGL2, or rather GLSL ES 3.0 you can have loops that are not based on constant values.
const startElem = document.querySelector('button');
startElem.addEventListener('click', main, {once: true});
function main() {
const gl = document.querySelector('canvas').getContext('webgl2');
if (!gl) {
return alert('need WebGL2')
}
let updateTexture = false;
const video = document.createElement('video');
video.crossOrigin = 'anonymous';
video.loop = true;
video.src = 'https://webglsamples.org/color-adjust/sample-video.mp4';
if (video.requestVideoFrameCallback) {
function update() {
draw();
video.requestVideoFrameCallback(update);
};
video.requestVideoFrameCallback(update);
} else {
function update() {
if (video.currentTime > 0) {
draw();
}
requestAnimationFrame(update);
}
requestAnimationFrame(update);
}
video.volume = 0;
video.play();
// create a 1x1 pixel RGBA/UNSIGNED_BYTE framebuffer
const fbi = twgl.createFramebufferInfo(gl, [
{ internalForamt: gl.RGBA },
], 1, 1);
const avgVS = `#version 300 es
in vec4 position;
void main() {
gl_Position = position;
}
`;
const avgFS = `#version 300 es
precision highp float;
uniform sampler2D tex;
out vec4 result;
void main() {
ivec2 size = textureSize(tex, 0);
float totalBrightness = 0.0;
float minBrightness = 1.0;
float maxBrightness = 0.0;
for (int y = 0; y < size.y; ++y) {
for (int x = 0; x < size.x; ++x) {
vec4 color = texelFetch(tex, ivec2(x, y), 0);
vec3 adjusted = color.rgb * vec3(0.2126, 0.7152, 0.0722);
float brightness = adjusted.r + adjusted.g + adjusted.b;
totalBrightness += brightness;
minBrightness = min(brightness, minBrightness);
maxBrightness = max(brightness, maxBrightness);
}
}
float averageBrightness = totalBrightness / float(size.x * size.y);
result = vec4(averageBrightness, minBrightness, maxBrightness, 0);
}
`;
// compile shaders, link program, look up uniforms
const averagePrgInfo = twgl.createProgramInfo(gl, [avgVS, avgFS]);
const tVS = `#version 300 es
in vec4 position;
in vec2 texcoord;
out vec2 v_texcoord;
void main() {
gl_Position = position;
v_texcoord = texcoord;
}
`;
const tFS = `#version 300 es
precision mediump float;
uniform sampler2D tex;
in vec2 v_texcoord;
out vec4 fragColor;
void main() {
fragColor = texture(tex, v_texcoord);
}
`;
// compile shaders, link program, look up uniforms
const textureProgInfo = twgl.createProgramInfo(gl, [tVS, tFS]);
const avgMinMaxVS = `#version 300 es
out float v_id;
uniform sampler2D avgMinMaxTex;
void main() {
vec4 avgMinMax = texelFetch(avgMinMaxTex, ivec2(0), 0);
float v = gl_VertexID == 0
? avgMinMax.x
: gl_VertexID == 1
? avgMinMax.y
: avgMinMax.z;
gl_Position = vec4(1. - (float(gl_VertexID) + 1.0) / 10., v * 2. - 1., 0, 1);
gl_PointSize = 10.0;
v_id = float(gl_VertexID);
}
`;
const avgMinMaxFS = `#version 300 es
precision mediump float;
in float v_id;
out vec4 fragColor;
void main() {
fragColor = vec4(1., v_id / 2., 1. - v_id / 2., 1);
}
`;
// compile shaders, link program, look up uniforms
const avgMinMaxPrgInfo = twgl.createProgramInfo(gl, [avgMinMaxVS, avgMinMaxFS]);
// creates buffers with positions and texcoords for a -1 to +1 quad
const planeBufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);
const videoTex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, videoTex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
function draw() {
// copy video to texture
gl.bindTexture(gl.TEXTURE_2D, videoTex);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
// --- [ compute average, min, max to single pixel ] ---
gl.useProgram(averagePrgInfo.program);
// calls gl.bindFramebuffer and gl.viewport
twgl.bindFramebufferInfo(gl, fbi);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, averagePrgInfo, planeBufferInfo);
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, planeBufferInfo);
// --- [ draw video to texture ] ---
// calls gl.bindFramebuffer and gl.viewport
twgl.bindFramebufferInfo(gl, null);
gl.useProgram(textureProgInfo.program);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, textureProgInfo, planeBufferInfo);
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, planeBufferInfo);
// -- [ draw 3 points showing avg, min, max] ---
gl.useProgram(avgMinMaxPrgInfo.program);
gl.bindTexture(gl.TEXTURE_2D, fbi.attachments[0]);
// draw 3 points
gl.drawArrays(gl.POINTS, 0, 3);
}
}
body {
background: #444;
}
canvas {
border: 1px solid black;
display: block;
}
<canvas></canvas>
<button type="button">start</button>
<span style="color: #FF0">■ max brightness</span>
<span style="color: #F80">■ min brightness, </span>
<span style="color: #F0F">■ average brightness, </span>
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>

Resources