get current pixel position on webGL2 fragment shader - webgl

I created a simple webGL script, it apply pixel color depending on (x,y) pixel position
What I get:
here's what I did:
#ifdef GL_ES
precision mediump float;
#endif
uniform float width;
uniform float height;
uniform float time;
void main() {
vec2 u_resolution = vec2(width, height);
vec2 st = gl_FragCoord.xy / u_resolution;
gl_FragColor = vec4(st.x, st.y, 0.5, 1.0);
}
Codepen: Hello WebGL
I'm trying to convert it to webGL2 but I don't know how to get current pixel position.
here's what I tried:
#version 300 es
#ifdef GL_ES
precision mediump float;
#endif
uniform float width;
uniform float height;
uniform float time;
out vec4 color;
void main() {
vec2 u_resolution = vec2(width, height);
vec2 st = color.xy / u_resolution;
color = vec4(st.x, st.y, 0.5, 1.0);
}
Codepen: Hello WebGL2
How to get current pixel position in webgl2?

gl_FragCoord is still the correct way in WebGL2
var canvas = document.body.appendChild(document.createElement("canvas"));
canvas.width = window.innerWidth;
canvas.height = window.innerHeight;
var gl = canvas.getContext("webgl2");
//************** Shader sources **************
var vertexSource = `
#version 300 es
in vec2 position;
void main() {
gl_Position = vec4(position, 0.0, 1.0);
}
`;
var fragmentSource = `
#version 300 es
#ifdef GL_ES
precision mediump float;
#endif
uniform float width;
uniform float height;
uniform float time;
out vec4 color;
void main() {
vec2 u_resolution = vec2(width, height);
vec2 st = gl_FragCoord.xy / u_resolution;
color = vec4(st.x, st.y, 0.5, 1.0);
}`;
window.addEventListener("resize", onWindowResize, false);
function onWindowResize() {
canvas.width = window.innerWidth;
canvas.height = window.innerHeight;
gl.viewport(0, 0, canvas.width, canvas.height);
gl.uniform1f(widthHandle, window.innerWidth);
gl.uniform1f(heightHandle, window.innerHeight);
}
//Compile shader and combine with source
function compileShader(shaderSource, shaderType) {
var shader = gl.createShader(shaderType);
gl.shaderSource(shader, shaderSource);
gl.compileShader(shader);
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
throw "Shader compile failed with: " + gl.getShaderInfoLog(shader);
}
return shader;
}
//From https://codepen.io/jlfwong/pen/GqmroZ
//Utility to complain loudly if we fail to find the attribute/uniform
function getAttribLocation(program, name) {
var attributeLocation = gl.getAttribLocation(program, name);
if (attributeLocation === -1) {
throw "Cannot find attribute " + name + ".";
}
return attributeLocation;
}
function getUniformLocation(program, name) {
var attributeLocation = gl.getUniformLocation(program, name);
if (attributeLocation === -1) {
throw "Cannot find uniform " + name + ".";
}
return attributeLocation;
}
//************** Create shaders **************
//Create vertex and fragment shaders
var vertexShader = compileShader(vertexSource.trim(), gl.VERTEX_SHADER);
var fragmentShader = compileShader(fragmentSource.trim(), gl.FRAGMENT_SHADER);
//Create shader programs
var program = gl.createProgram();
gl.attachShader(program, vertexShader);
gl.attachShader(program, fragmentShader);
gl.linkProgram(program);
gl.useProgram(program);
//Set up rectangle covering entire canvas
var vertexData = new Float32Array([
-1.0,
1.0, // top left
-1.0,
-1.0, // bottom left
1.0,
1.0, // top right
1.0,
-1.0 // bottom right
]);
//Create vertex buffer
var vertexDataBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, vertexDataBuffer);
gl.bufferData(gl.ARRAY_BUFFER, vertexData, gl.STATIC_DRAW);
// Layout of our data in the vertex buffer
var positionHandle = getAttribLocation(program, "position");
gl.enableVertexAttribArray(positionHandle);
gl.vertexAttribPointer(
positionHandle,
2, // position is a vec2 (2 values per component)
gl.FLOAT, // each component is a float
false, // don't normalize values
2 * 4, // two 4 byte float components per vertex (32 bit float is 4 bytes)
0 // how many bytes inside the buffer to start from
);
//Set uniform handle
var timeHandle = getUniformLocation(program, "time");
var widthHandle = getUniformLocation(program, "width");
var heightHandle = getUniformLocation(program, "height");
gl.uniform1f(widthHandle, window.innerWidth);
gl.uniform1f(heightHandle, window.innerHeight);
function draw() {
//Send uniforms to program
gl.uniform1f(timeHandle, performance.now());
//Draw a triangle strip connecting vertices 0-4
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
requestAnimationFrame(draw);
}
draw();
html {
overflow: hidden;
}
canvas {
display: block;
}
Some other random tips.
These ifdefs are irrelevant
#ifdef GL_ES
precision mediump float;
#endif
Just
precision mediump float;
is fine.
I'm guessing this obvious but why pass in width and height separate?
How about just
uniform vec2 u_resolution;
No reason to call performance.now. The time is passed to your requestAnimationFrame callback
function draw(time) {
//Send uniforms to program
gl.uniform1f(timeHandle, time);
...
requestAnimationFrame(draw);
}
requestAnimationFrame(draw);
The code checks for compile errors but not link errors
You should check for link errors
gl.linkProgram(program);
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
throw "Program link failed with: " + gl.getProgramInfoLog(program);
}
There will be link errors if your varyings don't match and further the spec doesn't require compiling to ever fail even on bad shaders. Rather it only requires if they were bad to fail to link.
window.innerWidth
see: this
gl.getUniformLocation returns null if the uniform does not exist
The code is checking for -1 which is correct for attributes but not for uniforms.
throwing on attributes and uniforms not existing
Of course it's helpful to know they don't exist but it's common to debug shaders by commenting things out or editing. For example lets say nothing appears on the screen. If it was me the first thing I'd do is change the fragment shader to this
const fragmentSource = `
#version 300 es
precision mediump float;
uniform vec2 u_resolution;
uniform float time;
out vec4 color;
void main() {
vec2 st = gl_FragCoord.xy / u_resolution;
color = vec4(st.x, st.y, 0.5, 1.0);
color = vec4(1, 0, 0, 1); // <----------------------
}`;
Just output a solid color to check if the issue is in the fragment shader or the vertex shader. The moment I do that most WebGL implentations will optimize out u_resolution and the code that throws when looking up locations effectively makes the program undebuggable.
In fact the code only runs currently because of the previous bug checking for -1 instead of null. With that bug fixed the code crashes beacuse time is optimized out.
var canvas = document.body.appendChild(document.createElement("canvas"));
var gl = canvas.getContext("webgl2");
//************** Shader sources **************
var vertexSource = `
#version 300 es
in vec2 position;
void main() {
gl_Position = vec4(position, 0.0, 1.0);
}
`;
var fragmentSource = `
#version 300 es
precision mediump float;
uniform vec2 u_resolution;
uniform float time;
out vec4 color;
void main() {
vec2 st = gl_FragCoord.xy / u_resolution;
color = vec4(st.x, st.y, 0.5, 1.0);
}`;
function resize() {
if (canvas.width !== canvas.clientWidth || canvas.height !== canvas.clientHeight) {
canvas.width = canvas.clientWidth;
canvas.height = canvas.clientHeight;
gl.viewport(0, 0, canvas.width, canvas.height);
gl.uniform2f(resHandle, canvas.width, canvas.height);
}
}
//Compile shader and combine with source
function compileShader(shaderSource, shaderType) {
var shader = gl.createShader(shaderType);
gl.shaderSource(shader, shaderSource);
gl.compileShader(shader);
if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
throw "Shader compile failed with: " + gl.getShaderInfoLog(shader);
}
return shader;
}
//From https://codepen.io/jlfwong/pen/GqmroZ
//Utility to complain loudly if we fail to find the attribute/uniform
function getAttribLocation(program, name) {
var attributeLocation = gl.getAttribLocation(program, name);
if (attributeLocation === -1) {
console.warn("Cannot find attribute", name);
}
return attributeLocation;
}
function getUniformLocation(program, name) {
var uniformLocation = gl.getUniformLocation(program, name);
if (uniformLocation === null) {
console.warn("Cannot find uniform", name);
}
return uniformLocation;
}
//************** Create shaders **************
//Create vertex and fragment shaders
var vertexShader = compileShader(vertexSource.trim(), gl.VERTEX_SHADER);
var fragmentShader = compileShader(fragmentSource.trim(), gl.FRAGMENT_SHADER);
//Create shader programs
var program = gl.createProgram();
gl.attachShader(program, vertexShader);
gl.attachShader(program, fragmentShader);
gl.linkProgram(program);
if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {
throw "Program link failed with: " + gl.getProgramInfoLog(program);
}
gl.useProgram(program);
//Set up rectangle covering entire canvas
var vertexData = new Float32Array([
-1.0,
1.0, // top left
-1.0,
-1.0, // bottom left
1.0,
1.0, // top right
1.0,
-1.0 // bottom right
]);
//Create vertex buffer
var vertexDataBuffer = gl.createBuffer();
gl.bindBuffer(gl.ARRAY_BUFFER, vertexDataBuffer);
gl.bufferData(gl.ARRAY_BUFFER, vertexData, gl.STATIC_DRAW);
// Layout of our data in the vertex buffer
var positionHandle = getAttribLocation(program, "position");
gl.enableVertexAttribArray(positionHandle);
gl.vertexAttribPointer(
positionHandle,
2, // position is a vec2 (2 values per component)
gl.FLOAT, // each component is a float
false, // don't normalize values
2 * 4, // two 4 byte float components per vertex (32 bit float is 4 bytes)
0 // how many bytes inside the buffer to start from
);
//Set uniform handle
var timeHandle = getUniformLocation(program, "time");
var resHandle = getUniformLocation(program, "u_resolution");
function draw(time) {
resize();
//Send uniforms to program
gl.uniform1f(timeHandle, time);
//Draw a triangle strip connecting vertices 0-4
gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
requestAnimationFrame(draw);
}
requestAnimationFrame(draw);
html,body {
height: 100%;
margin: 0;
}
canvas {
width: 100%;
height: 100%;
display: block;
}

Related

Framebuffer Attachments In WebGL Producing Unexpected Results

I have a problem with framebuffer attachments. Basically, my framebuffers always worked fine, but for my last project I needed to initialize them with some color values. So I created an attachment with a texture containing the color values I wanted. This leads to some really unexplainable (to me) behaviour, you can see in the code below that I create a framebuffer with an attachment, then have one shader which renders a very simple shape to the framebuffer and another shader which reads out the values with some noise added to the readout position.
The weird thing is that
The readouts seem to be weirdly all over the place, if you delete the framebuffer attachment in the source (just comment out the lines so that an empty object remains in the array) you will see how it is meant to look like (notice the noisy edges, so writing to and reading from framebuffer works as expected).
Instead it looks like this:
Also changing the values of the attachment texture changes the result, which is weird as I never read from the framebuffer before writing to it. It seems as if the readouts return the initial color value most of the time (in this case gray).
If you remove the noise term or attach a constant noise term (independent of position), the readouts seem to work just fine.
(function main() {
const dim = [512, 512];
twgl.setDefaults({ attribPrefix: "a_" });
const gl = twgl.getContext(document.querySelector("canvas"));
gl.canvas.width = dim[0];
gl.canvas.height = dim[1];
const bfi = twgl.primitives.createXYQuadBufferInfo(gl);
const pgi = {
cross: twgl.createProgramInfo(gl, ["vs", "fs_cross"]),
noise: twgl.createProgramInfo(gl, ["vs", "fs_noise"])
};
const fbi = twgl.createFramebufferInfo(
gl,
[
{
attachment: twgl.createTexture(gl, {
src: Array(dim[0] * dim[1])
.fill([128, 128, 0, 0])
.flat()
})
}
],
...dim
);
(function frame() {
twgl.bindFramebufferInfo(gl, fbi);
gl.useProgram(pgi.cross.program);
twgl.setUniforms(pgi.cross, {
u_resolution: dim
});
twgl.setBuffersAndAttributes(gl, pgi.cross, bfi);
twgl.drawBufferInfo(gl, bfi);
twgl.bindFramebufferInfo(gl, null);
gl.useProgram(pgi.noise.program);
twgl.setUniforms(pgi.noise, {
u_framebuffer: fbi.attachments[0],
u_pi: Math.PI,
u_resolution: dim,
u_seed: Array(24).fill().map(Math.random)
});
twgl.setBuffersAndAttributes(gl, pgi.noise, bfi);
twgl.drawBufferInfo(gl, bfi);
window.requestAnimationFrame(frame);
})();
})();
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<script id="vs" type="x-shader/x-vertex">
attribute vec4 a_position;
attribute vec2 a_texcoord;
varying vec2 v_texcoord;
void main() {
v_texcoord = a_texcoord;
gl_Position = a_position;
}
</script>
<script id="fs_cross" type="x-shader/x-fragment">
precision highp float;
varying vec2 v_texcoord;
uniform vec2 u_resolution;
void main() {
if(sign(v_texcoord.x - 0.5) * sign(v_texcoord.y - 0.5) < 0.0) {
gl_FragColor = vec4(0.0, 0.0, 0.0, 1.0);
} else {
gl_FragColor = vec4(1.0, 1.0, 1.0, 1.0);
}
}
</script>
<script id="fs_noise" type="x-shader/x-fragment">
precision highp float;
varying vec2 v_texcoord;
uniform sampler2D u_framebuffer;
uniform float u_pi;
uniform vec2 u_resolution;
uniform vec2 u_seed[12];
vec2 normal(vec2 uv) {
float scl = sqrt(-2.0 * log(uv.x));
float ang = 2.0 * u_pi * uv.y;
return vec2(scl * cos(ang), scl * sin(ang));
}
vec2 noisySample(sampler2D tex, vec2 coord) {
vec2 sum = vec2(0.0, 0.0);
vec2 uni = vec2(0.0, 0.0);
for(int i = 0; i < 6; i++) {
uni = fract(uni + vec2(
dot(gl_FragCoord.xy + sin(gl_FragCoord.xy), u_seed[i]),
dot(gl_FragCoord.xy + sin(gl_FragCoord.xy), u_seed[i + 6])
));
vec2 nc = coord + normal(uni) / u_resolution;
sum += texture2D(u_framebuffer, nc).rg;
}
return sum / 6.0;
}
void main() {
vec2 tmp = noisySample(u_framebuffer, v_texcoord);
gl_FragColor = vec4(tmp, 0.5, 1.0);
}
</script>
<canvas></canvas>

Webgl Texture Artifacts

I'm trying to create a simple page flip effect in WebGL through a vertex shader. If I use the following vertex shader code, the page turn and everything looks fine.
float y_rot = mix(uAnimationStep, ease_out, aTextureCoord.x) * -PI;
If however I add the following adjustment (to make the bottom part of the page rotate faster, I get very bad texture artifacts (see the picture below).
float curve = mix(0.0, 0.25, aTextureCoord.y);
float y_rot = mix(uAnimationStep, ease_out + curve, aTextureCoord.x) * -PI;
I'm sure I'm missing something basic here... any ideas? I've tried to turn mipmapping on, but it didnt help. Thanks!
There really isn't enough code to answer your question but copying your 2 lines into some random sample I see no issues so it seems like your issue is somewhere else.
'use strict';
/* global twgl, requestAnimationFrame, document */
const m4 = twgl.m4;
const gl = document.querySelector('canvas').getContext('webgl');
const vs = `
attribute vec4 position;
attribute vec3 normal;
attribute vec2 texcoord;
uniform mat4 projection;
uniform mat4 modelView;
uniform float uAnimationStep;
const float ease_out = 0.0;
varying vec3 v_normal;
varying vec2 v_texcoord;
#define PI radians(180.0)
mat4 rotY(float angleInRadians) {
float s = sin(angleInRadians);
float c = cos(angleInRadians);
return mat4(
c, 0,-s, 0,
0, 1, 0, 0,
s, 0, c, 0,
0, 0, 0, 1);
}
void main() {
vec2 aTextureCoord = texcoord;
float curve = mix(0.0, 0.25, aTextureCoord.y);
float y_rot = mix(uAnimationStep, ease_out + curve, aTextureCoord.x) * -PI;
mat4 effectiveModelView = modelView * rotY(y_rot);
gl_Position = projection * effectiveModelView * position;
v_normal = mat3(effectiveModelView) * normal;
v_texcoord = texcoord;
}
`;
const fs = `
precision highp float;
varying vec3 v_normal;
varying vec2 v_texcoord;
varying float v_modelId;
uniform sampler2D tex;
void main() {
vec3 lightDirection = normalize(vec3(1, 2, 30)); // arbitrary light direction
vec3 color = texture2D(tex, v_texcoord).rgb;
float l = dot(lightDirection, normalize(v_normal)) * .5 + .5;
gl_FragColor = vec4(color * l, 1);
}
`;
// compile shader, link, look up locations
const programInfo = twgl.createProgramInfo(gl, [vs, fs]);
// make some vertex data
// calls gl.createBuffer, gl.bindBuffer, gl.bufferData
const bufferInfo = twgl.primitives.createPlaneBufferInfo(
gl,
32, // width
32, // depth
32, // width subdivisions
32, // height subdivisions
m4.rotationX(Math.PI / 2), // matrix to apply (plane is XZ, make it XY)
);
const tex = twgl.createTexture(gl, {src: 'https://i.imgur.com/ZKMnXce.png'});
function render(time) {
time *= 0.001; // seconds
twgl.resizeCanvasToDisplaySize(gl.canvas);
gl.viewport(0, 0, gl.canvas.width, gl.canvas.height);
gl.enable(gl.DEPTH_TEST);
const fov = Math.PI * 0.25;
const aspect = gl.canvas.clientWidth / gl.canvas.clientHeight;
const near = 0.1;
const far = 100;
const projection = m4.perspective(fov, aspect, near, far);
const eye = [0, 30, 35];
const target = [0, 0, 0];
const up = [0, 1, 0];
const camera = m4.lookAt(eye, target, up);
const view = m4.inverse(camera);
let modelView = m4.rotateY(view, 0.2 * time);
modelView = m4.translate(modelView, [0, 0, 0]);
gl.useProgram(programInfo.program);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, programInfo, bufferInfo);
// calls gl.activeTexture, gl.bindTexture, gl.uniformXXX
twgl.setUniforms(programInfo, {
projection,
modelView,
tex,
uAnimationStep: Math.sin(time),
});
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, bufferInfo);
requestAnimationFrame(render);
}
requestAnimationFrame(render);
body { margin: 0; }
canvas { width: 100vw; height: 100vh; display: block; }
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<canvas></canvas>

Sampling integer texture in WebGL returns weird values

I'm trying to render an grayscale image from a 16-bit array buffer in WebGL2, by applying window leveling in the fragment shader. I'v generated the texture as below:
let typedArray = new Int16Array(data);
gl.texImage2D(
gl.TEXTURE_2D,
0,
gl.R16I,
w, h,
0,
gl.RED_INTEGER,
gl.SHORT,
typedArray);
and tried to use the data from the fragment shader below:
let fragmentShaderSource = `#version 300 es
precision highp float;
precision highp int;
precision highp isampler2D;
// our texture
uniform isampler2D u_image;
uniform highp float u_windowWidth;
uniform highp float u_windowCenter;
in vec2 v_texCoord;
out vec4 outColor;
void main() {
highp float f = float(texture(u_image, v_texCoord).r);
f = (f - (u_windowCenter - 0.5)) / max(u_windowWidth - 1.0, 1.0) + 0.5;
f = min(max(f, 0.0), 1.0);
outColor = vec4(vec3(f), 1.0);
}
`;
but this only renders a black screen. Actually, after some debugging, I found that texture(u_image, v_texCoord) had zero values in rgb across all pixels and a (alpha) field had very large (2^29 ~ 2^30) value. I've tried changing precisions in the shader but results were the same.
In order to narrow down the problem scope, I've tried a different approach by splitting the 16-bit integer into gl.RGBA4, which contains 4-bits in each RGBA channels:
let typedArray = new Uint16Array(data);
gl.texImage2D(
gl.TEXTURE_2D,
0,
gl.RGBA4,
w, h,
0,
gl.RGBA,
gl.UNSIGNED_SHORT_4_4_4_4,
typedArray);
and combined RGBA values back into 16-bit integer in the fragment shader.
let fragmentShaderSource = `#version 300 es
precision highp float;
precision highp int;
precision highp sampler2D;
// our texture
uniform sampler2D u_image;
uniform highp float u_windowWidth;
uniform highp float u_windowCenter;
in vec2 v_texCoord;
out vec4 outColor;
void main() {
highp vec4 rgba_map = texture(u_image, v_texCoord);
// Combining rgba4 back into int16
highp f = rgba_map.r * 65536.0 + rgba_map.g * 4096.0 + rgba_map.b * 256.0 + rgba_map.a * 16.0;
// signed value
if (f > 32768.0) {
f = 65536.0 - f;
}
f = (f - (u_windowCenter - 0.5)) / max(u_windowWidth - 1.0, 1.0) + 0.5;
f = min(max(f, 0.0), 1.0);
outColor = vec4(vec3(f), 1.0);
}
`;
and this version rendered the expected image quite well, although the result was a bit noisy due to the conversion. I've also tried some other formats, and those with float type were fine and the integer type formats were all not working. So I think the other parts of the program are fine. I wonder what is wrong with my program.
You haven't really posted enough code to debug so let's just make something that works.
function main() {
const gl = document.querySelector('canvas').getContext('webgl2');
if (!gl) {
return alert('need WebGL2');
}
const vs = `#version 300 es
void main() {
gl_PointSize = 300.0;
gl_Position = vec4(0, 0, 0, 1);
}
`;
const fs = `#version 300 es
precision highp float;
precision highp int;
precision highp isampler2D;
// our texture
uniform isampler2D u_image;
out vec4 color;
void main() {
ivec4 intColor = texture(u_image, gl_PointCoord.xy);
color = vec4(vec3(intColor.rrr) / 10000.0, 1);
}
`;
const program = twgl.createProgram(gl, [vs, fs]);
const tex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, tex);
gl.texImage2D(
gl.TEXTURE_2D,
0, // mip level
gl.R16I, // internal format
10, // width
1, // height
0, // border
gl.RED_INTEGER, // source format
gl.SHORT, // source type
new Int16Array([
1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000
]));
// can't filter integer textures
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.useProgram(program);
// no need to set any attributes or
// uniforms as we're not using attributes
// and uniforms default to zero so will use
// texture unit zero
gl.drawArrays(gl.POINTS, 0, 1);
console.log('max point size:', gl.getParameter(gl.ALIASED_POINT_SIZE_RANGE)[1]);
}
main();
canvas {
border: 1px solid black;
background: red;
}
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
<canvas></canvas>
Should look like this
but might have red borders if your GPUs max point size < 300
a few ideas
did you check the JavaScript console for errors?
did you turn off filtering for the texture?
integer texture can not be filtered
is your texture width an even number?
If not you probably need to set gl.pixelStorei(gl.UNPACK_ALIGNMENT, 1) though I'd have expected you to get an error here unless your Int16Array is larger than width * height

WebGL: fade drawing buffer

I've set preserveDrawingBuffer to true.
Doing this results in everything drawn on the buffer to be seen all at once, however,
I was wondering if there is a way to somehow fade the buffer as time goes on so that the old elements drawn disappear over time, and the newest drawn elements appear with a relatively high opacity until they also fade away.
Is there a better way to achieve such an effect?
I've tried to render previous elements again by lowering their opacity until it reaches 0 but it didn't seem like an efficient way of fading as once something is drawn I don't plan on changing it.
Thanks!
It's actually common it just redraw stuff which I went over here
WebGL: smoothly fade lines out of canvas
Redrawing stuff means you can keep some things from not fading out. For example if you're making a space shooting game and you only want explosions and missile trails to fade out but you don't want the spaceships and asteroids to fade out then you need to do it by redrawing everything and manually fading stuff out by drawn them while decreasing their alpha
If you just want everything to fade out then you can use a post processing type effect.
You make 2 textures and attach them to 2 framebuffers. You blend/fade the first framebuffer fadeFb1 into the second one fadeFb2 with a fadeColor using
gl_FragColor = mix(textureColor, fadeColor, mixAmount);
You then draw any new stuff to fadeFb2
Then finally draw fadeFb2 to the canvas so you can see the result.
The next frame you do the same thing except swap which buffer you're drawing to and which one you're fading to.
frame 0: mix(fadeFb1,fadeColor)->fadeFb2, draw->fadeFb2, fadeFB2->canvas
frame 1: mix(fadeFb2,fadeColor)->fadeFb1, draw->fadeFb1, fadeFB1->canvas
frame 2: mix(fadeFb1,fadeColor)->fadeFb2, draw->fadeFb2, fadeFB2->canvas
...
Note you don't clear when you draw since you need the result to be left behind
As for setting up framebuffers there's a tutorial here that might be useful
http://webglfundamentals.org/webgl/lessons/webgl-image-processing-continued.html
Here's an example using twgl since I'm too lazy for straight WebGL
var vs = `
attribute vec4 position;
uniform mat4 u_matrix;
void main() {
gl_Position = u_matrix * position;
}
`;
var fs = `
precision mediump float;
uniform vec4 u_color;
void main() {
gl_FragColor = u_color;
}
`;
var vsQuad = `
attribute vec4 position;
attribute vec2 texcoord;
varying vec2 v_texcoord;
void main() {
gl_Position = position;
v_texcoord = texcoord;
}
`;
var fsFade = `
precision mediump float;
varying vec2 v_texcoord;
uniform sampler2D u_texture;
uniform float u_mixAmount;
uniform vec4 u_fadeColor;
void main() {
vec4 color = texture2D(u_texture, v_texcoord);
gl_FragColor = mix(color, u_fadeColor, u_mixAmount);
}
`;
var fsCopy = `
precision mediump float;
varying vec2 v_texcoord;
uniform sampler2D u_texture;
void main() {
gl_FragColor = texture2D(u_texture, v_texcoord);
}
`;
var $ = document.querySelector.bind(document);
var mixAmount = 0.05;
var mixElem = $("#mix");
var mixValueElem = $("#mixValue");
mixElem.addEventListener('input', function(e) {
setMixAmount(e.target.value / 100);
});
function setMixAmount(value) {
mixAmount = value;
mixValueElem.innerHTML = mixAmount;
}
setMixAmount(mixAmount);
var gl = $("canvas").getContext("webgl");
var m4 = twgl.m4;
var programInfo = twgl.createProgramInfo(gl, [vs, fs]);
var fadeProgramInfo = twgl.createProgramInfo(gl, [vsQuad, fsFade]);
var copyProgramInfo = twgl.createProgramInfo(gl, [vsQuad, fsCopy]);
// Creates a -1 to +1 quad
var quadBufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);
// Creates an RGBA/UNSIGNED_BYTE texture and depth buffer framebuffer
var imgFbi = twgl.createFramebufferInfo(gl);
// Creates 2 RGBA texture + depth framebuffers
var fadeAttachments = [
{ format: gl.RGBA, min: gl.NEAREST, max: gl.NEAREST, wrap: gl.CLAMP_TO_EDGE, },
{ format: gl.DEPTH_STENCIL },
];
var fadeFbi1 = twgl.createFramebufferInfo(gl, fadeAttachments);
var fadeFbi2 = twgl.createFramebufferInfo(gl, fadeAttachments);
function drawThing(gl, x, y, rotation, scale, color) {
var matrix = m4.ortho(0, gl.canvas.width, gl.canvas.height, 0, -1, 1);
matrix = m4.translate(matrix, [x, y, 0]);
matrix = m4.rotateZ(matrix, rotation);
matrix = m4.scale(matrix, [scale, scale, 1]);
gl.useProgram(programInfo.program);
twgl.setBuffersAndAttributes(gl, programInfo, quadBufferInfo);
twgl.setUniforms(programInfo, {
u_matrix: matrix,
u_color: color,
});
twgl.drawBufferInfo(gl, gl.TRIANGLES, quadBufferInfo);
}
function rand(min, max) {
if (max === undefined) {
max = min;
min = 0;
}
return min + Math.random() * (max - min);
}
function render(time) {
if (twgl.resizeCanvasToDisplaySize(gl.canvas)) {
twgl.resizeFramebufferInfo(gl, fadeFbi1, fadeAttachments);
twgl.resizeFramebufferInfo(gl, fadeFbi2, fadeAttachments);
}
// fade by copying from fadeFbi1 into fabeFbi2 using mixAmount.
// fadeFbi2 will contain mix(fadeFb1, u_fadeColor, u_mixAmount)
twgl.bindFramebufferInfo(gl, fadeFbi2);
gl.useProgram(fadeProgramInfo.program);
twgl.setBuffersAndAttributes(gl, fadeProgramInfo, quadBufferInfo);
twgl.setUniforms(fadeProgramInfo, {
u_texture: fadeFbi1.attachments[0],
u_mixAmount: mixAmount,
u_fadeColor: [0, 0, 0, 0],
});
twgl.drawBufferInfo(gl, gl.TRIANGLES, quadBufferInfo);
// now draw new stuff to fadeFb2. Notice we don't clear!
twgl.bindFramebufferInfo(gl, fadeFbi2);
var x = rand(gl.canvas.width);
var y = rand(gl.canvas.height);
var rotation = rand(Math.PI);
var scale = rand(10, 20);
var color = [rand(1), rand(1), rand(1), 1];
drawThing(gl, x, y, rotation, scale, color);
// now copy fadeFbi2 to the canvas so we can see the result
twgl.bindFramebufferInfo(gl, null);
gl.useProgram(copyProgramInfo.program);
twgl.setBuffersAndAttributes(gl, copyProgramInfo, quadBufferInfo);
twgl.setUniforms(copyProgramInfo, {
u_texture: fadeFbi2.attachments[0],
});
twgl.drawBufferInfo(gl, gl.TRIANGLES, quadBufferInfo);
// swap the variables so we render to the opposite textures next time
var temp = fadeFbi1;
fadeFbi1 = fadeFbi2;
fadeFbi2 = temp;
requestAnimationFrame(render);
}
requestAnimationFrame(render);
body { margin: 0; }
canvas { display: block; width: 100vw; height: 100vh; }
#ui { position: absolute; top: 0 }
<script src="https://twgljs.org/dist/twgl-full.min.js"></script>
<canvas></canvas>
<div id="ui">
<span>mix:</span><input id="mix" type="range" min="0" max="100" value="5" /><span id="mixValue"></span>
</div>
The preserveDrawingBuffer flag is useful on a device with limited memory (mobile phones) as it allows those devices to reuse that chunk of memory.
The fading/ghosting effect is done in a different manner: you allocate a texture with the same size as the viewport and do the darkening on this texture instead. Every frame you re-render the contents of this texture to itself while multiplying the color value with a fading factor (say 0.9). Afterwards, on the same texture you render your new elements and finally you render the texture to the viewport (a simple "copy-render").

How do I determine the average scene brightness in WebGL?

I am currently doing straightforward direct-to-screen (no multiple passes or postprocessing) rendering in WebGL. I would like to determine the average brightness/luminance of the entire rendered image (i.e. a single number), in a way which is efficient enough to do every frame.
What I'm looking to accomplish is to implement “exposure” adjustment (as a video camera or the human eye would) in the scene, so as to view both indoor and outdoor scenes with realistic lighting and no transitions — the brightness of the current frame will be negative feedback to the brightness of the next frame.
I am currently calculating a very rough approximation on the CPU side by sending a few rays through my scene data to find the brightness at those points; this works, but has too few samples to be stable (brightness varies noticeably with view angle as the rays cross light sources). I would prefer to offload the work to the GPU if at all possible, as my application is typically CPU-bound.
I just thought of a horrible kludge, namely to render to texture and generateMipmaps on it, then read the smallest level. I hope there's a better way.
What's wrong with that? This way is almost entirely done on the GPU, can be worked nicely into an existing render pipeline, and should give reasonable results. I don't know of any reason to recommend against it.
I know this question is 8 years old but hey....
First off, WebGL1, generateMipmap only works for power of 2 images.
I'd suggest either (1) generating a simple shaders like this
function createShader(texWidth, texHeight) {
return `
precision mediump float;
uniform sampler2D tex;
void main() {
vec2 size = vec2(${texWidth}, ${texHeight});
float totalBrightness = 0.0;
float minBrightness = 1.0;
float maxBrightness = 0.0;
for (int y = 0; y < ${texHeight}; ++y) {
for (int x = 0; x < ${texWidth}; ++x) {
vec4 color = texture2D(tex, (vec2(x, y) + 0.5) / size);
vec3 adjusted = color.rgb * vec3(0.2126, 0.7152, 0.0722);
float brightness = adjusted.r + adjusted.g + adjusted.b;
totalBrightness += brightness;
minBrightness = min(brightness, minBrightness);
maxBrightness = max(brightness, maxBrightness);
}
}
float averageBrightness = totalBrightness / (size.x * size.y);
gl_FragColor = vec4(averageBrightness, minBrightness, maxBrightness, 0);
}
`;
}
const startElem = document.querySelector('button');
startElem.addEventListener('click', main, {once: true});
function createShader(texWidth, texHeight) {
return `
precision mediump float;
uniform sampler2D tex;
void main() {
vec2 size = vec2(${texWidth}, ${texHeight});
float totalBrightness = 0.0;
float minBrightness = 1.0;
float maxBrightness = 0.0;
for (int y = 0; y < ${texHeight}; ++y) {
for (int x = 0; x < ${texWidth}; ++x) {
vec4 color = texture2D(tex, (vec2(x, y) + 0.5) / size);
vec3 adjusted = color.rgb * vec3(0.2126, 0.7152, 0.0722);
float brightness = adjusted.r + adjusted.g + adjusted.b;
totalBrightness += brightness;
minBrightness = min(brightness, minBrightness);
maxBrightness = max(brightness, maxBrightness);
}
}
float averageBrightness = totalBrightness / (size.x * size.y);
gl_FragColor = vec4(averageBrightness, minBrightness, maxBrightness, 0);
}
`;
}
const prgs = {}
function getAverageProgram(gl, width, height) {
const id = `${width}x${height}`;
const prg = prgs[id];
if (prg) {
return prg;
}
const vs = `
attribute vec4 position;
void main() {
gl_Position = position;
}
`;
const fs = createShader(width, height);
// compile shaders, link program, look up uniforms
const newPrg = twgl.createProgramInfo(gl, [vs, fs]);
prgs[id] = newPrg;
return newPrg;
}
function main() {
const gl = document.querySelector('canvas').getContext('webgl');
let updateTexture = false;
const video = document.createElement('video');
video.crossOrigin = 'anonymous';
video.loop = true;
video.src = 'https://webglsamples.org/color-adjust/sample-video.mp4';
if (video.requestVideoFrameCallback) {
function update() {
draw();
video.requestVideoFrameCallback(update);
};
video.requestVideoFrameCallback(update);
} else {
function update() {
if (video.currentTime > 0) {
draw();
}
requestAnimationFrame(update);
}
requestAnimationFrame(update);
}
video.volume = 0;
video.play();
// create a 1x1 pixel RGBA/UNSIGNED_BYTE framebuffer
const fbi = twgl.createFramebufferInfo(gl, [
{ internalForamt: gl.RGBA },
], 1, 1);
const tVS = `
attribute vec4 position;
attribute vec2 texcoord;
varying vec2 v_texcoord;
void main() {
gl_Position = position;
v_texcoord = texcoord;
}
`;
const tFS = `
precision mediump float;
uniform sampler2D tex;
varying vec2 v_texcoord;
void main() {
gl_FragColor = texture2D(tex, v_texcoord);
}
`;
// compile shaders, link program, look up uniforms
const textureProgInfo = twgl.createProgramInfo(gl, [tVS, tFS]);
const avgMinMaxVS = `
attribute float id;
varying float v_id;
uniform sampler2D avgMinMaxTex;
void main() {
vec4 avgMinMax = texture2D(avgMinMaxTex, vec2(0.5));
float v = id < 1.0
? avgMinMax.x
: id < 2.0
? avgMinMax.y
: avgMinMax.z;
gl_Position = vec4(1. - (id + 1.0) / 10., v * 2. - 1., 0, 1);
gl_PointSize = 10.0;
v_id = id;
}
`;
const avgMinMaxFS = `
precision mediump float;
varying float v_id;
void main() {
gl_FragColor = vec4(1., v_id / 2., 1. - v_id / 2., 1);
}
`;
// compile shaders, link program, look up uniforms
const avgMinMaxPrgInfo = twgl.createProgramInfo(gl, [avgMinMaxVS, avgMinMaxFS]);
const planeBufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);
const idBufferInfo = twgl.createBufferInfoFromArrays(gl, {
id: {
data: [0, 1, 2],
numComponents: 1,
},
});
const videoTex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, videoTex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
function draw() {
// copy video to texture
gl.bindTexture(gl.TEXTURE_2D, videoTex);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
// --- [ compute average, min, max to single pixel ] ---
const averagePrgInfo = getAverageProgram(gl, video.videoWidth, video.videoHeight);
gl.useProgram(averagePrgInfo.program);
// calls gl.bindFramebuffer and gl.viewport
twgl.bindFramebufferInfo(gl, fbi);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, averagePrgInfo, planeBufferInfo);
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, planeBufferInfo);
// --- [ draw video to texture ] ---
// calls gl.bindFramebuffer and gl.viewport
twgl.bindFramebufferInfo(gl, null);
gl.useProgram(textureProgInfo.program);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, textureProgInfo, planeBufferInfo);
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, planeBufferInfo);
// -- [ draw 3 points showing avg, min, max] ---
gl.useProgram(avgMinMaxPrgInfo.program);
gl.bindTexture(gl.TEXTURE_2D, fbi.attachments[0]);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, avgMinMaxPrgInfo, idBufferInfo);
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, idBufferInfo, gl.POINTS);
}
}
body {
background: #444;
}
canvas {
border: 1px solid black;
display: block;
}
<canvas></canvas>
<button type="button">start</button>
<span style="color: #FF0">■ max brightness</span>
<span style="color: #F80">■ min brightness, </span>
<span style="color: #F0F">■ average brightness, </span>
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>
the only problem with this solution is that it can't be parallelized by the GPU AFAIK so (2) I might test doing something similar to generating mipmaps where I say make a shader that does 16x16 pixels and target it to generate a smaller texture and repeat until I get to 1x1. I'd have to test to see if that's actually faster and what size cell 2x2, 4x4, 16x16 etc is best.
Finally, if possible, like the example above, if I don't actually need the result on the CPU then just pass that 1x1 texture as input to some other shader. The example just draws 3 points but of course you could feed those values into the shader that's drawing the video to do some image processing like crank up the exposure if the brightness is low try to auto level the image based on the min and max brightness etc...
Note that in WebGL2 you wouldn't have to generate a different shader per size as WebGL2, or rather GLSL ES 3.0 you can have loops that are not based on constant values.
const startElem = document.querySelector('button');
startElem.addEventListener('click', main, {once: true});
function main() {
const gl = document.querySelector('canvas').getContext('webgl2');
if (!gl) {
return alert('need WebGL2')
}
let updateTexture = false;
const video = document.createElement('video');
video.crossOrigin = 'anonymous';
video.loop = true;
video.src = 'https://webglsamples.org/color-adjust/sample-video.mp4';
if (video.requestVideoFrameCallback) {
function update() {
draw();
video.requestVideoFrameCallback(update);
};
video.requestVideoFrameCallback(update);
} else {
function update() {
if (video.currentTime > 0) {
draw();
}
requestAnimationFrame(update);
}
requestAnimationFrame(update);
}
video.volume = 0;
video.play();
// create a 1x1 pixel RGBA/UNSIGNED_BYTE framebuffer
const fbi = twgl.createFramebufferInfo(gl, [
{ internalForamt: gl.RGBA },
], 1, 1);
const avgVS = `#version 300 es
in vec4 position;
void main() {
gl_Position = position;
}
`;
const avgFS = `#version 300 es
precision highp float;
uniform sampler2D tex;
out vec4 result;
void main() {
ivec2 size = textureSize(tex, 0);
float totalBrightness = 0.0;
float minBrightness = 1.0;
float maxBrightness = 0.0;
for (int y = 0; y < size.y; ++y) {
for (int x = 0; x < size.x; ++x) {
vec4 color = texelFetch(tex, ivec2(x, y), 0);
vec3 adjusted = color.rgb * vec3(0.2126, 0.7152, 0.0722);
float brightness = adjusted.r + adjusted.g + adjusted.b;
totalBrightness += brightness;
minBrightness = min(brightness, minBrightness);
maxBrightness = max(brightness, maxBrightness);
}
}
float averageBrightness = totalBrightness / float(size.x * size.y);
result = vec4(averageBrightness, minBrightness, maxBrightness, 0);
}
`;
// compile shaders, link program, look up uniforms
const averagePrgInfo = twgl.createProgramInfo(gl, [avgVS, avgFS]);
const tVS = `#version 300 es
in vec4 position;
in vec2 texcoord;
out vec2 v_texcoord;
void main() {
gl_Position = position;
v_texcoord = texcoord;
}
`;
const tFS = `#version 300 es
precision mediump float;
uniform sampler2D tex;
in vec2 v_texcoord;
out vec4 fragColor;
void main() {
fragColor = texture(tex, v_texcoord);
}
`;
// compile shaders, link program, look up uniforms
const textureProgInfo = twgl.createProgramInfo(gl, [tVS, tFS]);
const avgMinMaxVS = `#version 300 es
out float v_id;
uniform sampler2D avgMinMaxTex;
void main() {
vec4 avgMinMax = texelFetch(avgMinMaxTex, ivec2(0), 0);
float v = gl_VertexID == 0
? avgMinMax.x
: gl_VertexID == 1
? avgMinMax.y
: avgMinMax.z;
gl_Position = vec4(1. - (float(gl_VertexID) + 1.0) / 10., v * 2. - 1., 0, 1);
gl_PointSize = 10.0;
v_id = float(gl_VertexID);
}
`;
const avgMinMaxFS = `#version 300 es
precision mediump float;
in float v_id;
out vec4 fragColor;
void main() {
fragColor = vec4(1., v_id / 2., 1. - v_id / 2., 1);
}
`;
// compile shaders, link program, look up uniforms
const avgMinMaxPrgInfo = twgl.createProgramInfo(gl, [avgMinMaxVS, avgMinMaxFS]);
// creates buffers with positions and texcoords for a -1 to +1 quad
const planeBufferInfo = twgl.primitives.createXYQuadBufferInfo(gl);
const videoTex = gl.createTexture();
gl.bindTexture(gl.TEXTURE_2D, videoTex);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, true);
function draw() {
// copy video to texture
gl.bindTexture(gl.TEXTURE_2D, videoTex);
gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, video);
// --- [ compute average, min, max to single pixel ] ---
gl.useProgram(averagePrgInfo.program);
// calls gl.bindFramebuffer and gl.viewport
twgl.bindFramebufferInfo(gl, fbi);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, averagePrgInfo, planeBufferInfo);
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, planeBufferInfo);
// --- [ draw video to texture ] ---
// calls gl.bindFramebuffer and gl.viewport
twgl.bindFramebufferInfo(gl, null);
gl.useProgram(textureProgInfo.program);
// calls gl.bindBuffer, gl.enableVertexAttribArray, gl.vertexAttribPointer
twgl.setBuffersAndAttributes(gl, textureProgInfo, planeBufferInfo);
// calls gl.drawArrays or gl.drawElements
twgl.drawBufferInfo(gl, planeBufferInfo);
// -- [ draw 3 points showing avg, min, max] ---
gl.useProgram(avgMinMaxPrgInfo.program);
gl.bindTexture(gl.TEXTURE_2D, fbi.attachments[0]);
// draw 3 points
gl.drawArrays(gl.POINTS, 0, 3);
}
}
body {
background: #444;
}
canvas {
border: 1px solid black;
display: block;
}
<canvas></canvas>
<button type="button">start</button>
<span style="color: #FF0">■ max brightness</span>
<span style="color: #F80">■ min brightness, </span>
<span style="color: #F0F">■ average brightness, </span>
<script src="https://twgljs.org/dist/4.x/twgl-full.min.js"></script>

Resources