WebGPU example for zoom-in the assigned area of a video embedded in the webpage canvas #19

Kuo-TingKai · 2024-10-19T03:32:28Z

Kuo-TingKai
Oct 19, 2024

Dear maintainer,
Could you provide some example to realize this feature like below code snippet, with libmedia acceleration.
I saw some WebGPU render/shade and 2d canvas texture modules in libmedia but I don't know how to utilize them.
Thanks a lot!

index.html:

<!DOCTYPE html>
<html lang="zh-TW">
<head>
    <meta charset="UTF-8">
    <title>WebGPU Zoom-in</title>
    <style>
        #videoContainer, #zoomCanvas { 
            width: 640px; 
            height: 360px; 
        }
    </style>
</head>
<body>
    <video id="video" src="video.mp4" controls></video>
    <canvas id="zoomCanvas"></canvas>
    <script src="zoom.js"></script>
</body>
</html>

zoom.js:

async function initWebGPU() {
    if (!navigator.gpu) {
        console.error('WebGPU not supported');
        return;
    }

    const adapter = await navigator.gpu.requestAdapter();
    if (!adapter) {
        console.error('could not acquire GPU adapter');
        return;
    }

    const device = await adapter.requestDevice();
    return device;
}

async function createPipeline(device, shaderCode) {
    const shaderModule = device.createShaderModule({
        code: shaderCode
    });

    const pipelineLayout = device.createPipelineLayout({
        bindGroupLayouts: [
            device.createBindGroupLayout({
                entries: [
                    {
                        binding: 0,
                        visibility: GPUShaderStage.FRAGMENT,
                        sampler: {}
                    },
                    {
                        binding: 1,
                        visibility: GPUShaderStage.FRAGMENT,
                        texture: {}
                    }
                ]
            })
        ]
    });

    return device.createRenderPipeline({
        layout: pipelineLayout,
        vertex: {
            module: shaderModule,
            entryPoint: 'vertexMain',
            buffers: [
                {
                    arrayStride: 5 * 4,
                    attributes: [
                        { shaderLocation: 0, offset: 0, format: 'float32x2' },
                        { shaderLocation: 1, offset: 2 * 4, format: 'float32x2' }
                    ]
                }
            ]
        },
        fragment: {
            module: shaderModule,
            entryPoint: 'fragmentMain',
            targets: [{ format: 'bgra8unorm' }]
        },
        primitive: {
            topology: 'triangle-list'
        }
    });
}

async function main() {
    const video = document.getElementById('video');
    const canvas = document.getElementById('zoomCanvas');
    const device = await initWebGPU();

    if (!device) return;

    const context = canvas.getContext('webgpu');
    const canvasFormat = navigator.gpu.getPreferredCanvasFormat();
    context.configure({
        device: device,
        format: canvasFormat,
    });

    const shaderCode = `
        struct VertexOutput {
            @builtin(position) position: vec4f,
            @location(0) texCoord: vec2f,
        }

        @vertex
        fn vertexMain(@location(0) position: vec2f,
                      @location(1) texCoord: vec2f) -> VertexOutput {
            var output: VertexOutput;
            output.position = vec4f(position, 0.0, 1.0);
            output.texCoord = texCoord;
            return output;
        }

        @group(0) @binding(0) var texSampler: sampler;
        @group(0) @binding(1) var tex: texture_2d<f32>;

        @fragment
        fn fragmentMain(@location(0) texCoord: vec2f) -> @location(0) vec4f {
            return textureSample(tex, texSampler, texCoord);
        }
    `;

    const pipeline = await createPipeline(device, shaderCode);

    const vertices = new Float32Array([
        -1, -1, 0, 1,
         1, -1, 1, 1,
         1,  1, 1, 0,
        -1, -1, 0, 1,
         1,  1, 1, 0,
        -1,  1, 0, 0,
    ]);

    const vertexBuffer = device.createBuffer({
        size: vertices.byteLength,
        usage: GPUBufferUsage.VERTEX | GPUBufferUsage.COPY_DST,
    });
    device.queue.writeBuffer(vertexBuffer, 0, vertices);

    video.addEventListener('play', () => {
        function frame() {
            const texture = device.createTexture({
                size: [video.videoWidth, video.videoHeight],
                format: 'rgba8unorm',
                usage: GPUTextureUsage.TEXTURE_BINDING | GPUTextureUsage.COPY_DST | GPUTextureUsage.RENDER_ATTACHMENT
            });

            device.queue.copyExternalImageToTexture(
                { source: video },
                { texture: texture },
                [video.videoWidth, video.videoHeight]
            );

            const sampler = device.createSampler({
                magFilter: 'linear',
                minFilter: 'linear',
            });

            const bindGroup = device.createBindGroup({
                layout: pipeline.getBindGroupLayout(0),
                entries: [
                    { binding: 0, resource: sampler },
                    { binding: 1, resource: texture.createView() }
                ]
            });

            const commandEncoder = device.createCommandEncoder();
            const textureView = context.getCurrentTexture().createView();

            const renderPass = commandEncoder.beginRenderPass({
                colorAttachments: [{
                    view: textureView,
                    clearValue: { r: 0.0, g: 0.0, b: 0.0, a: 1.0 },
                    loadOp: 'clear',
                    storeOp: 'store',
                }]
            });

            renderPass.setPipeline(pipeline);
            renderPass.setBindGroup(0, bindGroup);
            renderPass.setVertexBuffer(0, vertexBuffer);
            renderPass.draw(6);
            renderPass.end();

            device.queue.submit([commandEncoder.finish()]);

            requestAnimationFrame(frame);
        }
        requestAnimationFrame(frame);
    });
}

main();

zhaohappy · 2024-10-21T07:41:29Z

zhaohappy
Oct 21, 2024
Maintainer

There are two ways to achieve it.

Method 1: You can set the texture coordinates of the four vertices as part of the entire texture.

you can change layout method of WebGPURender to compute WebGPURender.vertex , the format of vertex is [x, y, z, coord.x, coord.y, 0, 0]

Method 2: You can add scale matrix and translate matrix in vertex shader, for example:

precision highp float;

attribute vec3 point;
attribute vec4 color;
varying vec4 v_color;

uniform mat4 rotateMatrix;
uniform mat4 scaleMatrix;
uniform mat4 translateMatrix;

void main(void){
  gl_Position = rotateMatrix * scaleMatrix * translateMatrix * vec4(point, 1.0);
  v_color = color;
}

You can refer to rotateMatrix of WebGPURender to add and I recommend you use this one.

3 replies

Kuo-TingKai Oct 22, 2024
Author

Perhaps I misunderstood, but I think the method 2 you provided is rewritten from vertex.vert, which is the implementation of the vertex shader in WebGL. If I want to implement translation and scaling using WebGPU, should I rewrite vertex.wgsl:

struct PositionColorInput {
    @location(0) in_position_3d: vec3<f32>,
    @location(1) in_color_rgba: vec4<f32>
};

struct PositionColorOutput {
    @builtin(position) coords_output: vec4<f32>,
    @location(0) color_output: vec4<f32>
};

struct RotateMatrix {
    matrix: mat4x4<f32>
};

struct TranslationMatrix {
    matrix: mat4x4<f32>
};

struct ScaleMatrix {
    matrix: mat4x4<f32>
};

@group(0) @binding(0) var<uniform> rotateMatrix: RotateMatrix;
@group(0) @binding(1) var<uniform> translationMatrix: TranslationMatrix;
@group(0) @binding(2) var<uniform> scaleMatrix: ScaleMatrix;

@vertex
fn main(input: PositionColorInput) -> PositionColorOutput {
    var output: PositionColorOutput;
    output.color_output = input.in_color_rgba;
    output.coords_output = rotateMatrix.matrix * translationMatrix.matrix * scaleMatrix.matrix * vec4<f32>(input.in_position_3d, 1.0);
    return output;
}

zhaohappy Oct 22, 2024
Maintainer

You are right, I wrote a webgl shader with glsl. You wrote wgsl code above is correct.

ikigai-kevin-k Oct 24, 2024

Thanks for your useful and encourageable reply, what I want to do actually is a little bit different:
I want to create a enable/disable function to control the scale matrix applying to the video screen.
The code snippet above seems like it will simultaneously apply all the rotate matrix, the transition matrix and the scale matrix to my targeted rendered object.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

WebGPU example for zoom-in the assigned area of a video embedded in the webpage canvas #19

Uh oh!

{{title}}

Uh oh!

Replies: 1 comment 3 replies

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{title}}

Uh oh!

Select a reply

Uh oh!

WebGPU example for zoom-in the assigned area of a video embedded in the webpage canvas #19

Uh oh!

Kuo-TingKai Oct 19, 2024

Replies: 1 comment · 3 replies

Uh oh!

zhaohappy Oct 21, 2024 Maintainer

Uh oh!

Kuo-TingKai Oct 22, 2024 Author

Uh oh!

zhaohappy Oct 22, 2024 Maintainer

Uh oh!

ikigai-kevin-k Oct 24, 2024

Kuo-TingKai
Oct 19, 2024

Replies: 1 comment 3 replies

zhaohappy
Oct 21, 2024
Maintainer

Kuo-TingKai Oct 22, 2024
Author

zhaohappy Oct 22, 2024
Maintainer