From c1570128e37ac68b8cd2eda2422e8c439bf342f5 Mon Sep 17 00:00:00 2001 From: mmalavalli Date: Thu, 4 Jul 2024 03:53:31 -0500 Subject: [PATCH 01/28] Hybrid pipeline POC: Only the person mask upscale stage is based on a WebGL2-based fast bilateral filter implementation. --- lib/index.ts | 4 +- .../background/BackgroundProcessor.ts | 153 +++---- .../GaussianBlurBackgroundProcessor.ts | 5 - .../background/VirtualBackgroundProcessor.ts | 10 +- .../webgl2/helpers/backgroundHelper.ts | 4 - .../webgl2/helpers/postProcessingHelper.ts | 12 +- lib/processors/webgl2/helpers/sourceHelper.ts | 5 - lib/processors/webgl2/helpers/webglHelper.ts | 16 +- lib/processors/webgl2/index.ts | 4 +- .../pipelines/FastBilateralFilterStage.ts | 220 +++++++++++ .../pipelines/PersonMaskUpscalePipeline.ts | 47 +++ lib/processors/webgl2/pipelines/Pipeline.ts | 22 ++ .../webgl2/pipelines/WebGL2Pipeline.ts | 374 ++++++++++++++++++ .../webgl2/pipelines/backgroundBlurStage.ts | 292 -------------- .../webgl2/pipelines/backgroundImageStage.ts | 221 ----------- .../pipelines/fastBilateralFilterStage.ts | 222 ----------- .../webgl2/pipelines/loadSegmentationStage.ts | 99 ----- .../webgl2/pipelines/webgl2Pipeline.ts | 223 ----------- lib/types.ts | 29 -- 19 files changed, 729 insertions(+), 1233 deletions(-) delete mode 100644 lib/processors/webgl2/helpers/backgroundHelper.ts delete mode 100644 lib/processors/webgl2/helpers/sourceHelper.ts create mode 100644 lib/processors/webgl2/pipelines/FastBilateralFilterStage.ts create mode 100644 lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts create mode 100644 lib/processors/webgl2/pipelines/Pipeline.ts create mode 100644 lib/processors/webgl2/pipelines/WebGL2Pipeline.ts delete mode 100644 lib/processors/webgl2/pipelines/backgroundBlurStage.ts delete mode 100644 lib/processors/webgl2/pipelines/backgroundImageStage.ts delete mode 100644 lib/processors/webgl2/pipelines/fastBilateralFilterStage.ts delete mode 100644 lib/processors/webgl2/pipelines/loadSegmentationStage.ts delete mode 100644 lib/processors/webgl2/pipelines/webgl2Pipeline.ts diff --git a/lib/index.ts b/lib/index.ts index 2906696..3de9b13 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -1,6 +1,6 @@ import { GaussianBlurBackgroundProcessor, GaussianBlurBackgroundProcessorOptions } from './processors/background/GaussianBlurBackgroundProcessor'; import { VirtualBackgroundProcessor, VirtualBackgroundProcessorOptions } from './processors/background/VirtualBackgroundProcessor'; -import { ImageFit, Pipeline } from './types'; +import { ImageFit } from './types'; import { isSupported } from './utils/support'; import { version } from './utils/version'; @@ -10,7 +10,6 @@ if (typeof window !== 'undefined') { ...window.Twilio.VideoProcessors, GaussianBlurBackgroundProcessor, ImageFit, - Pipeline, isSupported, version, VirtualBackgroundProcessor, @@ -21,7 +20,6 @@ export { GaussianBlurBackgroundProcessor, GaussianBlurBackgroundProcessorOptions, ImageFit, - Pipeline, isSupported, version, VirtualBackgroundProcessor, diff --git a/lib/processors/background/BackgroundProcessor.ts b/lib/processors/background/BackgroundProcessor.ts index 30de120..955f5a8 100644 --- a/lib/processors/background/BackgroundProcessor.ts +++ b/lib/processors/background/BackgroundProcessor.ts @@ -2,8 +2,8 @@ import { Processor } from '../Processor'; import { Benchmark } from '../../utils/Benchmark'; import { TwilioTFLite } from '../../utils/TwilioTFLite'; import { isChromiumImageBitmap } from '../../utils/support'; -import { Dimensions, Pipeline, WebGL2PipelineType } from '../../types'; -import { buildWebGL2Pipeline } from '../webgl2'; +import { Dimensions } from '../../types'; +import { PersonMaskUpscalePipeline } from '../webgl2'; import { MASK_BLUR_RADIUS, @@ -82,15 +82,6 @@ export interface BackgroundProcessorOptions { * ``` */ maskBlurRadius?: number; - - /** - * Specifies which pipeline to use when processing video frames. - * @default - * ```html - * 'WebGL2' - * ``` - */ - pipeline?: Pipeline; } /** @@ -101,8 +92,8 @@ export abstract class BackgroundProcessor extends Processor { protected _backgroundImage: HTMLImageElement | null = null; protected _outputCanvas: HTMLCanvasElement | null = null; - protected _outputContext: CanvasRenderingContext2D | WebGL2RenderingContext | null = null; - protected _webgl2Pipeline: ReturnType | null = null; + protected _outputContext: CanvasRenderingContext2D | null = null; + protected _personMaskUpscalePipeline: PersonMaskUpscalePipeline | null = null; private _assetsPath: string; private _benchmark: Benchmark; @@ -119,8 +110,6 @@ export abstract class BackgroundProcessor extends Processor { private _isSimdEnabled: boolean | null; private _maskBlurRadius: number; private _maskCanvas: OffscreenCanvas | HTMLCanvasElement; - private _maskContext: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D; - private _pipeline: Pipeline; constructor(options: BackgroundProcessorOptions) { super(); @@ -142,7 +131,6 @@ export abstract class BackgroundProcessor extends Processor { ? options.inputResizeMode : (isChromiumImageBitmap() ? 'image-bitmap' : 'canvas'); - this._pipeline = options.pipeline! || Pipeline.WebGL2; this._benchmark = new Benchmark(); this._currentMask = null; this._isSimdEnabled = null; @@ -150,11 +138,8 @@ export abstract class BackgroundProcessor extends Processor { this._inferenceInputContext = this._inferenceInputCanvas.getContext('2d', { willReadFrequently: true }) as OffscreenCanvasRenderingContext2D; this._inputFrameCanvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); this._inputFrameContext = this._inputFrameCanvas.getContext('2d') as OffscreenCanvasRenderingContext2D; - this._maskBlurRadius = typeof options.maskBlurRadius === 'number' ? options.maskBlurRadius : ( - this._pipeline === Pipeline.WebGL2 ? MASK_BLUR_RADIUS : (MASK_BLUR_RADIUS / 2) - ); + this._maskBlurRadius = typeof options.maskBlurRadius === 'number' ? options.maskBlurRadius : MASK_BLUR_RADIUS; this._maskCanvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); - this._maskContext = this._maskCanvas.getContext('2d') as OffscreenCanvasRenderingContext2D; } /** @@ -174,10 +159,8 @@ export abstract class BackgroundProcessor extends Processor { } if (this._maskBlurRadius !== radius) { this._maskBlurRadius = radius; - this._webgl2Pipeline?.updatePostProcessingConfig({ - jointBilateralFilter: { - sigmaSpace: this._maskBlurRadius - } + this._personMaskUpscalePipeline?.updateFastBilateralFilterConfig({ + sigmaSpace: this._maskBlurRadius }); } } @@ -247,28 +230,18 @@ export abstract class BackgroundProcessor extends Processor { ? { width: inputFrameBuffer.videoWidth, height: inputFrameBuffer.videoHeight } : inputFrameBuffer; + const { + height: outputHeight, + width: outputWidth + } = outputFrameBuffer; + if (this._outputCanvas !== outputFrameBuffer) { this._outputCanvas = outputFrameBuffer; - this._outputContext = this._outputCanvas - .getContext(this._pipeline === Pipeline.Canvas2D ? '2d' : 'webgl2') as - CanvasRenderingContext2D | WebGL2RenderingContext; - this._webgl2Pipeline?.cleanUp(); - this._webgl2Pipeline = null; - } - - if (this._pipeline === Pipeline.WebGL2) { - if (!this._webgl2Pipeline) { - this._createWebGL2Pipeline( - inputFrameBuffer as HTMLVideoElement, - captureWidth, - captureHeight, - inferenceWidth, - inferenceHeight - ); - } - this._webgl2Pipeline?.sampleInputFrame(); + this._outputContext = this._outputCanvas.getContext('2d'); + this._maskCanvas.width = outputWidth; + this._maskCanvas.height = outputHeight; + this._cleanupPersonMaskUpscalePipeline(); } - // Only set the canvas' dimensions if they have changed to prevent unnecessary redraw if (this._inputFrameCanvas.width !== captureWidth) { this._inputFrameCanvas.width = captureWidth; @@ -278,20 +251,21 @@ export abstract class BackgroundProcessor extends Processor { } if (this._inferenceInputCanvas.width !== inferenceWidth) { this._inferenceInputCanvas.width = inferenceWidth; - this._maskCanvas.width = inferenceWidth; } if (this._inferenceInputCanvas.height !== inferenceHeight) { this._inferenceInputCanvas.height = inferenceHeight; - this._maskCanvas.height = inferenceHeight; } - let inputFrame: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement; + let inputFrame: OffscreenCanvas | HTMLCanvasElement; if (inputFrameBuffer instanceof HTMLVideoElement) { this._inputFrameContext.drawImage(inputFrameBuffer, 0, 0); inputFrame = this._inputFrameCanvas; } else { inputFrame = inputFrameBuffer; } + if (!this._personMaskUpscalePipeline) { + this._createPersonMaskUpscalePipeline(inputFrame); + } const personMask = await this._createPersonMask(inputFrame); if (this._debounce) { @@ -300,32 +274,23 @@ export abstract class BackgroundProcessor extends Processor { : personMask; } - if (this._pipeline === Pipeline.WebGL2) { - this._webgl2Pipeline?.render(personMask.data); - } - else { - this._benchmark.start('imageCompositionDelay'); - if (!this._debounce || this._currentMask) { - this._maskContext.putImageData(personMask, 0, 0); - } - const ctx = this._outputContext as CanvasRenderingContext2D; - const { - height: outputHeight, - width: outputWidth - } = this._outputCanvas; - ctx.save(); - ctx.filter = `blur(${this._maskBlurRadius}px)`; - ctx.globalCompositeOperation = 'copy'; - ctx.drawImage(this._maskCanvas, 0, 0, outputWidth, outputHeight); - ctx.filter = 'none'; - ctx.globalCompositeOperation = 'source-in'; - ctx.drawImage(inputFrame, 0, 0, outputWidth, outputHeight); - ctx.globalCompositeOperation = 'destination-over'; - this._setBackground(inputFrame); - ctx.restore(); - this._benchmark.end('imageCompositionDelay'); + this._benchmark.start('imageCompositionDelay'); + if (!this._debounce || this._currentMask) { + this._personMaskUpscalePipeline?.render(personMask); } + const ctx = this._outputContext as CanvasRenderingContext2D; + ctx.save(); + ctx.filter = 'none'; + ctx.globalCompositeOperation = 'copy'; + ctx.drawImage(this._maskCanvas, 0, 0, outputWidth, outputHeight); + ctx.globalCompositeOperation = 'source-in'; + ctx.drawImage(inputFrame, 0, 0, outputWidth, outputHeight); + ctx.globalCompositeOperation = 'destination-over'; + this._setBackground(inputFrame); + ctx.restore(); + this._benchmark.end('imageCompositionDelay'); + this._benchmark.end('processFrameDelay'); this._benchmark.end('totalProcessingDelay'); @@ -335,10 +300,13 @@ export abstract class BackgroundProcessor extends Processor { this._benchmark.start('captureFrameDelay'); } - protected abstract _getWebGL2PipelineType(): WebGL2PipelineType; - protected abstract _setBackground(inputFrame?: OffscreenCanvas | HTMLCanvasElement): void; + private _cleanupPersonMaskUpscalePipeline(): void { + this._personMaskUpscalePipeline?.cleanUp(); + this._personMaskUpscalePipeline = null; + } + private async _createPersonMask(inputFrame: OffscreenCanvas | HTMLCanvasElement): Promise { const { height, width } = this._inferenceDimensions; const stages = { @@ -367,41 +335,14 @@ export abstract class BackgroundProcessor extends Processor { return this._currentMask || new ImageData(personMaskBuffer, width, height); } - private _createWebGL2Pipeline( - inputFrame: HTMLVideoElement, - captureWidth: number, - captureHeight: number, - inferenceWidth: number, - inferenceHeight: number, - ): void { - this._webgl2Pipeline = buildWebGL2Pipeline( - { - htmlElement: inputFrame, - width: captureWidth, - height: captureHeight, - }, - this._backgroundImage, - { - type: this._getWebGL2PipelineType(), - }, - { - inputResolution: `${inferenceWidth}x${inferenceHeight}`, - }, - this._outputCanvas!, - this._benchmark, - this._debounce + private _createPersonMaskUpscalePipeline(inputFrame: OffscreenCanvas | HTMLCanvasElement): void { + this._personMaskUpscalePipeline = new PersonMaskUpscalePipeline( + inputFrame, + this._inferenceDimensions, + this._maskCanvas ); - this._webgl2Pipeline.updatePostProcessingConfig({ - jointBilateralFilter: { - sigmaSpace: this._maskBlurRadius, - sigmaColor: 0.1 - }, - coverage: [ - 0, - 0.99 - ], - lightWrapping: 0, - blendMode: 'screen' + this._personMaskUpscalePipeline?.updateFastBilateralFilterConfig({ + sigmaSpace: this._maskBlurRadius }); } diff --git a/lib/processors/background/GaussianBlurBackgroundProcessor.ts b/lib/processors/background/GaussianBlurBackgroundProcessor.ts index 93eeffc..c69c51e 100644 --- a/lib/processors/background/GaussianBlurBackgroundProcessor.ts +++ b/lib/processors/background/GaussianBlurBackgroundProcessor.ts @@ -1,6 +1,5 @@ import { BackgroundProcessor, BackgroundProcessorOptions } from './BackgroundProcessor'; import { BLUR_FILTER_RADIUS } from '../../constants'; -import { WebGL2PipelineType } from '../../types'; /** * Options passed to [[GaussianBlurBackgroundProcessor]] constructor. @@ -99,10 +98,6 @@ export class GaussianBlurBackgroundProcessor extends BackgroundProcessor { this._blurFilterRadius = radius; } - protected _getWebGL2PipelineType(): WebGL2PipelineType { - return WebGL2PipelineType.Blur; - } - protected _setBackground(inputFrame: OffscreenCanvas | HTMLCanvasElement): void { if (!this._outputContext) { return; diff --git a/lib/processors/background/VirtualBackgroundProcessor.ts b/lib/processors/background/VirtualBackgroundProcessor.ts index 2e8eeb4..4add076 100644 --- a/lib/processors/background/VirtualBackgroundProcessor.ts +++ b/lib/processors/background/VirtualBackgroundProcessor.ts @@ -1,5 +1,5 @@ import { BackgroundProcessor, BackgroundProcessorOptions } from './BackgroundProcessor'; -import { ImageFit, WebGL2PipelineType } from '../../types'; +import { ImageFit } from '../../types'; /** * Options passed to [[VirtualBackgroundProcessor]] constructor. @@ -112,10 +112,6 @@ export class VirtualBackgroundProcessor extends BackgroundProcessor { throw new Error('Invalid image. Make sure that the image is an HTMLImageElement and has been successfully loaded'); } this._backgroundImage = image; - - // Triggers recreation of the pipeline in the next processFrame call - this._webgl2Pipeline?.cleanUp(); - this._webgl2Pipeline = null; } /** @@ -137,10 +133,6 @@ export class VirtualBackgroundProcessor extends BackgroundProcessor { this._fitType = fitType; } - protected _getWebGL2PipelineType(): WebGL2PipelineType { - return WebGL2PipelineType.Image; - } - protected _setBackground(): void { if (!this._outputContext || !this._outputCanvas) { return; diff --git a/lib/processors/webgl2/helpers/backgroundHelper.ts b/lib/processors/webgl2/helpers/backgroundHelper.ts deleted file mode 100644 index e46dc12..0000000 --- a/lib/processors/webgl2/helpers/backgroundHelper.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type BackgroundConfig = { - type: 'none' | 'blur' | 'image' - url?: string -} diff --git a/lib/processors/webgl2/helpers/postProcessingHelper.ts b/lib/processors/webgl2/helpers/postProcessingHelper.ts index 2519ec9..9f6d772 100644 --- a/lib/processors/webgl2/helpers/postProcessingHelper.ts +++ b/lib/processors/webgl2/helpers/postProcessingHelper.ts @@ -1,13 +1,3 @@ -export type BlendMode = 'screen' | 'linearDodge' - -export type PostProcessingConfig = { - jointBilateralFilter?: JointBilateralFilterConfig - coverage?: [number, number] - lightWrapping?: number - blendMode?: BlendMode -} - -export type JointBilateralFilterConfig = { +export type FastBilateralFilterConfig = { sigmaSpace?: number - sigmaColor?: number } diff --git a/lib/processors/webgl2/helpers/sourceHelper.ts b/lib/processors/webgl2/helpers/sourceHelper.ts deleted file mode 100644 index 0ae4dcc..0000000 --- a/lib/processors/webgl2/helpers/sourceHelper.ts +++ /dev/null @@ -1,5 +0,0 @@ -export type SourcePlayback = { - htmlElement: HTMLImageElement | HTMLVideoElement - width: number - height: number -} diff --git a/lib/processors/webgl2/helpers/webglHelper.ts b/lib/processors/webgl2/helpers/webglHelper.ts index 3c3eed7..0efd484 100644 --- a/lib/processors/webgl2/helpers/webglHelper.ts +++ b/lib/processors/webgl2/helpers/webglHelper.ts @@ -9,7 +9,7 @@ */ export const glsl = String.raw -export function createPiplelineStageProgram( +export function createPipelineStageProgram( gl: WebGL2RenderingContext, vertexShader: WebGLShader, fragmentShader: WebGLShader, @@ -79,3 +79,17 @@ export function createTexture( gl.texStorage2D(gl.TEXTURE_2D, 1, internalformat, width, height) return texture } + +export function initBuffer( + gl: WebGL2RenderingContext, + data: number[] +) { + const buffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + gl.bufferData( + gl.ARRAY_BUFFER, + new Float32Array(data), + gl.STATIC_DRAW, + ); + return buffer; +} diff --git a/lib/processors/webgl2/index.ts b/lib/processors/webgl2/index.ts index faee7eb..4ebfb25 100644 --- a/lib/processors/webgl2/index.ts +++ b/lib/processors/webgl2/index.ts @@ -4,6 +4,4 @@ * It was modified and converted into a module to work with * Twilio's Video Processor */ -import { buildWebGL2Pipeline } from './pipelines/webgl2Pipeline' - -export { buildWebGL2Pipeline }; +export { PersonMaskUpscalePipeline } from './pipelines/PersonMaskUpscalePipeline'; diff --git a/lib/processors/webgl2/pipelines/FastBilateralFilterStage.ts b/lib/processors/webgl2/pipelines/FastBilateralFilterStage.ts new file mode 100644 index 0000000..018082e --- /dev/null +++ b/lib/processors/webgl2/pipelines/FastBilateralFilterStage.ts @@ -0,0 +1,220 @@ +import { Dimensions } from '../../../types' +import { WebGL2Pipeline } from './WebGL2Pipeline' + +/** + * @private + */ +export class FastBilateralFilterStage extends WebGL2Pipeline.ProcessingStage { + private _inputDimensions: Dimensions + + constructor( + glOut: WebGL2RenderingContext, + inputDimensions: Dimensions, + outputDimensions: Dimensions + ) { + const { + height, + width + } = outputDimensions + + super( + { + textureName: 'u_segmentationMask', + textureUnit: 1 + }, + { + // NOTE(mmalavalli): This is a faster approximation of the joint bilateral filter. + // For a given pixel, instead of calculating the space and color weights of all + // the pixels within the filter kernel, which would have a complexity of O(r^2), + // we calculate the space and color weights of only those pixels which form two + // diagonal lines between the two pairs of opposite corners of the filter kernel, + // which would have a complexity of O(r). This improves the overall complexity + // of this stage from O(w x h x r^2) to O(w x h x r), where: + // w => width of the output video frame + // h => height of the output video frame + // r => radius of the joint bilateral filter kernel + fragmentShaderSource: `#version 300 es + precision highp float; + + uniform sampler2D u_inputFrame; + uniform sampler2D u_segmentationMask; + uniform vec2 u_texelSize; + uniform float u_step; + uniform float u_radius; + uniform float u_offset; + uniform float u_sigmaTexel; + uniform float u_sigmaColor; + + in vec2 v_texCoord; + + out vec4 outColor; + + float gaussian(float x, float sigma) { + return exp(-0.5 * x * x / sigma / sigma); + } + + float calculateSpaceWeight(vec2 coord) { + float x = distance(v_texCoord, coord); + float sigma = u_sigmaTexel; + return gaussian(x, sigma); + } + + float calculateColorWeight(vec2 coord) { + vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb; + vec3 coordColor = texture(u_inputFrame, coord).rgb; + float x = distance(centerColor, coordColor); + float sigma = u_sigmaColor; + return gaussian(x, sigma); + } + + void main() { + vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb; + float newVal = 0.0; + float totalWeight = 0.0; + + vec2 leftTopCoord = vec2(v_texCoord + vec2(-u_radius, -u_radius) * u_texelSize); + vec2 rightTopCoord = vec2(v_texCoord + vec2(u_radius, -u_radius) * u_texelSize); + vec2 leftBottomCoord = vec2(v_texCoord + vec2(-u_radius, u_radius) * u_texelSize); + vec2 rightBottomCoord = vec2(v_texCoord + vec2(u_radius, u_radius) * u_texelSize); + + float leftTopSegAlpha = texture(u_segmentationMask, leftTopCoord).a; + float rightTopSegAlpha = texture(u_segmentationMask, rightTopCoord).a; + float leftBottomSegAlpha = texture(u_segmentationMask, leftBottomCoord).a; + float rightBottomSegAlpha = texture(u_segmentationMask, rightBottomCoord).a; + float totalSegAlpha = leftTopSegAlpha + rightTopSegAlpha + leftBottomSegAlpha + rightBottomSegAlpha; + + if (totalSegAlpha <= 0.0) { + newVal = 0.0; + } else if (totalSegAlpha >= 4.0) { + newVal = 1.0; + } else { + for (float i = 0.0; i <= u_radius - u_offset; i += u_step) { + vec2 shift = vec2(i, i) * u_texelSize; + vec2 coord = vec2(v_texCoord + shift); + float spaceWeight = calculateSpaceWeight(coord); + float colorWeight = calculateColorWeight(coord); + float weight = spaceWeight * colorWeight; + float alpha = texture(u_segmentationMask, coord).a; + totalWeight += weight; + newVal += weight * alpha; + + if (i != 0.0) { + shift = vec2(i, -i) * u_texelSize; + coord = vec2(v_texCoord + shift); + colorWeight = calculateColorWeight(coord); + weight = spaceWeight * colorWeight; + alpha = texture(u_segmentationMask, coord).a; + totalWeight += weight; + newVal += weight * texture(u_segmentationMask, coord).a; + + shift = vec2(-i, i) * u_texelSize; + coord = vec2(v_texCoord + shift); + colorWeight = calculateColorWeight(coord); + weight = spaceWeight * colorWeight; + alpha = texture(u_segmentationMask, coord).a; + totalWeight += weight; + newVal += weight * texture(u_segmentationMask, coord).a; + + shift = vec2(-i, -i) * u_texelSize; + coord = vec2(v_texCoord + shift); + colorWeight = calculateColorWeight(coord); + weight = spaceWeight * colorWeight; + alpha = texture(u_segmentationMask, coord).a; + totalWeight += weight; + newVal += weight * texture(u_segmentationMask, coord).a; + } + } + newVal /= totalWeight; + } + + outColor = vec4(vec3(0.0), newVal); + } + `, + glOut, + height, + type: 'canvas', + width, + uniformVars: [ + { + name: 'u_inputFrame', + type: 'int', + values: [0] + }, + { + name: 'u_texelSize', + type: 'float', + values: [1 / width, 1 / height] + } + ] + } + ) + + this._inputDimensions = inputDimensions + this.updateSigmaColor(0) + this.updateSigmaSpace(0) + } + + updateSigmaColor(sigmaColor: number): void { + this._setUniformVars([ + { + name: 'u_sigmaColor', + type: 'float', + values: [sigmaColor] + } + ]) + } + + updateSigmaSpace(sigmaSpace: number): void { + const { + height: inputHeight, + width: inputWidth + } = this._inputDimensions + + const { + height: outputHeight, + width: outputWidth + } = this._outputDimensions + + sigmaSpace *= Math.max( + outputWidth / inputWidth, + outputHeight / inputHeight + ) + + const kSparsityFactor = 0.66 + const sparsity = Math.max( + 1, + Math.sqrt(sigmaSpace) + * kSparsityFactor + ) + const step = sparsity + const radius = sigmaSpace + const offset = step > 1 ? step * 0.5 : 0 + const sigmaTexel = Math.max( + 1 / outputWidth, + 1 / outputHeight + ) * sigmaSpace + + this._setUniformVars([ + { + name: 'u_offset', + type: 'float', + values: [offset] + }, + { + name: 'u_radius', + type: 'float', + values: [radius] + }, + { + name: 'u_sigmaTexel', + type: 'float', + values: [sigmaTexel] + }, + { + name: 'u_step', + type: 'float', + values: [step] + }, + ]) + } +} diff --git a/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts b/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts new file mode 100644 index 0000000..4837a5d --- /dev/null +++ b/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts @@ -0,0 +1,47 @@ +import { Dimensions } from '../../../types'; +import { FastBilateralFilterConfig } from '../helpers/postProcessingHelper'; +import { FastBilateralFilterStage } from './FastBilateralFilterStage'; +import { WebGL2Pipeline } from './WebGL2Pipeline'; + +export class PersonMaskUpscalePipeline extends WebGL2Pipeline { + constructor( + inputCanvas: OffscreenCanvas | HTMLCanvasElement, + inputDimensions: Dimensions, + outputCanvas: OffscreenCanvas | HTMLCanvasElement + ) { + super() + const glOut = outputCanvas.getContext('webgl2')! as WebGL2RenderingContext + + const outputDimensions = { + height: outputCanvas.height, + width: outputCanvas.width + } + + this.addStage(new WebGL2Pipeline.InputStage( + glOut, + inputCanvas + )) + + this.addStage(new FastBilateralFilterStage( + glOut, + inputDimensions, + outputDimensions + )) + } + + updateFastBilateralFilterConfig(config: FastBilateralFilterConfig) { + const [ + /* inputStage */, + fastBilateralFilterStage + ] = this._stages as [ + any, + FastBilateralFilterStage + ] + + const { sigmaSpace } = config + if (typeof sigmaSpace === 'number') { + fastBilateralFilterStage.updateSigmaColor(0.1) + fastBilateralFilterStage.updateSigmaSpace(sigmaSpace) + } + } +} diff --git a/lib/processors/webgl2/pipelines/Pipeline.ts b/lib/processors/webgl2/pipelines/Pipeline.ts new file mode 100644 index 0000000..14abfc3 --- /dev/null +++ b/lib/processors/webgl2/pipelines/Pipeline.ts @@ -0,0 +1,22 @@ +/** + * @private + */ +export class Pipeline implements Pipeline.Stage { + protected _stages: Pipeline.Stage[] = [] + + addStage(stage: Pipeline.Stage): void { + this._stages.push(stage) + } + + render(...args: any[]): void { + this._stages.forEach((stage) => { + stage.render() + }) + } +} + +export namespace Pipeline { + export interface Stage { + render(...args: any[]): void + } +} diff --git a/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts b/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts new file mode 100644 index 0000000..78c74ea --- /dev/null +++ b/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts @@ -0,0 +1,374 @@ +import { Dimensions } from '../../../types' +import { + createPipelineStageProgram, + createTexture, + compileShader, + initBuffer +} from '../helpers/webglHelper' +import { Pipeline } from './Pipeline' + +interface InputConfig { + textureName: string + textureUnit: number +} + +interface OutputConfig { + fragmentShaderSource: string + glOut: WebGL2RenderingContext + height?: number + type: 'canvas' | 'texture' + uniformVars?: UniformVarInfo[] + vertexShaderSource?: string + width?: number +} + +interface UniformVarInfo { + name: string + type: 'float' | 'int' | 'uint' + values: number[] +} + +/** + * @private + */ +class WebGL2PipelineInputStage implements Pipeline.Stage { + private _glOut: WebGL2RenderingContext + private _inputFrame: OffscreenCanvas | HTMLCanvasElement + private _inputFrameTexture: WebGLTexture + private _personMaskTexture: WebGLTexture | null + + constructor( + glOut: WebGL2RenderingContext, + inputFrame: OffscreenCanvas | HTMLCanvasElement + ) { + const { height, width } = inputFrame + this._glOut = glOut + this._inputFrame = inputFrame + this._inputFrameTexture = createTexture( + glOut, + glOut.RGBA8, + width, + height, + glOut.NEAREST, + glOut.NEAREST + )! + this._personMaskTexture = null; + } + + cleanUp(): void { + const { + _glOut, + _inputFrameTexture, + _personMaskTexture + } = this + _glOut.deleteTexture(_inputFrameTexture) + _glOut.deleteTexture(_personMaskTexture) + } + + render(personMask: ImageData): void { + const { + _glOut, + _inputFrame, + _inputFrameTexture + } = this + + const { height, width } = _inputFrame + _glOut.viewport(0, 0, width, height) + _glOut.clearColor(0, 0, 0, 0) + _glOut.clear(_glOut.COLOR_BUFFER_BIT) + _glOut.activeTexture(_glOut.TEXTURE0) + + _glOut.bindTexture( + _glOut.TEXTURE_2D, + _inputFrameTexture + ) + _glOut.texSubImage2D( + _glOut.TEXTURE_2D, + 0, + 0, + 0, + width, + height, + _glOut.RGBA, + _glOut.UNSIGNED_BYTE, + _inputFrame + ) + + const { + data, + height: maskHeight, + width: maskWidth + } = personMask + + if (!this._personMaskTexture) { + this._personMaskTexture = createTexture( + _glOut, + _glOut.RGBA8, + maskWidth, + maskHeight, + _glOut.NEAREST, + _glOut.NEAREST + ) + } + + _glOut.viewport(0, 0, maskWidth, maskHeight) + _glOut.activeTexture(_glOut.TEXTURE1) + + _glOut.bindTexture( + _glOut.TEXTURE_2D, + this._personMaskTexture + ) + _glOut.texSubImage2D( + _glOut.TEXTURE_2D, + 0, + 0, + 0, + maskWidth, + maskHeight, + _glOut.RGBA, + _glOut.UNSIGNED_BYTE, + data + ) + } +} + +/** + * @private + */ +class WebGL2PipelineProcessingStage implements Pipeline.Stage { + protected _outputDimensions: Dimensions + private _fragmentShader: WebGLSampler + private _glOut: WebGL2RenderingContext + private _inputTextureUnit: number + private _outputFramebuffer: WebGLBuffer | null = null + private _outputTexture: WebGLTexture | null = null + private _positionBuffer: WebGLBuffer + private _program: WebGLProgram + private _texCoordBuffer: WebGLBuffer + private _vertexShader: WebGLShader + + constructor( + inputConfig: InputConfig, + outputConfig: OutputConfig + ) { + const { + textureName, + textureUnit, + } = inputConfig + + this._inputTextureUnit = textureUnit + + const { glOut } = outputConfig + this._glOut = glOut + + const { + fragmentShaderSource, + height = glOut.canvas.height, + type: outputType, + uniformVars = [], + vertexShaderSource = `#version 300 es + in vec2 a_position; + in vec2 a_texCoord; + + out vec2 v_texCoord; + + void main() { + gl_Position = vec4(a_position${ + outputType === 'canvas' + ? ' * vec2(1.0, -1.0)' + : '' + }, 0.0, 1.0); + v_texCoord = a_texCoord; + } + `, + width = glOut.canvas.width + } = outputConfig + + this._outputDimensions = { + height, + width + } + + this._fragmentShader = compileShader( + glOut, + glOut.FRAGMENT_SHADER, + fragmentShaderSource + ) + + this._vertexShader = compileShader( + glOut, + glOut.VERTEX_SHADER, + vertexShaderSource + ) + + this._positionBuffer = initBuffer( + glOut, + [ + -1.0, -1.0, + 1.0, -1.0, + -1.0, 1.0, + 1.0, 1.0, + ] + )! + + this._texCoordBuffer = initBuffer( + glOut, + [ + 0.0, 0.0, + 1.0, 0.0, + 0.0, 1.0, + 1.0, 1.0, + ] + )! + + if (outputType === 'texture') { + this._outputTexture = createTexture( + glOut, + glOut.RGBA8, + width, + height + ) + this._outputFramebuffer = glOut.createFramebuffer() + glOut.bindFramebuffer( + glOut.FRAMEBUFFER, + this._outputFramebuffer + ) + glOut.framebufferTexture2D( + glOut.FRAMEBUFFER, + glOut.COLOR_ATTACHMENT0, + glOut.TEXTURE_2D, + this._outputTexture, + 0 + ) + } + + const program = createPipelineStageProgram( + glOut, + this._vertexShader, + this._fragmentShader, + this._positionBuffer, + this._texCoordBuffer + ) + this._program = program + + this._setUniformVars([ + { + name: textureName, + type: 'int', + values: [textureUnit] + }, + ...uniformVars + ]) + } + + cleanUp(): void { + const { + _fragmentShader, + _glOut, + _positionBuffer, + _program, + _texCoordBuffer, + _vertexShader + } = this + _glOut.deleteProgram(_program) + _glOut.deleteBuffer(_texCoordBuffer) + _glOut.deleteBuffer(_positionBuffer) + _glOut.deleteShader(_vertexShader) + _glOut.deleteShader(_fragmentShader) + } + + render(): void { + const { + _glOut, + _inputTextureUnit, + _outputDimensions: { + height, + width + }, + _outputFramebuffer, + _outputTexture, + _program + } = this + + _glOut.viewport(0, 0, width, height) + _glOut.useProgram(_program) + + if (_outputTexture) { + _glOut.activeTexture( + _glOut.TEXTURE0 + + _inputTextureUnit + + 1 + ) + _glOut.bindTexture( + _glOut.TEXTURE_2D, + _outputTexture + ) + } + _glOut.bindFramebuffer( + _glOut.FRAMEBUFFER, + _outputFramebuffer + ) + _glOut.drawArrays( + _glOut.TRIANGLE_STRIP, + 0, + 4 + ) + } + + protected _setUniformVars(uniformVars: UniformVarInfo[]) { + const { + _glOut, + _program + } = this + + _glOut.useProgram(_program) + + uniformVars.forEach(({ + name, + type, + values + }) => { + const uniformVarLocation = _glOut + .getUniformLocation( + _program, + name + ) + + // @ts-ignore + _glOut[`uniform${values.length}${type[0]}`]( + uniformVarLocation, + ...values + ) + }) + } +} + +/** + * @private + */ +export class WebGL2Pipeline extends Pipeline { + static InputStage = WebGL2PipelineInputStage + static ProcessingStage = WebGL2PipelineProcessingStage + protected _stages: (WebGL2PipelineInputStage | WebGL2PipelineProcessingStage)[] = [] + + render(personMask: ImageData): void { + const [ + inputStage, + ...otherStages + ] = this._stages as [ + WebGL2PipelineInputStage, + ...WebGL2PipelineProcessingStage[] + ] + + inputStage.render(personMask) + otherStages.forEach( + (stage) => stage.render() + ) + } + + cleanUp(): void { + this._stages.forEach( + (stage) => stage.cleanUp() + ) + } +} diff --git a/lib/processors/webgl2/pipelines/backgroundBlurStage.ts b/lib/processors/webgl2/pipelines/backgroundBlurStage.ts deleted file mode 100644 index 5dda79a..0000000 --- a/lib/processors/webgl2/pipelines/backgroundBlurStage.ts +++ /dev/null @@ -1,292 +0,0 @@ -import { - compileShader, - createPiplelineStageProgram, - createTexture, - glsl, -} from '../helpers/webglHelper' - -export type BackgroundBlurStage = { - render(): void - updateCoverage(coverage: [number, number]): void - cleanUp(): void -} - -export function buildBackgroundBlurStage( - gl: WebGL2RenderingContext, - vertexShader: WebGLShader, - positionBuffer: WebGLBuffer, - texCoordBuffer: WebGLBuffer, - personMaskTexture: WebGLTexture, - canvas: HTMLCanvasElement -): BackgroundBlurStage { - const blurPass = buildBlurPass( - gl, - vertexShader, - positionBuffer, - texCoordBuffer, - personMaskTexture, - canvas - ) - const blendPass = buildBlendPass(gl, positionBuffer, texCoordBuffer, canvas) - - function render() { - blurPass.render() - blendPass.render() - } - - function updateCoverage(coverage: [number, number]) { - blendPass.updateCoverage(coverage) - } - - function cleanUp() { - blendPass.cleanUp() - blurPass.cleanUp() - } - - return { - render, - updateCoverage, - cleanUp, - } -} - -function buildBlurPass( - gl: WebGL2RenderingContext, - vertexShader: WebGLShader, - positionBuffer: WebGLBuffer, - texCoordBuffer: WebGLBuffer, - personMaskTexture: WebGLTexture, - canvas: HTMLCanvasElement -) { - const fragmentShaderSource = glsl`#version 300 es - - precision highp float; - - uniform sampler2D u_inputFrame; - uniform sampler2D u_personMask; - uniform vec2 u_texelSize; - - in vec2 v_texCoord; - - out vec4 outColor; - - const float offset[5] = float[](0.0, 1.0, 2.0, 3.0, 4.0); - const float weight[5] = float[](0.2270270270, 0.1945945946, 0.1216216216, - 0.0540540541, 0.0162162162); - - void main() { - vec4 centerColor = texture(u_inputFrame, v_texCoord); - float personMask = texture(u_personMask, v_texCoord).a; - - vec4 frameColor = centerColor * weight[0] * (1.0 - personMask); - - for (int i = 1; i < 5; i++) { - vec2 offset = vec2(offset[i]) * u_texelSize; - - vec2 texCoord = v_texCoord + offset; - frameColor += texture(u_inputFrame, texCoord) * weight[i] * - (1.0 - texture(u_personMask, texCoord).a); - - texCoord = v_texCoord - offset; - frameColor += texture(u_inputFrame, texCoord) * weight[i] * - (1.0 - texture(u_personMask, texCoord).a); - } - outColor = vec4(frameColor.rgb + (1.0 - frameColor.a) * centerColor.rgb, 1.0); - } - ` - - const scale = 0.5 - const outputWidth = canvas.width * scale - const outputHeight = canvas.height * scale - const texelWidth = 1 / outputWidth - const texelHeight = 1 / outputHeight - - const fragmentShader = compileShader( - gl, - gl.FRAGMENT_SHADER, - fragmentShaderSource - ) - const program = createPiplelineStageProgram( - gl, - vertexShader, - fragmentShader, - positionBuffer, - texCoordBuffer - ) - const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame') - const personMaskLocation = gl.getUniformLocation(program, 'u_personMask') - const texelSizeLocation = gl.getUniformLocation(program, 'u_texelSize') - const texture1 = createTexture( - gl, - gl.RGBA8, - outputWidth, - outputHeight, - gl.NEAREST, - gl.LINEAR - ) - const texture2 = createTexture( - gl, - gl.RGBA8, - outputWidth, - outputHeight, - gl.NEAREST, - gl.LINEAR - ) - - const frameBuffer1 = gl.createFramebuffer() - gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer1) - gl.framebufferTexture2D( - gl.FRAMEBUFFER, - gl.COLOR_ATTACHMENT0, - gl.TEXTURE_2D, - texture1, - 0 - ) - - const frameBuffer2 = gl.createFramebuffer() - gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer2) - gl.framebufferTexture2D( - gl.FRAMEBUFFER, - gl.COLOR_ATTACHMENT0, - gl.TEXTURE_2D, - texture2, - 0 - ) - - gl.useProgram(program) - gl.uniform1i(personMaskLocation, 1) - - function render() { - gl.viewport(0, 0, outputWidth, outputHeight) - gl.useProgram(program) - gl.uniform1i(inputFrameLocation, 0) - gl.activeTexture(gl.TEXTURE1) - gl.bindTexture(gl.TEXTURE_2D, personMaskTexture) - - for (let i = 0; i < 8; i++) { - gl.uniform2f(texelSizeLocation, 0, texelHeight) - gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer1) - gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4) - - gl.activeTexture(gl.TEXTURE2) - gl.bindTexture(gl.TEXTURE_2D, texture1) - gl.uniform1i(inputFrameLocation, 2) - - gl.uniform2f(texelSizeLocation, texelWidth, 0) - gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer2) - gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4) - - gl.bindTexture(gl.TEXTURE_2D, texture2) - } - } - - function cleanUp() { - gl.deleteFramebuffer(frameBuffer2) - gl.deleteFramebuffer(frameBuffer1) - gl.deleteTexture(texture2) - gl.deleteTexture(texture1) - gl.deleteProgram(program) - gl.deleteShader(fragmentShader) - } - - return { - render, - cleanUp, - } -} - -function buildBlendPass( - gl: WebGL2RenderingContext, - positionBuffer: WebGLBuffer, - texCoordBuffer: WebGLBuffer, - canvas: HTMLCanvasElement -) { - const vertexShaderSource = glsl`#version 300 es - - in vec2 a_position; - in vec2 a_texCoord; - - out vec2 v_texCoord; - - void main() { - // Flipping Y is required when rendering to canvas - gl_Position = vec4(a_position * vec2(1.0, -1.0), 0.0, 1.0); - v_texCoord = a_texCoord; - } - ` - - const fragmentShaderSource = glsl`#version 300 es - - precision highp float; - - uniform sampler2D u_inputFrame; - uniform sampler2D u_personMask; - uniform sampler2D u_blurredInputFrame; - uniform vec2 u_coverage; - - in vec2 v_texCoord; - - out vec4 outColor; - - void main() { - vec3 color = texture(u_inputFrame, v_texCoord).rgb; - vec3 blurredColor = texture(u_blurredInputFrame, v_texCoord).rgb; - float personMask = texture(u_personMask, v_texCoord).a; - personMask = smoothstep(u_coverage.x, u_coverage.y, personMask); - outColor = vec4(mix(blurredColor, color, personMask), 1.0); - } - ` - - const { width: outputWidth, height: outputHeight } = canvas - - const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource) - const fragmentShader = compileShader( - gl, - gl.FRAGMENT_SHADER, - fragmentShaderSource - ) - const program = createPiplelineStageProgram( - gl, - vertexShader, - fragmentShader, - positionBuffer, - texCoordBuffer - ) - const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame') - const personMaskLocation = gl.getUniformLocation(program, 'u_personMask') - const blurredInputFrame = gl.getUniformLocation( - program, - 'u_blurredInputFrame' - ) - const coverageLocation = gl.getUniformLocation(program, 'u_coverage') - - gl.useProgram(program) - gl.uniform1i(inputFrameLocation, 0) - gl.uniform1i(personMaskLocation, 1) - gl.uniform1i(blurredInputFrame, 2) - gl.uniform2f(coverageLocation, 0, 1) - - function render() { - gl.viewport(0, 0, outputWidth, outputHeight) - gl.useProgram(program) - gl.bindFramebuffer(gl.FRAMEBUFFER, null) - gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4) - } - - function updateCoverage(coverage: [number, number]) { - gl.useProgram(program) - gl.uniform2f(coverageLocation, coverage[0], coverage[1]) - } - - function cleanUp() { - gl.deleteProgram(program) - gl.deleteShader(fragmentShader) - gl.deleteShader(vertexShader) - } - - return { - render, - updateCoverage, - cleanUp, - } -} diff --git a/lib/processors/webgl2/pipelines/backgroundImageStage.ts b/lib/processors/webgl2/pipelines/backgroundImageStage.ts deleted file mode 100644 index 452b957..0000000 --- a/lib/processors/webgl2/pipelines/backgroundImageStage.ts +++ /dev/null @@ -1,221 +0,0 @@ -import { BlendMode } from '../helpers/postProcessingHelper' -import { - compileShader, - createPiplelineStageProgram, - createTexture, - glsl, -} from '../helpers/webglHelper' - -export type BackgroundImageStage = { - render(): void - updateCoverage(coverage: [number, number]): void - updateLightWrapping(lightWrapping: number): void - updateBlendMode(blendMode: BlendMode): void - cleanUp(): void -} - -export function buildBackgroundImageStage( - gl: WebGL2RenderingContext, - positionBuffer: WebGLBuffer, - texCoordBuffer: WebGLBuffer, - personMaskTexture: WebGLTexture, - backgroundImage: HTMLImageElement | null, - canvas: HTMLCanvasElement -): BackgroundImageStage { - const vertexShaderSource = glsl`#version 300 es - - uniform vec2 u_backgroundScale; - uniform vec2 u_backgroundOffset; - - in vec2 a_position; - in vec2 a_texCoord; - - out vec2 v_texCoord; - out vec2 v_backgroundCoord; - - void main() { - // Flipping Y is required when rendering to canvas - gl_Position = vec4(a_position * vec2(1.0, -1.0), 0.0, 1.0); - v_texCoord = a_texCoord; - v_backgroundCoord = a_texCoord * u_backgroundScale + u_backgroundOffset; - } - ` - - const fragmentShaderSource = glsl`#version 300 es - - precision highp float; - - uniform sampler2D u_inputFrame; - uniform sampler2D u_personMask; - uniform sampler2D u_background; - uniform vec2 u_coverage; - uniform float u_lightWrapping; - uniform float u_blendMode; - - in vec2 v_texCoord; - in vec2 v_backgroundCoord; - - out vec4 outColor; - - vec3 screen(vec3 a, vec3 b) { - return 1.0 - (1.0 - a) * (1.0 - b); - } - - vec3 linearDodge(vec3 a, vec3 b) { - return a + b; - } - - void main() { - vec3 frameColor = texture(u_inputFrame, v_texCoord).rgb; - vec3 backgroundColor = texture(u_background, v_backgroundCoord).rgb; - float personMask = texture(u_personMask, v_texCoord).a; - float lightWrapMask = 1.0 - max(0.0, personMask - u_coverage.y) / (1.0 - u_coverage.y); - vec3 lightWrap = u_lightWrapping * lightWrapMask * backgroundColor; - frameColor = u_blendMode * linearDodge(frameColor, lightWrap) + - (1.0 - u_blendMode) * screen(frameColor, lightWrap); - personMask = smoothstep(u_coverage.x, u_coverage.y, personMask); - outColor = vec4(frameColor * personMask + backgroundColor * (1.0 - personMask), 1.0); - } - ` - - const { width: outputWidth, height: outputHeight } = canvas - const outputRatio = outputWidth / outputHeight - - const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource) - const fragmentShader = compileShader( - gl, - gl.FRAGMENT_SHADER, - fragmentShaderSource - ) - const program = createPiplelineStageProgram( - gl, - vertexShader, - fragmentShader, - positionBuffer, - texCoordBuffer - ) - const backgroundScaleLocation = gl.getUniformLocation( - program, - 'u_backgroundScale' - ) - const backgroundOffsetLocation = gl.getUniformLocation( - program, - 'u_backgroundOffset' - ) - const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame') - const personMaskLocation = gl.getUniformLocation(program, 'u_personMask') - const backgroundLocation = gl.getUniformLocation(program, 'u_background') - const coverageLocation = gl.getUniformLocation(program, 'u_coverage') - const lightWrappingLocation = gl.getUniformLocation( - program, - 'u_lightWrapping' - ) - const blendModeLocation = gl.getUniformLocation(program, 'u_blendMode') - - gl.useProgram(program) - gl.uniform2f(backgroundScaleLocation, 1, 1) - gl.uniform2f(backgroundOffsetLocation, 0, 0) - gl.uniform1i(inputFrameLocation, 0) - gl.uniform1i(personMaskLocation, 1) - gl.uniform2f(coverageLocation, 0, 1) - gl.uniform1f(lightWrappingLocation, 0) - gl.uniform1f(blendModeLocation, 0) - - let backgroundTexture: WebGLTexture | null = null - // TODO Find a better to handle background being loaded - if (backgroundImage?.complete) { - updateBackgroundImage(backgroundImage) - } else if (backgroundImage) { - backgroundImage.onload = () => { - updateBackgroundImage(backgroundImage) - } - } - - function render() { - gl.viewport(0, 0, outputWidth, outputHeight) - gl.useProgram(program) - gl.activeTexture(gl.TEXTURE1) - gl.bindTexture(gl.TEXTURE_2D, personMaskTexture) - if (backgroundTexture !== null) { - gl.activeTexture(gl.TEXTURE2) - gl.bindTexture(gl.TEXTURE_2D, backgroundTexture) - // TODO Handle correctly the background not loaded yet - gl.uniform1i(backgroundLocation, 2) - } - gl.bindFramebuffer(gl.FRAMEBUFFER, null) - gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4) - } - - function updateBackgroundImage(backgroundImage: HTMLImageElement) { - backgroundTexture = createTexture( - gl, - gl.RGBA8, - backgroundImage.naturalWidth, - backgroundImage.naturalHeight, - gl.LINEAR, - gl.LINEAR - ) - gl.texSubImage2D( - gl.TEXTURE_2D, - 0, - 0, - 0, - backgroundImage.naturalWidth, - backgroundImage.naturalHeight, - gl.RGBA, - gl.UNSIGNED_BYTE, - backgroundImage - ) - - let xOffset = 0 - let yOffset = 0 - let backgroundWidth = backgroundImage.naturalWidth - let backgroundHeight = backgroundImage.naturalHeight - const backgroundRatio = backgroundWidth / backgroundHeight - if (backgroundRatio < outputRatio) { - backgroundHeight = backgroundWidth / outputRatio - yOffset = (backgroundImage.naturalHeight - backgroundHeight) / 2 - } else { - backgroundWidth = backgroundHeight * outputRatio - xOffset = (backgroundImage.naturalWidth - backgroundWidth) / 2 - } - - const xScale = backgroundWidth / backgroundImage.naturalWidth - const yScale = backgroundHeight / backgroundImage.naturalHeight - xOffset /= backgroundImage.naturalWidth - yOffset /= backgroundImage.naturalHeight - - gl.uniform2f(backgroundScaleLocation, xScale, yScale) - gl.uniform2f(backgroundOffsetLocation, xOffset, yOffset) - } - - function updateCoverage(coverage: [number, number]) { - gl.useProgram(program) - gl.uniform2f(coverageLocation, coverage[0], coverage[1]) - } - - function updateLightWrapping(lightWrapping: number) { - gl.useProgram(program) - gl.uniform1f(lightWrappingLocation, lightWrapping) - } - - function updateBlendMode(blendMode: BlendMode) { - gl.useProgram(program) - gl.uniform1f(blendModeLocation, blendMode === 'screen' ? 0 : 1) - } - - function cleanUp() { - gl.deleteTexture(backgroundTexture) - gl.deleteProgram(program) - gl.deleteShader(fragmentShader) - gl.deleteShader(vertexShader) - } - - return { - render, - updateCoverage, - updateLightWrapping, - updateBlendMode, - cleanUp, - } -} diff --git a/lib/processors/webgl2/pipelines/fastBilateralFilterStage.ts b/lib/processors/webgl2/pipelines/fastBilateralFilterStage.ts deleted file mode 100644 index e8f863e..0000000 --- a/lib/processors/webgl2/pipelines/fastBilateralFilterStage.ts +++ /dev/null @@ -1,222 +0,0 @@ -import { - inputResolutions, - SegmentationConfig, -} from '../helpers/segmentationHelper' -import { - compileShader, - createPiplelineStageProgram, - glsl, -} from '../helpers/webglHelper' - -export function buildFastBilateralFilterStage( - gl: WebGL2RenderingContext, - vertexShader: WebGLShader, - positionBuffer: WebGLBuffer, - texCoordBuffer: WebGLBuffer, - inputTexture: WebGLTexture, - segmentationConfig: SegmentationConfig, - outputTexture: WebGLTexture, - canvas: HTMLCanvasElement -) { - // NOTE(mmalavalli): This is a faster approximation of the joint bilateral filter. - // For a given pixel, instead of calculating the space and color weights of all - // the pixels within the filter kernel, which would have a complexity of O(r^2), - // we calculate the space and color weights of only those pixels which form two - // diagonal lines between the two pairs of opposite corners of the filter kernel, - // which would have a complexity of O(r). This improves the overall complexity - // of this stage from O(w x h x r^2) to O(w x h x r), where: - // w => width of the output video frame - // h => height of the output video frame - // r => radius of the joint bilateral filter kernel - const fragmentShaderSource = glsl`#version 300 es - - precision highp float; - - uniform sampler2D u_inputFrame; - uniform sampler2D u_segmentationMask; - uniform vec2 u_texelSize; - uniform float u_step; - uniform float u_radius; - uniform float u_offset; - uniform float u_sigmaTexel; - uniform float u_sigmaColor; - - in vec2 v_texCoord; - - out vec4 outColor; - - float gaussian(float x, float sigma) { - return exp(-0.5 * x * x / sigma / sigma); - } - - float calculateSpaceWeight(vec2 coord) { - float x = distance(v_texCoord, coord); - float sigma = u_sigmaTexel; - return gaussian(x, sigma); - } - - float calculateColorWeight(vec2 coord) { - vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb; - vec3 coordColor = texture(u_inputFrame, coord).rgb; - float x = distance(centerColor, coordColor); - float sigma = u_sigmaColor; - return gaussian(x, sigma); - } - - void main() { - vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb; - float newVal = 0.0; - float totalWeight = 0.0; - - vec2 leftTopCoord = vec2(v_texCoord + vec2(-u_radius, -u_radius) * u_texelSize); - vec2 rightTopCoord = vec2(v_texCoord + vec2(u_radius, -u_radius) * u_texelSize); - vec2 leftBottomCoord = vec2(v_texCoord + vec2(-u_radius, u_radius) * u_texelSize); - vec2 rightBottomCoord = vec2(v_texCoord + vec2(u_radius, u_radius) * u_texelSize); - - float leftTopSegAlpha = texture(u_segmentationMask, leftTopCoord).a; - float rightTopSegAlpha = texture(u_segmentationMask, rightTopCoord).a; - float leftBottomSegAlpha = texture(u_segmentationMask, leftBottomCoord).a; - float rightBottomSegAlpha = texture(u_segmentationMask, rightBottomCoord).a; - float totalSegAlpha = leftTopSegAlpha + rightTopSegAlpha + leftBottomSegAlpha + rightBottomSegAlpha; - - if (totalSegAlpha <= 0.0) { - newVal = 0.0; - } else if (totalSegAlpha >= 4.0) { - newVal = 1.0; - } else { - for (float i = 0.0; i <= u_radius - u_offset; i += u_step) { - vec2 shift = vec2(i, i) * u_texelSize; - vec2 coord = vec2(v_texCoord + shift); - float spaceWeight = calculateSpaceWeight(coord); - float colorWeight = calculateColorWeight(coord); - float weight = spaceWeight * colorWeight; - float alpha = texture(u_segmentationMask, coord).a; - totalWeight += weight; - newVal += weight * alpha; - - if (i != 0.0) { - shift = vec2(i, -i) * u_texelSize; - coord = vec2(v_texCoord + shift); - colorWeight = calculateColorWeight(coord); - weight = spaceWeight * colorWeight; - alpha = texture(u_segmentationMask, coord).a; - totalWeight += weight; - newVal += weight * texture(u_segmentationMask, coord).a; - - shift = vec2(-i, i) * u_texelSize; - coord = vec2(v_texCoord + shift); - colorWeight = calculateColorWeight(coord); - weight = spaceWeight * colorWeight; - alpha = texture(u_segmentationMask, coord).a; - totalWeight += weight; - newVal += weight * texture(u_segmentationMask, coord).a; - - shift = vec2(-i, -i) * u_texelSize; - coord = vec2(v_texCoord + shift); - colorWeight = calculateColorWeight(coord); - weight = spaceWeight * colorWeight; - alpha = texture(u_segmentationMask, coord).a; - totalWeight += weight; - newVal += weight * texture(u_segmentationMask, coord).a; - } - } - newVal /= totalWeight; - } - - outColor = vec4(vec3(0.0), newVal); - } - ` - - const [segmentationWidth, segmentationHeight] = inputResolutions[ - segmentationConfig.inputResolution - ] - const { width: outputWidth, height: outputHeight } = canvas - const texelWidth = 1 / outputWidth - const texelHeight = 1 / outputHeight - - const fragmentShader = compileShader( - gl, - gl.FRAGMENT_SHADER, - fragmentShaderSource - ) - const program = createPiplelineStageProgram( - gl, - vertexShader, - fragmentShader, - positionBuffer, - texCoordBuffer - ) - const inputFrameLocation = gl.getUniformLocation(program, 'u_inputFrame') - const segmentationMaskLocation = gl.getUniformLocation( - program, - 'u_segmentationMask' - ) - const texelSizeLocation = gl.getUniformLocation(program, 'u_texelSize') - const stepLocation = gl.getUniformLocation(program, 'u_step') - const radiusLocation = gl.getUniformLocation(program, 'u_radius') - const offsetLocation = gl.getUniformLocation(program, 'u_offset') - const sigmaTexelLocation = gl.getUniformLocation(program, 'u_sigmaTexel') - const sigmaColorLocation = gl.getUniformLocation(program, 'u_sigmaColor') - - const frameBuffer = gl.createFramebuffer() - gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer) - gl.framebufferTexture2D( - gl.FRAMEBUFFER, - gl.COLOR_ATTACHMENT0, - gl.TEXTURE_2D, - outputTexture, - 0 - ) - - gl.useProgram(program) - gl.uniform1i(inputFrameLocation, 0) - gl.uniform1i(segmentationMaskLocation, 1) - gl.uniform2f(texelSizeLocation, texelWidth, texelHeight) - - // Ensures default values are configured to prevent infinite - // loop in fragment shader - updateSigmaSpace(0) - updateSigmaColor(0) - - function render() { - gl.viewport(0, 0, outputWidth, outputHeight) - gl.useProgram(program) - gl.activeTexture(gl.TEXTURE1) - gl.bindTexture(gl.TEXTURE_2D, inputTexture) - gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer) - gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4) - } - - function updateSigmaSpace(sigmaSpace: number) { - sigmaSpace *= Math.max( - outputWidth / segmentationWidth, - outputHeight / segmentationHeight - ) - - const kSparsityFactor = 0.66 // Higher is more sparse. - const sparsity = Math.max(1, Math.sqrt(sigmaSpace) * kSparsityFactor) - const step = sparsity - const radius = sigmaSpace - const offset = step > 1 ? step * 0.5 : 0 - const sigmaTexel = Math.max(texelWidth, texelHeight) * sigmaSpace - - gl.useProgram(program) - gl.uniform1f(stepLocation, step) - gl.uniform1f(radiusLocation, radius) - gl.uniform1f(offsetLocation, offset) - gl.uniform1f(sigmaTexelLocation, sigmaTexel) - } - - function updateSigmaColor(sigmaColor: number) { - gl.useProgram(program) - gl.uniform1f(sigmaColorLocation, sigmaColor) - } - - function cleanUp() { - gl.deleteFramebuffer(frameBuffer) - gl.deleteProgram(program) - gl.deleteShader(fragmentShader) - } - - return { render, updateSigmaSpace, updateSigmaColor, cleanUp } -} diff --git a/lib/processors/webgl2/pipelines/loadSegmentationStage.ts b/lib/processors/webgl2/pipelines/loadSegmentationStage.ts deleted file mode 100644 index bef6398..0000000 --- a/lib/processors/webgl2/pipelines/loadSegmentationStage.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { - inputResolutions, - SegmentationConfig, -} from '../helpers/segmentationHelper' -import { - compileShader, - createPiplelineStageProgram, - createTexture, - glsl, -} from '../helpers/webglHelper' - -export function buildLoadSegmentationStage( - gl: WebGL2RenderingContext, - vertexShader: WebGLShader, - positionBuffer: WebGLBuffer, - texCoordBuffer: WebGLBuffer, - segmentationConfig: SegmentationConfig, - outputTexture: WebGLTexture -) { - const fragmentShaderSource = glsl`#version 300 es - - precision highp float; - - uniform sampler2D u_inputSegmentation; - - in vec2 v_texCoord; - - out vec4 outColor; - - void main() { - float segmentation = texture(u_inputSegmentation, v_texCoord).a; - outColor = vec4(vec3(0.0), segmentation); - } - ` - const [segmentationWidth, segmentationHeight] = inputResolutions[ - segmentationConfig.inputResolution - ] - const fragmentShader = compileShader( - gl, - gl.FRAGMENT_SHADER, - fragmentShaderSource - ) - const program = createPiplelineStageProgram( - gl, - vertexShader, - fragmentShader, - positionBuffer, - texCoordBuffer - ) - const inputLocation = gl.getUniformLocation(program, 'u_inputSegmentation') - const inputTexture = createTexture( - gl, - gl.RGBA8, - segmentationWidth, - segmentationHeight - ) - - const frameBuffer = gl.createFramebuffer() - gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer) - gl.framebufferTexture2D( - gl.FRAMEBUFFER, - gl.COLOR_ATTACHMENT0, - gl.TEXTURE_2D, - outputTexture, - 0 - ) - - gl.useProgram(program) - gl.uniform1i(inputLocation, 1) - - function render(segmentationData: Uint8ClampedArray) { - gl.viewport(0, 0, segmentationWidth, segmentationHeight) - gl.useProgram(program) - gl.activeTexture(gl.TEXTURE1) - gl.bindTexture(gl.TEXTURE_2D, inputTexture) - gl.texSubImage2D( - gl.TEXTURE_2D, - 0, - 0, - 0, - segmentationWidth, - segmentationHeight, - gl.RGBA, - gl.UNSIGNED_BYTE, - segmentationData - ) - gl.bindFramebuffer(gl.FRAMEBUFFER, frameBuffer) - gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4) - } - - function cleanUp() { - gl.deleteFramebuffer(frameBuffer) - gl.deleteTexture(inputTexture) - gl.deleteProgram(program) - gl.deleteShader(fragmentShader) - } - - return { render, cleanUp } -} diff --git a/lib/processors/webgl2/pipelines/webgl2Pipeline.ts b/lib/processors/webgl2/pipelines/webgl2Pipeline.ts deleted file mode 100644 index 40203ff..0000000 --- a/lib/processors/webgl2/pipelines/webgl2Pipeline.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { BackgroundConfig } from '../helpers/backgroundHelper' -import { PostProcessingConfig } from '../helpers/postProcessingHelper' -import { - inputResolutions, - SegmentationConfig, -} from '../helpers/segmentationHelper' -import { SourcePlayback } from '../helpers/sourceHelper' -import { compileShader, createTexture, glsl } from '../helpers/webglHelper' -import { - buildBackgroundBlurStage, -} from './backgroundBlurStage' -import { - BackgroundImageStage, - buildBackgroundImageStage, -} from './backgroundImageStage' -import { buildFastBilateralFilterStage } from './fastBilateralFilterStage' -import { buildLoadSegmentationStage } from './loadSegmentationStage' - -export function buildWebGL2Pipeline( - sourcePlayback: SourcePlayback, - backgroundImage: HTMLImageElement | null, - backgroundConfig: BackgroundConfig, - segmentationConfig: SegmentationConfig, - canvas: HTMLCanvasElement, - benchmark: any, - debounce: boolean -) { - let shouldUpscaleCurrentMask = true - - const vertexShaderSource = glsl`#version 300 es - - in vec2 a_position; - in vec2 a_texCoord; - - out vec2 v_texCoord; - - void main() { - gl_Position = vec4(a_position, 0.0, 1.0); - v_texCoord = a_texCoord; - } - ` - - const { width: outputWidth, height: outputHeight } = canvas; - const [segmentationWidth, segmentationHeight] = inputResolutions[ - segmentationConfig.inputResolution - ] - - const gl = canvas.getContext('webgl2')! - - const vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexShaderSource) - - const vertexArray = gl.createVertexArray() - gl.bindVertexArray(vertexArray) - - const positionBuffer = gl.createBuffer()! - gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer) - gl.bufferData( - gl.ARRAY_BUFFER, - new Float32Array([-1.0, -1.0, 1.0, -1.0, -1.0, 1.0, 1.0, 1.0]), - gl.STATIC_DRAW - ) - - const texCoordBuffer = gl.createBuffer()! - gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer) - gl.bufferData( - gl.ARRAY_BUFFER, - new Float32Array([0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0]), - gl.STATIC_DRAW - ) - - // We don't use texStorage2D here because texImage2D seems faster - // to upload video texture than texSubImage2D even though the latter - // is supposed to be the recommended way: - // https://developer.mozilla.org/en-US/docs/Web/API/WebGL_API/WebGL_best_practices#use_texstorage_to_create_textures - const inputFrameTexture = gl.createTexture() - gl.bindTexture(gl.TEXTURE_2D, inputFrameTexture) - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE) - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE) - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST) - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST) - - // TODO Rename segmentation and person mask to be more specific - const segmentationTexture = createTexture( - gl, - gl.RGBA8, - segmentationWidth, - segmentationHeight - )! - const personMaskTexture = createTexture( - gl, - gl.RGBA8, - outputWidth, - outputHeight - )! - const loadSegmentationStage = buildLoadSegmentationStage( - gl, - vertexShader, - positionBuffer, - texCoordBuffer, - segmentationConfig, - segmentationTexture - ) - const fastBilateralFilterStage = buildFastBilateralFilterStage( - gl, - vertexShader, - positionBuffer, - texCoordBuffer, - segmentationTexture, - segmentationConfig, - personMaskTexture, - canvas - ) - const backgroundStage = - backgroundConfig.type === 'blur' - ? buildBackgroundBlurStage( - gl, - vertexShader, - positionBuffer, - texCoordBuffer, - personMaskTexture, - canvas - ) - : buildBackgroundImageStage( - gl, - positionBuffer, - texCoordBuffer, - personMaskTexture, - backgroundImage, - canvas - ) - - async function sampleInputFrame() { - gl.clearColor(0, 0, 0, 0) - gl.clear(gl.COLOR_BUFFER_BIT) - - gl.activeTexture(gl.TEXTURE0) - gl.bindTexture(gl.TEXTURE_2D, inputFrameTexture) - - // texImage2D seems faster than texSubImage2D to upload - // video texture - gl.texImage2D( - gl.TEXTURE_2D, - 0, - gl.RGBA, - gl.RGBA, - gl.UNSIGNED_BYTE, - sourcePlayback.htmlElement - ) - - gl.bindVertexArray(vertexArray) - } - - async function render(segmentationData: Uint8ClampedArray) { - benchmark.start('imageCompositionDelay') - if (shouldUpscaleCurrentMask) { - loadSegmentationStage.render(segmentationData) - } - fastBilateralFilterStage.render() - backgroundStage.render() - if (debounce) { - shouldUpscaleCurrentMask = !shouldUpscaleCurrentMask - } - benchmark.end('imageCompositionDelay') - } - - function updatePostProcessingConfig( - postProcessingConfig: PostProcessingConfig - ) { - const { - blendMode, - coverage, - lightWrapping, - jointBilateralFilter = {} - } = postProcessingConfig - - const { - sigmaColor, - sigmaSpace - } = jointBilateralFilter - - if (typeof sigmaColor === 'number') { - fastBilateralFilterStage.updateSigmaColor(sigmaColor) - } - if (typeof sigmaSpace === 'number') { - fastBilateralFilterStage.updateSigmaSpace(sigmaSpace) - } - if (Array.isArray(coverage)) { - if (backgroundConfig.type === 'blur' || backgroundConfig.type === 'image') { - backgroundStage.updateCoverage(coverage) - } - } - if (backgroundConfig.type === 'image') { - const backgroundImageStage = backgroundStage as BackgroundImageStage - if (typeof lightWrapping === 'number') { - backgroundImageStage.updateLightWrapping(lightWrapping) - } - if (typeof blendMode === 'string') { - backgroundImageStage.updateBlendMode(blendMode) - } - } else if (backgroundConfig.type !== 'blur') { - // TODO Handle no background in a separate pipeline path - const backgroundImageStage = backgroundStage as BackgroundImageStage - backgroundImageStage.updateCoverage([0, 0.9999]) - backgroundImageStage.updateLightWrapping(0) - } - } - - function cleanUp() { - backgroundStage.cleanUp() - fastBilateralFilterStage.cleanUp() - loadSegmentationStage.cleanUp() - - gl.deleteTexture(personMaskTexture) - gl.deleteTexture(segmentationTexture) - gl.deleteTexture(inputFrameTexture) - gl.deleteBuffer(texCoordBuffer) - gl.deleteBuffer(positionBuffer) - gl.deleteVertexArray(vertexArray) - gl.deleteShader(vertexShader) - } - - return { render, sampleInputFrame, updatePostProcessingConfig, cleanUp } -} diff --git a/lib/types.ts b/lib/types.ts index 1ddb96d..40c46b1 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -11,14 +11,6 @@ } } -/** - * @private - */ -export enum WebGL2PipelineType { - Blur = 'blur', - Image = 'image', -} - /** * @private */ @@ -63,24 +55,3 @@ export enum ImageFit { */ None = 'None' } - -/** - * Specifies which pipeline to use when processing video frames. - */ -export enum Pipeline { - /** - * Use canvas 2d rendering context. Some browsers such as Safari do not - * have full support of this feature. Please test your application to make sure it works as intented. See - * [browser compatibility page](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D#browser_compatibility) - * for reference. - */ - Canvas2D = 'Canvas2D', - - /** - * Use canvas webgl2 rendering context. Major browsers have support for this feature. However, this does not work - * on some older versions of browsers. Please test your application to make sure it works as intented. See - * [browser compatibility page](https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext#browser_compatibility) - * for reference. - */ - WebGL2 = 'WebGL2' -} From 8e836c6072ba2350a9217656a86d3269334c2648 Mon Sep 17 00:00:00 2001 From: mmalavalli Date: Fri, 5 Jul 2024 15:41:22 -0500 Subject: [PATCH 02/28] Hybrid pipeline POC: Working around lack of Canvas2D blur filter feature support in Safari by using a WebGL2 alternative. --- .../background/BackgroundProcessor.ts | 21 ++-- .../GaussianBlurBackgroundProcessor.ts | 28 ++++- .../background/VirtualBackgroundProcessor.ts | 2 +- lib/processors/webgl2/index.ts | 1 + .../pipelines/GaussianBlurFilterPipeline.ts | 33 ++++++ lib/processors/webgl2/pipelines/Pipeline.ts | 4 +- .../SinglePassGaussianBlurFilterStage.ts | 104 ++++++++++++++++++ .../webgl2/pipelines/WebGL2Pipeline.ts | 74 +++++++------ lib/utils/support.ts | 26 +++++ 9 files changed, 240 insertions(+), 53 deletions(-) create mode 100644 lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts create mode 100644 lib/processors/webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts diff --git a/lib/processors/background/BackgroundProcessor.ts b/lib/processors/background/BackgroundProcessor.ts index 955f5a8..668864b 100644 --- a/lib/processors/background/BackgroundProcessor.ts +++ b/lib/processors/background/BackgroundProcessor.ts @@ -1,7 +1,7 @@ import { Processor } from '../Processor'; import { Benchmark } from '../../utils/Benchmark'; import { TwilioTFLite } from '../../utils/TwilioTFLite'; -import { isChromiumImageBitmap } from '../../utils/support'; +import { isCanvasBlurSupported, isChromiumImageBitmap } from '../../utils/support'; import { Dimensions } from '../../types'; import { PersonMaskUpscalePipeline } from '../webgl2'; @@ -78,7 +78,7 @@ export interface BackgroundProcessorOptions { * The blur radius to use when smoothing out the edges of the person's mask. * @default * ```html - * 8 for WebGL2 pipeline, 4 for Canvas2D pipeline + * 8 * ``` */ maskBlurRadius?: number; @@ -94,6 +94,7 @@ export abstract class BackgroundProcessor extends Processor { protected _outputCanvas: HTMLCanvasElement | null = null; protected _outputContext: CanvasRenderingContext2D | null = null; protected _personMaskUpscalePipeline: PersonMaskUpscalePipeline | null = null; + protected _webgl2Canvas: OffscreenCanvas | HTMLCanvasElement; private _assetsPath: string; private _benchmark: Benchmark; @@ -109,7 +110,6 @@ export abstract class BackgroundProcessor extends Processor { // tslint:disable-next-line no-unused-variable private _isSimdEnabled: boolean | null; private _maskBlurRadius: number; - private _maskCanvas: OffscreenCanvas | HTMLCanvasElement; constructor(options: BackgroundProcessorOptions) { super(); @@ -139,7 +139,7 @@ export abstract class BackgroundProcessor extends Processor { this._inputFrameCanvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); this._inputFrameContext = this._inputFrameCanvas.getContext('2d') as OffscreenCanvasRenderingContext2D; this._maskBlurRadius = typeof options.maskBlurRadius === 'number' ? options.maskBlurRadius : MASK_BLUR_RADIUS; - this._maskCanvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); + this._webgl2Canvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); } /** @@ -238,8 +238,8 @@ export abstract class BackgroundProcessor extends Processor { if (this._outputCanvas !== outputFrameBuffer) { this._outputCanvas = outputFrameBuffer; this._outputContext = this._outputCanvas.getContext('2d'); - this._maskCanvas.width = outputWidth; - this._maskCanvas.height = outputHeight; + this._webgl2Canvas.width = outputWidth; + this._webgl2Canvas.height = outputHeight; this._cleanupPersonMaskUpscalePipeline(); } // Only set the canvas' dimensions if they have changed to prevent unnecessary redraw @@ -275,15 +275,16 @@ export abstract class BackgroundProcessor extends Processor { } this._benchmark.start('imageCompositionDelay'); - if (!this._debounce || this._currentMask) { - this._personMaskUpscalePipeline?.render(personMask); + if (!this._debounce || this._currentMask || !isCanvasBlurSupported) { + this._personMaskUpscalePipeline?.setInputTextureData(personMask); + this._personMaskUpscalePipeline?.render(); } const ctx = this._outputContext as CanvasRenderingContext2D; ctx.save(); ctx.filter = 'none'; ctx.globalCompositeOperation = 'copy'; - ctx.drawImage(this._maskCanvas, 0, 0, outputWidth, outputHeight); + ctx.drawImage(this._webgl2Canvas, 0, 0, outputWidth, outputHeight); ctx.globalCompositeOperation = 'source-in'; ctx.drawImage(inputFrame, 0, 0, outputWidth, outputHeight); ctx.globalCompositeOperation = 'destination-over'; @@ -339,7 +340,7 @@ export abstract class BackgroundProcessor extends Processor { this._personMaskUpscalePipeline = new PersonMaskUpscalePipeline( inputFrame, this._inferenceDimensions, - this._maskCanvas + this._webgl2Canvas ); this._personMaskUpscalePipeline?.updateFastBilateralFilterConfig({ sigmaSpace: this._maskBlurRadius diff --git a/lib/processors/background/GaussianBlurBackgroundProcessor.ts b/lib/processors/background/GaussianBlurBackgroundProcessor.ts index c69c51e..23e5da5 100644 --- a/lib/processors/background/GaussianBlurBackgroundProcessor.ts +++ b/lib/processors/background/GaussianBlurBackgroundProcessor.ts @@ -1,5 +1,7 @@ -import { BackgroundProcessor, BackgroundProcessorOptions } from './BackgroundProcessor'; import { BLUR_FILTER_RADIUS } from '../../constants'; +import { isCanvasBlurSupported } from '../../utils/support'; +import { GaussianBlurFilterPipeline } from '../webgl2'; +import { BackgroundProcessor, BackgroundProcessorOptions } from './BackgroundProcessor'; /** * Options passed to [[GaussianBlurBackgroundProcessor]] constructor. @@ -67,6 +69,7 @@ export interface GaussianBlurBackgroundProcessorOptions extends BackgroundProces export class GaussianBlurBackgroundProcessor extends BackgroundProcessor { private _blurFilterRadius: number = BLUR_FILTER_RADIUS; + private _gaussianBlurFilterPipeline: GaussianBlurFilterPipeline | null; // tslint:disable-next-line no-unused-variable private readonly _name: string = 'GaussianBlurBackgroundProcessor'; @@ -77,6 +80,7 @@ export class GaussianBlurBackgroundProcessor extends BackgroundProcessor { */ constructor(options: GaussianBlurBackgroundProcessorOptions) { super(options); + this._gaussianBlurFilterPipeline = null; this.blurFilterRadius = options.blurFilterRadius!; } @@ -96,14 +100,28 @@ export class GaussianBlurBackgroundProcessor extends BackgroundProcessor { radius = BLUR_FILTER_RADIUS; } this._blurFilterRadius = radius; + this._gaussianBlurFilterPipeline?.updateRadius(this._blurFilterRadius); } protected _setBackground(inputFrame: OffscreenCanvas | HTMLCanvasElement): void { - if (!this._outputContext) { + const { + _outputContext: ctx, + _blurFilterRadius: radius, + _webgl2Canvas: canvas + } = this; + if (!ctx) { return; } - const ctx = this._outputContext as CanvasRenderingContext2D; - ctx.filter = `blur(${this._blurFilterRadius}px)`; - ctx.drawImage(inputFrame, 0, 0); + if (isCanvasBlurSupported) { + ctx.filter = `blur(${radius}px)`; + ctx.drawImage(inputFrame, 0, 0); + return; + } + if (!this._gaussianBlurFilterPipeline) { + this._gaussianBlurFilterPipeline = new GaussianBlurFilterPipeline(canvas); + this._gaussianBlurFilterPipeline.updateRadius(radius); + } + this._gaussianBlurFilterPipeline?.render(); + ctx.drawImage(canvas, 0, 0); } } diff --git a/lib/processors/background/VirtualBackgroundProcessor.ts b/lib/processors/background/VirtualBackgroundProcessor.ts index 4add076..7f379b7 100644 --- a/lib/processors/background/VirtualBackgroundProcessor.ts +++ b/lib/processors/background/VirtualBackgroundProcessor.ts @@ -142,7 +142,7 @@ export class VirtualBackgroundProcessor extends BackgroundProcessor { const imageHeight = img.naturalHeight; const canvasWidth = this._outputCanvas.width; const canvasHeight = this._outputCanvas.height; - const ctx = this._outputContext as CanvasRenderingContext2D; + const ctx = this._outputContext; if (this._fitType === ImageFit.Fill) { ctx.drawImage(img, 0, 0, imageWidth, imageHeight, 0, 0, canvasWidth, canvasHeight); diff --git a/lib/processors/webgl2/index.ts b/lib/processors/webgl2/index.ts index 4ebfb25..0cf258a 100644 --- a/lib/processors/webgl2/index.ts +++ b/lib/processors/webgl2/index.ts @@ -4,4 +4,5 @@ * It was modified and converted into a module to work with * Twilio's Video Processor */ +export { GaussianBlurFilterPipeline } from './pipelines/GaussianBlurFilterPipeline'; export { PersonMaskUpscalePipeline } from './pipelines/PersonMaskUpscalePipeline'; diff --git a/lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts b/lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts new file mode 100644 index 0000000..83af77e --- /dev/null +++ b/lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts @@ -0,0 +1,33 @@ +import { SinglePassGaussianBlurFilterStage } from './SinglePassGaussianBlurFilterStage'; +import { WebGL2Pipeline } from './WebGL2Pipeline'; + +/** + * @private + */ +export class GaussianBlurFilterPipeline extends WebGL2Pipeline { + constructor(outputCanvas: OffscreenCanvas | HTMLCanvasElement) { + super() + + const glOut = outputCanvas.getContext('webgl2')! as WebGL2RenderingContext + this.addStage(new SinglePassGaussianBlurFilterStage( + glOut, + 'horizontal', + 'texture', + 0, + 2 + )) + this.addStage(new SinglePassGaussianBlurFilterStage( + glOut, + 'vertical', + 'canvas', + 2 + )) + } + + updateRadius(radius: number): void { + this._stages.forEach( + (stage) => (stage as SinglePassGaussianBlurFilterStage) + .updateRadius(radius) + ) + } +} diff --git a/lib/processors/webgl2/pipelines/Pipeline.ts b/lib/processors/webgl2/pipelines/Pipeline.ts index 14abfc3..d34c92c 100644 --- a/lib/processors/webgl2/pipelines/Pipeline.ts +++ b/lib/processors/webgl2/pipelines/Pipeline.ts @@ -8,7 +8,7 @@ export class Pipeline implements Pipeline.Stage { this._stages.push(stage) } - render(...args: any[]): void { + render(): void { this._stages.forEach((stage) => { stage.render() }) @@ -17,6 +17,6 @@ export class Pipeline implements Pipeline.Stage { export namespace Pipeline { export interface Stage { - render(...args: any[]): void + render(): void } } diff --git a/lib/processors/webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts b/lib/processors/webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts new file mode 100644 index 0000000..ade6307 --- /dev/null +++ b/lib/processors/webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts @@ -0,0 +1,104 @@ +import { WebGL2Pipeline } from './WebGL2Pipeline' + +/** + * @private + */ +export class SinglePassGaussianBlurFilterStage extends WebGL2Pipeline.ProcessingStage { + constructor( + glOut: WebGL2RenderingContext, + direction: 'horizontal' | 'vertical', + outputType: 'canvas' | 'texture', + inputTextureUnit: number, + outputTextureUnit = inputTextureUnit + 1 + ) { + const { + height, + width + } = glOut.canvas + + super( + { + textureName: 'u_inputTexture', + textureUnit: inputTextureUnit + }, + { + fragmentShaderSource: `#version 300 es + precision highp float; + + uniform sampler2D u_inputTexture; + uniform vec2 u_texelSize; + uniform float u_direction; + uniform float u_radius; + + in vec2 v_texCoord; + + out vec4 outColor; + + const float PI = 3.14159265359; + + float gaussian(float x, float sigma) { + float coefficient = 1.0 / sqrt(2.0 * PI) / sigma; + float power = -0.5 * x * x / sigma / sigma; + return coefficient * exp(power); + } + + void main() { + vec3 newColor = vec3(0.0); + float totalWeight = 0.0; + + for (float i = 0.0; i <= u_radius; i += 1.0) { + float x = (1.0 - u_direction) * i; + float y = u_direction * i; + vec2 shift = vec2(x, y) * u_texelSize; + vec2 coord = vec2(v_texCoord + shift); + float weight = gaussian(i, u_radius); + newColor += weight * texture(u_inputTexture, coord).rgb; + totalWeight += weight; + + if (i != 0.0) { + x = (1.0 - u_direction) * -i; + y = u_direction * -i; + shift = vec2(x, y) * u_texelSize; + coord = vec2(v_texCoord + shift); + newColor += weight * texture(u_inputTexture, coord).rgb; + totalWeight += weight; + } + } + + newColor /= totalWeight; + outColor = vec4(newColor, 1.0); + } + `, + glOut, + height, + textureUnit: outputTextureUnit, + type: outputType, + width, + uniformVars: [ + { + name: 'u_direction', + type: 'float', + values: [direction === 'vertical' ? 1 : 0] + }, + { + name: 'u_texelSize', + type: 'float', + values: [1 / width, 1 / height] + } + ] + } + ) + + this.updateRadius(0) + } + + updateRadius(radius: number): void { + this._setUniformVars([ + { + name: 'u_radius', + type: 'float', + values: [radius] + } + ]) + } +} diff --git a/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts b/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts index 78c74ea..c0b2c47 100644 --- a/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts +++ b/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts @@ -16,6 +16,7 @@ interface OutputConfig { fragmentShaderSource: string glOut: WebGL2RenderingContext height?: number + textureUnit?: number type: 'canvas' | 'texture' uniformVars?: UniformVarInfo[] vertexShaderSource?: string @@ -35,7 +36,8 @@ class WebGL2PipelineInputStage implements Pipeline.Stage { private _glOut: WebGL2RenderingContext private _inputFrame: OffscreenCanvas | HTMLCanvasElement private _inputFrameTexture: WebGLTexture - private _personMaskTexture: WebGLTexture | null + private _inputTexture: WebGLTexture | null + private _inputTextureData: ImageData | null constructor( glOut: WebGL2RenderingContext, @@ -52,24 +54,26 @@ class WebGL2PipelineInputStage implements Pipeline.Stage { glOut.NEAREST, glOut.NEAREST )! - this._personMaskTexture = null; + this._inputTexture = null; + this._inputTextureData = null; } cleanUp(): void { const { _glOut, _inputFrameTexture, - _personMaskTexture + _inputTexture } = this _glOut.deleteTexture(_inputFrameTexture) - _glOut.deleteTexture(_personMaskTexture) + _glOut.deleteTexture(_inputTexture) } - render(personMask: ImageData): void { + render(): void { const { _glOut, _inputFrame, - _inputFrameTexture + _inputFrameTexture, + _inputTextureData } = this const { height, width } = _inputFrame @@ -94,42 +98,49 @@ class WebGL2PipelineInputStage implements Pipeline.Stage { _inputFrame ) + if (!_inputTextureData) { + return + } const { data, - height: maskHeight, - width: maskWidth - } = personMask + height: textureHeight, + width: textureWidth + } = _inputTextureData - if (!this._personMaskTexture) { - this._personMaskTexture = createTexture( + if (!this._inputTexture) { + this._inputTexture = createTexture( _glOut, _glOut.RGBA8, - maskWidth, - maskHeight, + textureWidth, + textureHeight, _glOut.NEAREST, _glOut.NEAREST ) } - _glOut.viewport(0, 0, maskWidth, maskHeight) + _glOut.viewport(0, 0, textureWidth, textureHeight) _glOut.activeTexture(_glOut.TEXTURE1) _glOut.bindTexture( _glOut.TEXTURE_2D, - this._personMaskTexture + this._inputTexture ) _glOut.texSubImage2D( _glOut.TEXTURE_2D, 0, 0, 0, - maskWidth, - maskHeight, + textureWidth, + textureHeight, _glOut.RGBA, _glOut.UNSIGNED_BYTE, data ) } + + setInputTextureData(inputTextureData: ImageData): void { + this._inputTextureData = inputTextureData + } } /** @@ -142,6 +153,7 @@ class WebGL2PipelineProcessingStage implements Pipeline.Stage { private _inputTextureUnit: number private _outputFramebuffer: WebGLBuffer | null = null private _outputTexture: WebGLTexture | null = null + private _outputTextureUnit: number private _positionBuffer: WebGLBuffer private _program: WebGLProgram private _texCoordBuffer: WebGLBuffer @@ -164,6 +176,7 @@ class WebGL2PipelineProcessingStage implements Pipeline.Stage { const { fragmentShaderSource, height = glOut.canvas.height, + textureUnit: outputTextureUnit = textureUnit + 1, type: outputType, uniformVars = [], vertexShaderSource = `#version 300 es @@ -189,6 +202,8 @@ class WebGL2PipelineProcessingStage implements Pipeline.Stage { width } + this._outputTextureUnit = outputTextureUnit + this._fragmentShader = compileShader( glOut, glOut.FRAGMENT_SHADER, @@ -280,13 +295,13 @@ class WebGL2PipelineProcessingStage implements Pipeline.Stage { render(): void { const { _glOut, - _inputTextureUnit, _outputDimensions: { height, width }, _outputFramebuffer, _outputTexture, + _outputTextureUnit, _program } = this @@ -296,8 +311,7 @@ class WebGL2PipelineProcessingStage implements Pipeline.Stage { if (_outputTexture) { _glOut.activeTexture( _glOut.TEXTURE0 - + _inputTextureUnit - + 1 + + _outputTextureUnit ) _glOut.bindTexture( _glOut.TEXTURE_2D, @@ -351,24 +365,14 @@ export class WebGL2Pipeline extends Pipeline { static ProcessingStage = WebGL2PipelineProcessingStage protected _stages: (WebGL2PipelineInputStage | WebGL2PipelineProcessingStage)[] = [] - render(personMask: ImageData): void { - const [ - inputStage, - ...otherStages - ] = this._stages as [ - WebGL2PipelineInputStage, - ...WebGL2PipelineProcessingStage[] - ] - - inputStage.render(personMask) - otherStages.forEach( - (stage) => stage.render() - ) - } - cleanUp(): void { this._stages.forEach( (stage) => stage.cleanUp() ) } + + setInputTextureData(inputTextureData: ImageData): void { + const [inputStage] = this._stages as [WebGL2PipelineInputStage] + inputStage.setInputTextureData(inputTextureData); + } } diff --git a/lib/utils/support.ts b/lib/utils/support.ts index a1dff00..a8d0327 100644 --- a/lib/utils/support.ts +++ b/lib/utils/support.ts @@ -27,6 +27,32 @@ export function isChromiumImageBitmap() { && typeof createImageBitmap === 'function'; } +/** + * @private + */ +export const isCanvasBlurSupported = (() => { + const blackPixel = [0, 0, 0, 255]; + const whitePixel = [255, 255, 255, 255]; + + const inputImageData = new ImageData(new Uint8ClampedArray([ + ...blackPixel, ...blackPixel, ...blackPixel, + ...blackPixel, ...whitePixel, ...blackPixel, + ...blackPixel, ...blackPixel, ...blackPixel + ]), 3, 3); + + const canvas = getCanvas(); + const context = canvas.getContext('2d'); + + canvas.width = 3; + canvas.height = 3; + context!.putImageData(inputImageData, 0, 0); + context!.filter = 'blur(1px)'; + context!.drawImage(canvas, 0, 0); + + const { data } = context!.getImageData(0, 0, 3, 3); + return data[0] > 0; +})(); + /** * Check if the current browser is officially supported by twilio-video-procesors.js. * This is set to `true` for browsers that supports canvas From 50569702d137492c0898f0fb0edc32563d6dd4b4 Mon Sep 17 00:00:00 2001 From: mmalavalli Date: Fri, 5 Jul 2024 19:03:42 -0500 Subject: [PATCH 03/28] Some refactoring. --- lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts | 3 ++- lib/processors/webgl2/pipelines/WebGL2Pipeline.ts | 3 --- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts b/lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts index 83af77e..a68bb0d 100644 --- a/lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts +++ b/lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts @@ -7,8 +7,8 @@ import { WebGL2Pipeline } from './WebGL2Pipeline'; export class GaussianBlurFilterPipeline extends WebGL2Pipeline { constructor(outputCanvas: OffscreenCanvas | HTMLCanvasElement) { super() - const glOut = outputCanvas.getContext('webgl2')! as WebGL2RenderingContext + this.addStage(new SinglePassGaussianBlurFilterStage( glOut, 'horizontal', @@ -16,6 +16,7 @@ export class GaussianBlurFilterPipeline extends WebGL2Pipeline { 0, 2 )) + this.addStage(new SinglePassGaussianBlurFilterStage( glOut, 'vertical', diff --git a/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts b/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts index c0b2c47..a448fd2 100644 --- a/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts +++ b/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts @@ -150,7 +150,6 @@ class WebGL2PipelineProcessingStage implements Pipeline.Stage { protected _outputDimensions: Dimensions private _fragmentShader: WebGLSampler private _glOut: WebGL2RenderingContext - private _inputTextureUnit: number private _outputFramebuffer: WebGLBuffer | null = null private _outputTexture: WebGLTexture | null = null private _outputTextureUnit: number @@ -168,8 +167,6 @@ class WebGL2PipelineProcessingStage implements Pipeline.Stage { textureUnit, } = inputConfig - this._inputTextureUnit = textureUnit - const { glOut } = outputConfig this._glOut = glOut From c7617ec49fe2d8724c236d28dcb9697fd3fcb84a Mon Sep 17 00:00:00 2001 From: mmalavalli Date: Wed, 10 Jul 2024 23:53:17 -0500 Subject: [PATCH 04/28] Refactor to implement a 2-pass bilateral filter. --- .../webgl2/helpers/postProcessingHelper.ts | 2 +- .../pipelines/FastBilateralFilterStage.ts | 220 ------------------ .../pipelines/PersonMaskUpscalePipeline.ts | 36 ++- .../SinglePassBilateralFilterStage.ts | 212 +++++++++++++++++ .../webgl2/pipelines/WebGL2Pipeline.ts | 21 +- 5 files changed, 254 insertions(+), 237 deletions(-) delete mode 100644 lib/processors/webgl2/pipelines/FastBilateralFilterStage.ts create mode 100644 lib/processors/webgl2/pipelines/SinglePassBilateralFilterStage.ts diff --git a/lib/processors/webgl2/helpers/postProcessingHelper.ts b/lib/processors/webgl2/helpers/postProcessingHelper.ts index 9f6d772..6a79c18 100644 --- a/lib/processors/webgl2/helpers/postProcessingHelper.ts +++ b/lib/processors/webgl2/helpers/postProcessingHelper.ts @@ -1,3 +1,3 @@ -export type FastBilateralFilterConfig = { +export type BilateralFilterConfig = { sigmaSpace?: number } diff --git a/lib/processors/webgl2/pipelines/FastBilateralFilterStage.ts b/lib/processors/webgl2/pipelines/FastBilateralFilterStage.ts deleted file mode 100644 index 018082e..0000000 --- a/lib/processors/webgl2/pipelines/FastBilateralFilterStage.ts +++ /dev/null @@ -1,220 +0,0 @@ -import { Dimensions } from '../../../types' -import { WebGL2Pipeline } from './WebGL2Pipeline' - -/** - * @private - */ -export class FastBilateralFilterStage extends WebGL2Pipeline.ProcessingStage { - private _inputDimensions: Dimensions - - constructor( - glOut: WebGL2RenderingContext, - inputDimensions: Dimensions, - outputDimensions: Dimensions - ) { - const { - height, - width - } = outputDimensions - - super( - { - textureName: 'u_segmentationMask', - textureUnit: 1 - }, - { - // NOTE(mmalavalli): This is a faster approximation of the joint bilateral filter. - // For a given pixel, instead of calculating the space and color weights of all - // the pixels within the filter kernel, which would have a complexity of O(r^2), - // we calculate the space and color weights of only those pixels which form two - // diagonal lines between the two pairs of opposite corners of the filter kernel, - // which would have a complexity of O(r). This improves the overall complexity - // of this stage from O(w x h x r^2) to O(w x h x r), where: - // w => width of the output video frame - // h => height of the output video frame - // r => radius of the joint bilateral filter kernel - fragmentShaderSource: `#version 300 es - precision highp float; - - uniform sampler2D u_inputFrame; - uniform sampler2D u_segmentationMask; - uniform vec2 u_texelSize; - uniform float u_step; - uniform float u_radius; - uniform float u_offset; - uniform float u_sigmaTexel; - uniform float u_sigmaColor; - - in vec2 v_texCoord; - - out vec4 outColor; - - float gaussian(float x, float sigma) { - return exp(-0.5 * x * x / sigma / sigma); - } - - float calculateSpaceWeight(vec2 coord) { - float x = distance(v_texCoord, coord); - float sigma = u_sigmaTexel; - return gaussian(x, sigma); - } - - float calculateColorWeight(vec2 coord) { - vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb; - vec3 coordColor = texture(u_inputFrame, coord).rgb; - float x = distance(centerColor, coordColor); - float sigma = u_sigmaColor; - return gaussian(x, sigma); - } - - void main() { - vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb; - float newVal = 0.0; - float totalWeight = 0.0; - - vec2 leftTopCoord = vec2(v_texCoord + vec2(-u_radius, -u_radius) * u_texelSize); - vec2 rightTopCoord = vec2(v_texCoord + vec2(u_radius, -u_radius) * u_texelSize); - vec2 leftBottomCoord = vec2(v_texCoord + vec2(-u_radius, u_radius) * u_texelSize); - vec2 rightBottomCoord = vec2(v_texCoord + vec2(u_radius, u_radius) * u_texelSize); - - float leftTopSegAlpha = texture(u_segmentationMask, leftTopCoord).a; - float rightTopSegAlpha = texture(u_segmentationMask, rightTopCoord).a; - float leftBottomSegAlpha = texture(u_segmentationMask, leftBottomCoord).a; - float rightBottomSegAlpha = texture(u_segmentationMask, rightBottomCoord).a; - float totalSegAlpha = leftTopSegAlpha + rightTopSegAlpha + leftBottomSegAlpha + rightBottomSegAlpha; - - if (totalSegAlpha <= 0.0) { - newVal = 0.0; - } else if (totalSegAlpha >= 4.0) { - newVal = 1.0; - } else { - for (float i = 0.0; i <= u_radius - u_offset; i += u_step) { - vec2 shift = vec2(i, i) * u_texelSize; - vec2 coord = vec2(v_texCoord + shift); - float spaceWeight = calculateSpaceWeight(coord); - float colorWeight = calculateColorWeight(coord); - float weight = spaceWeight * colorWeight; - float alpha = texture(u_segmentationMask, coord).a; - totalWeight += weight; - newVal += weight * alpha; - - if (i != 0.0) { - shift = vec2(i, -i) * u_texelSize; - coord = vec2(v_texCoord + shift); - colorWeight = calculateColorWeight(coord); - weight = spaceWeight * colorWeight; - alpha = texture(u_segmentationMask, coord).a; - totalWeight += weight; - newVal += weight * texture(u_segmentationMask, coord).a; - - shift = vec2(-i, i) * u_texelSize; - coord = vec2(v_texCoord + shift); - colorWeight = calculateColorWeight(coord); - weight = spaceWeight * colorWeight; - alpha = texture(u_segmentationMask, coord).a; - totalWeight += weight; - newVal += weight * texture(u_segmentationMask, coord).a; - - shift = vec2(-i, -i) * u_texelSize; - coord = vec2(v_texCoord + shift); - colorWeight = calculateColorWeight(coord); - weight = spaceWeight * colorWeight; - alpha = texture(u_segmentationMask, coord).a; - totalWeight += weight; - newVal += weight * texture(u_segmentationMask, coord).a; - } - } - newVal /= totalWeight; - } - - outColor = vec4(vec3(0.0), newVal); - } - `, - glOut, - height, - type: 'canvas', - width, - uniformVars: [ - { - name: 'u_inputFrame', - type: 'int', - values: [0] - }, - { - name: 'u_texelSize', - type: 'float', - values: [1 / width, 1 / height] - } - ] - } - ) - - this._inputDimensions = inputDimensions - this.updateSigmaColor(0) - this.updateSigmaSpace(0) - } - - updateSigmaColor(sigmaColor: number): void { - this._setUniformVars([ - { - name: 'u_sigmaColor', - type: 'float', - values: [sigmaColor] - } - ]) - } - - updateSigmaSpace(sigmaSpace: number): void { - const { - height: inputHeight, - width: inputWidth - } = this._inputDimensions - - const { - height: outputHeight, - width: outputWidth - } = this._outputDimensions - - sigmaSpace *= Math.max( - outputWidth / inputWidth, - outputHeight / inputHeight - ) - - const kSparsityFactor = 0.66 - const sparsity = Math.max( - 1, - Math.sqrt(sigmaSpace) - * kSparsityFactor - ) - const step = sparsity - const radius = sigmaSpace - const offset = step > 1 ? step * 0.5 : 0 - const sigmaTexel = Math.max( - 1 / outputWidth, - 1 / outputHeight - ) * sigmaSpace - - this._setUniformVars([ - { - name: 'u_offset', - type: 'float', - values: [offset] - }, - { - name: 'u_radius', - type: 'float', - values: [radius] - }, - { - name: 'u_sigmaTexel', - type: 'float', - values: [sigmaTexel] - }, - { - name: 'u_step', - type: 'float', - values: [step] - }, - ]) - } -} diff --git a/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts b/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts index 4837a5d..c60661c 100644 --- a/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts +++ b/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts @@ -1,6 +1,6 @@ import { Dimensions } from '../../../types'; -import { FastBilateralFilterConfig } from '../helpers/postProcessingHelper'; -import { FastBilateralFilterStage } from './FastBilateralFilterStage'; +import { BilateralFilterConfig } from '../helpers/postProcessingHelper'; +import { SinglePassBilateralFilterStage } from './SinglePassBilateralFilterStage'; import { WebGL2Pipeline } from './WebGL2Pipeline'; export class PersonMaskUpscalePipeline extends WebGL2Pipeline { @@ -22,26 +22,42 @@ export class PersonMaskUpscalePipeline extends WebGL2Pipeline { inputCanvas )) - this.addStage(new FastBilateralFilterStage( + this.addStage(new SinglePassBilateralFilterStage( glOut, + 'horizontal', + 'texture', inputDimensions, - outputDimensions + outputDimensions, + 1, + 2 + )) + + this.addStage(new SinglePassBilateralFilterStage( + glOut, + 'vertical', + 'canvas', + inputDimensions, + outputDimensions, + 2 )) } - updateFastBilateralFilterConfig(config: FastBilateralFilterConfig) { + updateFastBilateralFilterConfig(config: BilateralFilterConfig) { const [ /* inputStage */, - fastBilateralFilterStage + ...bilateralFilterStages ] = this._stages as [ any, - FastBilateralFilterStage + SinglePassBilateralFilterStage ] - const { sigmaSpace } = config if (typeof sigmaSpace === 'number') { - fastBilateralFilterStage.updateSigmaColor(0.1) - fastBilateralFilterStage.updateSigmaSpace(sigmaSpace) + bilateralFilterStages.forEach( + (stage: SinglePassBilateralFilterStage) => { + stage.updateSigmaColor(0.1) + stage.updateSigmaSpace(sigmaSpace) + } + ) } } } diff --git a/lib/processors/webgl2/pipelines/SinglePassBilateralFilterStage.ts b/lib/processors/webgl2/pipelines/SinglePassBilateralFilterStage.ts new file mode 100644 index 0000000..071bf43 --- /dev/null +++ b/lib/processors/webgl2/pipelines/SinglePassBilateralFilterStage.ts @@ -0,0 +1,212 @@ +import { Dimensions } from '../../../types' +import { WebGL2Pipeline } from './WebGL2Pipeline' + +function createSpaceWeights( + radius: number, + sigma: number, + texelSize: number +): number[] { + return '0'.repeat(radius).split('').map((zero, i) => { + const x = (i + 1) * texelSize + return Math.exp(-0.5 * x * x / sigma / sigma) + }) +} + +/** + * @private + */ +export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingStage { + private _direction: 'horizontal' | 'vertical' + private _inputDimensions: Dimensions + + constructor( + glOut: WebGL2RenderingContext, + direction: 'horizontal' | 'vertical', + outputType: 'canvas' | 'texture', + inputDimensions: Dimensions, + outputDimensions: Dimensions, + inputTextureUnit: number, + outputTextureUnit = inputTextureUnit + 1 + ) { + const { + height, + width + } = outputDimensions + + super( + { + textureName: 'u_segmentationMask', + textureUnit: inputTextureUnit + }, + { + fragmentShaderSource: `#version 300 es + precision highp float; + + uniform sampler2D u_inputFrame; + uniform sampler2D u_segmentationMask; + uniform vec2 u_texelSize; + uniform float u_direction; + uniform float u_radius; + uniform float u_sigmaTexel; + uniform float u_sigmaColor; + uniform float u_spaceWeights[128]; + + in vec2 v_texCoord; + + out vec4 outColor; + + float gaussian(float x, float sigma) { + return exp(-0.5 * x * x / sigma / sigma); + } + + float calculateColorWeight(vec2 coord) { + vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb; + vec3 coordColor = texture(u_inputFrame, coord).rgb; + float x = distance(centerColor, coordColor); + float sigma = u_sigmaColor; + return gaussian(x, sigma); + } + + float edgePixelsAverageAlpha() { + float totalAlpha = texture(u_segmentationMask, v_texCoord).a; + float totalPixels = 1.0; + + for (float i = -u_radius; u_radius > 0.0 && i <= u_radius; i += u_radius) { + for (float j = -u_radius; j <= u_radius; j += u_radius * (j == 0.0 ? 2.0 : 1.0)) { + vec2 shift = vec2(i, j) * u_texelSize; + vec2 coord = vec2(v_texCoord + shift); + totalAlpha += texture(u_segmentationMask, coord).a; + totalPixels++; + } + } + + return totalAlpha / totalPixels; + } + + void main() { + float averageAlpha = edgePixelsAverageAlpha(); + + if (averageAlpha == 0.0 || averageAlpha == 1.0) { + outColor = vec4(vec3(0.0), averageAlpha); + return; + } + + float outAlpha = texture(u_segmentationMask, v_texCoord).a; + float totalWeight = 1.0; + + for (float i = 1.0; i <= u_radius; i++) { + float x = (1.0 - u_direction) * i; + float y = u_direction * i; + vec2 shift = vec2(x, y) * u_texelSize; + vec2 coord = vec2(v_texCoord + shift); + float spaceWeight = u_spaceWeights[int(i - 1.0)]; + float colorWeight = calculateColorWeight(coord); + float weight = spaceWeight * colorWeight; + float alpha = texture(u_segmentationMask, coord).a; + totalWeight += weight; + outAlpha += weight * alpha; + + shift = vec2(-x, -y) * u_texelSize; + coord = vec2(v_texCoord + shift); + colorWeight = calculateColorWeight(coord); + weight = spaceWeight * colorWeight; + alpha = texture(u_segmentationMask, coord).a; + totalWeight += weight; + outAlpha += weight * alpha; + } + + outAlpha /= totalWeight; + outColor = vec4(vec3(0.0), outAlpha); + } + `, + glOut, + height, + textureUnit: outputTextureUnit, + type: outputType, + width, + uniformVars: [ + { + name: 'u_inputFrame', + type: 'int', + values: [0] + }, + { + name: 'u_direction', + type: 'float', + values: [direction === 'vertical' ? 1 : 0] + }, + { + name: 'u_texelSize', + type: 'float', + values: [1 / width, 1 / height] + } + ] + } + ) + + this._direction = direction + this._inputDimensions = inputDimensions + this.updateSigmaColor(0) + this.updateSigmaSpace(0) + } + + updateSigmaColor(sigmaColor: number): void { + this._setUniformVars([ + { + name: 'u_sigmaColor', + type: 'float', + values: [sigmaColor] + } + ]) + } + + updateSigmaSpace(sigmaSpace: number): void { + const { + height: inputHeight, + width: inputWidth + } = this._inputDimensions + + const { + height: outputHeight, + width: outputWidth + } = this._outputDimensions + + sigmaSpace *= Math.max( + outputWidth / inputWidth, + outputHeight / inputHeight + ) + + const sigmaTexel = Math.max( + 1 / outputWidth, + 1 / outputHeight + ) * sigmaSpace + + const texelSize = 1 / ( + this._direction === 'horizontal' + ? outputWidth + : outputHeight + ) + + this._setUniformVars([ + { + name: 'u_radius', + type: 'float', + values: [sigmaSpace] + }, + { + name: 'u_sigmaTexel', + type: 'float', + values: [sigmaTexel] + }, + { + name: 'u_spaceWeights', + type: 'float:v', + values: createSpaceWeights( + sigmaSpace, + sigmaTexel, + texelSize + ) + }, + ]) + } +} diff --git a/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts b/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts index a448fd2..bb4182d 100644 --- a/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts +++ b/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts @@ -25,7 +25,7 @@ interface OutputConfig { interface UniformVarInfo { name: string - type: 'float' | 'int' | 'uint' + type: 'float' | 'int' | 'uint' | 'float:v' values: number[] } @@ -345,11 +345,20 @@ class WebGL2PipelineProcessingStage implements Pipeline.Stage { name ) - // @ts-ignore - _glOut[`uniform${values.length}${type[0]}`]( - uniformVarLocation, - ...values - ) + const isVector = type.split(':')[1] === 'v'; + if (isVector) { + // @ts-ignore + _glOut[`uniform1${type[0]}v`]( + uniformVarLocation, + values + ) + } else { + // @ts-ignore + _glOut[`uniform${values.length}${type[0]}`]( + uniformVarLocation, + ...values + ) + } }) } } From a8bd62627d35a69acc5421b841cc74fa4989734f Mon Sep 17 00:00:00 2001 From: mmalavalli Date: Wed, 24 Jul 2024 22:29:05 -0500 Subject: [PATCH 05/28] Add logarithmic stepping to bilateral filter and pre-calculate color weights. --- .../background/BackgroundProcessor.ts | 4 +- .../pipelines/PersonMaskUpscalePipeline.ts | 2 +- .../SinglePassBilateralFilterStage.ts | 63 +++++++++++-------- 3 files changed, 39 insertions(+), 30 deletions(-) diff --git a/lib/processors/background/BackgroundProcessor.ts b/lib/processors/background/BackgroundProcessor.ts index 668864b..17e9f52 100644 --- a/lib/processors/background/BackgroundProcessor.ts +++ b/lib/processors/background/BackgroundProcessor.ts @@ -159,7 +159,7 @@ export abstract class BackgroundProcessor extends Processor { } if (this._maskBlurRadius !== radius) { this._maskBlurRadius = radius; - this._personMaskUpscalePipeline?.updateFastBilateralFilterConfig({ + this._personMaskUpscalePipeline?.updateBilateralFilterConfig({ sigmaSpace: this._maskBlurRadius }); } @@ -342,7 +342,7 @@ export abstract class BackgroundProcessor extends Processor { this._inferenceDimensions, this._webgl2Canvas ); - this._personMaskUpscalePipeline?.updateFastBilateralFilterConfig({ + this._personMaskUpscalePipeline?.updateBilateralFilterConfig({ sigmaSpace: this._maskBlurRadius }); } diff --git a/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts b/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts index c60661c..44904b1 100644 --- a/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts +++ b/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts @@ -42,7 +42,7 @@ export class PersonMaskUpscalePipeline extends WebGL2Pipeline { )) } - updateFastBilateralFilterConfig(config: BilateralFilterConfig) { + updateBilateralFilterConfig(config: BilateralFilterConfig) { const [ /* inputStage */, ...bilateralFilterStages diff --git a/lib/processors/webgl2/pipelines/SinglePassBilateralFilterStage.ts b/lib/processors/webgl2/pipelines/SinglePassBilateralFilterStage.ts index 071bf43..f361242 100644 --- a/lib/processors/webgl2/pipelines/SinglePassBilateralFilterStage.ts +++ b/lib/processors/webgl2/pipelines/SinglePassBilateralFilterStage.ts @@ -12,6 +12,15 @@ function createSpaceWeights( }) } +function createColorWeights( + sigma: number +): number[] { + return '0'.repeat(256).split('').map((zero, i) => { + const x = i / 255; + return Math.exp(-0.5 * x * x / sigma / sigma) + }) +} + /** * @private */ @@ -47,28 +56,22 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta uniform vec2 u_texelSize; uniform float u_direction; uniform float u_radius; - uniform float u_sigmaTexel; - uniform float u_sigmaColor; + uniform float u_step; uniform float u_spaceWeights[128]; + uniform float u_colorWeights[256]; in vec2 v_texCoord; out vec4 outColor; - float gaussian(float x, float sigma) { - return exp(-0.5 * x * x / sigma / sigma); - } - - float calculateColorWeight(vec2 coord) { - vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb; + float calculateColorWeight(vec2 coord, vec3 centerColor) { vec3 coordColor = texture(u_inputFrame, coord).rgb; float x = distance(centerColor, coordColor); - float sigma = u_sigmaColor; - return gaussian(x, sigma); + return u_colorWeights[int(x * 255.0)]; } - float edgePixelsAverageAlpha() { - float totalAlpha = texture(u_segmentationMask, v_texCoord).a; + float edgePixelsAverageAlpha(float outAlpha) { + float totalAlpha = outAlpha; float totalPixels = 1.0; for (float i = -u_radius; u_radius > 0.0 && i <= u_radius; i += u_radius) { @@ -84,23 +87,23 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta } void main() { - float averageAlpha = edgePixelsAverageAlpha(); + vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb; + float outAlpha = texture(u_segmentationMask, v_texCoord).a; + float averageAlpha = edgePixelsAverageAlpha(outAlpha); + float totalWeight = 1.0; if (averageAlpha == 0.0 || averageAlpha == 1.0) { outColor = vec4(vec3(0.0), averageAlpha); return; } - float outAlpha = texture(u_segmentationMask, v_texCoord).a; - float totalWeight = 1.0; - - for (float i = 1.0; i <= u_radius; i++) { + for (float i = 1.0; i <= u_radius; i += u_step) { float x = (1.0 - u_direction) * i; float y = u_direction * i; vec2 shift = vec2(x, y) * u_texelSize; vec2 coord = vec2(v_texCoord + shift); float spaceWeight = u_spaceWeights[int(i - 1.0)]; - float colorWeight = calculateColorWeight(coord); + float colorWeight = calculateColorWeight(coord, centerColor); float weight = spaceWeight * colorWeight; float alpha = texture(u_segmentationMask, coord).a; totalWeight += weight; @@ -108,7 +111,7 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta shift = vec2(-x, -y) * u_texelSize; coord = vec2(v_texCoord + shift); - colorWeight = calculateColorWeight(coord); + colorWeight = calculateColorWeight(coord, centerColor); weight = spaceWeight * colorWeight; alpha = texture(u_segmentationMask, coord).a; totalWeight += weight; @@ -153,9 +156,11 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta updateSigmaColor(sigmaColor: number): void { this._setUniformVars([ { - name: 'u_sigmaColor', - type: 'float', - values: [sigmaColor] + name: 'u_colorWeights', + type: 'float:v', + values: createColorWeights( + sigmaColor + ) } ]) } @@ -176,6 +181,10 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta outputHeight / inputHeight ) + const step = Math.floor( + 0.5 * sigmaSpace / Math.log(sigmaSpace) + ) + const sigmaTexel = Math.max( 1 / outputWidth, 1 / outputHeight @@ -193,11 +202,6 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta type: 'float', values: [sigmaSpace] }, - { - name: 'u_sigmaTexel', - type: 'float', - values: [sigmaTexel] - }, { name: 'u_spaceWeights', type: 'float:v', @@ -207,6 +211,11 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta texelSize ) }, + { + name: 'u_step', + type: 'float', + values: [step] + } ]) } } From fed0dea99e9caf15640af23d9d765239cd07e47e Mon Sep 17 00:00:00 2001 From: mmalavalli Date: Fri, 26 Jul 2024 02:27:02 -0500 Subject: [PATCH 06/28] Refactor gaussian blur fragment shader. --- .../webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/processors/webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts b/lib/processors/webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts index ade6307..31c3d84 100644 --- a/lib/processors/webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts +++ b/lib/processors/webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts @@ -56,9 +56,7 @@ export class SinglePassGaussianBlurFilterStage extends WebGL2Pipeline.Processing totalWeight += weight; if (i != 0.0) { - x = (1.0 - u_direction) * -i; - y = u_direction * -i; - shift = vec2(x, y) * u_texelSize; + shift = vec2(-x, -y) * u_texelSize; coord = vec2(v_texCoord + shift); newColor += weight * texture(u_inputTexture, coord).rgb; totalWeight += weight; From fff49bea0a35a3ed51a21a4322dd45418903707c Mon Sep 17 00:00:00 2001 From: mmalavalli Date: Sun, 28 Jul 2024 01:05:32 -0500 Subject: [PATCH 07/28] * Support bitmaprenderer output frame buffer context. * Support VideoFrame type for input frame buffer. --- examples/README.md | 3 +- examples/server.js | 1 + examples/virtualbackground/index.html | 2 +- examples/virtualbackground/index.js | 21 ++---- lib/processors/Processor.ts | 2 +- .../background/BackgroundProcessor.ts | 73 +++++++++++++++---- .../GaussianBlurBackgroundProcessor.ts | 4 +- .../pipelines/PersonMaskUpscalePipeline.ts | 6 +- lib/processors/webgl2/pipelines/Pipeline.ts | 6 +- .../webgl2/pipelines/WebGL2Pipeline.ts | 57 ++++++++------- 10 files changed, 103 insertions(+), 72 deletions(-) diff --git a/examples/README.md b/examples/README.md index e410d1c..ffd8a65 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,6 +11,5 @@ Open `http://localhost:3000` in a Chrome tab. The app captures your camera upon - `capFramerate` - Choose video capture frame rate (default: 30) - `capResolution=wxh` - Choose video capture resolution (default: 1280x720) - `debounce=true|false` - Whether to skip processing every other frame (default: false) -- `maskBlurRadius` - Radius of the mask blur filter (default: 8 for WebGL2, 4 for Canvas2D) -- `pipeline=Canvas2D|WebGL2` - Choose the canvas or webgl pipeline (default: WebGL2) +- `maskBlurRadius` - Radius of the mask blur filter (default: 8) - `stats=advanced|hide|show` - Show performance benchmarks (default: show) diff --git a/examples/server.js b/examples/server.js index 1701b05..bec509d 100644 --- a/examples/server.js +++ b/examples/server.js @@ -1,4 +1,5 @@ const express = require('express'); +const fs = require('fs'); const { resolve } = require('path'); const app = express(); diff --git a/examples/virtualbackground/index.html b/examples/virtualbackground/index.html index 7e3f74e..a71be3c 100644 --- a/examples/virtualbackground/index.html +++ b/examples/virtualbackground/index.html @@ -44,7 +44,7 @@ - + diff --git a/examples/virtualbackground/index.js b/examples/virtualbackground/index.js index 4467a35..8fa31cc 100644 --- a/examples/virtualbackground/index.js +++ b/examples/virtualbackground/index.js @@ -13,9 +13,8 @@ const defaultParams = { capFramerate: '30', capResolution: '1280x720', debounce: 'false', - pipeline: 'WebGL2', - maskBlurRadiusCanvas2D: '4', - maskBlurRadiusWebGL2: '8', + maskBlurRadius: '8', + stats: 'show' }; const params = { @@ -45,15 +44,10 @@ const loadImage = async (name) => { return bkgImage; } -params.maskBlurRadius = params.pipeline === 'WebGL2' - ? params.maskBlurRadiusWebGL2 - : params.maskBlurRadiusCanvas2D; - (async ({ Video: { createLocalVideoTrack }, VideoProcessors: { GaussianBlurBackgroundProcessor, - Pipeline: { Canvas2D, WebGL2 }, VirtualBackgroundProcessor, isSupported, version, @@ -62,18 +56,14 @@ params.maskBlurRadius = params.pipeline === 'WebGL2' const { capFramerate, capResolution, - pipeline, debounce, maskBlurRadius, - stats = 'show', + stats, } = params; const addProcessorOptions = { - inputFrameBufferType: 'video', - outputFrameBufferContextType: { - [Canvas2D]: '2d', - [WebGL2]: 'webgl2', - }[pipeline], + inputFrameBufferType: 'videoframe', + outputFrameBufferContextType: 'bitmaprenderer', }; const capDimensions = capResolution @@ -88,7 +78,6 @@ params.maskBlurRadius = params.pipeline === 'WebGL2' const processorOptions = { assetsPath, - pipeline, debounce: JSON.parse(debounce), maskBlurRadius: Number(maskBlurRadius), }; diff --git a/lib/processors/Processor.ts b/lib/processors/Processor.ts index 43777ae..ccae4f5 100644 --- a/lib/processors/Processor.ts +++ b/lib/processors/Processor.ts @@ -11,6 +11,6 @@ export abstract class Processor { * @param outputFrameBuffer - The output frame buffer to use to draw the processed frame. */ abstract processFrame( - inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement, + inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement | VideoFrame, outputFrameBuffer: HTMLCanvasElement): Promise | void; } diff --git a/lib/processors/background/BackgroundProcessor.ts b/lib/processors/background/BackgroundProcessor.ts index 17e9f52..79ec0cf 100644 --- a/lib/processors/background/BackgroundProcessor.ts +++ b/lib/processors/background/BackgroundProcessor.ts @@ -91,8 +91,8 @@ export abstract class BackgroundProcessor extends Processor { private static _tflite: TwilioTFLite | null = null; protected _backgroundImage: HTMLImageElement | null = null; - protected _outputCanvas: HTMLCanvasElement | null = null; - protected _outputContext: CanvasRenderingContext2D | null = null; + protected _outputCanvas: OffscreenCanvas | HTMLCanvasElement | null = null; + protected _outputContext: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D | null = null; protected _personMaskUpscalePipeline: PersonMaskUpscalePipeline | null = null; protected _webgl2Canvas: OffscreenCanvas | HTMLCanvasElement; @@ -110,6 +110,8 @@ export abstract class BackgroundProcessor extends Processor { // tslint:disable-next-line no-unused-variable private _isSimdEnabled: boolean | null; private _maskBlurRadius: number; + private _outputFrameBuffer: HTMLCanvasElement | null; + private _outputFrameBufferContext: CanvasRenderingContext2D | ImageBitmapRenderingContext | null; constructor(options: BackgroundProcessorOptions) { super(); @@ -139,6 +141,8 @@ export abstract class BackgroundProcessor extends Processor { this._inputFrameCanvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); this._inputFrameContext = this._inputFrameCanvas.getContext('2d') as OffscreenCanvasRenderingContext2D; this._maskBlurRadius = typeof options.maskBlurRadius === 'number' ? options.maskBlurRadius : MASK_BLUR_RADIUS; + this._outputFrameBuffer = null; + this._outputFrameBufferContext = null; this._webgl2Canvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); } @@ -206,7 +210,7 @@ export abstract class BackgroundProcessor extends Processor { * @param outputFrameBuffer - The output frame buffer to use to draw the processed frame. */ async processFrame( - inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement, + inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement | VideoFrame, outputFrameBuffer: HTMLCanvasElement ): Promise { if (!BackgroundProcessor._tflite) { @@ -228,20 +232,42 @@ export abstract class BackgroundProcessor extends Processor { height: captureHeight } = inputFrameBuffer instanceof HTMLVideoElement ? { width: inputFrameBuffer.videoWidth, height: inputFrameBuffer.videoHeight } - : inputFrameBuffer; + : typeof VideoFrame === 'function' && inputFrameBuffer instanceof VideoFrame + ? { width: inputFrameBuffer.displayWidth, height: inputFrameBuffer.displayHeight } + : inputFrameBuffer as (OffscreenCanvas | HTMLCanvasElement); const { height: outputHeight, width: outputWidth } = outputFrameBuffer; - if (this._outputCanvas !== outputFrameBuffer) { - this._outputCanvas = outputFrameBuffer; - this._outputContext = this._outputCanvas.getContext('2d'); + let shouldCleanUpPersonMaskUpscalePipeline = false; + if (this._outputFrameBuffer !== outputFrameBuffer) { + this._outputFrameBuffer = outputFrameBuffer; + this._outputFrameBufferContext = outputFrameBuffer.getContext('2d'); + if (this._outputFrameBufferContext) { + this._outputCanvas = outputFrameBuffer; + this._outputContext = this._outputFrameBufferContext; + } else { + this._outputFrameBufferContext = outputFrameBuffer.getContext('bitmaprenderer'); + this._outputCanvas ||= typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); + this._outputContext ||= this._outputCanvas.getContext('2d'); + } + } + if (this._webgl2Canvas.width !== outputWidth) { this._webgl2Canvas.width = outputWidth; + this._outputCanvas!.width = outputWidth; + shouldCleanUpPersonMaskUpscalePipeline = true; + } + if (this._webgl2Canvas.height !== outputHeight) { this._webgl2Canvas.height = outputHeight; + this._outputCanvas!.height = outputHeight; + shouldCleanUpPersonMaskUpscalePipeline = true; + } + if (shouldCleanUpPersonMaskUpscalePipeline) { this._cleanupPersonMaskUpscalePipeline(); } + // Only set the canvas' dimensions if they have changed to prevent unnecessary redraw if (this._inputFrameCanvas.width !== captureWidth) { this._inputFrameCanvas.width = captureWidth; @@ -256,7 +282,7 @@ export abstract class BackgroundProcessor extends Processor { this._inferenceInputCanvas.height = inferenceHeight; } - let inputFrame: OffscreenCanvas | HTMLCanvasElement; + let inputFrame: OffscreenCanvas | HTMLCanvasElement | VideoFrame; if (inputFrameBuffer instanceof HTMLVideoElement) { this._inputFrameContext.drawImage(inputFrameBuffer, 0, 0); inputFrame = this._inputFrameCanvas; @@ -264,7 +290,7 @@ export abstract class BackgroundProcessor extends Processor { inputFrame = inputFrameBuffer; } if (!this._personMaskUpscalePipeline) { - this._createPersonMaskUpscalePipeline(inputFrame); + this._createPersonMaskUpscalePipeline(); } const personMask = await this._createPersonMask(inputFrame); @@ -277,10 +303,10 @@ export abstract class BackgroundProcessor extends Processor { this._benchmark.start('imageCompositionDelay'); if (!this._debounce || this._currentMask || !isCanvasBlurSupported) { this._personMaskUpscalePipeline?.setInputTextureData(personMask); - this._personMaskUpscalePipeline?.render(); + this._personMaskUpscalePipeline?.render(inputFrame); } - const ctx = this._outputContext as CanvasRenderingContext2D; + const ctx = this._outputContext!; ctx.save(); ctx.filter = 'none'; ctx.globalCompositeOperation = 'copy'; @@ -290,6 +316,22 @@ export abstract class BackgroundProcessor extends Processor { ctx.globalCompositeOperation = 'destination-over'; this._setBackground(inputFrame); ctx.restore(); + + if (typeof VideoFrame === 'function' && inputFrame instanceof VideoFrame) { + inputFrame.close(); + } + const { + _outputCanvas: outputCanvas, + _outputFrameBufferContext: outputFrameBufferContext + } = this; + + if (outputFrameBufferContext instanceof ImageBitmapRenderingContext) { + const outputBitmap = this._outputCanvas instanceof OffscreenCanvas + ? (outputCanvas as OffscreenCanvas).transferToImageBitmap() + : await createImageBitmap(outputCanvas!); + outputFrameBufferContext.transferFromImageBitmap(outputBitmap); + } + this._benchmark.end('imageCompositionDelay'); this._benchmark.end('processFrameDelay'); @@ -301,14 +343,14 @@ export abstract class BackgroundProcessor extends Processor { this._benchmark.start('captureFrameDelay'); } - protected abstract _setBackground(inputFrame?: OffscreenCanvas | HTMLCanvasElement): void; + protected abstract _setBackground(inputFrame?: OffscreenCanvas | HTMLCanvasElement | VideoFrame): void; private _cleanupPersonMaskUpscalePipeline(): void { this._personMaskUpscalePipeline?.cleanUp(); this._personMaskUpscalePipeline = null; } - private async _createPersonMask(inputFrame: OffscreenCanvas | HTMLCanvasElement): Promise { + private async _createPersonMask(inputFrame: OffscreenCanvas | HTMLCanvasElement | VideoFrame): Promise { const { height, width } = this._inferenceDimensions; const stages = { inference: { @@ -336,9 +378,8 @@ export abstract class BackgroundProcessor extends Processor { return this._currentMask || new ImageData(personMaskBuffer, width, height); } - private _createPersonMaskUpscalePipeline(inputFrame: OffscreenCanvas | HTMLCanvasElement): void { + private _createPersonMaskUpscalePipeline(): void { this._personMaskUpscalePipeline = new PersonMaskUpscalePipeline( - inputFrame, this._inferenceDimensions, this._webgl2Canvas ); @@ -347,7 +388,7 @@ export abstract class BackgroundProcessor extends Processor { }); } - private async _resizeInputFrame(inputFrame: OffscreenCanvas | HTMLCanvasElement): Promise { + private async _resizeInputFrame(inputFrame: OffscreenCanvas | HTMLCanvasElement | VideoFrame): Promise { const { _inferenceInputCanvas: { width: resizeWidth, diff --git a/lib/processors/background/GaussianBlurBackgroundProcessor.ts b/lib/processors/background/GaussianBlurBackgroundProcessor.ts index 23e5da5..bf9cfaf 100644 --- a/lib/processors/background/GaussianBlurBackgroundProcessor.ts +++ b/lib/processors/background/GaussianBlurBackgroundProcessor.ts @@ -103,7 +103,7 @@ export class GaussianBlurBackgroundProcessor extends BackgroundProcessor { this._gaussianBlurFilterPipeline?.updateRadius(this._blurFilterRadius); } - protected _setBackground(inputFrame: OffscreenCanvas | HTMLCanvasElement): void { + protected _setBackground(inputFrame: OffscreenCanvas | HTMLCanvasElement | VideoFrame): void { const { _outputContext: ctx, _blurFilterRadius: radius, @@ -121,7 +121,7 @@ export class GaussianBlurBackgroundProcessor extends BackgroundProcessor { this._gaussianBlurFilterPipeline = new GaussianBlurFilterPipeline(canvas); this._gaussianBlurFilterPipeline.updateRadius(radius); } - this._gaussianBlurFilterPipeline?.render(); + this._gaussianBlurFilterPipeline!.render(); ctx.drawImage(canvas, 0, 0); } } diff --git a/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts b/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts index 44904b1..d707281 100644 --- a/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts +++ b/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts @@ -5,7 +5,6 @@ import { WebGL2Pipeline } from './WebGL2Pipeline'; export class PersonMaskUpscalePipeline extends WebGL2Pipeline { constructor( - inputCanvas: OffscreenCanvas | HTMLCanvasElement, inputDimensions: Dimensions, outputCanvas: OffscreenCanvas | HTMLCanvasElement ) { @@ -17,10 +16,7 @@ export class PersonMaskUpscalePipeline extends WebGL2Pipeline { width: outputCanvas.width } - this.addStage(new WebGL2Pipeline.InputStage( - glOut, - inputCanvas - )) + this.addStage(new WebGL2Pipeline.InputStage(glOut)) this.addStage(new SinglePassBilateralFilterStage( glOut, diff --git a/lib/processors/webgl2/pipelines/Pipeline.ts b/lib/processors/webgl2/pipelines/Pipeline.ts index d34c92c..df90f29 100644 --- a/lib/processors/webgl2/pipelines/Pipeline.ts +++ b/lib/processors/webgl2/pipelines/Pipeline.ts @@ -8,15 +8,15 @@ export class Pipeline implements Pipeline.Stage { this._stages.push(stage) } - render(): void { + render(...args: any[]): void { this._stages.forEach((stage) => { - stage.render() + stage.render(...args) }) } } export namespace Pipeline { export interface Stage { - render(): void + render(...args: any[]): void } } diff --git a/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts b/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts index bb4182d..aa94e58 100644 --- a/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts +++ b/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts @@ -34,18 +34,13 @@ interface UniformVarInfo { */ class WebGL2PipelineInputStage implements Pipeline.Stage { private _glOut: WebGL2RenderingContext - private _inputFrame: OffscreenCanvas | HTMLCanvasElement private _inputFrameTexture: WebGLTexture private _inputTexture: WebGLTexture | null private _inputTextureData: ImageData | null - constructor( - glOut: WebGL2RenderingContext, - inputFrame: OffscreenCanvas | HTMLCanvasElement - ) { - const { height, width } = inputFrame + constructor(glOut: WebGL2RenderingContext) { + const { height, width } = glOut.canvas this._glOut = glOut - this._inputFrame = inputFrame this._inputFrameTexture = createTexture( glOut, glOut.RGBA8, @@ -68,35 +63,36 @@ class WebGL2PipelineInputStage implements Pipeline.Stage { _glOut.deleteTexture(_inputTexture) } - render(): void { + render(inputFrame?: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement | VideoFrame): void { const { _glOut, - _inputFrame, _inputFrameTexture, _inputTextureData } = this - const { height, width } = _inputFrame + const { height, width } = _glOut.canvas; _glOut.viewport(0, 0, width, height) _glOut.clearColor(0, 0, 0, 0) _glOut.clear(_glOut.COLOR_BUFFER_BIT) - _glOut.activeTexture(_glOut.TEXTURE0) - _glOut.bindTexture( - _glOut.TEXTURE_2D, - _inputFrameTexture - ) - _glOut.texSubImage2D( - _glOut.TEXTURE_2D, - 0, - 0, - 0, - width, - height, - _glOut.RGBA, - _glOut.UNSIGNED_BYTE, - _inputFrame - ) + if (inputFrame) { + _glOut.activeTexture(_glOut.TEXTURE0) + _glOut.bindTexture( + _glOut.TEXTURE_2D, + _inputFrameTexture + ) + _glOut.texSubImage2D( + _glOut.TEXTURE_2D, + 0, + 0, + 0, + width, + height, + _glOut.RGBA, + _glOut.UNSIGNED_BYTE, + inputFrame + ) + } if (!_inputTextureData) { return @@ -377,6 +373,15 @@ export class WebGL2Pipeline extends Pipeline { ) } + render(inputFrame?: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement | VideoFrame): void { + const [inputStage, ...otherStages] = this._stages; + inputStage.render(inputFrame); + otherStages.forEach( + (stage) => (stage as WebGL2PipelineProcessingStage) + .render() + ); + } + setInputTextureData(inputTextureData: ImageData): void { const [inputStage] = this._stages as [WebGL2PipelineInputStage] inputStage.setInputTextureData(inputTextureData); From 2756056c070eac3b927877e520133eaccb36c3ca Mon Sep 17 00:00:00 2001 From: mmalavalli Date: Mon, 29 Jul 2024 15:26:44 -0500 Subject: [PATCH 08/28] Refactoring the processing code into a separate BackgroundProcessorPipeline class, which can be called either from the main thread or a web worker (to be implemented). The public classes invoke the pipeline's render() method to process every video frame. --- lib/constants.ts | 1 - .../background/BackgroundProcessor.ts | 352 ++++------------ .../GaussianBlurBackgroundProcessor.ts | 54 ++- .../background/VirtualBackgroundProcessor.ts | 95 ++--- .../BackgroundProcessorPipeline.ts | 158 +++++++ ...GaussianBlurBackgroundProcessorPipeline.ts | 65 +++ .../InputFrameDownscaleStage.ts | 67 +++ .../PostProcessingStage.ts | 85 ++++ .../VirtualBackgroundProcessorPipeline.ts | 136 ++++++ .../backgroundprocessorpipeline/index.ts | 3 + .../SinglePassGaussianBlurFilterStage.ts | 20 +- .../gaussianblurfilterpipeline/index.ts} | 13 +- lib/processors/background/pipelines/index.ts | 2 + .../SinglePassBilateralFilterStage.ts | 56 +-- .../personmaskupscalepipeline/index.ts} | 30 +- .../{webgl2 => }/pipelines/Pipeline.ts | 10 +- lib/processors/pipelines/index.ts | 2 + .../WebGL2PipelineInputStage.ts | 108 +++++ .../WebGL2PipelineProcessingStage.ts | 240 +++++++++++ .../pipelines/webgl2pipeline/index.ts | 31 ++ .../webgl2pipeline/webgl2PipelineHelpers.ts} | 82 ++-- .../webgl2/helpers/postProcessingHelper.ts | 3 - .../webgl2/helpers/segmentationHelper.ts | 14 - lib/processors/webgl2/index.ts | 8 - .../webgl2/pipelines/WebGL2Pipeline.ts | 389 ------------------ lib/types.ts | 12 + 26 files changed, 1156 insertions(+), 880 deletions(-) create mode 100644 lib/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.ts create mode 100644 lib/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.ts create mode 100644 lib/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.ts create mode 100644 lib/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.ts create mode 100644 lib/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.ts create mode 100644 lib/processors/background/pipelines/backgroundprocessorpipeline/index.ts rename lib/processors/{webgl2/pipelines => background/pipelines/gaussianblurfilterpipeline}/SinglePassGaussianBlurFilterStage.ts (94%) rename lib/processors/{webgl2/pipelines/GaussianBlurFilterPipeline.ts => background/pipelines/gaussianblurfilterpipeline/index.ts} (87%) create mode 100644 lib/processors/background/pipelines/index.ts rename lib/processors/{webgl2/pipelines => background/pipelines/personmaskupscalepipeline}/SinglePassBilateralFilterStage.ts (88%) rename lib/processors/{webgl2/pipelines/PersonMaskUpscalePipeline.ts => background/pipelines/personmaskupscalepipeline/index.ts} (73%) rename lib/processors/{webgl2 => }/pipelines/Pipeline.ts (62%) create mode 100644 lib/processors/pipelines/index.ts create mode 100644 lib/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.ts create mode 100644 lib/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.ts create mode 100644 lib/processors/pipelines/webgl2pipeline/index.ts rename lib/processors/{webgl2/helpers/webglHelper.ts => pipelines/webgl2pipeline/webgl2PipelineHelpers.ts} (61%) delete mode 100644 lib/processors/webgl2/helpers/postProcessingHelper.ts delete mode 100644 lib/processors/webgl2/helpers/segmentationHelper.ts delete mode 100644 lib/processors/webgl2/index.ts delete mode 100644 lib/processors/webgl2/pipelines/WebGL2Pipeline.ts diff --git a/lib/constants.ts b/lib/constants.ts index 39e005c..1f8e741 100644 --- a/lib/constants.ts +++ b/lib/constants.ts @@ -5,7 +5,6 @@ export const MASK_BLUR_RADIUS = 8; export const MODEL_NAME = 'selfie_segmentation_landscape.tflite'; export const TFLITE_LOADER_NAME = 'tflite-1-0-0.js'; export const TFLITE_SIMD_LOADER_NAME = 'tflite-simd-1-0-0.js'; -export const TFLITE_WORKER_NAME = 'tflite-worker.js'; export const WASM_INFERENCE_DIMENSIONS: Dimensions = { width: 256, diff --git a/lib/processors/background/BackgroundProcessor.ts b/lib/processors/background/BackgroundProcessor.ts index 79ec0cf..c882864 100644 --- a/lib/processors/background/BackgroundProcessor.ts +++ b/lib/processors/background/BackgroundProcessor.ts @@ -1,19 +1,8 @@ -import { Processor } from '../Processor'; +import { MASK_BLUR_RADIUS } from '../../constants'; import { Benchmark } from '../../utils/Benchmark'; -import { TwilioTFLite } from '../../utils/TwilioTFLite'; -import { isCanvasBlurSupported, isChromiumImageBitmap } from '../../utils/support'; -import { Dimensions } from '../../types'; -import { PersonMaskUpscalePipeline } from '../webgl2'; - -import { - MASK_BLUR_RADIUS, - MODEL_NAME, - TFLITE_LOADER_NAME, - TFLITE_SIMD_LOADER_NAME, - WASM_INFERENCE_DIMENSIONS, -} from '../../constants'; - -type InputResizeMode = 'canvas' | 'image-bitmap'; +import { InputFrame } from '../../types'; +import { Processor } from '../Processor'; +import { BackgroundProcessorPipeline } from './pipelines/backgroundprocessorpipeline'; /** * @private @@ -50,30 +39,6 @@ export interface BackgroundProcessorOptions { */ assetsPath: string; - /** - * Whether to skip processing every other frame to improve the output frame rate, but reducing accuracy in the process. - * @default - * ```html - * true - * ``` - */ - debounce?: boolean; - - /** - * @private - */ - deferInputResize?: boolean; - - /** - * @private - */ - inferenceDimensions?: Dimensions; - - /** - * @private - */ - inputResizeMode?: InputResizeMode; - /** * The blur radius to use when smoothing out the edges of the person's mask. * @default @@ -87,63 +52,39 @@ export interface BackgroundProcessorOptions { /** * @private */ -export abstract class BackgroundProcessor extends Processor { - private static _tflite: TwilioTFLite | null = null; - - protected _backgroundImage: HTMLImageElement | null = null; - protected _outputCanvas: OffscreenCanvas | HTMLCanvasElement | null = null; - protected _outputContext: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D | null = null; - protected _personMaskUpscalePipeline: PersonMaskUpscalePipeline | null = null; - protected _webgl2Canvas: OffscreenCanvas | HTMLCanvasElement; - - private _assetsPath: string; - private _benchmark: Benchmark; - private _currentMask: ImageData | null; - private _debounce: boolean; - private _deferInputResize: boolean; - private _inferenceDimensions: Dimensions = WASM_INFERENCE_DIMENSIONS; - private _inferenceInputCanvas: OffscreenCanvas | HTMLCanvasElement; - private _inferenceInputContext: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D; - private _inputFrameCanvas: OffscreenCanvas | HTMLCanvasElement; - private _inputFrameContext: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D; - private _inputResizeMode: InputResizeMode; - // tslint:disable-next-line no-unused-variable - private _isSimdEnabled: boolean | null; - private _maskBlurRadius: number; - private _outputFrameBuffer: HTMLCanvasElement | null; - private _outputFrameBufferContext: CanvasRenderingContext2D | ImageBitmapRenderingContext | null; - - constructor(options: BackgroundProcessorOptions) { +export class BackgroundProcessor extends Processor { + protected readonly _assetsPath: string; + protected readonly _backgroundProcessorPipeline: BackgroundProcessorPipeline; + + private readonly _benchmark: Benchmark; + private readonly _inputFrameCanvas: OffscreenCanvas = new OffscreenCanvas(1, 1); + private readonly _inputFrameContext: OffscreenCanvasRenderingContext2D = this._inputFrameCanvas.getContext('2d', { willReadFrequently: true })!; + private _isSimdEnabled: boolean | null = null; + private _maskBlurRadius: number = MASK_BLUR_RADIUS; + private _outputFrameBuffer: HTMLCanvasElement | null = null; + private _outputFrameBufferContext: CanvasRenderingContext2D | ImageBitmapRenderingContext | null = null; + + protected constructor( + backgroundProcessorPipeline: BackgroundProcessorPipeline, + options: BackgroundProcessorOptions + ) { super(); - if (typeof options.assetsPath !== 'string') { - throw new Error('assetsPath parameter is missing'); - } - let assetsPath = options.assetsPath; - if (assetsPath && assetsPath[assetsPath.length - 1] !== '/') { - assetsPath += '/'; - } - - this._assetsPath = assetsPath; - this._debounce = typeof options.debounce === 'boolean' ? options.debounce : true; - this._deferInputResize = typeof options.deferInputResize === 'boolean' ? options.deferInputResize : false; - this._inferenceDimensions = options.inferenceDimensions! || this._inferenceDimensions; + const { + assetsPath, + maskBlurRadius = this._maskBlurRadius + } = options; - this._inputResizeMode = typeof options.inputResizeMode === 'string' - ? options.inputResizeMode - : (isChromiumImageBitmap() ? 'image-bitmap' : 'canvas'); + if (typeof assetsPath !== 'string') { + throw new Error('assetsPath parameter must be a string'); + } - this._benchmark = new Benchmark(); - this._currentMask = null; - this._isSimdEnabled = null; - this._inferenceInputCanvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); - this._inferenceInputContext = this._inferenceInputCanvas.getContext('2d', { willReadFrequently: true }) as OffscreenCanvasRenderingContext2D; - this._inputFrameCanvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); - this._inputFrameContext = this._inputFrameCanvas.getContext('2d') as OffscreenCanvasRenderingContext2D; - this._maskBlurRadius = typeof options.maskBlurRadius === 'number' ? options.maskBlurRadius : MASK_BLUR_RADIUS; - this._outputFrameBuffer = null; - this._outputFrameBufferContext = null; - this._webgl2Canvas = typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); + this._assetsPath = assetsPath.replace(/([^/])$/, '$1/'); + this._backgroundProcessorPipeline = backgroundProcessorPipeline; + // TODO(mmalavalli): Remove the ts-ignore after refactoring to support web workers. + // @ts-ignore + this._benchmark = this._backgroundProcessorPipeline._benchmark; + this.maskBlurRadius = maskBlurRadius; } /** @@ -163,9 +104,11 @@ export abstract class BackgroundProcessor extends Processor { } if (this._maskBlurRadius !== radius) { this._maskBlurRadius = radius; - this._personMaskUpscalePipeline?.updateBilateralFilterConfig({ - sigmaSpace: this._maskBlurRadius - }); + this._backgroundProcessorPipeline + .setMaskBlurRadius(this._maskBlurRadius) + .catch(() => { + /* noop */ + }); } } @@ -175,18 +118,9 @@ export abstract class BackgroundProcessor extends Processor { * video frames are processed correctly. */ async loadModel(): Promise { - let { _tflite: tflite } = BackgroundProcessor; - if (!tflite) { - tflite = new TwilioTFLite(); - await tflite.initialize( - this._assetsPath, - MODEL_NAME, - TFLITE_LOADER_NAME, - TFLITE_SIMD_LOADER_NAME, - ); - BackgroundProcessor._tflite = tflite; - } - this._isSimdEnabled = tflite.isSimdEnabled; + this._isSimdEnabled = await this + ._backgroundProcessorPipeline + .loadTwilioTFLite(); } /** @@ -197,15 +131,18 @@ export abstract class BackgroundProcessor extends Processor { *
*
* [OffscreenCanvas](https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas) - Good for canvas-related processing - * that can be rendered off screen. Only works when using [[Pipeline.Canvas2D]]. + * that can be rendered off screen. *
*
* [HTMLCanvasElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement) - This is recommended on browsers - * that doesn't support `OffscreenCanvas`, or if you need to render the frame on the screen. Only works when using [[Pipeline.Canvas2D]]. + * that doesn't support `OffscreenCanvas`, or if you need to render the frame on the screen. + *
*
+ * [HTMLVideoElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement) *
- * [HTMLVideoElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement) - Recommended when using [[Pipeline.WebGL2]] but - * works for both [[Pipeline.Canvas2D]] and [[Pipeline.WebGL2]]. + *
+ * [VideoFrame](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame) - Recommended on browsers that support the + * [Insertable Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Insertable_Streams_for_MediaStreamTrack_API). *
* @param outputFrameBuffer - The output frame buffer to use to draw the processed frame. */ @@ -213,19 +150,20 @@ export abstract class BackgroundProcessor extends Processor { inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement | VideoFrame, outputFrameBuffer: HTMLCanvasElement ): Promise { - if (!BackgroundProcessor._tflite) { - return; - } if (!inputFrameBuffer || !outputFrameBuffer) { throw new Error('Missing input or output frame buffer'); } - this._benchmark.end('captureFrameDelay'); - this._benchmark.start('processFrameDelay'); const { - width: inferenceWidth, - height: inferenceHeight - } = this._inferenceDimensions; + _backgroundProcessorPipeline, + _benchmark, + _outputFrameBufferContext + } = this; + + _benchmark.end('captureFrameDelay'); + _benchmark.end('totalProcessingDelay'); + _benchmark.start('totalProcessingDelay'); + _benchmark.start('processFrameDelay'); const { width: captureWidth, @@ -236,179 +174,45 @@ export abstract class BackgroundProcessor extends Processor { ? { width: inputFrameBuffer.displayWidth, height: inputFrameBuffer.displayHeight } : inputFrameBuffer as (OffscreenCanvas | HTMLCanvasElement); - const { - height: outputHeight, - width: outputWidth - } = outputFrameBuffer; - - let shouldCleanUpPersonMaskUpscalePipeline = false; if (this._outputFrameBuffer !== outputFrameBuffer) { this._outputFrameBuffer = outputFrameBuffer; - this._outputFrameBufferContext = outputFrameBuffer.getContext('2d'); - if (this._outputFrameBufferContext) { - this._outputCanvas = outputFrameBuffer; - this._outputContext = this._outputFrameBufferContext; - } else { - this._outputFrameBufferContext = outputFrameBuffer.getContext('bitmaprenderer'); - this._outputCanvas ||= typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); - this._outputContext ||= this._outputCanvas.getContext('2d'); - } + this._outputFrameBufferContext = outputFrameBuffer.getContext('2d') + || outputFrameBuffer.getContext('bitmaprenderer'); } - if (this._webgl2Canvas.width !== outputWidth) { - this._webgl2Canvas.width = outputWidth; - this._outputCanvas!.width = outputWidth; - shouldCleanUpPersonMaskUpscalePipeline = true; - } - if (this._webgl2Canvas.height !== outputHeight) { - this._webgl2Canvas.height = outputHeight; - this._outputCanvas!.height = outputHeight; - shouldCleanUpPersonMaskUpscalePipeline = true; - } - if (shouldCleanUpPersonMaskUpscalePipeline) { - this._cleanupPersonMaskUpscalePipeline(); - } - - // Only set the canvas' dimensions if they have changed to prevent unnecessary redraw if (this._inputFrameCanvas.width !== captureWidth) { this._inputFrameCanvas.width = captureWidth; } if (this._inputFrameCanvas.height !== captureHeight) { this._inputFrameCanvas.height = captureHeight; } - if (this._inferenceInputCanvas.width !== inferenceWidth) { - this._inferenceInputCanvas.width = inferenceWidth; - } - if (this._inferenceInputCanvas.height !== inferenceHeight) { - this._inferenceInputCanvas.height = inferenceHeight; - } - let inputFrame: OffscreenCanvas | HTMLCanvasElement | VideoFrame; + let inputFrame: InputFrame; if (inputFrameBuffer instanceof HTMLVideoElement) { - this._inputFrameContext.drawImage(inputFrameBuffer, 0, 0); + this._inputFrameContext.drawImage( + inputFrameBuffer, + 0, + 0 + ); inputFrame = this._inputFrameCanvas; } else { inputFrame = inputFrameBuffer; } - if (!this._personMaskUpscalePipeline) { - this._createPersonMaskUpscalePipeline(); - } - - const personMask = await this._createPersonMask(inputFrame); - if (this._debounce) { - this._currentMask = this._currentMask === personMask - ? null - : personMask; - } - - this._benchmark.start('imageCompositionDelay'); - if (!this._debounce || this._currentMask || !isCanvasBlurSupported) { - this._personMaskUpscalePipeline?.setInputTextureData(personMask); - this._personMaskUpscalePipeline?.render(inputFrame); - } - const ctx = this._outputContext!; - ctx.save(); - ctx.filter = 'none'; - ctx.globalCompositeOperation = 'copy'; - ctx.drawImage(this._webgl2Canvas, 0, 0, outputWidth, outputHeight); - ctx.globalCompositeOperation = 'source-in'; - ctx.drawImage(inputFrame, 0, 0, outputWidth, outputHeight); - ctx.globalCompositeOperation = 'destination-over'; - this._setBackground(inputFrame); - ctx.restore(); - - if (typeof VideoFrame === 'function' && inputFrame instanceof VideoFrame) { - inputFrame.close(); - } - const { - _outputCanvas: outputCanvas, - _outputFrameBufferContext: outputFrameBufferContext - } = this; - - if (outputFrameBufferContext instanceof ImageBitmapRenderingContext) { - const outputBitmap = this._outputCanvas instanceof OffscreenCanvas - ? (outputCanvas as OffscreenCanvas).transferToImageBitmap() - : await createImageBitmap(outputCanvas!); - outputFrameBufferContext.transferFromImageBitmap(outputBitmap); - } + const outputFrame = await _backgroundProcessorPipeline.render(inputFrame); - this._benchmark.end('imageCompositionDelay'); - - this._benchmark.end('processFrameDelay'); - this._benchmark.end('totalProcessingDelay'); - - // NOTE (csantos): Start the benchmark from here so we can include the delay from the Video sdk - // for a more accurate fps - this._benchmark.start('totalProcessingDelay'); - this._benchmark.start('captureFrameDelay'); - } - - protected abstract _setBackground(inputFrame?: OffscreenCanvas | HTMLCanvasElement | VideoFrame): void; - - private _cleanupPersonMaskUpscalePipeline(): void { - this._personMaskUpscalePipeline?.cleanUp(); - this._personMaskUpscalePipeline = null; - } - - private async _createPersonMask(inputFrame: OffscreenCanvas | HTMLCanvasElement | VideoFrame): Promise { - const { height, width } = this._inferenceDimensions; - const stages = { - inference: { - false: () => BackgroundProcessor._tflite!.runInference(), - true: () => this._currentMask!.data - }, - resize: { - false: async () => this._resizeInputFrame(inputFrame), - true: async () => { /* noop */ } - } - }; - const shouldDebounce = !!this._currentMask; - const inferenceStage = stages.inference[`${shouldDebounce}`]; - const resizeStage = stages.resize[`${shouldDebounce}`]; - - this._benchmark.start('inputImageResizeDelay'); - const resizePromise = resizeStage(); - if (!this._deferInputResize) { - await resizePromise; + if (_outputFrameBufferContext instanceof ImageBitmapRenderingContext) { + _outputFrameBufferContext.transferFromImageBitmap( + outputFrame && outputFrame.transferToImageBitmap() + ); + } else if (_outputFrameBufferContext instanceof CanvasRenderingContext2D && outputFrame) { + _outputFrameBufferContext.drawImage( + outputFrame, + 0, + 0 + ); } - this._benchmark.end('inputImageResizeDelay'); - this._benchmark.start('segmentationDelay'); - const personMaskBuffer = inferenceStage(); - this._benchmark.end('segmentationDelay'); - return this._currentMask || new ImageData(personMaskBuffer, width, height); - } - - private _createPersonMaskUpscalePipeline(): void { - this._personMaskUpscalePipeline = new PersonMaskUpscalePipeline( - this._inferenceDimensions, - this._webgl2Canvas - ); - this._personMaskUpscalePipeline?.updateBilateralFilterConfig({ - sigmaSpace: this._maskBlurRadius - }); - } - private async _resizeInputFrame(inputFrame: OffscreenCanvas | HTMLCanvasElement | VideoFrame): Promise { - const { - _inferenceInputCanvas: { - width: resizeWidth, - height: resizeHeight - }, - _inferenceInputContext: ctx, - _inputResizeMode: resizeMode - } = this; - if (resizeMode === 'image-bitmap') { - const resizedInputFrameBitmap = await createImageBitmap(inputFrame, { - resizeWidth, - resizeHeight, - resizeQuality: 'pixelated' - }); - ctx.drawImage(resizedInputFrameBitmap, 0, 0, resizeWidth, resizeHeight); - resizedInputFrameBitmap.close(); - } else { - ctx.drawImage(inputFrame, 0, 0, resizeWidth, resizeHeight); - } - const imageData = ctx.getImageData(0, 0, resizeWidth, resizeHeight); - BackgroundProcessor._tflite!.loadInputBuffer(imageData.data); + _benchmark.end('processFrameDelay'); + _benchmark.start('captureFrameDelay'); } } diff --git a/lib/processors/background/GaussianBlurBackgroundProcessor.ts b/lib/processors/background/GaussianBlurBackgroundProcessor.ts index bf9cfaf..251534d 100644 --- a/lib/processors/background/GaussianBlurBackgroundProcessor.ts +++ b/lib/processors/background/GaussianBlurBackgroundProcessor.ts @@ -1,7 +1,6 @@ -import { BLUR_FILTER_RADIUS } from '../../constants'; -import { isCanvasBlurSupported } from '../../utils/support'; -import { GaussianBlurFilterPipeline } from '../webgl2'; +import { BLUR_FILTER_RADIUS, MASK_BLUR_RADIUS } from '../../constants'; import { BackgroundProcessor, BackgroundProcessorOptions } from './BackgroundProcessor'; +import { GaussianBlurBackgroundProcessorPipeline } from './pipelines/backgroundprocessorpipeline'; /** * Options passed to [[GaussianBlurBackgroundProcessor]] constructor. @@ -67,9 +66,7 @@ export interface GaussianBlurBackgroundProcessorOptions extends BackgroundProces * ``` */ export class GaussianBlurBackgroundProcessor extends BackgroundProcessor { - private _blurFilterRadius: number = BLUR_FILTER_RADIUS; - private _gaussianBlurFilterPipeline: GaussianBlurFilterPipeline | null; // tslint:disable-next-line no-unused-variable private readonly _name: string = 'GaussianBlurBackgroundProcessor'; @@ -79,8 +76,23 @@ export class GaussianBlurBackgroundProcessor extends BackgroundProcessor { * invalid properties will be ignored. */ constructor(options: GaussianBlurBackgroundProcessorOptions) { - super(options); - this._gaussianBlurFilterPipeline = null; + const { + assetsPath, + blurFilterRadius = BLUR_FILTER_RADIUS, + maskBlurRadius = MASK_BLUR_RADIUS + } = options; + + const backgroundProcessorPipeline = new GaussianBlurBackgroundProcessorPipeline({ + assetsPath: assetsPath.replace(/([^/])$/, '$1/'), + blurFilterRadius, + maskBlurRadius + }); + + super( + backgroundProcessorPipeline, + options + ); + this.blurFilterRadius = options.blurFilterRadius!; } @@ -100,28 +112,10 @@ export class GaussianBlurBackgroundProcessor extends BackgroundProcessor { radius = BLUR_FILTER_RADIUS; } this._blurFilterRadius = radius; - this._gaussianBlurFilterPipeline?.updateRadius(this._blurFilterRadius); - } - - protected _setBackground(inputFrame: OffscreenCanvas | HTMLCanvasElement | VideoFrame): void { - const { - _outputContext: ctx, - _blurFilterRadius: radius, - _webgl2Canvas: canvas - } = this; - if (!ctx) { - return; - } - if (isCanvasBlurSupported) { - ctx.filter = `blur(${radius}px)`; - ctx.drawImage(inputFrame, 0, 0); - return; - } - if (!this._gaussianBlurFilterPipeline) { - this._gaussianBlurFilterPipeline = new GaussianBlurFilterPipeline(canvas); - this._gaussianBlurFilterPipeline.updateRadius(radius); - } - this._gaussianBlurFilterPipeline!.render(); - ctx.drawImage(canvas, 0, 0); + (this._backgroundProcessorPipeline as GaussianBlurBackgroundProcessorPipeline) + .setBlurFilterRadius(this._blurFilterRadius) + .catch(() => { + /* noop */ + }); } } diff --git a/lib/processors/background/VirtualBackgroundProcessor.ts b/lib/processors/background/VirtualBackgroundProcessor.ts index 7f379b7..5a025b5 100644 --- a/lib/processors/background/VirtualBackgroundProcessor.ts +++ b/lib/processors/background/VirtualBackgroundProcessor.ts @@ -1,5 +1,7 @@ -import { BackgroundProcessor, BackgroundProcessorOptions } from './BackgroundProcessor'; import { ImageFit } from '../../types'; +import { MASK_BLUR_RADIUS } from '../../constants'; +import { BackgroundProcessor, BackgroundProcessorOptions } from './BackgroundProcessor'; +import { VirtualBackgroundProcessorPipeline } from './pipelines/backgroundprocessorpipeline'; /** * Options passed to [[VirtualBackgroundProcessor]] constructor. @@ -78,7 +80,7 @@ export interface VirtualBackgroundProcessorOptions extends BackgroundProcessorOp * ``` */ export class VirtualBackgroundProcessor extends BackgroundProcessor { - + private _backgroundImage!: HTMLImageElement; private _fitType!: ImageFit; // tslint:disable-next-line no-unused-variable private readonly _name: string = 'VirtualBackgroundProcessor'; @@ -89,16 +91,33 @@ export class VirtualBackgroundProcessor extends BackgroundProcessor { * and invalid properties will be ignored. */ constructor(options: VirtualBackgroundProcessorOptions) { - super(options); - this.backgroundImage = options.backgroundImage; - this.fitType = options.fitType!; + const { + assetsPath, + backgroundImage, + fitType = ImageFit.Fill, + maskBlurRadius = MASK_BLUR_RADIUS + } = options; + + const backgroundProcessorPipeline = new VirtualBackgroundProcessorPipeline({ + assetsPath: assetsPath.replace(/([^/])$/, '$1/'), + fitType, + maskBlurRadius + }); + + super( + backgroundProcessorPipeline, + options + ); + + this.backgroundImage = backgroundImage; + this.fitType = fitType; } /** * The HTMLImageElement representing the current background image. */ get backgroundImage(): HTMLImageElement { - return this._backgroundImage!; + return this._backgroundImage; } /** @@ -112,6 +131,12 @@ export class VirtualBackgroundProcessor extends BackgroundProcessor { throw new Error('Invalid image. Make sure that the image is an HTMLImageElement and has been successfully loaded'); } this._backgroundImage = image; + createImageBitmap(this._backgroundImage).then( + (imageBitmap) => (this._backgroundProcessorPipeline as VirtualBackgroundProcessorPipeline) + .setBackgroundImage(imageBitmap) + ).catch(() => { + /* noop */ + }); } /** @@ -131,58 +156,10 @@ export class VirtualBackgroundProcessor extends BackgroundProcessor { fitType = ImageFit.Fill; } this._fitType = fitType; - } - - protected _setBackground(): void { - if (!this._outputContext || !this._outputCanvas) { - return; - } - const img = this._backgroundImage!; - const imageWidth = img.naturalWidth; - const imageHeight = img.naturalHeight; - const canvasWidth = this._outputCanvas.width; - const canvasHeight = this._outputCanvas.height; - const ctx = this._outputContext; - - if (this._fitType === ImageFit.Fill) { - ctx.drawImage(img, 0, 0, imageWidth, imageHeight, 0, 0, canvasWidth, canvasHeight); - } else if (this._fitType === ImageFit.None) { - ctx.drawImage(img, 0, 0, imageWidth, imageHeight); - } else if (this._fitType === ImageFit.Contain) { - const { x, y, w, h } = this._getFitPosition(imageWidth, imageHeight, canvasWidth, canvasHeight, ImageFit.Contain); - ctx.drawImage(img, 0, 0, imageWidth, imageHeight, x, y, w, h); - } else if (this._fitType === ImageFit.Cover) { - const { x, y, w, h } = this._getFitPosition(imageWidth, imageHeight, canvasWidth, canvasHeight, ImageFit.Cover); - ctx.drawImage(img, 0, 0, imageWidth, imageHeight, x, y, w, h); - } - } - - private _getFitPosition(contentWidth: number, contentHeight: number, - viewportWidth: number, viewportHeight: number, type: ImageFit) - : { h: number, w: number, x: number, y: number } { - - // Calculate new content width to fit viewport width - let factor = viewportWidth / contentWidth; - let newContentWidth = viewportWidth; - let newContentHeight = factor * contentHeight; - - // Scale down the resulting height and width more - // to fit viewport height if the content still exceeds it - if ((type === ImageFit.Contain && newContentHeight > viewportHeight) - || (type === ImageFit.Cover && viewportHeight > newContentHeight)) { - factor = viewportHeight / newContentHeight; - newContentWidth = factor * newContentWidth; - newContentHeight = viewportHeight; - } - - // Calculate the destination top left corner to center the content - const x = (viewportWidth - newContentWidth) / 2; - const y = (viewportHeight - newContentHeight) / 2; - - return { - x, y, - w: newContentWidth, - h: newContentHeight, - }; + (this._backgroundProcessorPipeline as VirtualBackgroundProcessorPipeline) + .setFitType(this._fitType) + .catch(() => { + /* noop */ + }); } } diff --git a/lib/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.ts b/lib/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.ts new file mode 100644 index 0000000..04deb9e --- /dev/null +++ b/lib/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.ts @@ -0,0 +1,158 @@ +import { MODEL_NAME, TFLITE_LOADER_NAME, TFLITE_SIMD_LOADER_NAME, WASM_INFERENCE_DIMENSIONS } from '../../../../constants'; +import { Benchmark } from '../../../../utils/Benchmark'; +import { isChromiumImageBitmap } from '../../../../utils/support'; +import { TwilioTFLite } from '../../../../utils/TwilioTFLite'; +import { InputFrame } from '../../../../types'; +import { Pipeline } from '../../../pipelines'; +import { InputFrameDowscaleStage } from './InputFrameDownscaleStage'; +import { PostProcessingStage } from './PostProcessingStage'; + +export interface BackgroundProcessorPipelineOptions { + assetsPath: string; + maskBlurRadius: number; +} + +/** + * @private + */ +export abstract class BackgroundProcessorPipeline extends Pipeline { + private static _twilioTFLite: TwilioTFLite | null = null; + + protected readonly _outputCanvas = new OffscreenCanvas(1, 1); + private readonly _assetsPath: string; + private readonly _benchmark = new Benchmark(); + private readonly _deferInputFrameDownscale = false; + private readonly _inferenceInputCanvas = new OffscreenCanvas(WASM_INFERENCE_DIMENSIONS.width, WASM_INFERENCE_DIMENSIONS.height); + private readonly _inputFrameDownscaleMode = isChromiumImageBitmap() ? 'image-bitmap' : 'canvas'; + private readonly _webgl2Canvas = new OffscreenCanvas(1, 1); + + protected constructor( + options: BackgroundProcessorPipelineOptions + ) { + super(); + + const { + assetsPath, + maskBlurRadius + } = options; + + this._assetsPath = assetsPath; + + this.addStage(new InputFrameDowscaleStage( + this._inferenceInputCanvas, + this._inputFrameDownscaleMode + )); + + this.addStage(new PostProcessingStage( + WASM_INFERENCE_DIMENSIONS, + this._webgl2Canvas, + this._outputCanvas, + maskBlurRadius, + (inputFrame?: InputFrame, webgl2Canvas?: OffscreenCanvas): void => + this._setBackground(inputFrame, webgl2Canvas) + )); + } + + async loadTwilioTFLite(): Promise { + let { _twilioTFLite } = BackgroundProcessorPipeline; + if (!_twilioTFLite) { + _twilioTFLite = new TwilioTFLite(); + await _twilioTFLite.initialize( + this._assetsPath, + MODEL_NAME, + TFLITE_LOADER_NAME, + TFLITE_SIMD_LOADER_NAME + ); + BackgroundProcessorPipeline._twilioTFLite = _twilioTFLite; + } + return _twilioTFLite.isSimdEnabled!; + } + + async render(inputFrame: InputFrame): Promise { + if (!BackgroundProcessorPipeline._twilioTFLite) { + return null; + } + + const [ + inputFrameDownscaleStage, + postProcessingStage + ] = this._stages as [ + InputFrameDowscaleStage, + PostProcessingStage + ]; + + const { + _benchmark, + _deferInputFrameDownscale, + _inferenceInputCanvas: { + height: inferenceInputHeight, + width: inferenceInputWidth + }, + _outputCanvas, + _webgl2Canvas + } = this; + + const { + _twilioTFLite + } = BackgroundProcessorPipeline; + + const isInputVideoFrame = typeof VideoFrame === 'function' + && inputFrame instanceof VideoFrame; + + const { height, width } = isInputVideoFrame + ? { height: inputFrame.displayHeight, width: inputFrame.displayWidth } + : inputFrame as (OffscreenCanvas | HTMLCanvasElement); + + if (_outputCanvas.width !== width) { + _outputCanvas.width = width; + _webgl2Canvas.width = width; + } + if (_outputCanvas.height !== height) { + _outputCanvas.height = height; + _webgl2Canvas.height = height; + } + + _benchmark.start('inputImageResizeDelay'); + const downscalePromise = inputFrameDownscaleStage.render(inputFrame) + .then((downscaledFrameData) => { + _twilioTFLite!.loadInputBuffer(downscaledFrameData); + }); + + if (!_deferInputFrameDownscale) { + await downscalePromise; + } + _benchmark.end('inputImageResizeDelay'); + + _benchmark.start('segmentationDelay'); + const personMask = new ImageData( + _twilioTFLite!.runInference(), + inferenceInputWidth, + inferenceInputHeight + ); + _benchmark.end('segmentationDelay'); + + _benchmark.start('imageCompositionDelay'); + postProcessingStage.render( + inputFrame, + personMask + ); + _benchmark.end('imageCompositionDelay'); + + if (typeof VideoFrame === 'function' + && inputFrame instanceof VideoFrame) { + inputFrame.close(); + } + + return this._outputCanvas; + } + + async setMaskBlurRadius(radius: number): Promise { + (this._stages[1] as PostProcessingStage) + .updateMaskBlurRadius(radius); + } + + protected abstract _setBackground( + inputFrame?: InputFrame, + webgl2Canvas?: OffscreenCanvas + ): void; +} diff --git a/lib/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.ts b/lib/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.ts new file mode 100644 index 0000000..b89e0a0 --- /dev/null +++ b/lib/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.ts @@ -0,0 +1,65 @@ +import { InputFrame } from '../../../../types'; +import { isCanvasBlurSupported } from '../../../../utils/support'; +import { GaussianBlurFilterPipeline } from '../gaussianblurfilterpipeline'; +import { BackgroundProcessorPipeline, BackgroundProcessorPipelineOptions } from './BackgroundProcessorPipeline'; + +namespace GaussianBlurBackgroundProcessorPipeline { + export interface Options extends BackgroundProcessorPipelineOptions { + blurFilterRadius: number; + } +} + +export class GaussianBlurBackgroundProcessorPipeline extends BackgroundProcessorPipeline { + private _blurFilterRadius: number; + private _gaussianBlurFilterPipeline: GaussianBlurFilterPipeline | null; + + constructor(options: GaussianBlurBackgroundProcessorPipeline.Options) { + super(options); + + const { + blurFilterRadius + } = options; + + this._blurFilterRadius = blurFilterRadius; + this._gaussianBlurFilterPipeline = null; + } + + async setBlurFilterRadius(radius: number): Promise { + this._blurFilterRadius = radius; + } + + protected _setBackground( + inputFrame: InputFrame, + webgl2Canvas: OffscreenCanvas + ): void { + const { + _outputCanvas, + _blurFilterRadius + } = this; + + const ctx = _outputCanvas.getContext('2d')!; + if (isCanvasBlurSupported) { + ctx.filter = `blur(${_blurFilterRadius}px)`; + ctx.drawImage( + inputFrame, + 0, + 0 + ); + return; + } + if (!this._gaussianBlurFilterPipeline) { + this._gaussianBlurFilterPipeline = new GaussianBlurFilterPipeline( + webgl2Canvas + ); + this._gaussianBlurFilterPipeline.updateRadius( + _blurFilterRadius + ); + } + this._gaussianBlurFilterPipeline!.render(); + ctx.drawImage( + webgl2Canvas, + 0, + 0 + ); + } +} diff --git a/lib/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.ts b/lib/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.ts new file mode 100644 index 0000000..975316a --- /dev/null +++ b/lib/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.ts @@ -0,0 +1,67 @@ +import { InputFrame } from '../../../../types'; +import { Pipeline } from '../../../pipelines'; + +/** + * @private + */ +export class InputFrameDowscaleStage implements Pipeline.Stage { + private readonly _inputFrameDownscaleMode: 'canvas' | 'image-bitmap'; + private readonly _outputContext: OffscreenCanvasRenderingContext2D; + + constructor( + outputCanvas: OffscreenCanvas, + inputFrameDownscaleMode: 'canvas' | 'image-bitmap' + +) { + this._inputFrameDownscaleMode = inputFrameDownscaleMode; + this._outputContext = outputCanvas.getContext('2d', { willReadFrequently: true })!; + } + + async render(inputFrame: InputFrame): Promise { + const { + _outputContext, + _inputFrameDownscaleMode, + } = this; + + const { + canvas: { + height: resizeHeight, + width: resizeWidth + } + } = _outputContext; + + if (_inputFrameDownscaleMode === 'image-bitmap') { + const downscaledBitmap = await createImageBitmap( + inputFrame, + { + resizeWidth, + resizeHeight, + resizeQuality: 'pixelated' + } + ); + _outputContext.drawImage( + downscaledBitmap, + 0, + 0 + ); + downscaledBitmap.close(); + } else { + _outputContext.drawImage( + inputFrame, + 0, + 0, + resizeWidth, + resizeHeight + ); + } + + const { data } = _outputContext.getImageData( + 0, + 0, + resizeWidth, + resizeHeight + ); + + return data; + } +} diff --git a/lib/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.ts b/lib/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.ts new file mode 100644 index 0000000..6c9a8be --- /dev/null +++ b/lib/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.ts @@ -0,0 +1,85 @@ +import { Dimensions, InputFrame } from '../../../../types'; +import { Pipeline } from '../../../pipelines'; +import { PersonMaskUpscalePipeline } from '../personmaskupscalepipeline'; + +/** + * @private + */ +export class PostProcessingStage implements Pipeline.Stage { + private readonly _inputDimensions: Dimensions; + private _maskBlurRadius: number; + private readonly _outputContext: OffscreenCanvasRenderingContext2D; + private _personMaskUpscalePipeline: PersonMaskUpscalePipeline | null = null; + private readonly _setBackground: (inputFrame?: InputFrame, webgl2Canvas?: OffscreenCanvas) => void; + private readonly _webgl2Canvas: OffscreenCanvas; + + constructor( + inputDimensions: Dimensions, + webgl2Canvas: OffscreenCanvas, + outputCanvas: OffscreenCanvas, + maskBlurRadius: number, + setBackground: (inputFrame?: InputFrame, webgl2Canvas?: OffscreenCanvas) => void + ) { + this._inputDimensions = inputDimensions; + this._maskBlurRadius = maskBlurRadius; + this._outputContext = outputCanvas.getContext('2d')!; + this._webgl2Canvas = webgl2Canvas; + this._setBackground = setBackground; + } + + render( + inputFrame: InputFrame, + personMask: ImageData + ): void { + const { + _inputDimensions, + _maskBlurRadius, + _outputContext, + _setBackground, + _webgl2Canvas + } = this; + + if (!this._personMaskUpscalePipeline) { + this._personMaskUpscalePipeline = new PersonMaskUpscalePipeline( + _inputDimensions, + _webgl2Canvas + ); + this._personMaskUpscalePipeline.updateBilateralFilterConfig({ + sigmaSpace: _maskBlurRadius + }); + } + + this._personMaskUpscalePipeline.render( + inputFrame, + personMask + ); + + _outputContext.save(); + _outputContext.filter = 'none'; + _outputContext.globalCompositeOperation = 'copy'; + _outputContext.drawImage( + _webgl2Canvas, + 0, + 0 + ); + _outputContext.globalCompositeOperation = 'source-in'; + _outputContext.drawImage( + inputFrame, + 0, + 0 + ); + _outputContext.globalCompositeOperation = 'destination-over'; + _setBackground(inputFrame, _webgl2Canvas); + _outputContext.restore(); + } + + updateMaskBlurRadius(radius: number): void { + if (this._maskBlurRadius !== radius) { + this._maskBlurRadius = radius; + this._personMaskUpscalePipeline + ?.updateBilateralFilterConfig({ + sigmaSpace: radius + }); + } + } +} diff --git a/lib/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.ts b/lib/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.ts new file mode 100644 index 0000000..f39c134 --- /dev/null +++ b/lib/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.ts @@ -0,0 +1,136 @@ +import { ImageFit } from '../../../../types'; +import { BackgroundProcessorPipeline, BackgroundProcessorPipelineOptions } from './BackgroundProcessorPipeline'; + +namespace VirtualBackgroundProcessor { + export interface Options extends BackgroundProcessorPipelineOptions { + fitType: ImageFit; + } +} + +/** + * @private + */ +export class VirtualBackgroundProcessorPipeline extends BackgroundProcessorPipeline { + private _backgroundImage: ImageBitmap | null; + private _fitType: ImageFit; + + constructor(options: VirtualBackgroundProcessor.Options) { + super(options); + + const { + fitType + } = options; + + this._backgroundImage = null; + this._fitType = fitType; + } + + async setBackgroundImage( + backgroundImage: ImageBitmap + ): Promise { + this._backgroundImage?.close(); + this._backgroundImage = backgroundImage; + } + + async setFitType(fitType: ImageFit): Promise { + this._fitType = fitType; + } + + protected _setBackground(): void { + const { + _backgroundImage, + _fitType, + _outputCanvas + } = this; + + if (!_backgroundImage) { + return; + } + + const ctx = _outputCanvas.getContext('2d')!; + const imageWidth = _backgroundImage.width; + const imageHeight = _backgroundImage.height; + const canvasWidth = _outputCanvas.width; + const canvasHeight = _outputCanvas.height; + + if (_fitType === ImageFit.Fill) { + ctx.drawImage( + _backgroundImage, + 0, + 0, + imageWidth, + imageHeight, + 0, + 0, + canvasWidth, + canvasHeight + ); + } else if (_fitType === ImageFit.None) { + ctx.drawImage( + _backgroundImage, + 0, + 0, + imageWidth, + imageHeight + ); + } else { + const { x, y, w, h } = this._getFitPosition( + imageWidth, + imageHeight, + canvasWidth, + canvasHeight, + _fitType + ); + ctx.drawImage( + _backgroundImage, + 0, + 0, + imageWidth, + imageHeight, + x, + y, + w, + h + ); + } + } + + private _getFitPosition( + contentWidth: number, + contentHeight: number, + viewportWidth: number, + viewportHeight: number, + type: ImageFit + ): { + h: number, + w: number, + x: number, + y: number + } { + + // Calculate new content width to fit viewport width + let factor = viewportWidth / contentWidth; + let newContentWidth = viewportWidth; + let newContentHeight = factor * contentHeight; + + // Scale down the resulting height and width more + // to fit viewport height if the content still exceeds it + if ((type === ImageFit.Contain && newContentHeight > viewportHeight) + || (type === ImageFit.Cover && viewportHeight > newContentHeight)) { + factor = viewportHeight / newContentHeight; + newContentWidth = factor * newContentWidth; + newContentHeight = viewportHeight; + } + + // Calculate the destination top left corner to center the content + const x = (viewportWidth - newContentWidth) / 2; + const y = (viewportHeight - newContentHeight) / 2; + + return { + x, + y, + w: newContentWidth, + h: newContentHeight, + }; + } +} diff --git a/lib/processors/background/pipelines/backgroundprocessorpipeline/index.ts b/lib/processors/background/pipelines/backgroundprocessorpipeline/index.ts new file mode 100644 index 0000000..6d82815 --- /dev/null +++ b/lib/processors/background/pipelines/backgroundprocessorpipeline/index.ts @@ -0,0 +1,3 @@ +export { BackgroundProcessorPipeline } from './BackgroundProcessorPipeline'; +export { GaussianBlurBackgroundProcessorPipeline } from './GaussianBlurBackgroundProcessorPipeline'; +export { VirtualBackgroundProcessorPipeline } from './VirtualBackgroundProcessorPipeline'; diff --git a/lib/processors/webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts b/lib/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.ts similarity index 94% rename from lib/processors/webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts rename to lib/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.ts index 31c3d84..fb4061d 100644 --- a/lib/processors/webgl2/pipelines/SinglePassGaussianBlurFilterStage.ts +++ b/lib/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.ts @@ -1,4 +1,4 @@ -import { WebGL2Pipeline } from './WebGL2Pipeline' +import { WebGL2Pipeline } from '../../../pipelines'; /** * @private @@ -14,7 +14,7 @@ export class SinglePassGaussianBlurFilterStage extends WebGL2Pipeline.Processing const { height, width - } = glOut.canvas + } = glOut.canvas; super( { @@ -29,19 +29,19 @@ export class SinglePassGaussianBlurFilterStage extends WebGL2Pipeline.Processing uniform vec2 u_texelSize; uniform float u_direction; uniform float u_radius; - + in vec2 v_texCoord; out vec4 outColor; - + const float PI = 3.14159265359; - + float gaussian(float x, float sigma) { float coefficient = 1.0 / sqrt(2.0 * PI) / sigma; float power = -0.5 * x * x / sigma / sigma; return coefficient * exp(power); } - + void main() { vec3 newColor = vec3(0.0); float totalWeight = 0.0; @@ -54,7 +54,7 @@ export class SinglePassGaussianBlurFilterStage extends WebGL2Pipeline.Processing float weight = gaussian(i, u_radius); newColor += weight * texture(u_inputTexture, coord).rgb; totalWeight += weight; - + if (i != 0.0) { shift = vec2(-x, -y) * u_texelSize; coord = vec2(v_texCoord + shift); @@ -85,9 +85,9 @@ export class SinglePassGaussianBlurFilterStage extends WebGL2Pipeline.Processing } ] } - ) + ); - this.updateRadius(0) + this.updateRadius(0); } updateRadius(radius: number): void { @@ -97,6 +97,6 @@ export class SinglePassGaussianBlurFilterStage extends WebGL2Pipeline.Processing type: 'float', values: [radius] } - ]) + ]); } } diff --git a/lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts b/lib/processors/background/pipelines/gaussianblurfilterpipeline/index.ts similarity index 87% rename from lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts rename to lib/processors/background/pipelines/gaussianblurfilterpipeline/index.ts index a68bb0d..bf89fa5 100644 --- a/lib/processors/webgl2/pipelines/GaussianBlurFilterPipeline.ts +++ b/lib/processors/background/pipelines/gaussianblurfilterpipeline/index.ts @@ -1,13 +1,14 @@ +import { WebGL2Pipeline } from '../../../pipelines'; import { SinglePassGaussianBlurFilterStage } from './SinglePassGaussianBlurFilterStage'; -import { WebGL2Pipeline } from './WebGL2Pipeline'; /** * @private */ export class GaussianBlurFilterPipeline extends WebGL2Pipeline { constructor(outputCanvas: OffscreenCanvas | HTMLCanvasElement) { - super() - const glOut = outputCanvas.getContext('webgl2')! as WebGL2RenderingContext + super(); + + const glOut = outputCanvas.getContext('webgl2')! as WebGL2RenderingContext; this.addStage(new SinglePassGaussianBlurFilterStage( glOut, @@ -15,20 +16,20 @@ export class GaussianBlurFilterPipeline extends WebGL2Pipeline { 'texture', 0, 2 - )) + )); this.addStage(new SinglePassGaussianBlurFilterStage( glOut, 'vertical', 'canvas', 2 - )) + )); } updateRadius(radius: number): void { this._stages.forEach( (stage) => (stage as SinglePassGaussianBlurFilterStage) .updateRadius(radius) - ) + ); } } diff --git a/lib/processors/background/pipelines/index.ts b/lib/processors/background/pipelines/index.ts new file mode 100644 index 0000000..274342f --- /dev/null +++ b/lib/processors/background/pipelines/index.ts @@ -0,0 +1,2 @@ +export { GaussianBlurFilterPipeline } from './gaussianblurfilterpipeline'; +export { PersonMaskUpscalePipeline } from './personmaskupscalepipeline'; diff --git a/lib/processors/webgl2/pipelines/SinglePassBilateralFilterStage.ts b/lib/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.ts similarity index 88% rename from lib/processors/webgl2/pipelines/SinglePassBilateralFilterStage.ts rename to lib/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.ts index f361242..4d025e7 100644 --- a/lib/processors/webgl2/pipelines/SinglePassBilateralFilterStage.ts +++ b/lib/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.ts @@ -1,5 +1,5 @@ -import { Dimensions } from '../../../types' -import { WebGL2Pipeline } from './WebGL2Pipeline' +import { Dimensions } from '../../../../types'; +import { WebGL2Pipeline } from '../../../pipelines'; function createSpaceWeights( radius: number, @@ -7,9 +7,9 @@ function createSpaceWeights( texelSize: number ): number[] { return '0'.repeat(radius).split('').map((zero, i) => { - const x = (i + 1) * texelSize - return Math.exp(-0.5 * x * x / sigma / sigma) - }) + const x = (i + 1) * texelSize; + return Math.exp(-0.5 * x * x / sigma / sigma); + }); } function createColorWeights( @@ -17,16 +17,16 @@ function createColorWeights( ): number[] { return '0'.repeat(256).split('').map((zero, i) => { const x = i / 255; - return Math.exp(-0.5 * x * x / sigma / sigma) - }) + return Math.exp(-0.5 * x * x / sigma / sigma); + }); } /** * @private */ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingStage { - private _direction: 'horizontal' | 'vertical' - private _inputDimensions: Dimensions + private readonly _direction: 'horizontal' | 'vertical'; + private readonly _inputDimensions: Dimensions; constructor( glOut: WebGL2RenderingContext, @@ -40,7 +40,7 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta const { height, width - } = outputDimensions + } = outputDimensions; super( { @@ -59,11 +59,11 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta uniform float u_step; uniform float u_spaceWeights[128]; uniform float u_colorWeights[256]; - + in vec2 v_texCoord; out vec4 outColor; - + float calculateColorWeight(vec2 coord, vec3 centerColor) { vec3 coordColor = texture(u_inputFrame, coord).rgb; float x = distance(centerColor, coordColor); @@ -108,7 +108,7 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta float alpha = texture(u_segmentationMask, coord).a; totalWeight += weight; outAlpha += weight * alpha; - + shift = vec2(-x, -y) * u_texelSize; coord = vec2(v_texCoord + shift); colorWeight = calculateColorWeight(coord, centerColor); @@ -117,8 +117,8 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta totalWeight += weight; outAlpha += weight * alpha; } - - outAlpha /= totalWeight; + + outAlpha /= totalWeight; outColor = vec4(vec3(0.0), outAlpha); } `, @@ -145,12 +145,12 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta } ] } - ) + ); - this._direction = direction - this._inputDimensions = inputDimensions - this.updateSigmaColor(0) - this.updateSigmaSpace(0) + this._direction = direction; + this._inputDimensions = inputDimensions; + this.updateSigmaColor(0); + this.updateSigmaSpace(0); } updateSigmaColor(sigmaColor: number): void { @@ -162,39 +162,39 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta sigmaColor ) } - ]) + ]); } updateSigmaSpace(sigmaSpace: number): void { const { height: inputHeight, width: inputWidth - } = this._inputDimensions + } = this._inputDimensions; const { height: outputHeight, width: outputWidth - } = this._outputDimensions + } = this._outputDimensions; sigmaSpace *= Math.max( outputWidth / inputWidth, outputHeight / inputHeight - ) + ); const step = Math.floor( 0.5 * sigmaSpace / Math.log(sigmaSpace) - ) + ); const sigmaTexel = Math.max( 1 / outputWidth, 1 / outputHeight - ) * sigmaSpace + ) * sigmaSpace; const texelSize = 1 / ( this._direction === 'horizontal' ? outputWidth : outputHeight - ) + ); this._setUniformVars([ { @@ -216,6 +216,6 @@ export class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingSta type: 'float', values: [step] } - ]) + ]); } } diff --git a/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts b/lib/processors/background/pipelines/personmaskupscalepipeline/index.ts similarity index 73% rename from lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts rename to lib/processors/background/pipelines/personmaskupscalepipeline/index.ts index d707281..9177499 100644 --- a/lib/processors/webgl2/pipelines/PersonMaskUpscalePipeline.ts +++ b/lib/processors/background/pipelines/personmaskupscalepipeline/index.ts @@ -1,22 +1,24 @@ -import { Dimensions } from '../../../types'; -import { BilateralFilterConfig } from '../helpers/postProcessingHelper'; +import { BilateralFilterConfig, Dimensions } from '../../../../types'; +import { WebGL2Pipeline } from '../../../pipelines'; import { SinglePassBilateralFilterStage } from './SinglePassBilateralFilterStage'; -import { WebGL2Pipeline } from './WebGL2Pipeline'; +/** + * @private + */ export class PersonMaskUpscalePipeline extends WebGL2Pipeline { constructor( inputDimensions: Dimensions, outputCanvas: OffscreenCanvas | HTMLCanvasElement ) { - super() - const glOut = outputCanvas.getContext('webgl2')! as WebGL2RenderingContext + super(); + const glOut = outputCanvas.getContext('webgl2')! as WebGL2RenderingContext; const outputDimensions = { height: outputCanvas.height, width: outputCanvas.width - } + }; - this.addStage(new WebGL2Pipeline.InputStage(glOut)) + this.addStage(new WebGL2Pipeline.InputStage(glOut)); this.addStage(new SinglePassBilateralFilterStage( glOut, @@ -26,7 +28,7 @@ export class PersonMaskUpscalePipeline extends WebGL2Pipeline { outputDimensions, 1, 2 - )) + )); this.addStage(new SinglePassBilateralFilterStage( glOut, @@ -35,7 +37,7 @@ export class PersonMaskUpscalePipeline extends WebGL2Pipeline { inputDimensions, outputDimensions, 2 - )) + )); } updateBilateralFilterConfig(config: BilateralFilterConfig) { @@ -45,15 +47,15 @@ export class PersonMaskUpscalePipeline extends WebGL2Pipeline { ] = this._stages as [ any, SinglePassBilateralFilterStage - ] - const { sigmaSpace } = config + ]; + const { sigmaSpace } = config; if (typeof sigmaSpace === 'number') { bilateralFilterStages.forEach( (stage: SinglePassBilateralFilterStage) => { - stage.updateSigmaColor(0.1) - stage.updateSigmaSpace(sigmaSpace) + stage.updateSigmaColor(0.1); + stage.updateSigmaSpace(sigmaSpace); } - ) + ); } } } diff --git a/lib/processors/webgl2/pipelines/Pipeline.ts b/lib/processors/pipelines/Pipeline.ts similarity index 62% rename from lib/processors/webgl2/pipelines/Pipeline.ts rename to lib/processors/pipelines/Pipeline.ts index df90f29..0fd05ea 100644 --- a/lib/processors/webgl2/pipelines/Pipeline.ts +++ b/lib/processors/pipelines/Pipeline.ts @@ -2,21 +2,21 @@ * @private */ export class Pipeline implements Pipeline.Stage { - protected _stages: Pipeline.Stage[] = [] + protected readonly _stages: Pipeline.Stage[] = []; addStage(stage: Pipeline.Stage): void { - this._stages.push(stage) + this._stages.push(stage); } render(...args: any[]): void { this._stages.forEach((stage) => { - stage.render(...args) - }) + stage.render(...args); + }); } } export namespace Pipeline { export interface Stage { - render(...args: any[]): void + render(...args: any[]): void; } } diff --git a/lib/processors/pipelines/index.ts b/lib/processors/pipelines/index.ts new file mode 100644 index 0000000..7c722c6 --- /dev/null +++ b/lib/processors/pipelines/index.ts @@ -0,0 +1,2 @@ +export { Pipeline } from './Pipeline'; +export { WebGL2Pipeline } from './webgl2pipeline'; diff --git a/lib/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.ts b/lib/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.ts new file mode 100644 index 0000000..fa41391 --- /dev/null +++ b/lib/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.ts @@ -0,0 +1,108 @@ +import { InputFrame } from '../../../types'; +import { Pipeline } from '../Pipeline'; +import { createTexture } from './webgl2PipelineHelpers'; + +/** + * @private + */ +export class WebGL2PipelineInputStage implements Pipeline.Stage { + private readonly _glOut: WebGL2RenderingContext; + private readonly _inputFrameTexture: WebGLTexture; + private _inputTexture: WebGLTexture | null = null; + + constructor(glOut: WebGL2RenderingContext) { + const { height, width } = glOut.canvas; + this._glOut = glOut; + this._inputFrameTexture = createTexture( + glOut, + glOut.RGBA8, + width, + height, + glOut.NEAREST, + glOut.NEAREST + )!; + } + + cleanUp(): void { + const { + _glOut, + _inputFrameTexture, + _inputTexture + } = this; + _glOut.deleteTexture(_inputFrameTexture); + _glOut.deleteTexture(_inputTexture); + } + + render( + inputFrame?: InputFrame, + inputTextureData?: ImageData + ): void { + const { + _glOut, + _inputFrameTexture + } = this; + + const { height, width } = _glOut.canvas; + _glOut.viewport(0, 0, width, height); + _glOut.clearColor(0, 0, 0, 0); + _glOut.clear(_glOut.COLOR_BUFFER_BIT); + + if (inputFrame) { + _glOut.activeTexture(_glOut.TEXTURE0); + _glOut.bindTexture( + _glOut.TEXTURE_2D, + _inputFrameTexture + ); + _glOut.texSubImage2D( + _glOut.TEXTURE_2D, + 0, + 0, + 0, + width, + height, + _glOut.RGBA, + _glOut.UNSIGNED_BYTE, + inputFrame + ); + } + + if (!inputTextureData) { + return; + } + const { + data, + height: textureHeight, + width: textureWidth + } = inputTextureData; + + if (!this._inputTexture) { + this._inputTexture = createTexture( + _glOut, + _glOut.RGBA8, + textureWidth, + textureHeight, + _glOut.NEAREST, + _glOut.NEAREST + ); + } + + _glOut.viewport(0, 0, textureWidth, textureHeight); + _glOut.activeTexture(_glOut.TEXTURE1); + + _glOut.bindTexture( + _glOut.TEXTURE_2D, + this._inputTexture + ); + _glOut.texSubImage2D( + _glOut.TEXTURE_2D, + 0, + 0, + 0, + textureWidth, + textureHeight, + _glOut.RGBA, + _glOut.UNSIGNED_BYTE, + data + ); + } +} diff --git a/lib/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.ts b/lib/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.ts new file mode 100644 index 0000000..28b7791 --- /dev/null +++ b/lib/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.ts @@ -0,0 +1,240 @@ +import { Dimensions } from '../../../types'; +import { Pipeline } from '../Pipeline'; +import { compileShader, createPipelineStageProgram, createTexture, initBuffer } from './webgl2PipelineHelpers'; + +interface InputConfig { + textureName: string; + textureUnit: number; +} + +interface OutputConfig { + fragmentShaderSource: string; + glOut: WebGL2RenderingContext; + height?: number; + textureUnit?: number; + type: 'canvas' | 'texture'; + uniformVars?: UniformVarInfo[]; + vertexShaderSource?: string; + width?: number; +} + +interface UniformVarInfo { + name: string; + type: 'float' | 'int' | 'uint' | 'float:v'; + values: number[]; +} + +/**; + * @private + */ +export class WebGL2PipelineProcessingStage implements Pipeline.Stage { + protected readonly _outputDimensions: Dimensions; + private readonly _fragmentShader: WebGLSampler; + private readonly _glOut: WebGL2RenderingContext; + private readonly _outputFramebuffer: WebGLBuffer | null = null; + private readonly _outputTexture: WebGLTexture | null = null; + private readonly _outputTextureUnit: number; + private readonly _positionBuffer: WebGLBuffer; + private readonly _program: WebGLProgram; + private readonly _texCoordBuffer: WebGLBuffer; + private readonly _vertexShader: WebGLShader; + + constructor( + inputConfig: InputConfig, + outputConfig: OutputConfig + ) { + const { + textureName, + textureUnit, + } = inputConfig; + + const { glOut } = outputConfig; + this._glOut = glOut; + + const { + fragmentShaderSource, + height = glOut.canvas.height, + textureUnit: outputTextureUnit = textureUnit + 1, + type: outputType, + uniformVars = [], + vertexShaderSource = `#version 300 es + in vec2 a_position; + in vec2 a_texCoord; + + out vec2 v_texCoord; + + void main() { + gl_Position = vec4(a_position${ + outputType === 'canvas' + ? ' * vec2(1.0, -1.0)' + : '' + }, 0.0, 1.0); + v_texCoord = a_texCoord; + } + `, + width = glOut.canvas.width + } = outputConfig; + + this._outputDimensions = { + height, + width + }; + + this._outputTextureUnit = outputTextureUnit; + + this._fragmentShader = compileShader( + glOut, + glOut.FRAGMENT_SHADER, + fragmentShaderSource + ); + + this._vertexShader = compileShader( + glOut, + glOut.VERTEX_SHADER, + vertexShaderSource + ); + + this._positionBuffer = initBuffer( + glOut, + [ + -1.0, -1.0, + 1.0, -1.0, + -1.0, 1.0, + 1.0, 1.0, + ] + )!; + + this._texCoordBuffer = initBuffer( + glOut, + [ + 0.0, 0.0, + 1.0, 0.0, + 0.0, 1.0, + 1.0, 1.0, + ] + )!; + + if (outputType === 'texture') { + this._outputTexture = createTexture( + glOut, + glOut.RGBA8, + width, + height + ); + this._outputFramebuffer = glOut.createFramebuffer(); + glOut.bindFramebuffer( + glOut.FRAMEBUFFER, + this._outputFramebuffer + ); + glOut.framebufferTexture2D( + glOut.FRAMEBUFFER, + glOut.COLOR_ATTACHMENT0, + glOut.TEXTURE_2D, + this._outputTexture, + 0 + ); + } + + const program = createPipelineStageProgram( + glOut, + this._vertexShader, + this._fragmentShader, + this._positionBuffer, + this._texCoordBuffer + ); + this._program = program; + + this._setUniformVars([ + { + name: textureName, + type: 'int', + values: [textureUnit] + }, + ...uniformVars + ]); + } + + cleanUp(): void { + const { + _fragmentShader, + _glOut, + _positionBuffer, + _program, + _texCoordBuffer, + _vertexShader + } = this; + _glOut.deleteProgram(_program); + _glOut.deleteBuffer(_texCoordBuffer); + _glOut.deleteBuffer(_positionBuffer); + _glOut.deleteShader(_vertexShader); + _glOut.deleteShader(_fragmentShader); + } + + render(): void { + const { + _glOut, + _outputDimensions: { + height, + width + }, + _outputFramebuffer, + _outputTexture, + _outputTextureUnit, + _program + } = this; + + _glOut.viewport(0, 0, width, height); + _glOut.useProgram(_program); + + if (_outputTexture) { + _glOut.activeTexture( + _glOut.TEXTURE0 + + _outputTextureUnit + ); + _glOut.bindTexture( + _glOut.TEXTURE_2D, + _outputTexture + ); + } + _glOut.bindFramebuffer( + _glOut.FRAMEBUFFER, + _outputFramebuffer + ); + _glOut.drawArrays( + _glOut.TRIANGLE_STRIP, + 0, + 4 + ); + } + + protected _setUniformVars(uniformVars: UniformVarInfo[]) { + const { + _glOut, + _program + } = this; + + _glOut.useProgram(_program); + + uniformVars.forEach(({ name, type, values }) => { + const uniformVarLocation = _glOut + .getUniformLocation( + _program, + name + ); + const isVector = type.split(':')[1] === 'v'; + if (isVector) { + // @ts-ignore + _glOut[`uniform1${type[0]}v`]( + uniformVarLocation, + values + ); + } else { + // @ts-ignore + _glOut[`uniform${values.length}${type[0]}`]( + uniformVarLocation, + ...values + ); + } + }); + } +} diff --git a/lib/processors/pipelines/webgl2pipeline/index.ts b/lib/processors/pipelines/webgl2pipeline/index.ts new file mode 100644 index 0000000..e3d142e --- /dev/null +++ b/lib/processors/pipelines/webgl2pipeline/index.ts @@ -0,0 +1,31 @@ +import { InputFrame } from '../../../types'; +import { Pipeline } from '../Pipeline'; +import { WebGL2PipelineInputStage } from './WebGL2PipelineInputStage'; +import { WebGL2PipelineProcessingStage } from './WebGL2PipelineProcessingStage'; + +/** + * @private + */ +export class WebGL2Pipeline extends Pipeline { + static InputStage = WebGL2PipelineInputStage; + static ProcessingStage = WebGL2PipelineProcessingStage; + protected readonly _stages: (WebGL2PipelineInputStage | WebGL2PipelineProcessingStage)[] = []; + + cleanUp(): void { + this._stages.forEach( + (stage) => stage.cleanUp() + ); + } + + render( + inputFrame?: InputFrame, + inputTextureData?: ImageData + ): void { + const [inputStage, ...otherStages] = this._stages; + inputStage.render(inputFrame, inputTextureData); + otherStages.forEach( + (stage) => (stage as WebGL2PipelineProcessingStage) + .render() + ); + } +} diff --git a/lib/processors/webgl2/helpers/webglHelper.ts b/lib/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.ts similarity index 61% rename from lib/processors/webgl2/helpers/webglHelper.ts rename to lib/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.ts index 0efd484..0980941 100644 --- a/lib/processors/webgl2/helpers/webglHelper.ts +++ b/lib/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.ts @@ -1,14 +1,6 @@ /** - * Use it along with boyswan.glsl-literal VSCode extension - * to get GLSL syntax highlighting. - * https://marketplace.visualstudio.com/items?itemName=boyswan.glsl-literal - * - * On VSCode OSS, boyswan.glsl-literal requires slevesque.shader extension - * to be installed as well. - * https://marketplace.visualstudio.com/items?itemName=slevesque.shader + * @private */ -export const glsl = String.raw - export function createPipelineStageProgram( gl: WebGL2RenderingContext, vertexShader: WebGLShader, @@ -16,52 +8,61 @@ export function createPipelineStageProgram( positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer ) { - const program = createProgram(gl, vertexShader, fragmentShader) + const program = createProgram(gl, vertexShader, fragmentShader); - const positionAttributeLocation = gl.getAttribLocation(program, 'a_position') - gl.enableVertexAttribArray(positionAttributeLocation) - gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer) - gl.vertexAttribPointer(positionAttributeLocation, 2, gl.FLOAT, false, 0, 0) + const positionAttributeLocation = gl.getAttribLocation(program, 'a_position'); + gl.enableVertexAttribArray(positionAttributeLocation); + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + gl.vertexAttribPointer(positionAttributeLocation, 2, gl.FLOAT, false, 0, 0); - const texCoordAttributeLocation = gl.getAttribLocation(program, 'a_texCoord') - gl.enableVertexAttribArray(texCoordAttributeLocation) - gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer) - gl.vertexAttribPointer(texCoordAttributeLocation, 2, gl.FLOAT, false, 0, 0) + const texCoordAttributeLocation = gl.getAttribLocation(program, 'a_texCoord'); + gl.enableVertexAttribArray(texCoordAttributeLocation); + gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); + gl.vertexAttribPointer(texCoordAttributeLocation, 2, gl.FLOAT, false, 0, 0); - return program + return program; } +/** + * @private + */ export function createProgram( gl: WebGL2RenderingContext, vertexShader: WebGLShader, fragmentShader: WebGLShader ) { - const program = gl.createProgram()! - gl.attachShader(program, vertexShader) - gl.attachShader(program, fragmentShader) - gl.linkProgram(program) + const program = gl.createProgram()!; + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { throw new Error( `Could not link WebGL program: ${gl.getProgramInfoLog(program)}` - ) + ); } - return program + return program; } +/** + * @private + */ export function compileShader( gl: WebGL2RenderingContext, shaderType: number, shaderSource: string ) { - const shader = gl.createShader(shaderType)! - gl.shaderSource(shader, shaderSource) - gl.compileShader(shader) + const shader = gl.createShader(shaderType)!; + gl.shaderSource(shader, shaderSource); + gl.compileShader(shader); if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { - throw new Error(`Could not compile shader: ${gl.getShaderInfoLog(shader)}`) + throw new Error(`Could not compile shader: ${gl.getShaderInfoLog(shader)}`); } - return shader + return shader; } +/** + * @private + */ export function createTexture( gl: WebGL2RenderingContext, internalformat: number, @@ -70,16 +71,19 @@ export function createTexture( minFilter: GLint = gl.NEAREST, magFilter: GLint = gl.NEAREST ) { - const texture = gl.createTexture() - gl.bindTexture(gl.TEXTURE_2D, texture) - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE) - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE) - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, minFilter) - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, magFilter) - gl.texStorage2D(gl.TEXTURE_2D, 1, internalformat, width, height) - return texture + const texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, minFilter); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, magFilter); + gl.texStorage2D(gl.TEXTURE_2D, 1, internalformat, width, height); + return texture; } +/** + * @private + */ export function initBuffer( gl: WebGL2RenderingContext, data: number[] @@ -89,7 +93,7 @@ export function initBuffer( gl.bufferData( gl.ARRAY_BUFFER, new Float32Array(data), - gl.STATIC_DRAW, + gl.STATIC_DRAW ); return buffer; } diff --git a/lib/processors/webgl2/helpers/postProcessingHelper.ts b/lib/processors/webgl2/helpers/postProcessingHelper.ts deleted file mode 100644 index 6a79c18..0000000 --- a/lib/processors/webgl2/helpers/postProcessingHelper.ts +++ /dev/null @@ -1,3 +0,0 @@ -export type BilateralFilterConfig = { - sigmaSpace?: number -} diff --git a/lib/processors/webgl2/helpers/segmentationHelper.ts b/lib/processors/webgl2/helpers/segmentationHelper.ts deleted file mode 100644 index 9c58b02..0000000 --- a/lib/processors/webgl2/helpers/segmentationHelper.ts +++ /dev/null @@ -1,14 +0,0 @@ -export type InputResolution = '640x360' | '256x256' | '256x144' | '160x96' | string - -export const inputResolutions: { - [resolution in InputResolution]: [number, number] -} = { - '640x360': [640, 360], - '256x256': [256, 256], - '256x144': [256, 144], - '160x96': [160, 96], -} - -export type SegmentationConfig = { - inputResolution: InputResolution -} diff --git a/lib/processors/webgl2/index.ts b/lib/processors/webgl2/index.ts deleted file mode 100644 index 0cf258a..0000000 --- a/lib/processors/webgl2/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * This pipeline is based on Volcomix's react project. - * https://github.com/Volcomix/virtual-background - * It was modified and converted into a module to work with - * Twilio's Video Processor - */ -export { GaussianBlurFilterPipeline } from './pipelines/GaussianBlurFilterPipeline'; -export { PersonMaskUpscalePipeline } from './pipelines/PersonMaskUpscalePipeline'; diff --git a/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts b/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts deleted file mode 100644 index aa94e58..0000000 --- a/lib/processors/webgl2/pipelines/WebGL2Pipeline.ts +++ /dev/null @@ -1,389 +0,0 @@ -import { Dimensions } from '../../../types' -import { - createPipelineStageProgram, - createTexture, - compileShader, - initBuffer -} from '../helpers/webglHelper' -import { Pipeline } from './Pipeline' - -interface InputConfig { - textureName: string - textureUnit: number -} - -interface OutputConfig { - fragmentShaderSource: string - glOut: WebGL2RenderingContext - height?: number - textureUnit?: number - type: 'canvas' | 'texture' - uniformVars?: UniformVarInfo[] - vertexShaderSource?: string - width?: number -} - -interface UniformVarInfo { - name: string - type: 'float' | 'int' | 'uint' | 'float:v' - values: number[] -} - -/** - * @private - */ -class WebGL2PipelineInputStage implements Pipeline.Stage { - private _glOut: WebGL2RenderingContext - private _inputFrameTexture: WebGLTexture - private _inputTexture: WebGLTexture | null - private _inputTextureData: ImageData | null - - constructor(glOut: WebGL2RenderingContext) { - const { height, width } = glOut.canvas - this._glOut = glOut - this._inputFrameTexture = createTexture( - glOut, - glOut.RGBA8, - width, - height, - glOut.NEAREST, - glOut.NEAREST - )! - this._inputTexture = null; - this._inputTextureData = null; - } - - cleanUp(): void { - const { - _glOut, - _inputFrameTexture, - _inputTexture - } = this - _glOut.deleteTexture(_inputFrameTexture) - _glOut.deleteTexture(_inputTexture) - } - - render(inputFrame?: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement | VideoFrame): void { - const { - _glOut, - _inputFrameTexture, - _inputTextureData - } = this - - const { height, width } = _glOut.canvas; - _glOut.viewport(0, 0, width, height) - _glOut.clearColor(0, 0, 0, 0) - _glOut.clear(_glOut.COLOR_BUFFER_BIT) - - if (inputFrame) { - _glOut.activeTexture(_glOut.TEXTURE0) - _glOut.bindTexture( - _glOut.TEXTURE_2D, - _inputFrameTexture - ) - _glOut.texSubImage2D( - _glOut.TEXTURE_2D, - 0, - 0, - 0, - width, - height, - _glOut.RGBA, - _glOut.UNSIGNED_BYTE, - inputFrame - ) - } - - if (!_inputTextureData) { - return - } - const { - data, - height: textureHeight, - width: textureWidth - } = _inputTextureData - - if (!this._inputTexture) { - this._inputTexture = createTexture( - _glOut, - _glOut.RGBA8, - textureWidth, - textureHeight, - _glOut.NEAREST, - _glOut.NEAREST - ) - } - - _glOut.viewport(0, 0, textureWidth, textureHeight) - _glOut.activeTexture(_glOut.TEXTURE1) - - _glOut.bindTexture( - _glOut.TEXTURE_2D, - this._inputTexture - ) - _glOut.texSubImage2D( - _glOut.TEXTURE_2D, - 0, - 0, - 0, - textureWidth, - textureHeight, - _glOut.RGBA, - _glOut.UNSIGNED_BYTE, - data - ) - } - - setInputTextureData(inputTextureData: ImageData): void { - this._inputTextureData = inputTextureData - } -} - -/** - * @private - */ -class WebGL2PipelineProcessingStage implements Pipeline.Stage { - protected _outputDimensions: Dimensions - private _fragmentShader: WebGLSampler - private _glOut: WebGL2RenderingContext - private _outputFramebuffer: WebGLBuffer | null = null - private _outputTexture: WebGLTexture | null = null - private _outputTextureUnit: number - private _positionBuffer: WebGLBuffer - private _program: WebGLProgram - private _texCoordBuffer: WebGLBuffer - private _vertexShader: WebGLShader - - constructor( - inputConfig: InputConfig, - outputConfig: OutputConfig - ) { - const { - textureName, - textureUnit, - } = inputConfig - - const { glOut } = outputConfig - this._glOut = glOut - - const { - fragmentShaderSource, - height = glOut.canvas.height, - textureUnit: outputTextureUnit = textureUnit + 1, - type: outputType, - uniformVars = [], - vertexShaderSource = `#version 300 es - in vec2 a_position; - in vec2 a_texCoord; - - out vec2 v_texCoord; - - void main() { - gl_Position = vec4(a_position${ - outputType === 'canvas' - ? ' * vec2(1.0, -1.0)' - : '' - }, 0.0, 1.0); - v_texCoord = a_texCoord; - } - `, - width = glOut.canvas.width - } = outputConfig - - this._outputDimensions = { - height, - width - } - - this._outputTextureUnit = outputTextureUnit - - this._fragmentShader = compileShader( - glOut, - glOut.FRAGMENT_SHADER, - fragmentShaderSource - ) - - this._vertexShader = compileShader( - glOut, - glOut.VERTEX_SHADER, - vertexShaderSource - ) - - this._positionBuffer = initBuffer( - glOut, - [ - -1.0, -1.0, - 1.0, -1.0, - -1.0, 1.0, - 1.0, 1.0, - ] - )! - - this._texCoordBuffer = initBuffer( - glOut, - [ - 0.0, 0.0, - 1.0, 0.0, - 0.0, 1.0, - 1.0, 1.0, - ] - )! - - if (outputType === 'texture') { - this._outputTexture = createTexture( - glOut, - glOut.RGBA8, - width, - height - ) - this._outputFramebuffer = glOut.createFramebuffer() - glOut.bindFramebuffer( - glOut.FRAMEBUFFER, - this._outputFramebuffer - ) - glOut.framebufferTexture2D( - glOut.FRAMEBUFFER, - glOut.COLOR_ATTACHMENT0, - glOut.TEXTURE_2D, - this._outputTexture, - 0 - ) - } - - const program = createPipelineStageProgram( - glOut, - this._vertexShader, - this._fragmentShader, - this._positionBuffer, - this._texCoordBuffer - ) - this._program = program - - this._setUniformVars([ - { - name: textureName, - type: 'int', - values: [textureUnit] - }, - ...uniformVars - ]) - } - - cleanUp(): void { - const { - _fragmentShader, - _glOut, - _positionBuffer, - _program, - _texCoordBuffer, - _vertexShader - } = this - _glOut.deleteProgram(_program) - _glOut.deleteBuffer(_texCoordBuffer) - _glOut.deleteBuffer(_positionBuffer) - _glOut.deleteShader(_vertexShader) - _glOut.deleteShader(_fragmentShader) - } - - render(): void { - const { - _glOut, - _outputDimensions: { - height, - width - }, - _outputFramebuffer, - _outputTexture, - _outputTextureUnit, - _program - } = this - - _glOut.viewport(0, 0, width, height) - _glOut.useProgram(_program) - - if (_outputTexture) { - _glOut.activeTexture( - _glOut.TEXTURE0 - + _outputTextureUnit - ) - _glOut.bindTexture( - _glOut.TEXTURE_2D, - _outputTexture - ) - } - _glOut.bindFramebuffer( - _glOut.FRAMEBUFFER, - _outputFramebuffer - ) - _glOut.drawArrays( - _glOut.TRIANGLE_STRIP, - 0, - 4 - ) - } - - protected _setUniformVars(uniformVars: UniformVarInfo[]) { - const { - _glOut, - _program - } = this - - _glOut.useProgram(_program) - - uniformVars.forEach(({ - name, - type, - values - }) => { - const uniformVarLocation = _glOut - .getUniformLocation( - _program, - name - ) - - const isVector = type.split(':')[1] === 'v'; - if (isVector) { - // @ts-ignore - _glOut[`uniform1${type[0]}v`]( - uniformVarLocation, - values - ) - } else { - // @ts-ignore - _glOut[`uniform${values.length}${type[0]}`]( - uniformVarLocation, - ...values - ) - } - }) - } -} - -/** - * @private - */ -export class WebGL2Pipeline extends Pipeline { - static InputStage = WebGL2PipelineInputStage - static ProcessingStage = WebGL2PipelineProcessingStage - protected _stages: (WebGL2PipelineInputStage | WebGL2PipelineProcessingStage)[] = [] - - cleanUp(): void { - this._stages.forEach( - (stage) => stage.cleanUp() - ) - } - - render(inputFrame?: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement | VideoFrame): void { - const [inputStage, ...otherStages] = this._stages; - inputStage.render(inputFrame); - otherStages.forEach( - (stage) => (stage as WebGL2PipelineProcessingStage) - .render() - ); - } - - setInputTextureData(inputTextureData: ImageData): void { - const [inputStage] = this._stages as [WebGL2PipelineInputStage] - inputStage.setInputTextureData(inputTextureData); - } -} diff --git a/lib/types.ts b/lib/types.ts index 40c46b1..88283e5 100644 --- a/lib/types.ts +++ b/lib/types.ts @@ -28,6 +28,18 @@ export interface Dimensions { width: number; } +/** + * @private + */ +export type BilateralFilterConfig = { + sigmaSpace?: number +}; + +/** + * @private + */ +export type InputFrame = OffscreenCanvas | HTMLCanvasElement | VideoFrame; + /** * ImageFit specifies the positioning of an image inside a viewport. */ From 2bc2467f370e5e003f07bdcc7a01c3e243356339 Mon Sep 17 00:00:00 2001 From: mmalavalli Date: Tue, 30 Jul 2024 00:30:27 -0500 Subject: [PATCH 09/28] Implementing configurable web workers for the background processing pipelines. --- examples/README.md | 3 +- examples/server.js | 1 - examples/virtualbackground/index.js | 15 ++++-- lib/constants.ts | 3 ++ .../background/BackgroundProcessor.ts | 34 ++++++++++---- .../GaussianBlurBackgroundProcessor.ts | 21 ++++++--- .../background/VirtualBackgroundProcessor.ts | 23 ++++++--- .../BackgroundProcessorPipeline.proxy.ts | 30 ++++++++++++ .../BackgroundProcessorPipeline.ts | 2 +- ...anBlurBackgroundProcessorPipeline.proxy.ts | 39 +++++++++++++++ ...GaussianBlurBackgroundProcessorPipeline.ts | 8 ++-- ...nBlurBackgroundProcessorPipeline.worker.ts | 19 ++++++++ ...irtualBackgroundProcessorPipeline.proxy.ts | 47 +++++++++++++++++++ .../VirtualBackgroundProcessorPipeline.ts | 8 ++-- ...rtualBackgroundProcessorPipeline.worker.ts | 19 ++++++++ .../backgroundprocessorpipeline/index.ts | 3 ++ lib/utils/TwilioTFLite.ts | 10 ++++ lib/utils/support.ts | 3 +- package.json | 8 +++- 19 files changed, 254 insertions(+), 42 deletions(-) create mode 100644 lib/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.proxy.ts create mode 100644 lib/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.proxy.ts create mode 100644 lib/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.worker.ts create mode 100644 lib/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.proxy.ts create mode 100644 lib/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.worker.ts diff --git a/examples/README.md b/examples/README.md index ffd8a65..d395278 100644 --- a/examples/README.md +++ b/examples/README.md @@ -8,8 +8,9 @@ Go to `${PROJECT_ROOT}/examples` and run `npm install`. It will install the depe Open `http://localhost:3000` in a Chrome tab. The app captures your camera upon loading and plays it in a `