diff --git a/dist/build/selfie_segmentation_landscape.tflite b/dist/build/selfie_segmentation_landscape.tflite new file mode 100644 index 0000000..3a508f8 Binary files /dev/null and b/dist/build/selfie_segmentation_landscape.tflite differ diff --git a/dist/build/tflite-1-0-0.js b/dist/build/tflite-1-0-0.js new file mode 100644 index 0000000..7d2e41c --- /dev/null +++ b/dist/build/tflite-1-0-0.js @@ -0,0 +1,21 @@ + +var createTwilioTFLiteModule = (function() { + var _scriptDir = typeof document !== 'undefined' && document.currentScript ? document.currentScript.src : undefined; + if (typeof __filename !== 'undefined') _scriptDir = _scriptDir || __filename; + return ( +function(createTwilioTFLiteModule) { + createTwilioTFLiteModule = createTwilioTFLiteModule || {}; + +var Module=typeof createTwilioTFLiteModule!=="undefined"?createTwilioTFLiteModule:{};var readyPromiseResolve,readyPromiseReject;Module["ready"]=new Promise(function(resolve,reject){readyPromiseResolve=resolve;readyPromiseReject=reject});var moduleOverrides={};var key;for(key in Module){if(Module.hasOwnProperty(key)){moduleOverrides[key]=Module[key]}}var arguments_=[];var thisProgram="./this.program";var quit_=function(status,toThrow){throw toThrow};var ENVIRONMENT_IS_WEB=false;var ENVIRONMENT_IS_WORKER=false;var ENVIRONMENT_IS_NODE=false;var ENVIRONMENT_IS_SHELL=false;ENVIRONMENT_IS_WEB=typeof window==="object";ENVIRONMENT_IS_WORKER=typeof importScripts==="function";ENVIRONMENT_IS_NODE=typeof process==="object"&&typeof process.versions==="object"&&typeof process.versions.node==="string";ENVIRONMENT_IS_SHELL=!ENVIRONMENT_IS_WEB&&!ENVIRONMENT_IS_NODE&&!ENVIRONMENT_IS_WORKER;var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var read_,readAsync,readBinary,setWindowTitle;var nodeFS;var nodePath;if(ENVIRONMENT_IS_NODE){if(ENVIRONMENT_IS_WORKER){scriptDirectory=require("path").dirname(scriptDirectory)+"/"}else{scriptDirectory=__dirname+"/"}read_=function shell_read(filename,binary){if(!nodeFS)nodeFS=require("fs");if(!nodePath)nodePath=require("path");filename=nodePath["normalize"](filename);return nodeFS["readFileSync"](filename,binary?null:"utf8")};readBinary=function readBinary(filename){var ret=read_(filename,true);if(!ret.buffer){ret=new Uint8Array(ret)}assert(ret.buffer);return ret};if(process["argv"].length>1){thisProgram=process["argv"][1].replace(/\\/g,"/")}arguments_=process["argv"].slice(2);process["on"]("uncaughtException",function(ex){if(!(ex instanceof ExitStatus)){throw ex}});process["on"]("unhandledRejection",abort);quit_=function(status){process["exit"](status)};Module["inspect"]=function(){return"[Emscripten Module object]"}}else if(ENVIRONMENT_IS_SHELL){if(typeof read!="undefined"){read_=function shell_read(f){return read(f)}}readBinary=function readBinary(f){var data;if(typeof readbuffer==="function"){return new Uint8Array(readbuffer(f))}data=read(f,"binary");assert(typeof data==="object");return data};if(typeof scriptArgs!="undefined"){arguments_=scriptArgs}else if(typeof arguments!="undefined"){arguments_=arguments}if(typeof quit==="function"){quit_=function(status){quit(status)}}if(typeof print!=="undefined"){if(typeof console==="undefined")console={};console.log=print;console.warn=console.error=typeof printErr!=="undefined"?printErr:print}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){if(ENVIRONMENT_IS_WORKER){scriptDirectory=self.location.href}else if(typeof document!=="undefined"&&document.currentScript){scriptDirectory=document.currentScript.src}if(_scriptDir){scriptDirectory=_scriptDir}if(scriptDirectory.indexOf("blob:")!==0){scriptDirectory=scriptDirectory.substr(0,scriptDirectory.lastIndexOf("/")+1)}else{scriptDirectory=""}{read_=function(url){var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.send(null);return xhr.responseText};if(ENVIRONMENT_IS_WORKER){readBinary=function(url){var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=function(url,onload,onerror){var xhr=new XMLHttpRequest;xhr.open("GET",url,true);xhr.responseType="arraybuffer";xhr.onload=function(){if(xhr.status==200||xhr.status==0&&xhr.response){onload(xhr.response);return}onerror()};xhr.onerror=onerror;xhr.send(null)}}setWindowTitle=function(title){document.title=title}}else{}var out=Module["print"]||console.log.bind(console);var err=Module["printErr"]||console.warn.bind(console);for(key in moduleOverrides){if(moduleOverrides.hasOwnProperty(key)){Module[key]=moduleOverrides[key]}}moduleOverrides=null;if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];if(Module["quit"])quit_=Module["quit"];var wasmBinary;if(Module["wasmBinary"])wasmBinary=Module["wasmBinary"];var noExitRuntime=Module["noExitRuntime"]||true;if(typeof WebAssembly!=="object"){abort("no native wasm support detected")}var wasmMemory;var ABORT=false;var EXITSTATUS;function assert(condition,text){if(!condition){abort("Assertion failed: "+text)}}var UTF8Decoder=typeof TextDecoder!=="undefined"?new TextDecoder("utf8"):undefined;function UTF8ArrayToString(heap,idx,maxBytesToRead){var endIdx=idx+maxBytesToRead;var endPtr=idx;while(heap[endPtr]&&!(endPtr>=endIdx))++endPtr;if(endPtr-idx>16&&heap.subarray&&UTF8Decoder){return UTF8Decoder.decode(heap.subarray(idx,endPtr))}else{var str="";while(idx>10,56320|ch&1023)}}}return str}function UTF8ToString(ptr,maxBytesToRead){return ptr?UTF8ArrayToString(HEAPU8,ptr,maxBytesToRead):""}function writeAsciiToMemory(str,buffer,dontAddNull){for(var i=0;i>0]=str.charCodeAt(i)}if(!dontAddNull)HEAP8[buffer>>0]=0}function alignUp(x,multiple){if(x%multiple>0){x+=multiple-x%multiple}return x}var buffer,HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;function updateGlobalBufferAndViews(buf){buffer=buf;Module["HEAP8"]=HEAP8=new Int8Array(buf);Module["HEAP16"]=HEAP16=new Int16Array(buf);Module["HEAP32"]=HEAP32=new Int32Array(buf);Module["HEAPU8"]=HEAPU8=new Uint8Array(buf);Module["HEAPU16"]=HEAPU16=new Uint16Array(buf);Module["HEAPU32"]=HEAPU32=new Uint32Array(buf);Module["HEAPF32"]=HEAPF32=new Float32Array(buf);Module["HEAPF64"]=HEAPF64=new Float64Array(buf)}var INITIAL_MEMORY=Module["INITIAL_MEMORY"]||16777216;var wasmTable;var __ATPRERUN__=[];var __ATINIT__=[];var __ATMAIN__=[];var __ATPOSTRUN__=[];var runtimeInitialized=false;var runtimeExited=false;function preRun(){if(Module["preRun"]){if(typeof Module["preRun"]=="function")Module["preRun"]=[Module["preRun"]];while(Module["preRun"].length){addOnPreRun(Module["preRun"].shift())}}callRuntimeCallbacks(__ATPRERUN__)}function initRuntime(){runtimeInitialized=true;callRuntimeCallbacks(__ATINIT__)}function preMain(){callRuntimeCallbacks(__ATMAIN__)}function exitRuntime(){runtimeExited=true}function postRun(){if(Module["postRun"]){if(typeof Module["postRun"]=="function")Module["postRun"]=[Module["postRun"]];while(Module["postRun"].length){addOnPostRun(Module["postRun"].shift())}}callRuntimeCallbacks(__ATPOSTRUN__)}function addOnPreRun(cb){__ATPRERUN__.unshift(cb)}function addOnInit(cb){__ATINIT__.unshift(cb)}function addOnPostRun(cb){__ATPOSTRUN__.unshift(cb)}var runDependencies=0;var runDependencyWatcher=null;var dependenciesFulfilled=null;function addRunDependency(id){runDependencies++;if(Module["monitorRunDependencies"]){Module["monitorRunDependencies"](runDependencies)}}function removeRunDependency(id){runDependencies--;if(Module["monitorRunDependencies"]){Module["monitorRunDependencies"](runDependencies)}if(runDependencies==0){if(runDependencyWatcher!==null){clearInterval(runDependencyWatcher);runDependencyWatcher=null}if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}}Module["preloadedImages"]={};Module["preloadedAudios"]={};function abort(what){if(Module["onAbort"]){Module["onAbort"](what)}what+="";err(what);ABORT=true;EXITSTATUS=1;what="abort("+what+"). Build with -s ASSERTIONS=1 for more info.";var e=new WebAssembly.RuntimeError(what);readyPromiseReject(e);throw e}function hasPrefix(str,prefix){return String.prototype.startsWith?str.startsWith(prefix):str.indexOf(prefix)===0}var dataURIPrefix="data:application/octet-stream;base64,";function isDataURI(filename){return hasPrefix(filename,dataURIPrefix)}var fileURIPrefix="file://";function isFileURI(filename){return hasPrefix(filename,fileURIPrefix)}var wasmBinaryFile="tflite-1-0-0.wasm";if(!isDataURI(wasmBinaryFile)){wasmBinaryFile=locateFile(wasmBinaryFile)}function getBinary(file){try{if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}else{throw"both async and sync fetching of the wasm failed"}}catch(err){abort(err)}}function getBinaryPromise(){if(!wasmBinary&&(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER)){if(typeof fetch==="function"&&!isFileURI(wasmBinaryFile)){return fetch(wasmBinaryFile,{credentials:"same-origin"}).then(function(response){if(!response["ok"]){throw"failed to load wasm binary file at '"+wasmBinaryFile+"'"}return response["arrayBuffer"]()}).catch(function(){return getBinary(wasmBinaryFile)})}else{if(readAsync){return new Promise(function(resolve,reject){readAsync(wasmBinaryFile,function(response){resolve(new Uint8Array(response))},reject)})}}}return Promise.resolve().then(function(){return getBinary(wasmBinaryFile)})}function createWasm(){var info={"a":asmLibraryArg};function receiveInstance(instance,module){var exports=instance.exports;Module["asm"]=exports;wasmMemory=Module["asm"]["q"];updateGlobalBufferAndViews(wasmMemory.buffer);wasmTable=Module["asm"]["D"];addOnInit(Module["asm"]["r"]);removeRunDependency("wasm-instantiate")}addRunDependency("wasm-instantiate");function receiveInstantiatedSource(output){receiveInstance(output["instance"])}function instantiateArrayBuffer(receiver){return getBinaryPromise().then(function(binary){var result=WebAssembly.instantiate(binary,info);return result}).then(receiver,function(reason){err("failed to asynchronously prepare wasm: "+reason);abort(reason)})}function instantiateAsync(){if(!wasmBinary&&typeof WebAssembly.instantiateStreaming==="function"&&!isDataURI(wasmBinaryFile)&&!isFileURI(wasmBinaryFile)&&typeof fetch==="function"){return fetch(wasmBinaryFile,{credentials:"same-origin"}).then(function(response){var result=WebAssembly.instantiateStreaming(response,info);return result.then(receiveInstantiatedSource,function(reason){err("wasm streaming compile failed: "+reason);err("falling back to ArrayBuffer instantiation");return instantiateArrayBuffer(receiveInstantiatedSource)})})}else{return instantiateArrayBuffer(receiveInstantiatedSource)}}if(Module["instantiateWasm"]){try{var exports=Module["instantiateWasm"](info,receiveInstance);return exports}catch(e){err("Module.instantiateWasm callback failed with error: "+e);return false}}instantiateAsync().catch(readyPromiseReject);return{}}function callRuntimeCallbacks(callbacks){while(callbacks.length>0){var callback=callbacks.shift();if(typeof callback=="function"){callback(Module);continue}var func=callback.func;if(typeof func==="number"){if(callback.arg===undefined){wasmTable.get(func)()}else{wasmTable.get(func)(callback.arg)}}else{func(callback.arg===undefined?null:callback.arg)}}}var runtimeKeepaliveCounter=0;function keepRuntimeAlive(){return noExitRuntime||runtimeKeepaliveCounter>0}function _abort(){abort()}var _emscripten_get_now;if(ENVIRONMENT_IS_NODE){_emscripten_get_now=function(){var t=process["hrtime"]();return t[0]*1e3+t[1]/1e6}}else if(typeof dateNow!=="undefined"){_emscripten_get_now=dateNow}else _emscripten_get_now=function(){return performance.now()};var _emscripten_get_now_is_monotonic=true;function setErrNo(value){HEAP32[___errno_location()>>2]=value;return value}function _clock_gettime(clk_id,tp){var now;if(clk_id===0){now=Date.now()}else if((clk_id===1||clk_id===4)&&_emscripten_get_now_is_monotonic){now=_emscripten_get_now()}else{setErrNo(28);return-1}HEAP32[tp>>2]=now/1e3|0;HEAP32[tp+4>>2]=now%1e3*1e3*1e3|0;return 0}function _dlopen(filename,flag){abort("To use dlopen, you need to use Emscripten's linking support, see https://github.com/emscripten-core/emscripten/wiki/Linking")}function _dlsym(handle,symbol){abort("To use dlopen, you need to use Emscripten's linking support, see https://github.com/emscripten-core/emscripten/wiki/Linking")}function _emscripten_get_heap_max(){return 2147483648}function _emscripten_memcpy_big(dest,src,num){HEAPU8.copyWithin(dest,src,src+num)}function emscripten_realloc_buffer(size){try{wasmMemory.grow(size-buffer.byteLength+65535>>>16);updateGlobalBufferAndViews(wasmMemory.buffer);return 1}catch(e){}}function _emscripten_resize_heap(requestedSize){var oldSize=HEAPU8.length;var maxHeapSize=2147483648;if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignUp(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=emscripten_realloc_buffer(newSize);if(replacement){return true}}return false}function _emscripten_thread_sleep(msecs){var start=_emscripten_get_now();while(_emscripten_get_now()-start0){return}preRun();if(runDependencies>0){return}function doRun(){if(calledRun)return;calledRun=true;Module["calledRun"]=true;if(ABORT)return;initRuntime();preMain();readyPromiseResolve(Module);if(Module["onRuntimeInitialized"])Module["onRuntimeInitialized"]();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(function(){setTimeout(function(){Module["setStatus"]("")},1);doRun()},1)}else{doRun()}}Module["run"]=run;function exit(status,implicit){EXITSTATUS=status;if(implicit&&keepRuntimeAlive()&&status===0){return}if(keepRuntimeAlive()){}else{exitRuntime();if(Module["onExit"])Module["onExit"](status);ABORT=true}quit_(status,new ExitStatus(status))}if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].pop()()}}run(); + + + return createTwilioTFLiteModule.ready +} +); +})(); +if (typeof exports === 'object' && typeof module === 'object') + module.exports = createTwilioTFLiteModule; +else if (typeof define === 'function' && define['amd']) + define([], function() { return createTwilioTFLiteModule; }); +else if (typeof exports === 'object') + exports["createTwilioTFLiteModule"] = createTwilioTFLiteModule; diff --git a/dist/build/tflite-1-0-0.wasm b/dist/build/tflite-1-0-0.wasm new file mode 100755 index 0000000..ead8e52 Binary files /dev/null and b/dist/build/tflite-1-0-0.wasm differ diff --git a/dist/build/tflite-simd-1-0-0.js b/dist/build/tflite-simd-1-0-0.js new file mode 100644 index 0000000..d68e168 --- /dev/null +++ b/dist/build/tflite-simd-1-0-0.js @@ -0,0 +1,21 @@ + +var createTwilioTFLiteSIMDModule = (function() { + var _scriptDir = typeof document !== 'undefined' && document.currentScript ? document.currentScript.src : undefined; + if (typeof __filename !== 'undefined') _scriptDir = _scriptDir || __filename; + return ( +function(createTwilioTFLiteSIMDModule) { + createTwilioTFLiteSIMDModule = createTwilioTFLiteSIMDModule || {}; + +var Module=typeof createTwilioTFLiteSIMDModule!=="undefined"?createTwilioTFLiteSIMDModule:{};var readyPromiseResolve,readyPromiseReject;Module["ready"]=new Promise(function(resolve,reject){readyPromiseResolve=resolve;readyPromiseReject=reject});var moduleOverrides={};var key;for(key in Module){if(Module.hasOwnProperty(key)){moduleOverrides[key]=Module[key]}}var arguments_=[];var thisProgram="./this.program";var quit_=function(status,toThrow){throw toThrow};var ENVIRONMENT_IS_WEB=false;var ENVIRONMENT_IS_WORKER=false;var ENVIRONMENT_IS_NODE=false;var ENVIRONMENT_IS_SHELL=false;ENVIRONMENT_IS_WEB=typeof window==="object";ENVIRONMENT_IS_WORKER=typeof importScripts==="function";ENVIRONMENT_IS_NODE=typeof process==="object"&&typeof process.versions==="object"&&typeof process.versions.node==="string";ENVIRONMENT_IS_SHELL=!ENVIRONMENT_IS_WEB&&!ENVIRONMENT_IS_NODE&&!ENVIRONMENT_IS_WORKER;var scriptDirectory="";function locateFile(path){if(Module["locateFile"]){return Module["locateFile"](path,scriptDirectory)}return scriptDirectory+path}var read_,readAsync,readBinary,setWindowTitle;var nodeFS;var nodePath;if(ENVIRONMENT_IS_NODE){if(ENVIRONMENT_IS_WORKER){scriptDirectory=require("path").dirname(scriptDirectory)+"/"}else{scriptDirectory=__dirname+"/"}read_=function shell_read(filename,binary){if(!nodeFS)nodeFS=require("fs");if(!nodePath)nodePath=require("path");filename=nodePath["normalize"](filename);return nodeFS["readFileSync"](filename,binary?null:"utf8")};readBinary=function readBinary(filename){var ret=read_(filename,true);if(!ret.buffer){ret=new Uint8Array(ret)}assert(ret.buffer);return ret};if(process["argv"].length>1){thisProgram=process["argv"][1].replace(/\\/g,"/")}arguments_=process["argv"].slice(2);process["on"]("uncaughtException",function(ex){if(!(ex instanceof ExitStatus)){throw ex}});process["on"]("unhandledRejection",abort);quit_=function(status){process["exit"](status)};Module["inspect"]=function(){return"[Emscripten Module object]"}}else if(ENVIRONMENT_IS_SHELL){if(typeof read!="undefined"){read_=function shell_read(f){return read(f)}}readBinary=function readBinary(f){var data;if(typeof readbuffer==="function"){return new Uint8Array(readbuffer(f))}data=read(f,"binary");assert(typeof data==="object");return data};if(typeof scriptArgs!="undefined"){arguments_=scriptArgs}else if(typeof arguments!="undefined"){arguments_=arguments}if(typeof quit==="function"){quit_=function(status){quit(status)}}if(typeof print!=="undefined"){if(typeof console==="undefined")console={};console.log=print;console.warn=console.error=typeof printErr!=="undefined"?printErr:print}}else if(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER){if(ENVIRONMENT_IS_WORKER){scriptDirectory=self.location.href}else if(typeof document!=="undefined"&&document.currentScript){scriptDirectory=document.currentScript.src}if(_scriptDir){scriptDirectory=_scriptDir}if(scriptDirectory.indexOf("blob:")!==0){scriptDirectory=scriptDirectory.substr(0,scriptDirectory.lastIndexOf("/")+1)}else{scriptDirectory=""}{read_=function(url){var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.send(null);return xhr.responseText};if(ENVIRONMENT_IS_WORKER){readBinary=function(url){var xhr=new XMLHttpRequest;xhr.open("GET",url,false);xhr.responseType="arraybuffer";xhr.send(null);return new Uint8Array(xhr.response)}}readAsync=function(url,onload,onerror){var xhr=new XMLHttpRequest;xhr.open("GET",url,true);xhr.responseType="arraybuffer";xhr.onload=function(){if(xhr.status==200||xhr.status==0&&xhr.response){onload(xhr.response);return}onerror()};xhr.onerror=onerror;xhr.send(null)}}setWindowTitle=function(title){document.title=title}}else{}var out=Module["print"]||console.log.bind(console);var err=Module["printErr"]||console.warn.bind(console);for(key in moduleOverrides){if(moduleOverrides.hasOwnProperty(key)){Module[key]=moduleOverrides[key]}}moduleOverrides=null;if(Module["arguments"])arguments_=Module["arguments"];if(Module["thisProgram"])thisProgram=Module["thisProgram"];if(Module["quit"])quit_=Module["quit"];var wasmBinary;if(Module["wasmBinary"])wasmBinary=Module["wasmBinary"];var noExitRuntime=Module["noExitRuntime"]||true;if(typeof WebAssembly!=="object"){abort("no native wasm support detected")}var wasmMemory;var ABORT=false;var EXITSTATUS;function assert(condition,text){if(!condition){abort("Assertion failed: "+text)}}var UTF8Decoder=typeof TextDecoder!=="undefined"?new TextDecoder("utf8"):undefined;function UTF8ArrayToString(heap,idx,maxBytesToRead){var endIdx=idx+maxBytesToRead;var endPtr=idx;while(heap[endPtr]&&!(endPtr>=endIdx))++endPtr;if(endPtr-idx>16&&heap.subarray&&UTF8Decoder){return UTF8Decoder.decode(heap.subarray(idx,endPtr))}else{var str="";while(idx>10,56320|ch&1023)}}}return str}function UTF8ToString(ptr,maxBytesToRead){return ptr?UTF8ArrayToString(HEAPU8,ptr,maxBytesToRead):""}function writeAsciiToMemory(str,buffer,dontAddNull){for(var i=0;i>0]=str.charCodeAt(i)}if(!dontAddNull)HEAP8[buffer>>0]=0}function alignUp(x,multiple){if(x%multiple>0){x+=multiple-x%multiple}return x}var buffer,HEAP8,HEAPU8,HEAP16,HEAPU16,HEAP32,HEAPU32,HEAPF32,HEAPF64;function updateGlobalBufferAndViews(buf){buffer=buf;Module["HEAP8"]=HEAP8=new Int8Array(buf);Module["HEAP16"]=HEAP16=new Int16Array(buf);Module["HEAP32"]=HEAP32=new Int32Array(buf);Module["HEAPU8"]=HEAPU8=new Uint8Array(buf);Module["HEAPU16"]=HEAPU16=new Uint16Array(buf);Module["HEAPU32"]=HEAPU32=new Uint32Array(buf);Module["HEAPF32"]=HEAPF32=new Float32Array(buf);Module["HEAPF64"]=HEAPF64=new Float64Array(buf)}var INITIAL_MEMORY=Module["INITIAL_MEMORY"]||16777216;var wasmTable;var __ATPRERUN__=[];var __ATINIT__=[];var __ATMAIN__=[];var __ATPOSTRUN__=[];var runtimeInitialized=false;var runtimeExited=false;function preRun(){if(Module["preRun"]){if(typeof Module["preRun"]=="function")Module["preRun"]=[Module["preRun"]];while(Module["preRun"].length){addOnPreRun(Module["preRun"].shift())}}callRuntimeCallbacks(__ATPRERUN__)}function initRuntime(){runtimeInitialized=true;callRuntimeCallbacks(__ATINIT__)}function preMain(){callRuntimeCallbacks(__ATMAIN__)}function exitRuntime(){runtimeExited=true}function postRun(){if(Module["postRun"]){if(typeof Module["postRun"]=="function")Module["postRun"]=[Module["postRun"]];while(Module["postRun"].length){addOnPostRun(Module["postRun"].shift())}}callRuntimeCallbacks(__ATPOSTRUN__)}function addOnPreRun(cb){__ATPRERUN__.unshift(cb)}function addOnInit(cb){__ATINIT__.unshift(cb)}function addOnPostRun(cb){__ATPOSTRUN__.unshift(cb)}var runDependencies=0;var runDependencyWatcher=null;var dependenciesFulfilled=null;function addRunDependency(id){runDependencies++;if(Module["monitorRunDependencies"]){Module["monitorRunDependencies"](runDependencies)}}function removeRunDependency(id){runDependencies--;if(Module["monitorRunDependencies"]){Module["monitorRunDependencies"](runDependencies)}if(runDependencies==0){if(runDependencyWatcher!==null){clearInterval(runDependencyWatcher);runDependencyWatcher=null}if(dependenciesFulfilled){var callback=dependenciesFulfilled;dependenciesFulfilled=null;callback()}}}Module["preloadedImages"]={};Module["preloadedAudios"]={};function abort(what){if(Module["onAbort"]){Module["onAbort"](what)}what+="";err(what);ABORT=true;EXITSTATUS=1;what="abort("+what+"). Build with -s ASSERTIONS=1 for more info.";var e=new WebAssembly.RuntimeError(what);readyPromiseReject(e);throw e}function hasPrefix(str,prefix){return String.prototype.startsWith?str.startsWith(prefix):str.indexOf(prefix)===0}var dataURIPrefix="data:application/octet-stream;base64,";function isDataURI(filename){return hasPrefix(filename,dataURIPrefix)}var fileURIPrefix="file://";function isFileURI(filename){return hasPrefix(filename,fileURIPrefix)}var wasmBinaryFile="tflite-simd-1-0-0.wasm";if(!isDataURI(wasmBinaryFile)){wasmBinaryFile=locateFile(wasmBinaryFile)}function getBinary(file){try{if(file==wasmBinaryFile&&wasmBinary){return new Uint8Array(wasmBinary)}if(readBinary){return readBinary(file)}else{throw"both async and sync fetching of the wasm failed"}}catch(err){abort(err)}}function getBinaryPromise(){if(!wasmBinary&&(ENVIRONMENT_IS_WEB||ENVIRONMENT_IS_WORKER)){if(typeof fetch==="function"&&!isFileURI(wasmBinaryFile)){return fetch(wasmBinaryFile,{credentials:"same-origin"}).then(function(response){if(!response["ok"]){throw"failed to load wasm binary file at '"+wasmBinaryFile+"'"}return response["arrayBuffer"]()}).catch(function(){return getBinary(wasmBinaryFile)})}else{if(readAsync){return new Promise(function(resolve,reject){readAsync(wasmBinaryFile,function(response){resolve(new Uint8Array(response))},reject)})}}}return Promise.resolve().then(function(){return getBinary(wasmBinaryFile)})}function createWasm(){var info={"a":asmLibraryArg};function receiveInstance(instance,module){var exports=instance.exports;Module["asm"]=exports;wasmMemory=Module["asm"]["q"];updateGlobalBufferAndViews(wasmMemory.buffer);wasmTable=Module["asm"]["D"];addOnInit(Module["asm"]["r"]);removeRunDependency("wasm-instantiate")}addRunDependency("wasm-instantiate");function receiveInstantiatedSource(output){receiveInstance(output["instance"])}function instantiateArrayBuffer(receiver){return getBinaryPromise().then(function(binary){var result=WebAssembly.instantiate(binary,info);return result}).then(receiver,function(reason){err("failed to asynchronously prepare wasm: "+reason);abort(reason)})}function instantiateAsync(){if(!wasmBinary&&typeof WebAssembly.instantiateStreaming==="function"&&!isDataURI(wasmBinaryFile)&&!isFileURI(wasmBinaryFile)&&typeof fetch==="function"){return fetch(wasmBinaryFile,{credentials:"same-origin"}).then(function(response){var result=WebAssembly.instantiateStreaming(response,info);return result.then(receiveInstantiatedSource,function(reason){err("wasm streaming compile failed: "+reason);err("falling back to ArrayBuffer instantiation");return instantiateArrayBuffer(receiveInstantiatedSource)})})}else{return instantiateArrayBuffer(receiveInstantiatedSource)}}if(Module["instantiateWasm"]){try{var exports=Module["instantiateWasm"](info,receiveInstance);return exports}catch(e){err("Module.instantiateWasm callback failed with error: "+e);return false}}instantiateAsync().catch(readyPromiseReject);return{}}function callRuntimeCallbacks(callbacks){while(callbacks.length>0){var callback=callbacks.shift();if(typeof callback=="function"){callback(Module);continue}var func=callback.func;if(typeof func==="number"){if(callback.arg===undefined){wasmTable.get(func)()}else{wasmTable.get(func)(callback.arg)}}else{func(callback.arg===undefined?null:callback.arg)}}}var runtimeKeepaliveCounter=0;function keepRuntimeAlive(){return noExitRuntime||runtimeKeepaliveCounter>0}function _abort(){abort()}var _emscripten_get_now;if(ENVIRONMENT_IS_NODE){_emscripten_get_now=function(){var t=process["hrtime"]();return t[0]*1e3+t[1]/1e6}}else if(typeof dateNow!=="undefined"){_emscripten_get_now=dateNow}else _emscripten_get_now=function(){return performance.now()};var _emscripten_get_now_is_monotonic=true;function setErrNo(value){HEAP32[___errno_location()>>2]=value;return value}function _clock_gettime(clk_id,tp){var now;if(clk_id===0){now=Date.now()}else if((clk_id===1||clk_id===4)&&_emscripten_get_now_is_monotonic){now=_emscripten_get_now()}else{setErrNo(28);return-1}HEAP32[tp>>2]=now/1e3|0;HEAP32[tp+4>>2]=now%1e3*1e3*1e3|0;return 0}function _dlopen(filename,flag){abort("To use dlopen, you need to use Emscripten's linking support, see https://github.com/emscripten-core/emscripten/wiki/Linking")}function _dlsym(handle,symbol){abort("To use dlopen, you need to use Emscripten's linking support, see https://github.com/emscripten-core/emscripten/wiki/Linking")}function _emscripten_get_heap_max(){return 2147483648}function _emscripten_memcpy_big(dest,src,num){HEAPU8.copyWithin(dest,src,src+num)}function emscripten_realloc_buffer(size){try{wasmMemory.grow(size-buffer.byteLength+65535>>>16);updateGlobalBufferAndViews(wasmMemory.buffer);return 1}catch(e){}}function _emscripten_resize_heap(requestedSize){var oldSize=HEAPU8.length;var maxHeapSize=2147483648;if(requestedSize>maxHeapSize){return false}for(var cutDown=1;cutDown<=4;cutDown*=2){var overGrownHeapSize=oldSize*(1+.2/cutDown);overGrownHeapSize=Math.min(overGrownHeapSize,requestedSize+100663296);var newSize=Math.min(maxHeapSize,alignUp(Math.max(requestedSize,overGrownHeapSize),65536));var replacement=emscripten_realloc_buffer(newSize);if(replacement){return true}}return false}function _emscripten_thread_sleep(msecs){var start=_emscripten_get_now();while(_emscripten_get_now()-start0){return}preRun();if(runDependencies>0){return}function doRun(){if(calledRun)return;calledRun=true;Module["calledRun"]=true;if(ABORT)return;initRuntime();preMain();readyPromiseResolve(Module);if(Module["onRuntimeInitialized"])Module["onRuntimeInitialized"]();postRun()}if(Module["setStatus"]){Module["setStatus"]("Running...");setTimeout(function(){setTimeout(function(){Module["setStatus"]("")},1);doRun()},1)}else{doRun()}}Module["run"]=run;function exit(status,implicit){EXITSTATUS=status;if(implicit&&keepRuntimeAlive()&&status===0){return}if(keepRuntimeAlive()){}else{exitRuntime();if(Module["onExit"])Module["onExit"](status);ABORT=true}quit_(status,new ExitStatus(status))}if(Module["preInit"]){if(typeof Module["preInit"]=="function")Module["preInit"]=[Module["preInit"]];while(Module["preInit"].length>0){Module["preInit"].pop()()}}run(); + + + return createTwilioTFLiteSIMDModule.ready +} +); +})(); +if (typeof exports === 'object' && typeof module === 'object') + module.exports = createTwilioTFLiteSIMDModule; +else if (typeof define === 'function' && define['amd']) + define([], function() { return createTwilioTFLiteSIMDModule; }); +else if (typeof exports === 'object') + exports["createTwilioTFLiteSIMDModule"] = createTwilioTFLiteSIMDModule; diff --git a/dist/build/tflite-simd-1-0-0.wasm b/dist/build/tflite-simd-1-0-0.wasm new file mode 100755 index 0000000..bb24d2d Binary files /dev/null and b/dist/build/tflite-simd-1-0-0.wasm differ diff --git a/dist/build/twilio-gaussian-blur-background-processor-pipeline-worker.js b/dist/build/twilio-gaussian-blur-background-processor-pipeline-worker.js new file mode 100644 index 0000000..89431b4 --- /dev/null +++ b/dist/build/twilio-gaussian-blur-background-processor-pipeline-worker.js @@ -0,0 +1,1877 @@ +/*! twilio-video-processors.js 3.0.0-preview.1 + +The following license applies to all parts of this software except as +documented below. + + Copyright (C) 2022 Twilio Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + 3. Neither the name of Twilio nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ +(function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.BackgroundProcessorPipeline = void 0; +var constants_1 = require("../../../../constants"); +var Benchmark_1 = require("../../../../utils/Benchmark"); +var support_1 = require("../../../../utils/support"); +var TwilioTFLite_1 = require("../../../../utils/TwilioTFLite"); +var pipelines_1 = require("../../../pipelines"); +var InputFrameDownscaleStage_1 = require("./InputFrameDownscaleStage"); +var PostProcessingStage_1 = require("./PostProcessingStage"); +/** + * @private + */ +var BackgroundProcessorPipeline = /** @class */ (function (_super) { + __extends(BackgroundProcessorPipeline, _super); + function BackgroundProcessorPipeline(options, onResizeWebGL2Canvas) { + if (onResizeWebGL2Canvas === void 0) { onResizeWebGL2Canvas = function () { }; } + var _this = _super.call(this) || this; + _this._outputCanvas = new OffscreenCanvas(1, 1); + _this._webgl2Canvas = new OffscreenCanvas(1, 1); + _this._benchmark = new Benchmark_1.Benchmark(); + _this._inferenceInputCanvas = new OffscreenCanvas(constants_1.WASM_INFERENCE_DIMENSIONS.width, constants_1.WASM_INFERENCE_DIMENSIONS.height); + _this._inputFrameDownscaleMode = (0, support_1.isChromiumImageBitmap)() ? 'image-bitmap' : 'canvas'; + var assetsPath = options.assetsPath, deferInputFrameDownscale = options.deferInputFrameDownscale, maskBlurRadius = options.maskBlurRadius; + _this._assetsPath = assetsPath; + _this._deferInputFrameDownscale = deferInputFrameDownscale; + _this._onResizeWebGL2Canvas = onResizeWebGL2Canvas; + _this.addStage(new InputFrameDownscaleStage_1.InputFrameDowscaleStage(_this._inferenceInputCanvas, _this._inputFrameDownscaleMode)); + _this.addStage(new PostProcessingStage_1.PostProcessingStage(constants_1.WASM_INFERENCE_DIMENSIONS, _this._webgl2Canvas, _this._outputCanvas, maskBlurRadius, function (inputFrame) { return _this._setBackground(inputFrame); })); + return _this; + } + BackgroundProcessorPipeline.prototype.loadTwilioTFLite = function () { + return __awaiter(this, void 0, void 0, function () { + var _twilioTFLite; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + _twilioTFLite = BackgroundProcessorPipeline._twilioTFLite; + if (!!_twilioTFLite) return [3 /*break*/, 2]; + _twilioTFLite = new TwilioTFLite_1.TwilioTFLite(); + return [4 /*yield*/, _twilioTFLite.initialize(this._assetsPath, constants_1.MODEL_NAME, constants_1.TFLITE_LOADER_NAME, constants_1.TFLITE_SIMD_LOADER_NAME)]; + case 1: + _a.sent(); + BackgroundProcessorPipeline._twilioTFLite = _twilioTFLite; + _a.label = 2; + case 2: return [2 /*return*/, _twilioTFLite.isSimdEnabled]; + } + }); + }); + }; + BackgroundProcessorPipeline.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var _a, inputFrameDownscaleStage, postProcessingStage, _b, _benchmark, _deferInputFrameDownscale, _c, inferenceInputHeight, inferenceInputWidth, _outputCanvas, _webgl2Canvas, _twilioTFLite, isInputVideoFrame, _d, height, width, didResizeWebGL2Canvas, downscalePromise, personMask; + return __generator(this, function (_e) { + switch (_e.label) { + case 0: + if (!BackgroundProcessorPipeline._twilioTFLite) { + return [2 /*return*/, null]; + } + _a = this._stages, inputFrameDownscaleStage = _a[0], postProcessingStage = _a[1]; + _b = this, _benchmark = _b._benchmark, _deferInputFrameDownscale = _b._deferInputFrameDownscale, _c = _b._inferenceInputCanvas, inferenceInputHeight = _c.height, inferenceInputWidth = _c.width, _outputCanvas = _b._outputCanvas, _webgl2Canvas = _b._webgl2Canvas; + _twilioTFLite = BackgroundProcessorPipeline._twilioTFLite; + isInputVideoFrame = typeof VideoFrame === 'function' + && inputFrame instanceof VideoFrame; + _d = isInputVideoFrame + ? { height: inputFrame.displayHeight, width: inputFrame.displayWidth } + : inputFrame, height = _d.height, width = _d.width; + didResizeWebGL2Canvas = false; + if (_outputCanvas.width !== width) { + _outputCanvas.width = width; + _webgl2Canvas.width = width; + didResizeWebGL2Canvas = true; + } + if (_outputCanvas.height !== height) { + _outputCanvas.height = height; + _webgl2Canvas.height = height; + didResizeWebGL2Canvas = true; + } + if (didResizeWebGL2Canvas) { + postProcessingStage.resetPersonMaskUpscalePipeline(); + this._onResizeWebGL2Canvas(); + } + _benchmark.start('inputImageResizeDelay'); + downscalePromise = inputFrameDownscaleStage.render(inputFrame) + .then(function (downscaledFrameData) { + _twilioTFLite.loadInputBuffer(downscaledFrameData); + }); + if (!!_deferInputFrameDownscale) return [3 /*break*/, 2]; + return [4 /*yield*/, downscalePromise]; + case 1: + _e.sent(); + _e.label = 2; + case 2: + _benchmark.end('inputImageResizeDelay'); + _benchmark.start('segmentationDelay'); + personMask = new ImageData(_twilioTFLite.runInference(), inferenceInputWidth, inferenceInputHeight); + _benchmark.end('segmentationDelay'); + _benchmark.start('imageCompositionDelay'); + postProcessingStage.render(inputFrame, personMask); + _benchmark.end('imageCompositionDelay'); + if (typeof VideoFrame === 'function' + && inputFrame instanceof VideoFrame) { + inputFrame.close(); + } + return [2 /*return*/, this._outputCanvas]; + } + }); + }); + }; + BackgroundProcessorPipeline.prototype.setDeferInputFrameDownscale = function (defer) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + this._deferInputFrameDownscale = defer; + return [2 /*return*/]; + }); + }); + }; + BackgroundProcessorPipeline.prototype.setMaskBlurRadius = function (radius) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + this._stages[1] + .updateMaskBlurRadius(radius); + return [2 /*return*/]; + }); + }); + }; + BackgroundProcessorPipeline._twilioTFLite = null; + return BackgroundProcessorPipeline; +}(pipelines_1.Pipeline)); +exports.BackgroundProcessorPipeline = BackgroundProcessorPipeline; + +},{"../../../../constants":1,"../../../../utils/Benchmark":17,"../../../../utils/TwilioTFLite":18,"../../../../utils/support":19,"../../../pipelines":12,"./InputFrameDownscaleStage":5,"./PostProcessingStage":6}],3:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurBackgroundProcessorPipeline = void 0; +var support_1 = require("../../../../utils/support"); +var gaussianblurfilterpipeline_1 = require("../gaussianblurfilterpipeline"); +var BackgroundProcessorPipeline_1 = require("./BackgroundProcessorPipeline"); +/** + * @private + */ +var GaussianBlurBackgroundProcessorPipeline = /** @class */ (function (_super) { + __extends(GaussianBlurBackgroundProcessorPipeline, _super); + function GaussianBlurBackgroundProcessorPipeline(options) { + var _this = _super.call(this, options, function () { return _this._resetGaussianBlurFilterPipeline(); }) || this; + _this._gaussianBlurFilterPipeline = null; + var blurFilterRadius = options.blurFilterRadius; + _this._blurFilterRadius = blurFilterRadius; + return _this; + } + GaussianBlurBackgroundProcessorPipeline.prototype.setBlurFilterRadius = function (radius) { + var _a; + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_b) { + this._blurFilterRadius = radius; + return [2 /*return*/, (_a = this._gaussianBlurFilterPipeline) === null || _a === void 0 ? void 0 : _a.updateRadius(this._blurFilterRadius)]; + }); + }); + }; + GaussianBlurBackgroundProcessorPipeline.prototype._setBackground = function (inputFrame) { + var _a = this, _blurFilterRadius = _a._blurFilterRadius, _outputCanvas = _a._outputCanvas, _webgl2Canvas = _a._webgl2Canvas; + var ctx = _outputCanvas.getContext('2d'); + if (support_1.isCanvasBlurSupported) { + ctx.filter = "blur(".concat(_blurFilterRadius, "px)"); + ctx.drawImage(inputFrame, 0, 0); + ctx.filter = 'none'; + return; + } + if (!this._gaussianBlurFilterPipeline) { + this._resetGaussianBlurFilterPipeline(); + } + this._gaussianBlurFilterPipeline.render(); + ctx.drawImage(_webgl2Canvas, 0, 0); + }; + GaussianBlurBackgroundProcessorPipeline.prototype._resetGaussianBlurFilterPipeline = function () { + var _a; + var _b = this, _blurFilterRadius = _b._blurFilterRadius, _webgl2Canvas = _b._webgl2Canvas; + (_a = this._gaussianBlurFilterPipeline) === null || _a === void 0 ? void 0 : _a.cleanUp(); + this._gaussianBlurFilterPipeline = new gaussianblurfilterpipeline_1.GaussianBlurFilterPipeline(_webgl2Canvas); + this._gaussianBlurFilterPipeline.updateRadius(_blurFilterRadius); + }; + return GaussianBlurBackgroundProcessorPipeline; +}(BackgroundProcessorPipeline_1.BackgroundProcessorPipeline)); +exports.GaussianBlurBackgroundProcessorPipeline = GaussianBlurBackgroundProcessorPipeline; + +},{"../../../../utils/support":19,"../gaussianblurfilterpipeline":8,"./BackgroundProcessorPipeline":2}],4:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurBackgroundProcessorPipelineWorker = void 0; +var comlink_1 = require("comlink"); +var GaussianBlurBackgroundProcessorPipeline_1 = require("./GaussianBlurBackgroundProcessorPipeline"); +/** + * @private + */ +var GaussianBlurBackgroundProcessorPipelineWorker = /** @class */ (function (_super) { + __extends(GaussianBlurBackgroundProcessorPipelineWorker, _super); + function GaussianBlurBackgroundProcessorPipelineWorker() { + return _super !== null && _super.apply(this, arguments) || this; + } + GaussianBlurBackgroundProcessorPipelineWorker.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var outputFrame, outputBitmap; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, _super.prototype.render.call(this, inputFrame)]; + case 1: + outputFrame = _a.sent(); + outputBitmap = outputFrame instanceof OffscreenCanvas + ? outputFrame.transferToImageBitmap() + : outputFrame; + return [2 /*return*/, outputBitmap && (0, comlink_1.transfer)(outputBitmap, [outputBitmap])]; + } + }); + }); + }; + return GaussianBlurBackgroundProcessorPipelineWorker; +}(GaussianBlurBackgroundProcessorPipeline_1.GaussianBlurBackgroundProcessorPipeline)); +exports.GaussianBlurBackgroundProcessorPipelineWorker = GaussianBlurBackgroundProcessorPipelineWorker; +(0, comlink_1.expose)(GaussianBlurBackgroundProcessorPipelineWorker); + +},{"./GaussianBlurBackgroundProcessorPipeline":3,"comlink":20}],5:[function(require,module,exports){ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.InputFrameDowscaleStage = void 0; +/** + * @private + */ +var InputFrameDowscaleStage = /** @class */ (function () { + function InputFrameDowscaleStage(outputCanvas, inputFrameDownscaleMode) { + this._inputFrameDownscaleMode = inputFrameDownscaleMode; + this._outputContext = outputCanvas.getContext('2d', { willReadFrequently: true }); + } + InputFrameDowscaleStage.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var _a, _outputContext, _inputFrameDownscaleMode, _b, resizeHeight, resizeWidth, downscaledBitmap, data; + return __generator(this, function (_c) { + switch (_c.label) { + case 0: + _a = this, _outputContext = _a._outputContext, _inputFrameDownscaleMode = _a._inputFrameDownscaleMode; + _b = _outputContext.canvas, resizeHeight = _b.height, resizeWidth = _b.width; + if (!(_inputFrameDownscaleMode === 'image-bitmap')) return [3 /*break*/, 2]; + return [4 /*yield*/, createImageBitmap(inputFrame, { + resizeWidth: resizeWidth, + resizeHeight: resizeHeight, + resizeQuality: 'pixelated' + })]; + case 1: + downscaledBitmap = _c.sent(); + _outputContext.drawImage(downscaledBitmap, 0, 0); + downscaledBitmap.close(); + return [3 /*break*/, 3]; + case 2: + _outputContext.drawImage(inputFrame, 0, 0, resizeWidth, resizeHeight); + _c.label = 3; + case 3: + data = _outputContext.getImageData(0, 0, resizeWidth, resizeHeight).data; + return [2 /*return*/, data]; + } + }); + }); + }; + return InputFrameDowscaleStage; +}()); +exports.InputFrameDowscaleStage = InputFrameDowscaleStage; + +},{}],6:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PostProcessingStage = void 0; +var personmaskupscalepipeline_1 = require("../personmaskupscalepipeline"); +/** + * @private + */ +var PostProcessingStage = /** @class */ (function () { + function PostProcessingStage(inputDimensions, webgl2Canvas, outputCanvas, maskBlurRadius, setBackground) { + this._personMaskUpscalePipeline = null; + this._inputDimensions = inputDimensions; + this._maskBlurRadius = maskBlurRadius; + this._outputContext = outputCanvas.getContext('2d'); + this._webgl2Canvas = webgl2Canvas; + this._setBackground = setBackground; + } + PostProcessingStage.prototype.render = function (inputFrame, personMask) { + var _a = this, _outputContext = _a._outputContext, _setBackground = _a._setBackground, _webgl2Canvas = _a._webgl2Canvas; + if (!this._personMaskUpscalePipeline) { + this.resetPersonMaskUpscalePipeline(); + } + this._personMaskUpscalePipeline.render(inputFrame, personMask); + _outputContext.save(); + _outputContext.globalCompositeOperation = 'copy'; + _outputContext.drawImage(_webgl2Canvas, 0, 0); + _outputContext.globalCompositeOperation = 'destination-over'; + _setBackground(inputFrame); + _outputContext.restore(); + }; + PostProcessingStage.prototype.resetPersonMaskUpscalePipeline = function () { + var _a; + var _b = this, _inputDimensions = _b._inputDimensions, _maskBlurRadius = _b._maskBlurRadius, _webgl2Canvas = _b._webgl2Canvas; + (_a = this._personMaskUpscalePipeline) === null || _a === void 0 ? void 0 : _a.cleanUp(); + this._personMaskUpscalePipeline = new personmaskupscalepipeline_1.PersonMaskUpscalePipeline(_inputDimensions, _webgl2Canvas); + this._personMaskUpscalePipeline.updateBilateralFilterConfig({ + sigmaSpace: _maskBlurRadius + }); + }; + PostProcessingStage.prototype.updateMaskBlurRadius = function (radius) { + var _a; + if (this._maskBlurRadius !== radius) { + this._maskBlurRadius = radius; + (_a = this._personMaskUpscalePipeline) === null || _a === void 0 ? void 0 : _a.updateBilateralFilterConfig({ + sigmaSpace: radius + }); + } + }; + return PostProcessingStage; +}()); +exports.PostProcessingStage = PostProcessingStage; + +},{"../personmaskupscalepipeline":10}],7:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.SinglePassGaussianBlurFilterStage = void 0; +var pipelines_1 = require("../../../pipelines"); +/** + * @private + */ +function createGaussianBlurWeights(radius) { + var coeff = 1.0 / Math.sqrt(2.0 * Math.PI) / radius; + return '0'.repeat(radius + 1).split('').map(function (zero, x) { + return coeff * Math.exp(-0.5 * x * x / radius / radius); + }); +} +/** + * @private + */ +var SinglePassGaussianBlurFilterStage = /** @class */ (function (_super) { + __extends(SinglePassGaussianBlurFilterStage, _super); + function SinglePassGaussianBlurFilterStage(glOut, direction, outputType, inputTextureUnit, outputTextureUnit) { + var _this = this; + if (outputTextureUnit === void 0) { outputTextureUnit = inputTextureUnit + 1; } + var _a = glOut.canvas, height = _a.height, width = _a.width; + _this = _super.call(this, { + textureName: 'u_inputTexture', + textureUnit: inputTextureUnit + }, { + fragmentShaderSource: "#version 300 es\n precision highp float;\n\n uniform sampler2D u_inputTexture;\n uniform vec2 u_texelSize;\n uniform float u_direction;\n uniform float u_radius;\n uniform float u_gaussianBlurWeights[128];\n\n in vec2 v_texCoord;\n\n out vec4 outColor;\n\n void main() {\n float totalWeight = u_gaussianBlurWeights[0];\n vec3 newColor = totalWeight * texture(u_inputTexture, v_texCoord).rgb;\n\n for (float i = 1.0; i <= u_radius; i += 1.0) {\n float x = (1.0 - u_direction) * i;\n float y = u_direction * i;\n\n vec2 shift = vec2(x, y) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n float weight = u_gaussianBlurWeights[int(i)];\n newColor += weight * texture(u_inputTexture, coord).rgb;\n totalWeight += weight;\n\n shift = vec2(-x, -y) * u_texelSize;\n coord = vec2(v_texCoord + shift);\n newColor += weight * texture(u_inputTexture, coord).rgb;\n totalWeight += weight;\n }\n\n newColor /= totalWeight;\n outColor = vec4(newColor, 1.0);\n }\n ", + glOut: glOut, + height: height, + textureUnit: outputTextureUnit, + type: outputType, + width: width, + uniformVars: [ + { + name: 'u_direction', + type: 'float', + values: [direction === 'vertical' ? 1 : 0] + }, + { + name: 'u_texelSize', + type: 'float', + values: [1 / width, 1 / height] + } + ] + }) || this; + _this.updateRadius(0); + return _this; + } + SinglePassGaussianBlurFilterStage.prototype.updateRadius = function (radius) { + this._setUniformVars([ + { + name: 'u_radius', + type: 'float', + values: [radius] + }, + { + name: 'u_gaussianBlurWeights', + type: 'float:v', + values: createGaussianBlurWeights(radius) + } + ]); + }; + return SinglePassGaussianBlurFilterStage; +}(pipelines_1.WebGL2Pipeline.ProcessingStage)); +exports.SinglePassGaussianBlurFilterStage = SinglePassGaussianBlurFilterStage; + +},{"../../../pipelines":12}],8:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurFilterPipeline = void 0; +var pipelines_1 = require("../../../pipelines"); +var SinglePassGaussianBlurFilterStage_1 = require("./SinglePassGaussianBlurFilterStage"); +/** + * @private + */ +var GaussianBlurFilterPipeline = /** @class */ (function (_super) { + __extends(GaussianBlurFilterPipeline, _super); + function GaussianBlurFilterPipeline(outputCanvas) { + var _this = _super.call(this) || this; + var glOut = outputCanvas.getContext('webgl2'); + _this.addStage(new SinglePassGaussianBlurFilterStage_1.SinglePassGaussianBlurFilterStage(glOut, 'horizontal', 'texture', 0, 2)); + _this.addStage(new SinglePassGaussianBlurFilterStage_1.SinglePassGaussianBlurFilterStage(glOut, 'vertical', 'canvas', 2)); + return _this; + } + GaussianBlurFilterPipeline.prototype.updateRadius = function (radius) { + this._stages.forEach(function (stage) { return stage + .updateRadius(radius); }); + }; + return GaussianBlurFilterPipeline; +}(pipelines_1.WebGL2Pipeline)); +exports.GaussianBlurFilterPipeline = GaussianBlurFilterPipeline; + +},{"../../../pipelines":12,"./SinglePassGaussianBlurFilterStage":7}],9:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.SinglePassBilateralFilterStage = void 0; +var pipelines_1 = require("../../../pipelines"); +/** + * @private + */ +function createSpaceWeights(radius, sigma, texelSize) { + return '0'.repeat(radius).split('').map(function (zero, i) { + var x = (i + 1) * texelSize; + return Math.exp(-0.5 * x * x / sigma / sigma); + }); +} +/** + * @private + */ +function createColorWeights(sigma) { + return '0'.repeat(256).split('').map(function (zero, i) { + var x = i / 255; + return Math.exp(-0.5 * x * x / sigma / sigma); + }); +} +/** + * @private + */ +var SinglePassBilateralFilterStage = /** @class */ (function (_super) { + __extends(SinglePassBilateralFilterStage, _super); + function SinglePassBilateralFilterStage(glOut, direction, outputType, inputDimensions, outputDimensions, inputTextureUnit, outputTextureUnit) { + var _this = this; + if (outputTextureUnit === void 0) { outputTextureUnit = inputTextureUnit + 1; } + var height = outputDimensions.height, width = outputDimensions.width; + _this = _super.call(this, { + textureName: 'u_segmentationMask', + textureUnit: inputTextureUnit + }, { + fragmentShaderSource: "#version 300 es\n precision highp float;\n\n uniform sampler2D u_inputFrame;\n uniform sampler2D u_segmentationMask;\n uniform vec2 u_texelSize;\n uniform float u_direction;\n uniform float u_radius;\n uniform float u_step;\n uniform float u_spaceWeights[128];\n uniform float u_colorWeights[256];\n\n in vec2 v_texCoord;\n\n out vec4 outColor;\n\n float calculateColorWeight(vec2 coord, vec3 centerColor) {\n vec3 coordColor = texture(u_inputFrame, coord).rgb;\n float x = distance(centerColor, coordColor);\n return u_colorWeights[int(x * 255.0)];\n }\n\n float edgePixelsAverageAlpha(float outAlpha) {\n float totalAlpha = outAlpha;\n float totalPixels = 1.0;\n\n for (float i = -u_radius; u_radius > 0.0 && i <= u_radius; i += u_radius) {\n for (float j = -u_radius; j <= u_radius; j += u_radius * (j == 0.0 ? 2.0 : 1.0)) {\n vec2 shift = vec2(i, j) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n totalAlpha += texture(u_segmentationMask, coord).a;\n totalPixels++;\n }\n }\n\n return totalAlpha / totalPixels;\n }\n\n void main() {\n vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb;\n float outAlpha = texture(u_segmentationMask, v_texCoord).a;\n float averageAlpha = edgePixelsAverageAlpha(outAlpha);\n float totalWeight = 1.0;\n\n if (averageAlpha == 0.0 || averageAlpha == 1.0) {\n outColor = vec4(averageAlpha * centerColor, averageAlpha);\n return;\n }\n\n for (float i = 1.0; i <= u_radius; i += u_step) {\n float x = (1.0 - u_direction) * i;\n float y = u_direction * i;\n vec2 shift = vec2(x, y) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n float spaceWeight = u_spaceWeights[int(i - 1.0)];\n float colorWeight = calculateColorWeight(coord, centerColor);\n float weight = spaceWeight * colorWeight;\n float alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n\n shift = vec2(-x, -y) * u_texelSize;\n coord = vec2(v_texCoord + shift);\n colorWeight = calculateColorWeight(coord, centerColor);\n weight = spaceWeight * colorWeight;\n alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n }\n\n outAlpha /= totalWeight;\n outColor = vec4(outAlpha * centerColor, outAlpha);\n }\n ", + glOut: glOut, + height: height, + textureUnit: outputTextureUnit, + type: outputType, + width: width, + uniformVars: [ + { + name: 'u_inputFrame', + type: 'int', + values: [0] + }, + { + name: 'u_direction', + type: 'float', + values: [direction === 'vertical' ? 1 : 0] + }, + { + name: 'u_texelSize', + type: 'float', + values: [1 / width, 1 / height] + } + ] + }) || this; + _this._direction = direction; + _this._inputDimensions = inputDimensions; + _this.updateSigmaColor(0); + _this.updateSigmaSpace(0); + return _this; + } + SinglePassBilateralFilterStage.prototype.updateSigmaColor = function (sigmaColor) { + this._setUniformVars([ + { + name: 'u_colorWeights', + type: 'float:v', + values: createColorWeights(sigmaColor) + } + ]); + }; + SinglePassBilateralFilterStage.prototype.updateSigmaSpace = function (sigmaSpace) { + var _a = this._inputDimensions, inputHeight = _a.height, inputWidth = _a.width; + var _b = this._outputDimensions, outputHeight = _b.height, outputWidth = _b.width; + sigmaSpace *= Math.max(outputWidth / inputWidth, outputHeight / inputHeight); + var step = Math.floor(0.5 * sigmaSpace / Math.log(sigmaSpace)); + var sigmaTexel = Math.max(1 / outputWidth, 1 / outputHeight) * sigmaSpace; + var texelSize = 1 / (this._direction === 'horizontal' + ? outputWidth + : outputHeight); + this._setUniformVars([ + { + name: 'u_radius', + type: 'float', + values: [sigmaSpace] + }, + { + name: 'u_spaceWeights', + type: 'float:v', + values: createSpaceWeights(sigmaSpace, sigmaTexel, texelSize) + }, + { + name: 'u_step', + type: 'float', + values: [step] + } + ]); + }; + return SinglePassBilateralFilterStage; +}(pipelines_1.WebGL2Pipeline.ProcessingStage)); +exports.SinglePassBilateralFilterStage = SinglePassBilateralFilterStage; + +},{"../../../pipelines":12}],10:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PersonMaskUpscalePipeline = void 0; +var pipelines_1 = require("../../../pipelines"); +var SinglePassBilateralFilterStage_1 = require("./SinglePassBilateralFilterStage"); +/** + * @private + */ +var PersonMaskUpscalePipeline = /** @class */ (function (_super) { + __extends(PersonMaskUpscalePipeline, _super); + function PersonMaskUpscalePipeline(inputDimensions, outputCanvas) { + var _this = _super.call(this) || this; + var glOut = outputCanvas.getContext('webgl2'); + var outputDimensions = { + height: outputCanvas.height, + width: outputCanvas.width + }; + _this.addStage(new pipelines_1.WebGL2Pipeline.InputStage(glOut)); + _this.addStage(new SinglePassBilateralFilterStage_1.SinglePassBilateralFilterStage(glOut, 'horizontal', 'texture', inputDimensions, outputDimensions, 1, 2)); + _this.addStage(new SinglePassBilateralFilterStage_1.SinglePassBilateralFilterStage(glOut, 'vertical', 'canvas', inputDimensions, outputDimensions, 2)); + return _this; + } + PersonMaskUpscalePipeline.prototype.updateBilateralFilterConfig = function (config) { + var _a = this._stages, bilateralFilterStages = _a.slice(1); + var sigmaSpace = config.sigmaSpace; + if (typeof sigmaSpace === 'number') { + bilateralFilterStages.forEach(function (stage) { + stage.updateSigmaColor(0.1); + stage.updateSigmaSpace(sigmaSpace); + }); + } + }; + return PersonMaskUpscalePipeline; +}(pipelines_1.WebGL2Pipeline)); +exports.PersonMaskUpscalePipeline = PersonMaskUpscalePipeline; + +},{"../../../pipelines":12,"./SinglePassBilateralFilterStage":9}],11:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Pipeline = void 0; +/** + * @private + */ +var Pipeline = /** @class */ (function () { + function Pipeline() { + this._stages = []; + } + Pipeline.prototype.addStage = function (stage) { + this._stages.push(stage); + }; + Pipeline.prototype.render = function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + this._stages.forEach(function (stage) { + stage.render.apply(stage, args); + }); + }; + return Pipeline; +}()); +exports.Pipeline = Pipeline; + +},{}],12:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2Pipeline = exports.Pipeline = void 0; +var Pipeline_1 = require("./Pipeline"); +Object.defineProperty(exports, "Pipeline", { enumerable: true, get: function () { return Pipeline_1.Pipeline; } }); +var webgl2pipeline_1 = require("./webgl2pipeline"); +Object.defineProperty(exports, "WebGL2Pipeline", { enumerable: true, get: function () { return webgl2pipeline_1.WebGL2Pipeline; } }); + +},{"./Pipeline":11,"./webgl2pipeline":15}],13:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2PipelineInputStage = void 0; +var webgl2PipelineHelpers_1 = require("./webgl2PipelineHelpers"); +/** + * @private + */ +var WebGL2PipelineInputStage = /** @class */ (function () { + function WebGL2PipelineInputStage(glOut) { + this._inputTexture = null; + var _a = glOut.canvas, height = _a.height, width = _a.width; + this._glOut = glOut; + this._inputFrameTexture = (0, webgl2PipelineHelpers_1.createTexture)(glOut, glOut.RGBA8, width, height, glOut.NEAREST, glOut.NEAREST); + } + WebGL2PipelineInputStage.prototype.cleanUp = function () { + var _a = this, _glOut = _a._glOut, _inputFrameTexture = _a._inputFrameTexture, _inputTexture = _a._inputTexture; + _glOut.deleteTexture(_inputFrameTexture); + _glOut.deleteTexture(_inputTexture); + }; + WebGL2PipelineInputStage.prototype.render = function (inputFrame, inputTextureData) { + var _a = this, _glOut = _a._glOut, _inputFrameTexture = _a._inputFrameTexture; + var _b = _glOut.canvas, height = _b.height, width = _b.width; + _glOut.viewport(0, 0, width, height); + _glOut.clearColor(0, 0, 0, 0); + _glOut.clear(_glOut.COLOR_BUFFER_BIT); + if (inputFrame) { + _glOut.activeTexture(_glOut.TEXTURE0); + _glOut.bindTexture(_glOut.TEXTURE_2D, _inputFrameTexture); + _glOut.texSubImage2D(_glOut.TEXTURE_2D, 0, 0, 0, width, height, _glOut.RGBA, _glOut.UNSIGNED_BYTE, inputFrame); + } + if (!inputTextureData) { + return; + } + var data = inputTextureData.data, textureHeight = inputTextureData.height, textureWidth = inputTextureData.width; + if (!this._inputTexture) { + this._inputTexture = (0, webgl2PipelineHelpers_1.createTexture)(_glOut, _glOut.RGBA8, textureWidth, textureHeight, _glOut.NEAREST, _glOut.NEAREST); + } + _glOut.viewport(0, 0, textureWidth, textureHeight); + _glOut.activeTexture(_glOut.TEXTURE1); + _glOut.bindTexture(_glOut.TEXTURE_2D, this._inputTexture); + _glOut.texSubImage2D(_glOut.TEXTURE_2D, 0, 0, 0, textureWidth, textureHeight, _glOut.RGBA, _glOut.UNSIGNED_BYTE, data); + }; + return WebGL2PipelineInputStage; +}()); +exports.WebGL2PipelineInputStage = WebGL2PipelineInputStage; + +},{"./webgl2PipelineHelpers":16}],14:[function(require,module,exports){ +"use strict"; +var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) { + if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) { + if (ar || !(i in from)) { + if (!ar) ar = Array.prototype.slice.call(from, 0, i); + ar[i] = from[i]; + } + } + return to.concat(ar || Array.prototype.slice.call(from)); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2PipelineProcessingStage = void 0; +var webgl2PipelineHelpers_1 = require("./webgl2PipelineHelpers"); +/**; + * @private + */ +var WebGL2PipelineProcessingStage = /** @class */ (function () { + function WebGL2PipelineProcessingStage(inputConfig, outputConfig) { + this._outputFramebuffer = null; + this._outputTexture = null; + var textureName = inputConfig.textureName, textureUnit = inputConfig.textureUnit; + var glOut = outputConfig.glOut; + this._glOut = glOut; + var fragmentShaderSource = outputConfig.fragmentShaderSource, _a = outputConfig.height, height = _a === void 0 ? glOut.canvas.height : _a, _b = outputConfig.textureUnit, outputTextureUnit = _b === void 0 ? textureUnit + 1 : _b, outputType = outputConfig.type, _c = outputConfig.uniformVars, uniformVars = _c === void 0 ? [] : _c, _d = outputConfig.vertexShaderSource, vertexShaderSource = _d === void 0 ? "#version 300 es\n in vec2 a_position;\n in vec2 a_texCoord;\n\n out vec2 v_texCoord;\n\n void main() {\n gl_Position = vec4(a_position".concat(outputType === 'canvas' + ? ' * vec2(1.0, -1.0)' + : '', ", 0.0, 1.0);\n v_texCoord = a_texCoord;\n }\n ") : _d, _e = outputConfig.width, width = _e === void 0 ? glOut.canvas.width : _e; + this._outputDimensions = { + height: height, + width: width + }; + this._outputTextureUnit = outputTextureUnit; + this._fragmentShader = (0, webgl2PipelineHelpers_1.compileShader)(glOut, glOut.FRAGMENT_SHADER, fragmentShaderSource); + this._vertexShader = (0, webgl2PipelineHelpers_1.compileShader)(glOut, glOut.VERTEX_SHADER, vertexShaderSource); + this._positionBuffer = (0, webgl2PipelineHelpers_1.initBuffer)(glOut, [ + -1.0, -1.0, + 1.0, -1.0, + -1.0, 1.0, + 1.0, 1.0, + ]); + this._texCoordBuffer = (0, webgl2PipelineHelpers_1.initBuffer)(glOut, [ + 0.0, 0.0, + 1.0, 0.0, + 0.0, 1.0, + 1.0, 1.0, + ]); + if (outputType === 'texture') { + this._outputTexture = (0, webgl2PipelineHelpers_1.createTexture)(glOut, glOut.RGBA8, width, height); + this._outputFramebuffer = glOut.createFramebuffer(); + glOut.bindFramebuffer(glOut.FRAMEBUFFER, this._outputFramebuffer); + glOut.framebufferTexture2D(glOut.FRAMEBUFFER, glOut.COLOR_ATTACHMENT0, glOut.TEXTURE_2D, this._outputTexture, 0); + } + var program = (0, webgl2PipelineHelpers_1.createPipelineStageProgram)(glOut, this._vertexShader, this._fragmentShader, this._positionBuffer, this._texCoordBuffer); + this._program = program; + this._setUniformVars(__spreadArray([ + { + name: textureName, + type: 'int', + values: [textureUnit] + } + ], uniformVars, true)); + } + WebGL2PipelineProcessingStage.prototype.cleanUp = function () { + var _a = this, _fragmentShader = _a._fragmentShader, _glOut = _a._glOut, _positionBuffer = _a._positionBuffer, _program = _a._program, _texCoordBuffer = _a._texCoordBuffer, _vertexShader = _a._vertexShader; + _glOut.deleteProgram(_program); + _glOut.deleteBuffer(_texCoordBuffer); + _glOut.deleteBuffer(_positionBuffer); + _glOut.deleteShader(_vertexShader); + _glOut.deleteShader(_fragmentShader); + }; + WebGL2PipelineProcessingStage.prototype.render = function () { + var _a = this, _glOut = _a._glOut, _b = _a._outputDimensions, height = _b.height, width = _b.width, _outputFramebuffer = _a._outputFramebuffer, _outputTexture = _a._outputTexture, _outputTextureUnit = _a._outputTextureUnit, _program = _a._program; + _glOut.viewport(0, 0, width, height); + _glOut.useProgram(_program); + if (_outputTexture) { + _glOut.activeTexture(_glOut.TEXTURE0 + + _outputTextureUnit); + _glOut.bindTexture(_glOut.TEXTURE_2D, _outputTexture); + } + _glOut.bindFramebuffer(_glOut.FRAMEBUFFER, _outputFramebuffer); + _glOut.drawArrays(_glOut.TRIANGLE_STRIP, 0, 4); + }; + WebGL2PipelineProcessingStage.prototype._setUniformVars = function (uniformVars) { + var _a = this, _glOut = _a._glOut, _program = _a._program; + _glOut.useProgram(_program); + uniformVars.forEach(function (_a) { + var name = _a.name, type = _a.type, values = _a.values; + var uniformVarLocation = _glOut + .getUniformLocation(_program, name); + var isVector = type.split(':')[1] === 'v'; + if (isVector) { + // @ts-ignore + _glOut["uniform1".concat(type[0], "v")](uniformVarLocation, values); + } + else { + // @ts-ignore + _glOut["uniform".concat(values.length).concat(type[0])].apply(_glOut, __spreadArray([uniformVarLocation], values, false)); + } + }); + }; + return WebGL2PipelineProcessingStage; +}()); +exports.WebGL2PipelineProcessingStage = WebGL2PipelineProcessingStage; + +},{"./webgl2PipelineHelpers":16}],15:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2Pipeline = void 0; +var Pipeline_1 = require("../Pipeline"); +var WebGL2PipelineInputStage_1 = require("./WebGL2PipelineInputStage"); +var WebGL2PipelineProcessingStage_1 = require("./WebGL2PipelineProcessingStage"); +/** + * @private + */ +var WebGL2Pipeline = /** @class */ (function (_super) { + __extends(WebGL2Pipeline, _super); + function WebGL2Pipeline() { + var _this = _super !== null && _super.apply(this, arguments) || this; + _this._stages = []; + return _this; + } + WebGL2Pipeline.prototype.cleanUp = function () { + this._stages.forEach(function (stage) { return stage.cleanUp(); }); + }; + WebGL2Pipeline.prototype.render = function (inputFrame, inputTextureData) { + var _a = this._stages, inputStage = _a[0], otherStages = _a.slice(1); + inputStage.render(inputFrame, inputTextureData); + otherStages.forEach(function (stage) { return stage + .render(); }); + }; + WebGL2Pipeline.InputStage = WebGL2PipelineInputStage_1.WebGL2PipelineInputStage; + WebGL2Pipeline.ProcessingStage = WebGL2PipelineProcessingStage_1.WebGL2PipelineProcessingStage; + return WebGL2Pipeline; +}(Pipeline_1.Pipeline)); +exports.WebGL2Pipeline = WebGL2Pipeline; + +},{"../Pipeline":11,"./WebGL2PipelineInputStage":13,"./WebGL2PipelineProcessingStage":14}],16:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.initBuffer = exports.createTexture = exports.compileShader = exports.createProgram = exports.createPipelineStageProgram = void 0; +/** + * @private + */ +function createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer) { + var program = createProgram(gl, vertexShader, fragmentShader); + var positionAttributeLocation = gl.getAttribLocation(program, 'a_position'); + gl.enableVertexAttribArray(positionAttributeLocation); + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + gl.vertexAttribPointer(positionAttributeLocation, 2, gl.FLOAT, false, 0, 0); + var texCoordAttributeLocation = gl.getAttribLocation(program, 'a_texCoord'); + gl.enableVertexAttribArray(texCoordAttributeLocation); + gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); + gl.vertexAttribPointer(texCoordAttributeLocation, 2, gl.FLOAT, false, 0, 0); + return program; +} +exports.createPipelineStageProgram = createPipelineStageProgram; +/** + * @private + */ +function createProgram(gl, vertexShader, fragmentShader) { + var program = gl.createProgram(); + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + throw new Error("Could not link WebGL program: ".concat(gl.getProgramInfoLog(program))); + } + return program; +} +exports.createProgram = createProgram; +/** + * @private + */ +function compileShader(gl, shaderType, shaderSource) { + var shader = gl.createShader(shaderType); + gl.shaderSource(shader, shaderSource); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + throw new Error("Could not compile shader: ".concat(gl.getShaderInfoLog(shader))); + } + return shader; +} +exports.compileShader = compileShader; +/** + * @private + */ +function createTexture(gl, internalformat, width, height, minFilter, magFilter) { + if (minFilter === void 0) { minFilter = gl.NEAREST; } + if (magFilter === void 0) { magFilter = gl.NEAREST; } + var texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, minFilter); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, magFilter); + gl.texStorage2D(gl.TEXTURE_2D, 1, internalformat, width, height); + return texture; +} +exports.createTexture = createTexture; +/** + * @private + */ +function initBuffer(gl, data) { + var buffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(data), gl.STATIC_DRAW); + return buffer; +} +exports.initBuffer = initBuffer; + +},{}],17:[function(require,module,exports){ +"use strict"; +var __assign = (this && this.__assign) || function () { + __assign = Object.assign || function(t) { + for (var s, i = 1, n = arguments.length; i < n; i++) { + s = arguments[i]; + for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) + t[p] = s[p]; + } + return t; + }; + return __assign.apply(this, arguments); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Benchmark = void 0; +/** + * @private + */ +var Benchmark = /** @class */ (function () { + function Benchmark() { + this._timingCache = new Map(); + this._timings = new Map(); + } + Benchmark.prototype.end = function (name) { + var timing = this._timings.get(name); + if (!timing) { + return; + } + timing.end = Date.now(); + timing.delay = timing.end - timing.start; + this._save(name, __assign({}, timing)); + }; + Benchmark.prototype.getAverageDelay = function (name) { + var timingCache = this._timingCache.get(name); + if (!timingCache || !timingCache.length) { + return; + } + return timingCache.map(function (timing) { return timing.delay; }) + .reduce(function (total, value) { return total += value; }, 0) / timingCache.length; + }; + Benchmark.prototype.getNames = function () { + return Array.from(this._timingCache.keys()); + }; + Benchmark.prototype.getRate = function (name) { + var timingCache = this._timingCache.get(name); + if (!timingCache || timingCache.length < 2) { + return; + } + var totalDelay = timingCache[timingCache.length - 1].end - timingCache[0].start; + return (timingCache.length / totalDelay) * 1000; + }; + Benchmark.prototype.merge = function (benchmark) { + var _this = this; + var _timingCache = benchmark._timingCache, _timings = benchmark._timings; + _timingCache.forEach(function (cache, name) { return _this._timingCache.set(name, cache); }); + _timings.forEach(function (timing, name) { return _this._timings.set(name, timing); }); + }; + Benchmark.prototype.start = function (name) { + var timing = this._timings.get(name); + if (!timing) { + timing = {}; + this._timings.set(name, timing); + } + timing.start = Date.now(); + delete timing.end; + delete timing.delay; + }; + Benchmark.prototype._save = function (name, timing) { + var timingCache = this._timingCache.get(name); + if (!timingCache) { + timingCache = []; + this._timingCache.set(name, timingCache); + } + timingCache.push(timing); + if (timingCache.length > Benchmark.cacheSize) { + timingCache.splice(0, timingCache.length - Benchmark.cacheSize); + } + }; + // NOTE (csantos): How many timing information to save per benchmark. + // This is about the amount of timing info generated on a 24fps input. + // Enough samples to calculate fps + Benchmark.cacheSize = 41; + return Benchmark; +}()); +exports.Benchmark = Benchmark; + +},{}],18:[function(require,module,exports){ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.TwilioTFLite = void 0; +var isWebWorker = typeof WorkerGlobalScope !== 'undefined' + && self instanceof WorkerGlobalScope; +var loadedScripts = new Set(); +var model; +/** + * @private + */ +var TwilioTFLite = /** @class */ (function () { + function TwilioTFLite() { + this._inputBuffer = null; + this._isSimdEnabled = null; + this._tflite = null; + } + Object.defineProperty(TwilioTFLite.prototype, "isSimdEnabled", { + get: function () { + return this._isSimdEnabled; + }, + enumerable: false, + configurable: true + }); + TwilioTFLite.prototype.initialize = function (assetsPath, modelName, moduleLoaderName, moduleSimdLoaderName) { + return __awaiter(this, void 0, void 0, function () { + var _a, modelResponse, _b, tflite, modelBufferOffset; + return __generator(this, function (_c) { + switch (_c.label) { + case 0: + if (this._tflite) { + return [2 /*return*/]; + } + if (isWebWorker) { + // NOTE(mmalavalli): In a web worker, paths to other dependencies + // are determined relative to the assets path, so no need to append + // it to the file names of the dependencies. + assetsPath = ''; + } + return [4 /*yield*/, Promise.all([ + this._loadWasmModule(assetsPath, moduleLoaderName, moduleSimdLoaderName), + fetch("".concat(assetsPath).concat(modelName)), + ])]; + case 1: + _a = _c.sent(), modelResponse = _a[1]; + _b = model; + if (_b) return [3 /*break*/, 3]; + return [4 /*yield*/, modelResponse.arrayBuffer()]; + case 2: + _b = (_c.sent()); + _c.label = 3; + case 3: + model = _b; + tflite = this._tflite; + modelBufferOffset = tflite._getModelBufferMemoryOffset(); + tflite.HEAPU8.set(new Uint8Array(model), modelBufferOffset); + tflite._loadModel(model.byteLength); + return [2 /*return*/]; + } + }); + }); + }; + TwilioTFLite.prototype.loadInputBuffer = function (inputBuffer) { + var tflite = this._tflite; + var height = tflite._getInputHeight(); + var width = tflite._getInputWidth(); + var pixels = width * height; + var tfliteInputMemoryOffset = tflite._getInputMemoryOffset() / 4; + for (var i = 0; i < pixels; i++) { + var curTFLiteOffset = tfliteInputMemoryOffset + i * 3; + var curImageBufferOffset = i * 4; + tflite.HEAPF32[curTFLiteOffset] = inputBuffer[curImageBufferOffset] / 255; + tflite.HEAPF32[curTFLiteOffset + 1] = inputBuffer[curImageBufferOffset + 1] / 255; + tflite.HEAPF32[curTFLiteOffset + 2] = inputBuffer[curImageBufferOffset + 2] / 255; + } + this._inputBuffer = inputBuffer; + }; + TwilioTFLite.prototype.runInference = function () { + var tflite = this._tflite; + var height = tflite._getInputHeight(); + var width = tflite._getInputWidth(); + var pixels = width * height; + var tfliteOutputMemoryOffset = tflite._getOutputMemoryOffset() / 4; + tflite._runInference(); + var inputBuffer = this._inputBuffer || new Uint8ClampedArray(pixels * 4); + for (var i = 0; i < pixels; i++) { + inputBuffer[i * 4 + 3] = Math.round(tflite.HEAPF32[tfliteOutputMemoryOffset + i] * 255); + } + return inputBuffer; + }; + TwilioTFLite.prototype._loadScript = function (path) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + if (loadedScripts.has(path)) { + return [2 /*return*/]; + } + if (isWebWorker) { + importScripts(path); + loadedScripts.add(path); + return [2 /*return*/]; + } + return [2 /*return*/, new Promise(function (resolve, reject) { + var script = document.createElement('script'); + script.onload = function () { + loadedScripts.add(path); + resolve(); + }; + script.onerror = function () { + reject(); + }; + document.head.append(script); + script.src = path; + })]; + }); + }); + }; + TwilioTFLite.prototype._loadWasmModule = function (assetsPath, moduleLoaderName, moduleSimdLoaderName) { + return __awaiter(this, void 0, void 0, function () { + var _a, _b, _c; + return __generator(this, function (_d) { + switch (_d.label) { + case 0: + _d.trys.push([0, 3, , 6]); + return [4 /*yield*/, this._loadScript("".concat(assetsPath).concat(moduleSimdLoaderName))]; + case 1: + _d.sent(); + _a = this; + return [4 /*yield*/, createTwilioTFLiteSIMDModule()]; + case 2: + _a._tflite = _d.sent(); + this._isSimdEnabled = true; + return [3 /*break*/, 6]; + case 3: + _b = _d.sent(); + return [4 /*yield*/, this._loadScript("".concat(assetsPath).concat(moduleLoaderName))]; + case 4: + _d.sent(); + _c = this; + return [4 /*yield*/, createTwilioTFLiteModule()]; + case 5: + _c._tflite = _d.sent(); + this._isSimdEnabled = false; + return [3 /*break*/, 6]; + case 6: return [2 /*return*/]; + } + }); + }); + }; + return TwilioTFLite; +}()); +exports.TwilioTFLite = TwilioTFLite; + +},{}],19:[function(require,module,exports){ +"use strict"; +var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) { + if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) { + if (ar || !(i in from)) { + if (!ar) ar = Array.prototype.slice.call(from, 0, i); + ar[i] = from[i]; + } + } + return to.concat(ar || Array.prototype.slice.call(from)); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.isSupported = exports.isCanvasBlurSupported = exports.isChromiumImageBitmap = exports.isBrowserSupported = void 0; +/** + * @private + */ +function getCanvas() { + return typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); +} +/** + * @private + */ +function isBrowserSupported() { + if (typeof window !== 'undefined' && typeof document !== 'undefined') { + return !!(getCanvas().getContext('2d') || getCanvas().getContext('webgl2')); + } + else { + return false; + } +} +exports.isBrowserSupported = isBrowserSupported; +/** + * @private + */ +function isChromiumImageBitmap() { + return /Chrome/.test(navigator.userAgent) + && typeof createImageBitmap === 'function'; +} +exports.isChromiumImageBitmap = isChromiumImageBitmap; +/** + * @private + */ +exports.isCanvasBlurSupported = (function () { + var blackPixel = [0, 0, 0, 255]; + var whitePixel = [255, 255, 255, 255]; + var inputImageData = new ImageData(new Uint8ClampedArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], blackPixel, true), blackPixel, true), blackPixel, true), blackPixel, true), whitePixel, true), blackPixel, true), blackPixel, true), blackPixel, true), blackPixel, true)), 3, 3); + var canvas = getCanvas(); + var context = canvas.getContext('2d'); + canvas.width = 3; + canvas.height = 3; + context.putImageData(inputImageData, 0, 0); + context.filter = 'blur(1px)'; + context.drawImage(canvas, 0, 0); + var data = context.getImageData(0, 0, 3, 3).data; + return data[0] > 0; +})(); +/** + * Check if the current browser is officially supported by twilio-video-procesors.js. + * This is set to `true` for browsers that supports canvas + * [2D](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D) or + * [webgl2](https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext) + * rendering context. + * @example + * ```ts + * import { isSupported } from '@twilio/video-processors'; + * + * if (isSupported) { + * // Initialize the background processors + * } + * ``` + */ +exports.isSupported = isBrowserSupported(); + +},{}],20:[function(require,module,exports){ +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : + typeof define === 'function' && define.amd ? define(['exports'], factory) : + (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.Comlink = {})); +})(this, (function (exports) { 'use strict'; + + /** + * @license + * Copyright 2019 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + const proxyMarker = Symbol("Comlink.proxy"); + const createEndpoint = Symbol("Comlink.endpoint"); + const releaseProxy = Symbol("Comlink.releaseProxy"); + const finalizer = Symbol("Comlink.finalizer"); + const throwMarker = Symbol("Comlink.thrown"); + const isObject = (val) => (typeof val === "object" && val !== null) || typeof val === "function"; + /** + * Internal transfer handle to handle objects marked to proxy. + */ + const proxyTransferHandler = { + canHandle: (val) => isObject(val) && val[proxyMarker], + serialize(obj) { + const { port1, port2 } = new MessageChannel(); + expose(obj, port1); + return [port2, [port2]]; + }, + deserialize(port) { + port.start(); + return wrap(port); + }, + }; + /** + * Internal transfer handler to handle thrown exceptions. + */ + const throwTransferHandler = { + canHandle: (value) => isObject(value) && throwMarker in value, + serialize({ value }) { + let serialized; + if (value instanceof Error) { + serialized = { + isError: true, + value: { + message: value.message, + name: value.name, + stack: value.stack, + }, + }; + } + else { + serialized = { isError: false, value }; + } + return [serialized, []]; + }, + deserialize(serialized) { + if (serialized.isError) { + throw Object.assign(new Error(serialized.value.message), serialized.value); + } + throw serialized.value; + }, + }; + /** + * Allows customizing the serialization of certain values. + */ + const transferHandlers = new Map([ + ["proxy", proxyTransferHandler], + ["throw", throwTransferHandler], + ]); + function isAllowedOrigin(allowedOrigins, origin) { + for (const allowedOrigin of allowedOrigins) { + if (origin === allowedOrigin || allowedOrigin === "*") { + return true; + } + if (allowedOrigin instanceof RegExp && allowedOrigin.test(origin)) { + return true; + } + } + return false; + } + function expose(obj, ep = globalThis, allowedOrigins = ["*"]) { + ep.addEventListener("message", function callback(ev) { + if (!ev || !ev.data) { + return; + } + if (!isAllowedOrigin(allowedOrigins, ev.origin)) { + console.warn(`Invalid origin '${ev.origin}' for comlink proxy`); + return; + } + const { id, type, path } = Object.assign({ path: [] }, ev.data); + const argumentList = (ev.data.argumentList || []).map(fromWireValue); + let returnValue; + try { + const parent = path.slice(0, -1).reduce((obj, prop) => obj[prop], obj); + const rawValue = path.reduce((obj, prop) => obj[prop], obj); + switch (type) { + case "GET" /* MessageType.GET */: + { + returnValue = rawValue; + } + break; + case "SET" /* MessageType.SET */: + { + parent[path.slice(-1)[0]] = fromWireValue(ev.data.value); + returnValue = true; + } + break; + case "APPLY" /* MessageType.APPLY */: + { + returnValue = rawValue.apply(parent, argumentList); + } + break; + case "CONSTRUCT" /* MessageType.CONSTRUCT */: + { + const value = new rawValue(...argumentList); + returnValue = proxy(value); + } + break; + case "ENDPOINT" /* MessageType.ENDPOINT */: + { + const { port1, port2 } = new MessageChannel(); + expose(obj, port2); + returnValue = transfer(port1, [port1]); + } + break; + case "RELEASE" /* MessageType.RELEASE */: + { + returnValue = undefined; + } + break; + default: + return; + } + } + catch (value) { + returnValue = { value, [throwMarker]: 0 }; + } + Promise.resolve(returnValue) + .catch((value) => { + return { value, [throwMarker]: 0 }; + }) + .then((returnValue) => { + const [wireValue, transferables] = toWireValue(returnValue); + ep.postMessage(Object.assign(Object.assign({}, wireValue), { id }), transferables); + if (type === "RELEASE" /* MessageType.RELEASE */) { + // detach and deactive after sending release response above. + ep.removeEventListener("message", callback); + closeEndPoint(ep); + if (finalizer in obj && typeof obj[finalizer] === "function") { + obj[finalizer](); + } + } + }) + .catch((error) => { + // Send Serialization Error To Caller + const [wireValue, transferables] = toWireValue({ + value: new TypeError("Unserializable return value"), + [throwMarker]: 0, + }); + ep.postMessage(Object.assign(Object.assign({}, wireValue), { id }), transferables); + }); + }); + if (ep.start) { + ep.start(); + } + } + function isMessagePort(endpoint) { + return endpoint.constructor.name === "MessagePort"; + } + function closeEndPoint(endpoint) { + if (isMessagePort(endpoint)) + endpoint.close(); + } + function wrap(ep, target) { + return createProxy(ep, [], target); + } + function throwIfProxyReleased(isReleased) { + if (isReleased) { + throw new Error("Proxy has been released and is not useable"); + } + } + function releaseEndpoint(ep) { + return requestResponseMessage(ep, { + type: "RELEASE" /* MessageType.RELEASE */, + }).then(() => { + closeEndPoint(ep); + }); + } + const proxyCounter = new WeakMap(); + const proxyFinalizers = "FinalizationRegistry" in globalThis && + new FinalizationRegistry((ep) => { + const newCount = (proxyCounter.get(ep) || 0) - 1; + proxyCounter.set(ep, newCount); + if (newCount === 0) { + releaseEndpoint(ep); + } + }); + function registerProxy(proxy, ep) { + const newCount = (proxyCounter.get(ep) || 0) + 1; + proxyCounter.set(ep, newCount); + if (proxyFinalizers) { + proxyFinalizers.register(proxy, ep, proxy); + } + } + function unregisterProxy(proxy) { + if (proxyFinalizers) { + proxyFinalizers.unregister(proxy); + } + } + function createProxy(ep, path = [], target = function () { }) { + let isProxyReleased = false; + const proxy = new Proxy(target, { + get(_target, prop) { + throwIfProxyReleased(isProxyReleased); + if (prop === releaseProxy) { + return () => { + unregisterProxy(proxy); + releaseEndpoint(ep); + isProxyReleased = true; + }; + } + if (prop === "then") { + if (path.length === 0) { + return { then: () => proxy }; + } + const r = requestResponseMessage(ep, { + type: "GET" /* MessageType.GET */, + path: path.map((p) => p.toString()), + }).then(fromWireValue); + return r.then.bind(r); + } + return createProxy(ep, [...path, prop]); + }, + set(_target, prop, rawValue) { + throwIfProxyReleased(isProxyReleased); + // FIXME: ES6 Proxy Handler `set` methods are supposed to return a + // boolean. To show good will, we return true asynchronously ¯\_(ツ)_/¯ + const [value, transferables] = toWireValue(rawValue); + return requestResponseMessage(ep, { + type: "SET" /* MessageType.SET */, + path: [...path, prop].map((p) => p.toString()), + value, + }, transferables).then(fromWireValue); + }, + apply(_target, _thisArg, rawArgumentList) { + throwIfProxyReleased(isProxyReleased); + const last = path[path.length - 1]; + if (last === createEndpoint) { + return requestResponseMessage(ep, { + type: "ENDPOINT" /* MessageType.ENDPOINT */, + }).then(fromWireValue); + } + // We just pretend that `bind()` didn’t happen. + if (last === "bind") { + return createProxy(ep, path.slice(0, -1)); + } + const [argumentList, transferables] = processArguments(rawArgumentList); + return requestResponseMessage(ep, { + type: "APPLY" /* MessageType.APPLY */, + path: path.map((p) => p.toString()), + argumentList, + }, transferables).then(fromWireValue); + }, + construct(_target, rawArgumentList) { + throwIfProxyReleased(isProxyReleased); + const [argumentList, transferables] = processArguments(rawArgumentList); + return requestResponseMessage(ep, { + type: "CONSTRUCT" /* MessageType.CONSTRUCT */, + path: path.map((p) => p.toString()), + argumentList, + }, transferables).then(fromWireValue); + }, + }); + registerProxy(proxy, ep); + return proxy; + } + function myFlat(arr) { + return Array.prototype.concat.apply([], arr); + } + function processArguments(argumentList) { + const processed = argumentList.map(toWireValue); + return [processed.map((v) => v[0]), myFlat(processed.map((v) => v[1]))]; + } + const transferCache = new WeakMap(); + function transfer(obj, transfers) { + transferCache.set(obj, transfers); + return obj; + } + function proxy(obj) { + return Object.assign(obj, { [proxyMarker]: true }); + } + function windowEndpoint(w, context = globalThis, targetOrigin = "*") { + return { + postMessage: (msg, transferables) => w.postMessage(msg, targetOrigin, transferables), + addEventListener: context.addEventListener.bind(context), + removeEventListener: context.removeEventListener.bind(context), + }; + } + function toWireValue(value) { + for (const [name, handler] of transferHandlers) { + if (handler.canHandle(value)) { + const [serializedValue, transferables] = handler.serialize(value); + return [ + { + type: "HANDLER" /* WireValueType.HANDLER */, + name, + value: serializedValue, + }, + transferables, + ]; + } + } + return [ + { + type: "RAW" /* WireValueType.RAW */, + value, + }, + transferCache.get(value) || [], + ]; + } + function fromWireValue(value) { + switch (value.type) { + case "HANDLER" /* WireValueType.HANDLER */: + return transferHandlers.get(value.name).deserialize(value.value); + case "RAW" /* WireValueType.RAW */: + return value.value; + } + } + function requestResponseMessage(ep, msg, transfers) { + return new Promise((resolve) => { + const id = generateUUID(); + ep.addEventListener("message", function l(ev) { + if (!ev.data || !ev.data.id || ev.data.id !== id) { + return; + } + ep.removeEventListener("message", l); + resolve(ev.data); + }); + if (ep.start) { + ep.start(); + } + ep.postMessage(Object.assign({ id }, msg), transfers); + }); + } + function generateUUID() { + return new Array(4) + .fill(0) + .map(() => Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(16)) + .join("-"); + } + + exports.createEndpoint = createEndpoint; + exports.expose = expose; + exports.finalizer = finalizer; + exports.proxy = proxy; + exports.proxyMarker = proxyMarker; + exports.releaseProxy = releaseProxy; + exports.transfer = transfer; + exports.transferHandlers = transferHandlers; + exports.windowEndpoint = windowEndpoint; + exports.wrap = wrap; + +})); + + +},{}]},{},[4]); diff --git a/dist/build/twilio-video-processors.js b/dist/build/twilio-video-processors.js new file mode 100644 index 0000000..0c221eb --- /dev/null +++ b/dist/build/twilio-video-processors.js @@ -0,0 +1,2863 @@ +/*! twilio-video-processors.js 3.0.0-preview.1 + +The following license applies to all parts of this software except as +documented below. + + Copyright (C) 2022 Twilio Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + 3. Neither the name of Twilio nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ +(function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.BackgroundProcessor = void 0; +var constants_1 = require("../../constants"); +var Processor_1 = require("../Processor"); +var backgroundprocessorpipeline_1 = require("./pipelines/backgroundprocessorpipeline"); +/** + * @private + */ +var BackgroundProcessor = /** @class */ (function (_super) { + __extends(BackgroundProcessor, _super); + function BackgroundProcessor(backgroundProcessorPipeline, options) { + var _this = _super.call(this) || this; + _this._deferInputFrameDownscale = false; + _this._inputFrameCanvas = new OffscreenCanvas(1, 1); + _this._inputFrameContext = _this._inputFrameCanvas.getContext('2d', { willReadFrequently: true }); + _this._isSimdEnabled = null; + _this._maskBlurRadius = constants_1.MASK_BLUR_RADIUS; + _this._outputFrameBuffer = null; + _this._outputFrameBufferContext = null; + var assetsPath = options.assetsPath, _a = options.deferInputFrameDownscale, deferInputFrameDownscale = _a === void 0 ? _this._deferInputFrameDownscale : _a, _b = options.maskBlurRadius, maskBlurRadius = _b === void 0 ? _this._maskBlurRadius : _b; + if (typeof assetsPath !== 'string') { + throw new Error('assetsPath parameter must be a string'); + } + _this._assetsPath = assetsPath.replace(/([^/])$/, '$1/'); + _this._backgroundProcessorPipeline = backgroundProcessorPipeline; + // @ts-ignore + _this._benchmark = _this._backgroundProcessorPipeline._benchmark; + _this.deferInputFrameDownscale = deferInputFrameDownscale; + _this.maskBlurRadius = maskBlurRadius; + return _this; + } + Object.defineProperty(BackgroundProcessor.prototype, "deferInputFrameDownscale", { + /** + * Whether the pipeline is calculating the person mask without + * waiting for the current input frame to be downscaled (Chrome only). + */ + get: function () { + return this._deferInputFrameDownscale; + }, + /** + * Toggle whether the pipeline should calculate the person mask + * without waiting for the current input frame to be downscaled + * (Chrome only). + */ + set: function (defer) { + if (typeof defer !== 'boolean') { + console.warn('Provided deferInputFrameDownscale is not a boolean.'); + defer = this._deferInputFrameDownscale; + } + if (this._deferInputFrameDownscale !== defer) { + this._deferInputFrameDownscale = defer; + this._backgroundProcessorPipeline.setDeferInputFrameDownscale(this._deferInputFrameDownscale).catch(function () { + /* noop */ + }); + } + }, + enumerable: false, + configurable: true + }); + Object.defineProperty(BackgroundProcessor.prototype, "maskBlurRadius", { + /** + * The current blur radius when smoothing out the edges of the person's mask. + */ + get: function () { + return this._maskBlurRadius; + }, + /** + * Set a new blur radius to be used when smoothing out the edges of the person's mask. + */ + set: function (radius) { + if (typeof radius !== 'number' || radius < 0) { + console.warn("Valid mask blur radius not found. Using ".concat(constants_1.MASK_BLUR_RADIUS, " as default.")); + radius = constants_1.MASK_BLUR_RADIUS; + } + if (this._maskBlurRadius !== radius) { + this._maskBlurRadius = radius; + this._backgroundProcessorPipeline + .setMaskBlurRadius(this._maskBlurRadius) + .catch(function () { + /* noop */ + }); + } + }, + enumerable: false, + configurable: true + }); + /** + * Load the segmentation model. + * Call this method before attaching the processor to ensure + * video frames are processed correctly. + */ + BackgroundProcessor.prototype.loadModel = function () { + return __awaiter(this, void 0, void 0, function () { + var _a; + return __generator(this, function (_b) { + switch (_b.label) { + case 0: + _a = this; + return [4 /*yield*/, this + ._backgroundProcessorPipeline + .loadTwilioTFLite()]; + case 1: + _a._isSimdEnabled = _b.sent(); + return [2 /*return*/]; + } + }); + }); + }; + /** + * Apply a transform to the background of an input video frame and leaving + * the foreground (person(s)) untouched. Any exception detected will + * result in the frame being dropped. + * @param inputFrameBuffer - The source of the input frame to process. + *
+ *
+ * [OffscreenCanvas](https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas) - Good for canvas-related processing + * that can be rendered off screen. + *
+ *
+ * [HTMLCanvasElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement) - This is recommended on browsers + * that doesn't support `OffscreenCanvas`, or if you need to render the frame on the screen. + *
+ *
+ * [HTMLVideoElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement) + *
+ *
+ * [VideoFrame](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame) - Recommended on browsers that support the + * [Insertable Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Insertable_Streams_for_MediaStreamTrack_API). + *
+ * @param outputFrameBuffer - The output frame buffer to use to draw the processed frame. + */ + BackgroundProcessor.prototype.processFrame = function (inputFrameBuffer, outputFrameBuffer) { + return __awaiter(this, void 0, void 0, function () { + var _a, _backgroundProcessorPipeline, _benchmark, _outputFrameBufferContext, _b, captureWidth, captureHeight, inputFrame, outputFrame, _c, outputBitmap; + return __generator(this, function (_d) { + switch (_d.label) { + case 0: + if (!inputFrameBuffer || !outputFrameBuffer) { + throw new Error('Missing input or output frame buffer'); + } + _a = this, _backgroundProcessorPipeline = _a._backgroundProcessorPipeline, _benchmark = _a._benchmark, _outputFrameBufferContext = _a._outputFrameBufferContext; + _benchmark.end('captureFrameDelay'); + _benchmark.end('totalProcessingDelay'); + _benchmark.start('totalProcessingDelay'); + _benchmark.start('processFrameDelay'); + _b = inputFrameBuffer instanceof HTMLVideoElement + ? { width: inputFrameBuffer.videoWidth, height: inputFrameBuffer.videoHeight } + : typeof VideoFrame === 'function' && inputFrameBuffer instanceof VideoFrame + ? { width: inputFrameBuffer.displayWidth, height: inputFrameBuffer.displayHeight } + : inputFrameBuffer, captureWidth = _b.width, captureHeight = _b.height; + if (this._outputFrameBuffer !== outputFrameBuffer) { + this._outputFrameBuffer = outputFrameBuffer; + this._outputFrameBufferContext = outputFrameBuffer.getContext('2d') + || outputFrameBuffer.getContext('bitmaprenderer'); + } + if (this._inputFrameCanvas.width !== captureWidth) { + this._inputFrameCanvas.width = captureWidth; + } + if (this._inputFrameCanvas.height !== captureHeight) { + this._inputFrameCanvas.height = captureHeight; + } + if (inputFrameBuffer instanceof HTMLVideoElement) { + this._inputFrameContext.drawImage(inputFrameBuffer, 0, 0); + inputFrame = this._inputFrameCanvas; + } + else { + inputFrame = inputFrameBuffer; + } + if (!(_backgroundProcessorPipeline instanceof backgroundprocessorpipeline_1.BackgroundProcessorPipeline)) return [3 /*break*/, 2]; + return [4 /*yield*/, _backgroundProcessorPipeline + .render(inputFrame)]; + case 1: + _c = _d.sent(); + return [3 /*break*/, 4]; + case 2: return [4 /*yield*/, _backgroundProcessorPipeline + .render(inputFrame)]; + case 3: + _c = _d.sent(); + _d.label = 4; + case 4: + outputFrame = _c; + if (_outputFrameBufferContext instanceof ImageBitmapRenderingContext) { + outputBitmap = outputFrame instanceof OffscreenCanvas + ? outputFrame.transferToImageBitmap() + : outputFrame; + _outputFrameBufferContext.transferFromImageBitmap(outputBitmap); + } + else if (_outputFrameBufferContext instanceof CanvasRenderingContext2D && outputFrame) { + _outputFrameBufferContext.drawImage(outputFrame, 0, 0); + } + _benchmark.end('processFrameDelay'); + _benchmark.start('captureFrameDelay'); + return [2 /*return*/]; + } + }); + }); + }; + return BackgroundProcessor; +}(Processor_1.Processor)); +exports.BackgroundProcessor = BackgroundProcessor; + +},{"../../constants":1,"../Processor":3,"./pipelines/backgroundprocessorpipeline":15}],5:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurBackgroundProcessor = void 0; +var constants_1 = require("../../constants"); +var support_1 = require("../../utils/support"); +var BackgroundProcessor_1 = require("./BackgroundProcessor"); +var backgroundprocessorpipeline_1 = require("./pipelines/backgroundprocessorpipeline"); +/** + * The GaussianBlurBackgroundProcessor, when added to a VideoTrack, + * applies a gaussian blur filter on the background in each video frame + * and leaves the foreground (person(s)) untouched. Each instance of + * GaussianBlurBackgroundProcessor should be added to only one VideoTrack + * at a time to prevent overlapping of image data from multiple VideoTracks. + * + * @example + * + * ```ts + * import { createLocalVideoTrack } from 'twilio-video'; + * import { GaussianBlurBackgroundProcessor } from '@twilio/video-processors'; + * + * let blurBackground: GaussianBlurBackgroundProcessor; + * + * (async() => { + * blurBackground = new GaussianBlurBackgroundProcessor({ + * assetsPath: 'https://my-server-path/assets' + * }); + * await blurBackground.loadModel(); + * + * const track = await createLocalVideoTrack({ + * // Increasing the capture resolution decreases the output FPS + * // especially on browsers that do not support SIMD + * // such as desktop Safari and iOS browsers, or on Chrome + * // with capture resolutions above 640x480 for webgl2. + * width: 640, + * height: 480, + * + * // Any frame rate above 24 fps on desktop browsers increase CPU + * // usage without noticeable increase in quality. + * frameRate: 24 + * }); + * track.addProcessor(virtualBackground, { + * inputFrameBufferType: 'videoframe', + * outputFrameBufferContextType: 'bitmaprenderer' + * }); + * })(); + * ``` + */ +var GaussianBlurBackgroundProcessor = /** @class */ (function (_super) { + __extends(GaussianBlurBackgroundProcessor, _super); + /** + * Construct a GaussianBlurBackgroundProcessor. Default values will be used for + * any missing properties in [[GaussianBlurBackgroundProcessorOptions]], and + * invalid properties will be ignored. + */ + function GaussianBlurBackgroundProcessor(options) { + var _this = this; + var _a = options.blurFilterRadius, blurFilterRadius = _a === void 0 ? constants_1.BLUR_FILTER_RADIUS : _a, _b = options.deferInputFrameDownscale, deferInputFrameDownscale = _b === void 0 ? false : _b, _c = options.maskBlurRadius, maskBlurRadius = _c === void 0 ? constants_1.MASK_BLUR_RADIUS : _c, _d = options.useWebWorker, useWebWorker = _d === void 0 ? true : _d; + var assetsPath = options + .assetsPath + .replace(/([^/])$/, '$1/'); + var BackgroundProcessorPipelineOrProxy = useWebWorker && (0, support_1.isChromiumImageBitmap)() + ? backgroundprocessorpipeline_1.GaussianBlurBackgroundProcessorPipelineProxy + : backgroundprocessorpipeline_1.GaussianBlurBackgroundProcessorPipeline; + var backgroundProcessorPipeline = new BackgroundProcessorPipelineOrProxy({ + assetsPath: assetsPath, + blurFilterRadius: blurFilterRadius, + deferInputFrameDownscale: deferInputFrameDownscale, + maskBlurRadius: maskBlurRadius + }); + _this = _super.call(this, backgroundProcessorPipeline, options) || this; + _this._blurFilterRadius = constants_1.BLUR_FILTER_RADIUS; + // tslint:disable-next-line no-unused-variable + _this._name = 'GaussianBlurBackgroundProcessor'; + _this.blurFilterRadius = options.blurFilterRadius; + return _this; + } + Object.defineProperty(GaussianBlurBackgroundProcessor.prototype, "blurFilterRadius", { + /** + * The current background blur filter radius in pixels. + */ + get: function () { + return this._blurFilterRadius; + }, + /** + * Set a new background blur filter radius in pixels. + */ + set: function (radius) { + if (!radius) { + console.warn("Valid blur filter radius not found. Using ".concat(constants_1.BLUR_FILTER_RADIUS, " as default.")); + radius = constants_1.BLUR_FILTER_RADIUS; + } + this._blurFilterRadius = radius; + this._backgroundProcessorPipeline + .setBlurFilterRadius(this._blurFilterRadius) + .catch(function () { + /* noop */ + }); + }, + enumerable: false, + configurable: true + }); + return GaussianBlurBackgroundProcessor; +}(BackgroundProcessor_1.BackgroundProcessor)); +exports.GaussianBlurBackgroundProcessor = GaussianBlurBackgroundProcessor; + +},{"../../constants":1,"../../utils/support":29,"./BackgroundProcessor":4,"./pipelines/backgroundprocessorpipeline":15}],6:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessor = void 0; +var types_1 = require("../../types"); +var constants_1 = require("../../constants"); +var support_1 = require("../../utils/support"); +var BackgroundProcessor_1 = require("./BackgroundProcessor"); +var backgroundprocessorpipeline_1 = require("./pipelines/backgroundprocessorpipeline"); +/** + * The VirtualBackgroundProcessor, when added to a VideoTrack, + * replaces the background in each video frame with a given image, + * and leaves the foreground (person(s)) untouched. Each instance of + * VirtualBackgroundProcessor should be added to only one VideoTrack + * at a time to prevent overlapping of image data from multiple VideoTracks. + * + * @example + * + * ```ts + * import { createLocalVideoTrack } from 'twilio-video'; + * import { VirtualBackgroundProcessor } from '@twilio/video-processors'; + * + * let virtualBackground: VirtualBackgroundProcessor; + * const img = new Image(); + * + * img.onload = async () => { + * virtualBackground = new VirtualBackgroundProcessor({ + * assetsPath: 'https://my-server-path/assets', + * backgroundImage: img + * }); + * await virtualBackground.loadModel(); + * + * const track = await createLocalVideoTrack({ + * // Increasing the capture resolution decreases the output FPS + * // especially on browsers that do not support SIMD + * // such as desktop Safari and iOS browsers, or on Chrome + * // with capture resolutions above 640x480 for webgl2. + * width: 640, + * height: 480, + * + * // Any frame rate above 24 fps on desktop browsers increase CPU + * // usage without noticeable increase in quality. + * frameRate: 24 + * }); + * track.addProcessor(virtualBackground, { + * inputFrameBufferType: 'videoframe', + * outputFrameBufferContextType: 'bitmaprenderer' + * }); + * }; + * + * img.src = '/background.jpg'; + * ``` + */ +var VirtualBackgroundProcessor = /** @class */ (function (_super) { + __extends(VirtualBackgroundProcessor, _super); + /** + * Construct a VirtualBackgroundProcessor. Default values will be used for + * any missing optional properties in [[VirtualBackgroundProcessorOptions]], + * and invalid properties will be ignored. + */ + function VirtualBackgroundProcessor(options) { + var _this = this; + var backgroundImage = options.backgroundImage, _a = options.deferInputFrameDownscale, deferInputFrameDownscale = _a === void 0 ? false : _a, _b = options.fitType, fitType = _b === void 0 ? types_1.ImageFit.Fill : _b, _c = options.maskBlurRadius, maskBlurRadius = _c === void 0 ? constants_1.MASK_BLUR_RADIUS : _c, _d = options.useWebWorker, useWebWorker = _d === void 0 ? true : _d; + var assetsPath = options + .assetsPath + .replace(/([^/])$/, '$1/'); + var VirtualBackgroundProcessorPipelineOrProxy = useWebWorker && (0, support_1.isChromiumImageBitmap)() + ? backgroundprocessorpipeline_1.VirtualBackgroundProcessorPipelineProxy + : backgroundprocessorpipeline_1.VirtualBackgroundProcessorPipeline; + var backgroundProcessorPipeline = new VirtualBackgroundProcessorPipelineOrProxy({ + assetsPath: assetsPath, + deferInputFrameDownscale: deferInputFrameDownscale, + fitType: fitType, + maskBlurRadius: maskBlurRadius + }); + _this = _super.call(this, backgroundProcessorPipeline, options) || this; + // tslint:disable-next-line no-unused-variable + _this._name = 'VirtualBackgroundProcessor'; + _this.backgroundImage = backgroundImage; + _this.fitType = fitType; + return _this; + } + Object.defineProperty(VirtualBackgroundProcessor.prototype, "backgroundImage", { + /** + * The HTMLImageElement representing the current background image. + */ + get: function () { + return this._backgroundImage; + }, + /** + * Set an HTMLImageElement as the new background image. + * An error will be raised if the image hasn't been fully loaded yet. Additionally, the image must follow + * [security guidelines](https://developer.mozilla.org/en-US/docs/Web/HTML/CORS_enabled_image) + * when loading the image from a different origin. Failing to do so will result to an empty output frame. + */ + set: function (image) { + var _this = this; + if (!image || !image.complete || !image.naturalHeight) { + throw new Error('Invalid image. Make sure that the image is an HTMLImageElement and has been successfully loaded'); + } + this._backgroundImage = image; + createImageBitmap(this._backgroundImage).then(function (imageBitmap) { return _this._backgroundProcessorPipeline + .setBackgroundImage(imageBitmap); }).catch(function () { + /* noop */ + }); + }, + enumerable: false, + configurable: true + }); + Object.defineProperty(VirtualBackgroundProcessor.prototype, "fitType", { + /** + * The current [[ImageFit]] for positioning of the background image in the viewport. + */ + get: function () { + return this._fitType; + }, + /** + * Set a new [[ImageFit]] to be used for positioning the background image in the viewport. + */ + set: function (fitType) { + var validTypes = Object.keys(types_1.ImageFit); + if (!validTypes.includes(fitType)) { + console.warn("Valid fitType not found. Using '".concat(types_1.ImageFit.Fill, "' as default.")); + fitType = types_1.ImageFit.Fill; + } + this._fitType = fitType; + this._backgroundProcessorPipeline + .setFitType(this._fitType) + .catch(function () { + /* noop */ + }); + }, + enumerable: false, + configurable: true + }); + return VirtualBackgroundProcessor; +}(BackgroundProcessor_1.BackgroundProcessor)); +exports.VirtualBackgroundProcessor = VirtualBackgroundProcessor; + +},{"../../constants":1,"../../types":26,"../../utils/support":29,"./BackgroundProcessor":4,"./pipelines/backgroundprocessorpipeline":15}],7:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.BackgroundProcessorPipeline = void 0; +var constants_1 = require("../../../../constants"); +var Benchmark_1 = require("../../../../utils/Benchmark"); +var support_1 = require("../../../../utils/support"); +var TwilioTFLite_1 = require("../../../../utils/TwilioTFLite"); +var pipelines_1 = require("../../../pipelines"); +var InputFrameDownscaleStage_1 = require("./InputFrameDownscaleStage"); +var PostProcessingStage_1 = require("./PostProcessingStage"); +/** + * @private + */ +var BackgroundProcessorPipeline = /** @class */ (function (_super) { + __extends(BackgroundProcessorPipeline, _super); + function BackgroundProcessorPipeline(options, onResizeWebGL2Canvas) { + if (onResizeWebGL2Canvas === void 0) { onResizeWebGL2Canvas = function () { }; } + var _this = _super.call(this) || this; + _this._outputCanvas = new OffscreenCanvas(1, 1); + _this._webgl2Canvas = new OffscreenCanvas(1, 1); + _this._benchmark = new Benchmark_1.Benchmark(); + _this._inferenceInputCanvas = new OffscreenCanvas(constants_1.WASM_INFERENCE_DIMENSIONS.width, constants_1.WASM_INFERENCE_DIMENSIONS.height); + _this._inputFrameDownscaleMode = (0, support_1.isChromiumImageBitmap)() ? 'image-bitmap' : 'canvas'; + var assetsPath = options.assetsPath, deferInputFrameDownscale = options.deferInputFrameDownscale, maskBlurRadius = options.maskBlurRadius; + _this._assetsPath = assetsPath; + _this._deferInputFrameDownscale = deferInputFrameDownscale; + _this._onResizeWebGL2Canvas = onResizeWebGL2Canvas; + _this.addStage(new InputFrameDownscaleStage_1.InputFrameDowscaleStage(_this._inferenceInputCanvas, _this._inputFrameDownscaleMode)); + _this.addStage(new PostProcessingStage_1.PostProcessingStage(constants_1.WASM_INFERENCE_DIMENSIONS, _this._webgl2Canvas, _this._outputCanvas, maskBlurRadius, function (inputFrame) { return _this._setBackground(inputFrame); })); + return _this; + } + BackgroundProcessorPipeline.prototype.loadTwilioTFLite = function () { + return __awaiter(this, void 0, void 0, function () { + var _twilioTFLite; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + _twilioTFLite = BackgroundProcessorPipeline._twilioTFLite; + if (!!_twilioTFLite) return [3 /*break*/, 2]; + _twilioTFLite = new TwilioTFLite_1.TwilioTFLite(); + return [4 /*yield*/, _twilioTFLite.initialize(this._assetsPath, constants_1.MODEL_NAME, constants_1.TFLITE_LOADER_NAME, constants_1.TFLITE_SIMD_LOADER_NAME)]; + case 1: + _a.sent(); + BackgroundProcessorPipeline._twilioTFLite = _twilioTFLite; + _a.label = 2; + case 2: return [2 /*return*/, _twilioTFLite.isSimdEnabled]; + } + }); + }); + }; + BackgroundProcessorPipeline.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var _a, inputFrameDownscaleStage, postProcessingStage, _b, _benchmark, _deferInputFrameDownscale, _c, inferenceInputHeight, inferenceInputWidth, _outputCanvas, _webgl2Canvas, _twilioTFLite, isInputVideoFrame, _d, height, width, didResizeWebGL2Canvas, downscalePromise, personMask; + return __generator(this, function (_e) { + switch (_e.label) { + case 0: + if (!BackgroundProcessorPipeline._twilioTFLite) { + return [2 /*return*/, null]; + } + _a = this._stages, inputFrameDownscaleStage = _a[0], postProcessingStage = _a[1]; + _b = this, _benchmark = _b._benchmark, _deferInputFrameDownscale = _b._deferInputFrameDownscale, _c = _b._inferenceInputCanvas, inferenceInputHeight = _c.height, inferenceInputWidth = _c.width, _outputCanvas = _b._outputCanvas, _webgl2Canvas = _b._webgl2Canvas; + _twilioTFLite = BackgroundProcessorPipeline._twilioTFLite; + isInputVideoFrame = typeof VideoFrame === 'function' + && inputFrame instanceof VideoFrame; + _d = isInputVideoFrame + ? { height: inputFrame.displayHeight, width: inputFrame.displayWidth } + : inputFrame, height = _d.height, width = _d.width; + didResizeWebGL2Canvas = false; + if (_outputCanvas.width !== width) { + _outputCanvas.width = width; + _webgl2Canvas.width = width; + didResizeWebGL2Canvas = true; + } + if (_outputCanvas.height !== height) { + _outputCanvas.height = height; + _webgl2Canvas.height = height; + didResizeWebGL2Canvas = true; + } + if (didResizeWebGL2Canvas) { + postProcessingStage.resetPersonMaskUpscalePipeline(); + this._onResizeWebGL2Canvas(); + } + _benchmark.start('inputImageResizeDelay'); + downscalePromise = inputFrameDownscaleStage.render(inputFrame) + .then(function (downscaledFrameData) { + _twilioTFLite.loadInputBuffer(downscaledFrameData); + }); + if (!!_deferInputFrameDownscale) return [3 /*break*/, 2]; + return [4 /*yield*/, downscalePromise]; + case 1: + _e.sent(); + _e.label = 2; + case 2: + _benchmark.end('inputImageResizeDelay'); + _benchmark.start('segmentationDelay'); + personMask = new ImageData(_twilioTFLite.runInference(), inferenceInputWidth, inferenceInputHeight); + _benchmark.end('segmentationDelay'); + _benchmark.start('imageCompositionDelay'); + postProcessingStage.render(inputFrame, personMask); + _benchmark.end('imageCompositionDelay'); + if (typeof VideoFrame === 'function' + && inputFrame instanceof VideoFrame) { + inputFrame.close(); + } + return [2 /*return*/, this._outputCanvas]; + } + }); + }); + }; + BackgroundProcessorPipeline.prototype.setDeferInputFrameDownscale = function (defer) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + this._deferInputFrameDownscale = defer; + return [2 /*return*/]; + }); + }); + }; + BackgroundProcessorPipeline.prototype.setMaskBlurRadius = function (radius) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + this._stages[1] + .updateMaskBlurRadius(radius); + return [2 /*return*/]; + }); + }); + }; + BackgroundProcessorPipeline._twilioTFLite = null; + return BackgroundProcessorPipeline; +}(pipelines_1.Pipeline)); +exports.BackgroundProcessorPipeline = BackgroundProcessorPipeline; + +},{"../../../../constants":1,"../../../../utils/Benchmark":27,"../../../../utils/TwilioTFLite":28,"../../../../utils/support":29,"../../../pipelines":21,"./InputFrameDownscaleStage":11,"./PostProcessingStage":12}],8:[function(require,module,exports){ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.BackgroundProcessorPipelineProxy = void 0; +var comlink_1 = require("comlink"); +var Benchmark_1 = require("../../../../utils/Benchmark"); +/** + * @private + */ +var BackgroundProcessorPipelineProxy = /** @class */ (function () { + function BackgroundProcessorPipelineProxy(pipelineWorkerPromise) { + this._benchmark = new Benchmark_1.Benchmark(); + this._pipelineWorkerPromise = pipelineWorkerPromise; + } + BackgroundProcessorPipelineProxy.prototype.loadTwilioTFLite = function () { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.loadTwilioTFLite()]; + } + }); + }); + }; + BackgroundProcessorPipelineProxy.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker, outputFrame, _a, _b; + return __generator(this, function (_c) { + switch (_c.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _c.sent(); + return [4 /*yield*/, pipelineWorker.render((0, comlink_1.transfer)(inputFrame, [inputFrame]))]; + case 2: + outputFrame = _c.sent(); + // @ts-ignore + _b = (_a = this._benchmark).merge; + return [4 /*yield*/, pipelineWorker._benchmark]; + case 3: + // @ts-ignore + _b.apply(_a, [_c.sent()]); + return [2 /*return*/, outputFrame]; + } + }); + }); + }; + BackgroundProcessorPipelineProxy.prototype.setDeferInputFrameDownscale = function (defer) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.setDeferInputFrameDownscale(defer)]; + } + }); + }); + }; + BackgroundProcessorPipelineProxy.prototype.setMaskBlurRadius = function (radius) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.setMaskBlurRadius(radius)]; + } + }); + }); + }; + return BackgroundProcessorPipelineProxy; +}()); +exports.BackgroundProcessorPipelineProxy = BackgroundProcessorPipelineProxy; + +},{"../../../../utils/Benchmark":27,"comlink":31}],9:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurBackgroundProcessorPipeline = void 0; +var support_1 = require("../../../../utils/support"); +var gaussianblurfilterpipeline_1 = require("../gaussianblurfilterpipeline"); +var BackgroundProcessorPipeline_1 = require("./BackgroundProcessorPipeline"); +/** + * @private + */ +var GaussianBlurBackgroundProcessorPipeline = /** @class */ (function (_super) { + __extends(GaussianBlurBackgroundProcessorPipeline, _super); + function GaussianBlurBackgroundProcessorPipeline(options) { + var _this = _super.call(this, options, function () { return _this._resetGaussianBlurFilterPipeline(); }) || this; + _this._gaussianBlurFilterPipeline = null; + var blurFilterRadius = options.blurFilterRadius; + _this._blurFilterRadius = blurFilterRadius; + return _this; + } + GaussianBlurBackgroundProcessorPipeline.prototype.setBlurFilterRadius = function (radius) { + var _a; + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_b) { + this._blurFilterRadius = radius; + return [2 /*return*/, (_a = this._gaussianBlurFilterPipeline) === null || _a === void 0 ? void 0 : _a.updateRadius(this._blurFilterRadius)]; + }); + }); + }; + GaussianBlurBackgroundProcessorPipeline.prototype._setBackground = function (inputFrame) { + var _a = this, _blurFilterRadius = _a._blurFilterRadius, _outputCanvas = _a._outputCanvas, _webgl2Canvas = _a._webgl2Canvas; + var ctx = _outputCanvas.getContext('2d'); + if (support_1.isCanvasBlurSupported) { + ctx.filter = "blur(".concat(_blurFilterRadius, "px)"); + ctx.drawImage(inputFrame, 0, 0); + ctx.filter = 'none'; + return; + } + if (!this._gaussianBlurFilterPipeline) { + this._resetGaussianBlurFilterPipeline(); + } + this._gaussianBlurFilterPipeline.render(); + ctx.drawImage(_webgl2Canvas, 0, 0); + }; + GaussianBlurBackgroundProcessorPipeline.prototype._resetGaussianBlurFilterPipeline = function () { + var _a; + var _b = this, _blurFilterRadius = _b._blurFilterRadius, _webgl2Canvas = _b._webgl2Canvas; + (_a = this._gaussianBlurFilterPipeline) === null || _a === void 0 ? void 0 : _a.cleanUp(); + this._gaussianBlurFilterPipeline = new gaussianblurfilterpipeline_1.GaussianBlurFilterPipeline(_webgl2Canvas); + this._gaussianBlurFilterPipeline.updateRadius(_blurFilterRadius); + }; + return GaussianBlurBackgroundProcessorPipeline; +}(BackgroundProcessorPipeline_1.BackgroundProcessorPipeline)); +exports.GaussianBlurBackgroundProcessorPipeline = GaussianBlurBackgroundProcessorPipeline; + +},{"../../../../utils/support":29,"../gaussianblurfilterpipeline":17,"./BackgroundProcessorPipeline":7}],10:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurBackgroundProcessorPipelineProxy = void 0; +var comlink_1 = require("comlink"); +var constants_1 = require("../../../../constants"); +var BackgroundProcessorPipeline_proxy_1 = require("./BackgroundProcessorPipeline.proxy"); +var GaussianBlurBackgroundProcessorPipelineWorker; +/** + * @private + */ +var GaussianBlurBackgroundProcessorPipelineProxy = /** @class */ (function (_super) { + __extends(GaussianBlurBackgroundProcessorPipelineProxy, _super); + function GaussianBlurBackgroundProcessorPipelineProxy(options) { + var _this = this; + GaussianBlurBackgroundProcessorPipelineWorker || (GaussianBlurBackgroundProcessorPipelineWorker = (0, comlink_1.wrap)(new Worker("".concat(options.assetsPath).concat(constants_1.TWILIO_GAUSSIAN_BLUR_BACKGROUND_PROCESSOR_PIPELINE_WORKER)))); + var pipelineWorkerPromise = new GaussianBlurBackgroundProcessorPipelineWorker(options); + _this = _super.call(this, pipelineWorkerPromise) || this; + _this._pipelineWorkerPromise = pipelineWorkerPromise; + return _this; + } + GaussianBlurBackgroundProcessorPipelineProxy.prototype.setBlurFilterRadius = function (radius) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.setBlurFilterRadius(radius)]; + } + }); + }); + }; + return GaussianBlurBackgroundProcessorPipelineProxy; +}(BackgroundProcessorPipeline_proxy_1.BackgroundProcessorPipelineProxy)); +exports.GaussianBlurBackgroundProcessorPipelineProxy = GaussianBlurBackgroundProcessorPipelineProxy; + +},{"../../../../constants":1,"./BackgroundProcessorPipeline.proxy":8,"comlink":31}],11:[function(require,module,exports){ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.InputFrameDowscaleStage = void 0; +/** + * @private + */ +var InputFrameDowscaleStage = /** @class */ (function () { + function InputFrameDowscaleStage(outputCanvas, inputFrameDownscaleMode) { + this._inputFrameDownscaleMode = inputFrameDownscaleMode; + this._outputContext = outputCanvas.getContext('2d', { willReadFrequently: true }); + } + InputFrameDowscaleStage.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var _a, _outputContext, _inputFrameDownscaleMode, _b, resizeHeight, resizeWidth, downscaledBitmap, data; + return __generator(this, function (_c) { + switch (_c.label) { + case 0: + _a = this, _outputContext = _a._outputContext, _inputFrameDownscaleMode = _a._inputFrameDownscaleMode; + _b = _outputContext.canvas, resizeHeight = _b.height, resizeWidth = _b.width; + if (!(_inputFrameDownscaleMode === 'image-bitmap')) return [3 /*break*/, 2]; + return [4 /*yield*/, createImageBitmap(inputFrame, { + resizeWidth: resizeWidth, + resizeHeight: resizeHeight, + resizeQuality: 'pixelated' + })]; + case 1: + downscaledBitmap = _c.sent(); + _outputContext.drawImage(downscaledBitmap, 0, 0); + downscaledBitmap.close(); + return [3 /*break*/, 3]; + case 2: + _outputContext.drawImage(inputFrame, 0, 0, resizeWidth, resizeHeight); + _c.label = 3; + case 3: + data = _outputContext.getImageData(0, 0, resizeWidth, resizeHeight).data; + return [2 /*return*/, data]; + } + }); + }); + }; + return InputFrameDowscaleStage; +}()); +exports.InputFrameDowscaleStage = InputFrameDowscaleStage; + +},{}],12:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PostProcessingStage = void 0; +var personmaskupscalepipeline_1 = require("../personmaskupscalepipeline"); +/** + * @private + */ +var PostProcessingStage = /** @class */ (function () { + function PostProcessingStage(inputDimensions, webgl2Canvas, outputCanvas, maskBlurRadius, setBackground) { + this._personMaskUpscalePipeline = null; + this._inputDimensions = inputDimensions; + this._maskBlurRadius = maskBlurRadius; + this._outputContext = outputCanvas.getContext('2d'); + this._webgl2Canvas = webgl2Canvas; + this._setBackground = setBackground; + } + PostProcessingStage.prototype.render = function (inputFrame, personMask) { + var _a = this, _outputContext = _a._outputContext, _setBackground = _a._setBackground, _webgl2Canvas = _a._webgl2Canvas; + if (!this._personMaskUpscalePipeline) { + this.resetPersonMaskUpscalePipeline(); + } + this._personMaskUpscalePipeline.render(inputFrame, personMask); + _outputContext.save(); + _outputContext.globalCompositeOperation = 'copy'; + _outputContext.drawImage(_webgl2Canvas, 0, 0); + _outputContext.globalCompositeOperation = 'destination-over'; + _setBackground(inputFrame); + _outputContext.restore(); + }; + PostProcessingStage.prototype.resetPersonMaskUpscalePipeline = function () { + var _a; + var _b = this, _inputDimensions = _b._inputDimensions, _maskBlurRadius = _b._maskBlurRadius, _webgl2Canvas = _b._webgl2Canvas; + (_a = this._personMaskUpscalePipeline) === null || _a === void 0 ? void 0 : _a.cleanUp(); + this._personMaskUpscalePipeline = new personmaskupscalepipeline_1.PersonMaskUpscalePipeline(_inputDimensions, _webgl2Canvas); + this._personMaskUpscalePipeline.updateBilateralFilterConfig({ + sigmaSpace: _maskBlurRadius + }); + }; + PostProcessingStage.prototype.updateMaskBlurRadius = function (radius) { + var _a; + if (this._maskBlurRadius !== radius) { + this._maskBlurRadius = radius; + (_a = this._personMaskUpscalePipeline) === null || _a === void 0 ? void 0 : _a.updateBilateralFilterConfig({ + sigmaSpace: radius + }); + } + }; + return PostProcessingStage; +}()); +exports.PostProcessingStage = PostProcessingStage; + +},{"../personmaskupscalepipeline":19}],13:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessorPipeline = void 0; +var types_1 = require("../../../../types"); +var BackgroundProcessorPipeline_1 = require("./BackgroundProcessorPipeline"); +/** + * @private + */ +var VirtualBackgroundProcessorPipeline = /** @class */ (function (_super) { + __extends(VirtualBackgroundProcessorPipeline, _super); + function VirtualBackgroundProcessorPipeline(options) { + var _this = _super.call(this, options) || this; + var fitType = options.fitType; + _this._backgroundImage = null; + _this._fitType = fitType; + return _this; + } + VirtualBackgroundProcessorPipeline.prototype.setBackgroundImage = function (backgroundImage) { + var _a; + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_b) { + (_a = this._backgroundImage) === null || _a === void 0 ? void 0 : _a.close(); + this._backgroundImage = backgroundImage; + return [2 /*return*/]; + }); + }); + }; + VirtualBackgroundProcessorPipeline.prototype.setFitType = function (fitType) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + this._fitType = fitType; + return [2 /*return*/]; + }); + }); + }; + VirtualBackgroundProcessorPipeline.prototype._setBackground = function () { + var _a = this, _backgroundImage = _a._backgroundImage, _fitType = _a._fitType, _outputCanvas = _a._outputCanvas; + if (!_backgroundImage) { + return; + } + var ctx = _outputCanvas.getContext('2d'); + var imageWidth = _backgroundImage.width; + var imageHeight = _backgroundImage.height; + var canvasWidth = _outputCanvas.width; + var canvasHeight = _outputCanvas.height; + if (_fitType === types_1.ImageFit.Fill) { + ctx.drawImage(_backgroundImage, 0, 0, imageWidth, imageHeight, 0, 0, canvasWidth, canvasHeight); + } + else if (_fitType === types_1.ImageFit.None) { + ctx.drawImage(_backgroundImage, 0, 0, imageWidth, imageHeight); + } + else { + var _b = this._getFitPosition(imageWidth, imageHeight, canvasWidth, canvasHeight, _fitType), x = _b.x, y = _b.y, w = _b.w, h = _b.h; + ctx.drawImage(_backgroundImage, 0, 0, imageWidth, imageHeight, x, y, w, h); + } + }; + VirtualBackgroundProcessorPipeline.prototype._getFitPosition = function (contentWidth, contentHeight, viewportWidth, viewportHeight, type) { + // Calculate new content width to fit viewport width + var factor = viewportWidth / contentWidth; + var newContentWidth = viewportWidth; + var newContentHeight = factor * contentHeight; + // Scale down the resulting height and width more + // to fit viewport height if the content still exceeds it + if ((type === types_1.ImageFit.Contain && newContentHeight > viewportHeight) + || (type === types_1.ImageFit.Cover && viewportHeight > newContentHeight)) { + factor = viewportHeight / newContentHeight; + newContentWidth = factor * newContentWidth; + newContentHeight = viewportHeight; + } + // Calculate the destination top left corner to center the content + var x = (viewportWidth - newContentWidth) / 2; + var y = (viewportHeight - newContentHeight) / 2; + return { + x: x, + y: y, + w: newContentWidth, + h: newContentHeight, + }; + }; + return VirtualBackgroundProcessorPipeline; +}(BackgroundProcessorPipeline_1.BackgroundProcessorPipeline)); +exports.VirtualBackgroundProcessorPipeline = VirtualBackgroundProcessorPipeline; + +},{"../../../../types":26,"./BackgroundProcessorPipeline":7}],14:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessorPipelineProxy = void 0; +var comlink_1 = require("comlink"); +var constants_1 = require("../../../../constants"); +var BackgroundProcessorPipeline_proxy_1 = require("./BackgroundProcessorPipeline.proxy"); +var VirtualBackgroundProcessorPipelineWorker; +/** + * @private + */ +var VirtualBackgroundProcessorPipelineProxy = /** @class */ (function (_super) { + __extends(VirtualBackgroundProcessorPipelineProxy, _super); + function VirtualBackgroundProcessorPipelineProxy(options) { + var _this = this; + VirtualBackgroundProcessorPipelineWorker || (VirtualBackgroundProcessorPipelineWorker = (0, comlink_1.wrap)(new Worker("".concat(options.assetsPath).concat(constants_1.TWILIO_VIRTUAL_BACKGROUND_PROCESSOR_PIPELINE_WORKER)))); + var pipelineWorkerPromise = new VirtualBackgroundProcessorPipelineWorker(options); + _this = _super.call(this, pipelineWorkerPromise) || this; + _this._pipelineWorkerPromise = pipelineWorkerPromise; + return _this; + } + VirtualBackgroundProcessorPipelineProxy.prototype.setBackgroundImage = function (backgroundImage) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.setBackgroundImage((0, comlink_1.transfer)(backgroundImage, [backgroundImage]))]; + } + }); + }); + }; + VirtualBackgroundProcessorPipelineProxy.prototype.setFitType = function (fitType) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.setFitType(fitType)]; + } + }); + }); + }; + return VirtualBackgroundProcessorPipelineProxy; +}(BackgroundProcessorPipeline_proxy_1.BackgroundProcessorPipelineProxy)); +exports.VirtualBackgroundProcessorPipelineProxy = VirtualBackgroundProcessorPipelineProxy; + +},{"../../../../constants":1,"./BackgroundProcessorPipeline.proxy":8,"comlink":31}],15:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessorPipelineProxy = exports.VirtualBackgroundProcessorPipeline = exports.GaussianBlurBackgroundProcessorPipelineProxy = exports.GaussianBlurBackgroundProcessorPipeline = exports.BackgroundProcessorPipelineProxy = exports.BackgroundProcessorPipeline = void 0; +var BackgroundProcessorPipeline_1 = require("./BackgroundProcessorPipeline"); +Object.defineProperty(exports, "BackgroundProcessorPipeline", { enumerable: true, get: function () { return BackgroundProcessorPipeline_1.BackgroundProcessorPipeline; } }); +var BackgroundProcessorPipeline_proxy_1 = require("./BackgroundProcessorPipeline.proxy"); +Object.defineProperty(exports, "BackgroundProcessorPipelineProxy", { enumerable: true, get: function () { return BackgroundProcessorPipeline_proxy_1.BackgroundProcessorPipelineProxy; } }); +var GaussianBlurBackgroundProcessorPipeline_1 = require("./GaussianBlurBackgroundProcessorPipeline"); +Object.defineProperty(exports, "GaussianBlurBackgroundProcessorPipeline", { enumerable: true, get: function () { return GaussianBlurBackgroundProcessorPipeline_1.GaussianBlurBackgroundProcessorPipeline; } }); +var GaussianBlurBackgroundProcessorPipeline_proxy_1 = require("./GaussianBlurBackgroundProcessorPipeline.proxy"); +Object.defineProperty(exports, "GaussianBlurBackgroundProcessorPipelineProxy", { enumerable: true, get: function () { return GaussianBlurBackgroundProcessorPipeline_proxy_1.GaussianBlurBackgroundProcessorPipelineProxy; } }); +var VirtualBackgroundProcessorPipeline_1 = require("./VirtualBackgroundProcessorPipeline"); +Object.defineProperty(exports, "VirtualBackgroundProcessorPipeline", { enumerable: true, get: function () { return VirtualBackgroundProcessorPipeline_1.VirtualBackgroundProcessorPipeline; } }); +var VirtualBackgroundProcessorPipeline_proxy_1 = require("./VirtualBackgroundProcessorPipeline.proxy"); +Object.defineProperty(exports, "VirtualBackgroundProcessorPipelineProxy", { enumerable: true, get: function () { return VirtualBackgroundProcessorPipeline_proxy_1.VirtualBackgroundProcessorPipelineProxy; } }); + +},{"./BackgroundProcessorPipeline":7,"./BackgroundProcessorPipeline.proxy":8,"./GaussianBlurBackgroundProcessorPipeline":9,"./GaussianBlurBackgroundProcessorPipeline.proxy":10,"./VirtualBackgroundProcessorPipeline":13,"./VirtualBackgroundProcessorPipeline.proxy":14}],16:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.SinglePassGaussianBlurFilterStage = void 0; +var pipelines_1 = require("../../../pipelines"); +/** + * @private + */ +function createGaussianBlurWeights(radius) { + var coeff = 1.0 / Math.sqrt(2.0 * Math.PI) / radius; + return '0'.repeat(radius + 1).split('').map(function (zero, x) { + return coeff * Math.exp(-0.5 * x * x / radius / radius); + }); +} +/** + * @private + */ +var SinglePassGaussianBlurFilterStage = /** @class */ (function (_super) { + __extends(SinglePassGaussianBlurFilterStage, _super); + function SinglePassGaussianBlurFilterStage(glOut, direction, outputType, inputTextureUnit, outputTextureUnit) { + var _this = this; + if (outputTextureUnit === void 0) { outputTextureUnit = inputTextureUnit + 1; } + var _a = glOut.canvas, height = _a.height, width = _a.width; + _this = _super.call(this, { + textureName: 'u_inputTexture', + textureUnit: inputTextureUnit + }, { + fragmentShaderSource: "#version 300 es\n precision highp float;\n\n uniform sampler2D u_inputTexture;\n uniform vec2 u_texelSize;\n uniform float u_direction;\n uniform float u_radius;\n uniform float u_gaussianBlurWeights[128];\n\n in vec2 v_texCoord;\n\n out vec4 outColor;\n\n void main() {\n float totalWeight = u_gaussianBlurWeights[0];\n vec3 newColor = totalWeight * texture(u_inputTexture, v_texCoord).rgb;\n\n for (float i = 1.0; i <= u_radius; i += 1.0) {\n float x = (1.0 - u_direction) * i;\n float y = u_direction * i;\n\n vec2 shift = vec2(x, y) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n float weight = u_gaussianBlurWeights[int(i)];\n newColor += weight * texture(u_inputTexture, coord).rgb;\n totalWeight += weight;\n\n shift = vec2(-x, -y) * u_texelSize;\n coord = vec2(v_texCoord + shift);\n newColor += weight * texture(u_inputTexture, coord).rgb;\n totalWeight += weight;\n }\n\n newColor /= totalWeight;\n outColor = vec4(newColor, 1.0);\n }\n ", + glOut: glOut, + height: height, + textureUnit: outputTextureUnit, + type: outputType, + width: width, + uniformVars: [ + { + name: 'u_direction', + type: 'float', + values: [direction === 'vertical' ? 1 : 0] + }, + { + name: 'u_texelSize', + type: 'float', + values: [1 / width, 1 / height] + } + ] + }) || this; + _this.updateRadius(0); + return _this; + } + SinglePassGaussianBlurFilterStage.prototype.updateRadius = function (radius) { + this._setUniformVars([ + { + name: 'u_radius', + type: 'float', + values: [radius] + }, + { + name: 'u_gaussianBlurWeights', + type: 'float:v', + values: createGaussianBlurWeights(radius) + } + ]); + }; + return SinglePassGaussianBlurFilterStage; +}(pipelines_1.WebGL2Pipeline.ProcessingStage)); +exports.SinglePassGaussianBlurFilterStage = SinglePassGaussianBlurFilterStage; + +},{"../../../pipelines":21}],17:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurFilterPipeline = void 0; +var pipelines_1 = require("../../../pipelines"); +var SinglePassGaussianBlurFilterStage_1 = require("./SinglePassGaussianBlurFilterStage"); +/** + * @private + */ +var GaussianBlurFilterPipeline = /** @class */ (function (_super) { + __extends(GaussianBlurFilterPipeline, _super); + function GaussianBlurFilterPipeline(outputCanvas) { + var _this = _super.call(this) || this; + var glOut = outputCanvas.getContext('webgl2'); + _this.addStage(new SinglePassGaussianBlurFilterStage_1.SinglePassGaussianBlurFilterStage(glOut, 'horizontal', 'texture', 0, 2)); + _this.addStage(new SinglePassGaussianBlurFilterStage_1.SinglePassGaussianBlurFilterStage(glOut, 'vertical', 'canvas', 2)); + return _this; + } + GaussianBlurFilterPipeline.prototype.updateRadius = function (radius) { + this._stages.forEach(function (stage) { return stage + .updateRadius(radius); }); + }; + return GaussianBlurFilterPipeline; +}(pipelines_1.WebGL2Pipeline)); +exports.GaussianBlurFilterPipeline = GaussianBlurFilterPipeline; + +},{"../../../pipelines":21,"./SinglePassGaussianBlurFilterStage":16}],18:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.SinglePassBilateralFilterStage = void 0; +var pipelines_1 = require("../../../pipelines"); +/** + * @private + */ +function createSpaceWeights(radius, sigma, texelSize) { + return '0'.repeat(radius).split('').map(function (zero, i) { + var x = (i + 1) * texelSize; + return Math.exp(-0.5 * x * x / sigma / sigma); + }); +} +/** + * @private + */ +function createColorWeights(sigma) { + return '0'.repeat(256).split('').map(function (zero, i) { + var x = i / 255; + return Math.exp(-0.5 * x * x / sigma / sigma); + }); +} +/** + * @private + */ +var SinglePassBilateralFilterStage = /** @class */ (function (_super) { + __extends(SinglePassBilateralFilterStage, _super); + function SinglePassBilateralFilterStage(glOut, direction, outputType, inputDimensions, outputDimensions, inputTextureUnit, outputTextureUnit) { + var _this = this; + if (outputTextureUnit === void 0) { outputTextureUnit = inputTextureUnit + 1; } + var height = outputDimensions.height, width = outputDimensions.width; + _this = _super.call(this, { + textureName: 'u_segmentationMask', + textureUnit: inputTextureUnit + }, { + fragmentShaderSource: "#version 300 es\n precision highp float;\n\n uniform sampler2D u_inputFrame;\n uniform sampler2D u_segmentationMask;\n uniform vec2 u_texelSize;\n uniform float u_direction;\n uniform float u_radius;\n uniform float u_step;\n uniform float u_spaceWeights[128];\n uniform float u_colorWeights[256];\n\n in vec2 v_texCoord;\n\n out vec4 outColor;\n\n float calculateColorWeight(vec2 coord, vec3 centerColor) {\n vec3 coordColor = texture(u_inputFrame, coord).rgb;\n float x = distance(centerColor, coordColor);\n return u_colorWeights[int(x * 255.0)];\n }\n\n float edgePixelsAverageAlpha(float outAlpha) {\n float totalAlpha = outAlpha;\n float totalPixels = 1.0;\n\n for (float i = -u_radius; u_radius > 0.0 && i <= u_radius; i += u_radius) {\n for (float j = -u_radius; j <= u_radius; j += u_radius * (j == 0.0 ? 2.0 : 1.0)) {\n vec2 shift = vec2(i, j) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n totalAlpha += texture(u_segmentationMask, coord).a;\n totalPixels++;\n }\n }\n\n return totalAlpha / totalPixels;\n }\n\n void main() {\n vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb;\n float outAlpha = texture(u_segmentationMask, v_texCoord).a;\n float averageAlpha = edgePixelsAverageAlpha(outAlpha);\n float totalWeight = 1.0;\n\n if (averageAlpha == 0.0 || averageAlpha == 1.0) {\n outColor = vec4(averageAlpha * centerColor, averageAlpha);\n return;\n }\n\n for (float i = 1.0; i <= u_radius; i += u_step) {\n float x = (1.0 - u_direction) * i;\n float y = u_direction * i;\n vec2 shift = vec2(x, y) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n float spaceWeight = u_spaceWeights[int(i - 1.0)];\n float colorWeight = calculateColorWeight(coord, centerColor);\n float weight = spaceWeight * colorWeight;\n float alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n\n shift = vec2(-x, -y) * u_texelSize;\n coord = vec2(v_texCoord + shift);\n colorWeight = calculateColorWeight(coord, centerColor);\n weight = spaceWeight * colorWeight;\n alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n }\n\n outAlpha /= totalWeight;\n outColor = vec4(outAlpha * centerColor, outAlpha);\n }\n ", + glOut: glOut, + height: height, + textureUnit: outputTextureUnit, + type: outputType, + width: width, + uniformVars: [ + { + name: 'u_inputFrame', + type: 'int', + values: [0] + }, + { + name: 'u_direction', + type: 'float', + values: [direction === 'vertical' ? 1 : 0] + }, + { + name: 'u_texelSize', + type: 'float', + values: [1 / width, 1 / height] + } + ] + }) || this; + _this._direction = direction; + _this._inputDimensions = inputDimensions; + _this.updateSigmaColor(0); + _this.updateSigmaSpace(0); + return _this; + } + SinglePassBilateralFilterStage.prototype.updateSigmaColor = function (sigmaColor) { + this._setUniformVars([ + { + name: 'u_colorWeights', + type: 'float:v', + values: createColorWeights(sigmaColor) + } + ]); + }; + SinglePassBilateralFilterStage.prototype.updateSigmaSpace = function (sigmaSpace) { + var _a = this._inputDimensions, inputHeight = _a.height, inputWidth = _a.width; + var _b = this._outputDimensions, outputHeight = _b.height, outputWidth = _b.width; + sigmaSpace *= Math.max(outputWidth / inputWidth, outputHeight / inputHeight); + var step = Math.floor(0.5 * sigmaSpace / Math.log(sigmaSpace)); + var sigmaTexel = Math.max(1 / outputWidth, 1 / outputHeight) * sigmaSpace; + var texelSize = 1 / (this._direction === 'horizontal' + ? outputWidth + : outputHeight); + this._setUniformVars([ + { + name: 'u_radius', + type: 'float', + values: [sigmaSpace] + }, + { + name: 'u_spaceWeights', + type: 'float:v', + values: createSpaceWeights(sigmaSpace, sigmaTexel, texelSize) + }, + { + name: 'u_step', + type: 'float', + values: [step] + } + ]); + }; + return SinglePassBilateralFilterStage; +}(pipelines_1.WebGL2Pipeline.ProcessingStage)); +exports.SinglePassBilateralFilterStage = SinglePassBilateralFilterStage; + +},{"../../../pipelines":21}],19:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PersonMaskUpscalePipeline = void 0; +var pipelines_1 = require("../../../pipelines"); +var SinglePassBilateralFilterStage_1 = require("./SinglePassBilateralFilterStage"); +/** + * @private + */ +var PersonMaskUpscalePipeline = /** @class */ (function (_super) { + __extends(PersonMaskUpscalePipeline, _super); + function PersonMaskUpscalePipeline(inputDimensions, outputCanvas) { + var _this = _super.call(this) || this; + var glOut = outputCanvas.getContext('webgl2'); + var outputDimensions = { + height: outputCanvas.height, + width: outputCanvas.width + }; + _this.addStage(new pipelines_1.WebGL2Pipeline.InputStage(glOut)); + _this.addStage(new SinglePassBilateralFilterStage_1.SinglePassBilateralFilterStage(glOut, 'horizontal', 'texture', inputDimensions, outputDimensions, 1, 2)); + _this.addStage(new SinglePassBilateralFilterStage_1.SinglePassBilateralFilterStage(glOut, 'vertical', 'canvas', inputDimensions, outputDimensions, 2)); + return _this; + } + PersonMaskUpscalePipeline.prototype.updateBilateralFilterConfig = function (config) { + var _a = this._stages, bilateralFilterStages = _a.slice(1); + var sigmaSpace = config.sigmaSpace; + if (typeof sigmaSpace === 'number') { + bilateralFilterStages.forEach(function (stage) { + stage.updateSigmaColor(0.1); + stage.updateSigmaSpace(sigmaSpace); + }); + } + }; + return PersonMaskUpscalePipeline; +}(pipelines_1.WebGL2Pipeline)); +exports.PersonMaskUpscalePipeline = PersonMaskUpscalePipeline; + +},{"../../../pipelines":21,"./SinglePassBilateralFilterStage":18}],20:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Pipeline = void 0; +/** + * @private + */ +var Pipeline = /** @class */ (function () { + function Pipeline() { + this._stages = []; + } + Pipeline.prototype.addStage = function (stage) { + this._stages.push(stage); + }; + Pipeline.prototype.render = function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + this._stages.forEach(function (stage) { + stage.render.apply(stage, args); + }); + }; + return Pipeline; +}()); +exports.Pipeline = Pipeline; + +},{}],21:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2Pipeline = exports.Pipeline = void 0; +var Pipeline_1 = require("./Pipeline"); +Object.defineProperty(exports, "Pipeline", { enumerable: true, get: function () { return Pipeline_1.Pipeline; } }); +var webgl2pipeline_1 = require("./webgl2pipeline"); +Object.defineProperty(exports, "WebGL2Pipeline", { enumerable: true, get: function () { return webgl2pipeline_1.WebGL2Pipeline; } }); + +},{"./Pipeline":20,"./webgl2pipeline":24}],22:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2PipelineInputStage = void 0; +var webgl2PipelineHelpers_1 = require("./webgl2PipelineHelpers"); +/** + * @private + */ +var WebGL2PipelineInputStage = /** @class */ (function () { + function WebGL2PipelineInputStage(glOut) { + this._inputTexture = null; + var _a = glOut.canvas, height = _a.height, width = _a.width; + this._glOut = glOut; + this._inputFrameTexture = (0, webgl2PipelineHelpers_1.createTexture)(glOut, glOut.RGBA8, width, height, glOut.NEAREST, glOut.NEAREST); + } + WebGL2PipelineInputStage.prototype.cleanUp = function () { + var _a = this, _glOut = _a._glOut, _inputFrameTexture = _a._inputFrameTexture, _inputTexture = _a._inputTexture; + _glOut.deleteTexture(_inputFrameTexture); + _glOut.deleteTexture(_inputTexture); + }; + WebGL2PipelineInputStage.prototype.render = function (inputFrame, inputTextureData) { + var _a = this, _glOut = _a._glOut, _inputFrameTexture = _a._inputFrameTexture; + var _b = _glOut.canvas, height = _b.height, width = _b.width; + _glOut.viewport(0, 0, width, height); + _glOut.clearColor(0, 0, 0, 0); + _glOut.clear(_glOut.COLOR_BUFFER_BIT); + if (inputFrame) { + _glOut.activeTexture(_glOut.TEXTURE0); + _glOut.bindTexture(_glOut.TEXTURE_2D, _inputFrameTexture); + _glOut.texSubImage2D(_glOut.TEXTURE_2D, 0, 0, 0, width, height, _glOut.RGBA, _glOut.UNSIGNED_BYTE, inputFrame); + } + if (!inputTextureData) { + return; + } + var data = inputTextureData.data, textureHeight = inputTextureData.height, textureWidth = inputTextureData.width; + if (!this._inputTexture) { + this._inputTexture = (0, webgl2PipelineHelpers_1.createTexture)(_glOut, _glOut.RGBA8, textureWidth, textureHeight, _glOut.NEAREST, _glOut.NEAREST); + } + _glOut.viewport(0, 0, textureWidth, textureHeight); + _glOut.activeTexture(_glOut.TEXTURE1); + _glOut.bindTexture(_glOut.TEXTURE_2D, this._inputTexture); + _glOut.texSubImage2D(_glOut.TEXTURE_2D, 0, 0, 0, textureWidth, textureHeight, _glOut.RGBA, _glOut.UNSIGNED_BYTE, data); + }; + return WebGL2PipelineInputStage; +}()); +exports.WebGL2PipelineInputStage = WebGL2PipelineInputStage; + +},{"./webgl2PipelineHelpers":25}],23:[function(require,module,exports){ +"use strict"; +var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) { + if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) { + if (ar || !(i in from)) { + if (!ar) ar = Array.prototype.slice.call(from, 0, i); + ar[i] = from[i]; + } + } + return to.concat(ar || Array.prototype.slice.call(from)); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2PipelineProcessingStage = void 0; +var webgl2PipelineHelpers_1 = require("./webgl2PipelineHelpers"); +/**; + * @private + */ +var WebGL2PipelineProcessingStage = /** @class */ (function () { + function WebGL2PipelineProcessingStage(inputConfig, outputConfig) { + this._outputFramebuffer = null; + this._outputTexture = null; + var textureName = inputConfig.textureName, textureUnit = inputConfig.textureUnit; + var glOut = outputConfig.glOut; + this._glOut = glOut; + var fragmentShaderSource = outputConfig.fragmentShaderSource, _a = outputConfig.height, height = _a === void 0 ? glOut.canvas.height : _a, _b = outputConfig.textureUnit, outputTextureUnit = _b === void 0 ? textureUnit + 1 : _b, outputType = outputConfig.type, _c = outputConfig.uniformVars, uniformVars = _c === void 0 ? [] : _c, _d = outputConfig.vertexShaderSource, vertexShaderSource = _d === void 0 ? "#version 300 es\n in vec2 a_position;\n in vec2 a_texCoord;\n\n out vec2 v_texCoord;\n\n void main() {\n gl_Position = vec4(a_position".concat(outputType === 'canvas' + ? ' * vec2(1.0, -1.0)' + : '', ", 0.0, 1.0);\n v_texCoord = a_texCoord;\n }\n ") : _d, _e = outputConfig.width, width = _e === void 0 ? glOut.canvas.width : _e; + this._outputDimensions = { + height: height, + width: width + }; + this._outputTextureUnit = outputTextureUnit; + this._fragmentShader = (0, webgl2PipelineHelpers_1.compileShader)(glOut, glOut.FRAGMENT_SHADER, fragmentShaderSource); + this._vertexShader = (0, webgl2PipelineHelpers_1.compileShader)(glOut, glOut.VERTEX_SHADER, vertexShaderSource); + this._positionBuffer = (0, webgl2PipelineHelpers_1.initBuffer)(glOut, [ + -1.0, -1.0, + 1.0, -1.0, + -1.0, 1.0, + 1.0, 1.0, + ]); + this._texCoordBuffer = (0, webgl2PipelineHelpers_1.initBuffer)(glOut, [ + 0.0, 0.0, + 1.0, 0.0, + 0.0, 1.0, + 1.0, 1.0, + ]); + if (outputType === 'texture') { + this._outputTexture = (0, webgl2PipelineHelpers_1.createTexture)(glOut, glOut.RGBA8, width, height); + this._outputFramebuffer = glOut.createFramebuffer(); + glOut.bindFramebuffer(glOut.FRAMEBUFFER, this._outputFramebuffer); + glOut.framebufferTexture2D(glOut.FRAMEBUFFER, glOut.COLOR_ATTACHMENT0, glOut.TEXTURE_2D, this._outputTexture, 0); + } + var program = (0, webgl2PipelineHelpers_1.createPipelineStageProgram)(glOut, this._vertexShader, this._fragmentShader, this._positionBuffer, this._texCoordBuffer); + this._program = program; + this._setUniformVars(__spreadArray([ + { + name: textureName, + type: 'int', + values: [textureUnit] + } + ], uniformVars, true)); + } + WebGL2PipelineProcessingStage.prototype.cleanUp = function () { + var _a = this, _fragmentShader = _a._fragmentShader, _glOut = _a._glOut, _positionBuffer = _a._positionBuffer, _program = _a._program, _texCoordBuffer = _a._texCoordBuffer, _vertexShader = _a._vertexShader; + _glOut.deleteProgram(_program); + _glOut.deleteBuffer(_texCoordBuffer); + _glOut.deleteBuffer(_positionBuffer); + _glOut.deleteShader(_vertexShader); + _glOut.deleteShader(_fragmentShader); + }; + WebGL2PipelineProcessingStage.prototype.render = function () { + var _a = this, _glOut = _a._glOut, _b = _a._outputDimensions, height = _b.height, width = _b.width, _outputFramebuffer = _a._outputFramebuffer, _outputTexture = _a._outputTexture, _outputTextureUnit = _a._outputTextureUnit, _program = _a._program; + _glOut.viewport(0, 0, width, height); + _glOut.useProgram(_program); + if (_outputTexture) { + _glOut.activeTexture(_glOut.TEXTURE0 + + _outputTextureUnit); + _glOut.bindTexture(_glOut.TEXTURE_2D, _outputTexture); + } + _glOut.bindFramebuffer(_glOut.FRAMEBUFFER, _outputFramebuffer); + _glOut.drawArrays(_glOut.TRIANGLE_STRIP, 0, 4); + }; + WebGL2PipelineProcessingStage.prototype._setUniformVars = function (uniformVars) { + var _a = this, _glOut = _a._glOut, _program = _a._program; + _glOut.useProgram(_program); + uniformVars.forEach(function (_a) { + var name = _a.name, type = _a.type, values = _a.values; + var uniformVarLocation = _glOut + .getUniformLocation(_program, name); + var isVector = type.split(':')[1] === 'v'; + if (isVector) { + // @ts-ignore + _glOut["uniform1".concat(type[0], "v")](uniformVarLocation, values); + } + else { + // @ts-ignore + _glOut["uniform".concat(values.length).concat(type[0])].apply(_glOut, __spreadArray([uniformVarLocation], values, false)); + } + }); + }; + return WebGL2PipelineProcessingStage; +}()); +exports.WebGL2PipelineProcessingStage = WebGL2PipelineProcessingStage; + +},{"./webgl2PipelineHelpers":25}],24:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2Pipeline = void 0; +var Pipeline_1 = require("../Pipeline"); +var WebGL2PipelineInputStage_1 = require("./WebGL2PipelineInputStage"); +var WebGL2PipelineProcessingStage_1 = require("./WebGL2PipelineProcessingStage"); +/** + * @private + */ +var WebGL2Pipeline = /** @class */ (function (_super) { + __extends(WebGL2Pipeline, _super); + function WebGL2Pipeline() { + var _this = _super !== null && _super.apply(this, arguments) || this; + _this._stages = []; + return _this; + } + WebGL2Pipeline.prototype.cleanUp = function () { + this._stages.forEach(function (stage) { return stage.cleanUp(); }); + }; + WebGL2Pipeline.prototype.render = function (inputFrame, inputTextureData) { + var _a = this._stages, inputStage = _a[0], otherStages = _a.slice(1); + inputStage.render(inputFrame, inputTextureData); + otherStages.forEach(function (stage) { return stage + .render(); }); + }; + WebGL2Pipeline.InputStage = WebGL2PipelineInputStage_1.WebGL2PipelineInputStage; + WebGL2Pipeline.ProcessingStage = WebGL2PipelineProcessingStage_1.WebGL2PipelineProcessingStage; + return WebGL2Pipeline; +}(Pipeline_1.Pipeline)); +exports.WebGL2Pipeline = WebGL2Pipeline; + +},{"../Pipeline":20,"./WebGL2PipelineInputStage":22,"./WebGL2PipelineProcessingStage":23}],25:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.initBuffer = exports.createTexture = exports.compileShader = exports.createProgram = exports.createPipelineStageProgram = void 0; +/** + * @private + */ +function createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer) { + var program = createProgram(gl, vertexShader, fragmentShader); + var positionAttributeLocation = gl.getAttribLocation(program, 'a_position'); + gl.enableVertexAttribArray(positionAttributeLocation); + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + gl.vertexAttribPointer(positionAttributeLocation, 2, gl.FLOAT, false, 0, 0); + var texCoordAttributeLocation = gl.getAttribLocation(program, 'a_texCoord'); + gl.enableVertexAttribArray(texCoordAttributeLocation); + gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); + gl.vertexAttribPointer(texCoordAttributeLocation, 2, gl.FLOAT, false, 0, 0); + return program; +} +exports.createPipelineStageProgram = createPipelineStageProgram; +/** + * @private + */ +function createProgram(gl, vertexShader, fragmentShader) { + var program = gl.createProgram(); + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + throw new Error("Could not link WebGL program: ".concat(gl.getProgramInfoLog(program))); + } + return program; +} +exports.createProgram = createProgram; +/** + * @private + */ +function compileShader(gl, shaderType, shaderSource) { + var shader = gl.createShader(shaderType); + gl.shaderSource(shader, shaderSource); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + throw new Error("Could not compile shader: ".concat(gl.getShaderInfoLog(shader))); + } + return shader; +} +exports.compileShader = compileShader; +/** + * @private + */ +function createTexture(gl, internalformat, width, height, minFilter, magFilter) { + if (minFilter === void 0) { minFilter = gl.NEAREST; } + if (magFilter === void 0) { magFilter = gl.NEAREST; } + var texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, minFilter); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, magFilter); + gl.texStorage2D(gl.TEXTURE_2D, 1, internalformat, width, height); + return texture; +} +exports.createTexture = createTexture; +/** + * @private + */ +function initBuffer(gl, data) { + var buffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(data), gl.STATIC_DRAW); + return buffer; +} +exports.initBuffer = initBuffer; + +},{}],26:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.ImageFit = void 0; +/** + * ImageFit specifies the positioning of an image inside a viewport. + */ +var ImageFit; +(function (ImageFit) { + /** + * Scale the image up or down to fill the viewport while preserving the aspect ratio. + * The image will be fully visible but will add empty space in the viewport if + * aspect ratios do not match. + */ + ImageFit["Contain"] = "Contain"; + /** + * Scale the image to fill both height and width of the viewport while preserving + * the aspect ratio, but will crop the image if aspect ratios do not match. + */ + ImageFit["Cover"] = "Cover"; + /** + * Stretches the image to fill the viewport regardless of aspect ratio. + */ + ImageFit["Fill"] = "Fill"; + /** + * Ignore height and width and use the original size. + */ + ImageFit["None"] = "None"; +})(ImageFit || (exports.ImageFit = ImageFit = {})); + +},{}],27:[function(require,module,exports){ +"use strict"; +var __assign = (this && this.__assign) || function () { + __assign = Object.assign || function(t) { + for (var s, i = 1, n = arguments.length; i < n; i++) { + s = arguments[i]; + for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) + t[p] = s[p]; + } + return t; + }; + return __assign.apply(this, arguments); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Benchmark = void 0; +/** + * @private + */ +var Benchmark = /** @class */ (function () { + function Benchmark() { + this._timingCache = new Map(); + this._timings = new Map(); + } + Benchmark.prototype.end = function (name) { + var timing = this._timings.get(name); + if (!timing) { + return; + } + timing.end = Date.now(); + timing.delay = timing.end - timing.start; + this._save(name, __assign({}, timing)); + }; + Benchmark.prototype.getAverageDelay = function (name) { + var timingCache = this._timingCache.get(name); + if (!timingCache || !timingCache.length) { + return; + } + return timingCache.map(function (timing) { return timing.delay; }) + .reduce(function (total, value) { return total += value; }, 0) / timingCache.length; + }; + Benchmark.prototype.getNames = function () { + return Array.from(this._timingCache.keys()); + }; + Benchmark.prototype.getRate = function (name) { + var timingCache = this._timingCache.get(name); + if (!timingCache || timingCache.length < 2) { + return; + } + var totalDelay = timingCache[timingCache.length - 1].end - timingCache[0].start; + return (timingCache.length / totalDelay) * 1000; + }; + Benchmark.prototype.merge = function (benchmark) { + var _this = this; + var _timingCache = benchmark._timingCache, _timings = benchmark._timings; + _timingCache.forEach(function (cache, name) { return _this._timingCache.set(name, cache); }); + _timings.forEach(function (timing, name) { return _this._timings.set(name, timing); }); + }; + Benchmark.prototype.start = function (name) { + var timing = this._timings.get(name); + if (!timing) { + timing = {}; + this._timings.set(name, timing); + } + timing.start = Date.now(); + delete timing.end; + delete timing.delay; + }; + Benchmark.prototype._save = function (name, timing) { + var timingCache = this._timingCache.get(name); + if (!timingCache) { + timingCache = []; + this._timingCache.set(name, timingCache); + } + timingCache.push(timing); + if (timingCache.length > Benchmark.cacheSize) { + timingCache.splice(0, timingCache.length - Benchmark.cacheSize); + } + }; + // NOTE (csantos): How many timing information to save per benchmark. + // This is about the amount of timing info generated on a 24fps input. + // Enough samples to calculate fps + Benchmark.cacheSize = 41; + return Benchmark; +}()); +exports.Benchmark = Benchmark; + +},{}],28:[function(require,module,exports){ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.TwilioTFLite = void 0; +var isWebWorker = typeof WorkerGlobalScope !== 'undefined' + && self instanceof WorkerGlobalScope; +var loadedScripts = new Set(); +var model; +/** + * @private + */ +var TwilioTFLite = /** @class */ (function () { + function TwilioTFLite() { + this._inputBuffer = null; + this._isSimdEnabled = null; + this._tflite = null; + } + Object.defineProperty(TwilioTFLite.prototype, "isSimdEnabled", { + get: function () { + return this._isSimdEnabled; + }, + enumerable: false, + configurable: true + }); + TwilioTFLite.prototype.initialize = function (assetsPath, modelName, moduleLoaderName, moduleSimdLoaderName) { + return __awaiter(this, void 0, void 0, function () { + var _a, modelResponse, _b, tflite, modelBufferOffset; + return __generator(this, function (_c) { + switch (_c.label) { + case 0: + if (this._tflite) { + return [2 /*return*/]; + } + if (isWebWorker) { + // NOTE(mmalavalli): In a web worker, paths to other dependencies + // are determined relative to the assets path, so no need to append + // it to the file names of the dependencies. + assetsPath = ''; + } + return [4 /*yield*/, Promise.all([ + this._loadWasmModule(assetsPath, moduleLoaderName, moduleSimdLoaderName), + fetch("".concat(assetsPath).concat(modelName)), + ])]; + case 1: + _a = _c.sent(), modelResponse = _a[1]; + _b = model; + if (_b) return [3 /*break*/, 3]; + return [4 /*yield*/, modelResponse.arrayBuffer()]; + case 2: + _b = (_c.sent()); + _c.label = 3; + case 3: + model = _b; + tflite = this._tflite; + modelBufferOffset = tflite._getModelBufferMemoryOffset(); + tflite.HEAPU8.set(new Uint8Array(model), modelBufferOffset); + tflite._loadModel(model.byteLength); + return [2 /*return*/]; + } + }); + }); + }; + TwilioTFLite.prototype.loadInputBuffer = function (inputBuffer) { + var tflite = this._tflite; + var height = tflite._getInputHeight(); + var width = tflite._getInputWidth(); + var pixels = width * height; + var tfliteInputMemoryOffset = tflite._getInputMemoryOffset() / 4; + for (var i = 0; i < pixels; i++) { + var curTFLiteOffset = tfliteInputMemoryOffset + i * 3; + var curImageBufferOffset = i * 4; + tflite.HEAPF32[curTFLiteOffset] = inputBuffer[curImageBufferOffset] / 255; + tflite.HEAPF32[curTFLiteOffset + 1] = inputBuffer[curImageBufferOffset + 1] / 255; + tflite.HEAPF32[curTFLiteOffset + 2] = inputBuffer[curImageBufferOffset + 2] / 255; + } + this._inputBuffer = inputBuffer; + }; + TwilioTFLite.prototype.runInference = function () { + var tflite = this._tflite; + var height = tflite._getInputHeight(); + var width = tflite._getInputWidth(); + var pixels = width * height; + var tfliteOutputMemoryOffset = tflite._getOutputMemoryOffset() / 4; + tflite._runInference(); + var inputBuffer = this._inputBuffer || new Uint8ClampedArray(pixels * 4); + for (var i = 0; i < pixels; i++) { + inputBuffer[i * 4 + 3] = Math.round(tflite.HEAPF32[tfliteOutputMemoryOffset + i] * 255); + } + return inputBuffer; + }; + TwilioTFLite.prototype._loadScript = function (path) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + if (loadedScripts.has(path)) { + return [2 /*return*/]; + } + if (isWebWorker) { + importScripts(path); + loadedScripts.add(path); + return [2 /*return*/]; + } + return [2 /*return*/, new Promise(function (resolve, reject) { + var script = document.createElement('script'); + script.onload = function () { + loadedScripts.add(path); + resolve(); + }; + script.onerror = function () { + reject(); + }; + document.head.append(script); + script.src = path; + })]; + }); + }); + }; + TwilioTFLite.prototype._loadWasmModule = function (assetsPath, moduleLoaderName, moduleSimdLoaderName) { + return __awaiter(this, void 0, void 0, function () { + var _a, _b, _c; + return __generator(this, function (_d) { + switch (_d.label) { + case 0: + _d.trys.push([0, 3, , 6]); + return [4 /*yield*/, this._loadScript("".concat(assetsPath).concat(moduleSimdLoaderName))]; + case 1: + _d.sent(); + _a = this; + return [4 /*yield*/, createTwilioTFLiteSIMDModule()]; + case 2: + _a._tflite = _d.sent(); + this._isSimdEnabled = true; + return [3 /*break*/, 6]; + case 3: + _b = _d.sent(); + return [4 /*yield*/, this._loadScript("".concat(assetsPath).concat(moduleLoaderName))]; + case 4: + _d.sent(); + _c = this; + return [4 /*yield*/, createTwilioTFLiteModule()]; + case 5: + _c._tflite = _d.sent(); + this._isSimdEnabled = false; + return [3 /*break*/, 6]; + case 6: return [2 /*return*/]; + } + }); + }); + }; + return TwilioTFLite; +}()); +exports.TwilioTFLite = TwilioTFLite; + +},{}],29:[function(require,module,exports){ +"use strict"; +var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) { + if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) { + if (ar || !(i in from)) { + if (!ar) ar = Array.prototype.slice.call(from, 0, i); + ar[i] = from[i]; + } + } + return to.concat(ar || Array.prototype.slice.call(from)); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.isSupported = exports.isCanvasBlurSupported = exports.isChromiumImageBitmap = exports.isBrowserSupported = void 0; +/** + * @private + */ +function getCanvas() { + return typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); +} +/** + * @private + */ +function isBrowserSupported() { + if (typeof window !== 'undefined' && typeof document !== 'undefined') { + return !!(getCanvas().getContext('2d') || getCanvas().getContext('webgl2')); + } + else { + return false; + } +} +exports.isBrowserSupported = isBrowserSupported; +/** + * @private + */ +function isChromiumImageBitmap() { + return /Chrome/.test(navigator.userAgent) + && typeof createImageBitmap === 'function'; +} +exports.isChromiumImageBitmap = isChromiumImageBitmap; +/** + * @private + */ +exports.isCanvasBlurSupported = (function () { + var blackPixel = [0, 0, 0, 255]; + var whitePixel = [255, 255, 255, 255]; + var inputImageData = new ImageData(new Uint8ClampedArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], blackPixel, true), blackPixel, true), blackPixel, true), blackPixel, true), whitePixel, true), blackPixel, true), blackPixel, true), blackPixel, true), blackPixel, true)), 3, 3); + var canvas = getCanvas(); + var context = canvas.getContext('2d'); + canvas.width = 3; + canvas.height = 3; + context.putImageData(inputImageData, 0, 0); + context.filter = 'blur(1px)'; + context.drawImage(canvas, 0, 0); + var data = context.getImageData(0, 0, 3, 3).data; + return data[0] > 0; +})(); +/** + * Check if the current browser is officially supported by twilio-video-procesors.js. + * This is set to `true` for browsers that supports canvas + * [2D](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D) or + * [webgl2](https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext) + * rendering context. + * @example + * ```ts + * import { isSupported } from '@twilio/video-processors'; + * + * if (isSupported) { + * // Initialize the background processors + * } + * ``` + */ +exports.isSupported = isBrowserSupported(); + +},{}],30:[function(require,module,exports){ +"use strict"; +// This file is generated on build. To make changes, see scripts/version.js +Object.defineProperty(exports, "__esModule", { value: true }); +exports.version = void 0; +/** + * The current version of the library. + */ +exports.version = '3.0.0-preview.1'; + +},{}],31:[function(require,module,exports){ +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : + typeof define === 'function' && define.amd ? define(['exports'], factory) : + (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.Comlink = {})); +})(this, (function (exports) { 'use strict'; + + /** + * @license + * Copyright 2019 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + const proxyMarker = Symbol("Comlink.proxy"); + const createEndpoint = Symbol("Comlink.endpoint"); + const releaseProxy = Symbol("Comlink.releaseProxy"); + const finalizer = Symbol("Comlink.finalizer"); + const throwMarker = Symbol("Comlink.thrown"); + const isObject = (val) => (typeof val === "object" && val !== null) || typeof val === "function"; + /** + * Internal transfer handle to handle objects marked to proxy. + */ + const proxyTransferHandler = { + canHandle: (val) => isObject(val) && val[proxyMarker], + serialize(obj) { + const { port1, port2 } = new MessageChannel(); + expose(obj, port1); + return [port2, [port2]]; + }, + deserialize(port) { + port.start(); + return wrap(port); + }, + }; + /** + * Internal transfer handler to handle thrown exceptions. + */ + const throwTransferHandler = { + canHandle: (value) => isObject(value) && throwMarker in value, + serialize({ value }) { + let serialized; + if (value instanceof Error) { + serialized = { + isError: true, + value: { + message: value.message, + name: value.name, + stack: value.stack, + }, + }; + } + else { + serialized = { isError: false, value }; + } + return [serialized, []]; + }, + deserialize(serialized) { + if (serialized.isError) { + throw Object.assign(new Error(serialized.value.message), serialized.value); + } + throw serialized.value; + }, + }; + /** + * Allows customizing the serialization of certain values. + */ + const transferHandlers = new Map([ + ["proxy", proxyTransferHandler], + ["throw", throwTransferHandler], + ]); + function isAllowedOrigin(allowedOrigins, origin) { + for (const allowedOrigin of allowedOrigins) { + if (origin === allowedOrigin || allowedOrigin === "*") { + return true; + } + if (allowedOrigin instanceof RegExp && allowedOrigin.test(origin)) { + return true; + } + } + return false; + } + function expose(obj, ep = globalThis, allowedOrigins = ["*"]) { + ep.addEventListener("message", function callback(ev) { + if (!ev || !ev.data) { + return; + } + if (!isAllowedOrigin(allowedOrigins, ev.origin)) { + console.warn(`Invalid origin '${ev.origin}' for comlink proxy`); + return; + } + const { id, type, path } = Object.assign({ path: [] }, ev.data); + const argumentList = (ev.data.argumentList || []).map(fromWireValue); + let returnValue; + try { + const parent = path.slice(0, -1).reduce((obj, prop) => obj[prop], obj); + const rawValue = path.reduce((obj, prop) => obj[prop], obj); + switch (type) { + case "GET" /* MessageType.GET */: + { + returnValue = rawValue; + } + break; + case "SET" /* MessageType.SET */: + { + parent[path.slice(-1)[0]] = fromWireValue(ev.data.value); + returnValue = true; + } + break; + case "APPLY" /* MessageType.APPLY */: + { + returnValue = rawValue.apply(parent, argumentList); + } + break; + case "CONSTRUCT" /* MessageType.CONSTRUCT */: + { + const value = new rawValue(...argumentList); + returnValue = proxy(value); + } + break; + case "ENDPOINT" /* MessageType.ENDPOINT */: + { + const { port1, port2 } = new MessageChannel(); + expose(obj, port2); + returnValue = transfer(port1, [port1]); + } + break; + case "RELEASE" /* MessageType.RELEASE */: + { + returnValue = undefined; + } + break; + default: + return; + } + } + catch (value) { + returnValue = { value, [throwMarker]: 0 }; + } + Promise.resolve(returnValue) + .catch((value) => { + return { value, [throwMarker]: 0 }; + }) + .then((returnValue) => { + const [wireValue, transferables] = toWireValue(returnValue); + ep.postMessage(Object.assign(Object.assign({}, wireValue), { id }), transferables); + if (type === "RELEASE" /* MessageType.RELEASE */) { + // detach and deactive after sending release response above. + ep.removeEventListener("message", callback); + closeEndPoint(ep); + if (finalizer in obj && typeof obj[finalizer] === "function") { + obj[finalizer](); + } + } + }) + .catch((error) => { + // Send Serialization Error To Caller + const [wireValue, transferables] = toWireValue({ + value: new TypeError("Unserializable return value"), + [throwMarker]: 0, + }); + ep.postMessage(Object.assign(Object.assign({}, wireValue), { id }), transferables); + }); + }); + if (ep.start) { + ep.start(); + } + } + function isMessagePort(endpoint) { + return endpoint.constructor.name === "MessagePort"; + } + function closeEndPoint(endpoint) { + if (isMessagePort(endpoint)) + endpoint.close(); + } + function wrap(ep, target) { + return createProxy(ep, [], target); + } + function throwIfProxyReleased(isReleased) { + if (isReleased) { + throw new Error("Proxy has been released and is not useable"); + } + } + function releaseEndpoint(ep) { + return requestResponseMessage(ep, { + type: "RELEASE" /* MessageType.RELEASE */, + }).then(() => { + closeEndPoint(ep); + }); + } + const proxyCounter = new WeakMap(); + const proxyFinalizers = "FinalizationRegistry" in globalThis && + new FinalizationRegistry((ep) => { + const newCount = (proxyCounter.get(ep) || 0) - 1; + proxyCounter.set(ep, newCount); + if (newCount === 0) { + releaseEndpoint(ep); + } + }); + function registerProxy(proxy, ep) { + const newCount = (proxyCounter.get(ep) || 0) + 1; + proxyCounter.set(ep, newCount); + if (proxyFinalizers) { + proxyFinalizers.register(proxy, ep, proxy); + } + } + function unregisterProxy(proxy) { + if (proxyFinalizers) { + proxyFinalizers.unregister(proxy); + } + } + function createProxy(ep, path = [], target = function () { }) { + let isProxyReleased = false; + const proxy = new Proxy(target, { + get(_target, prop) { + throwIfProxyReleased(isProxyReleased); + if (prop === releaseProxy) { + return () => { + unregisterProxy(proxy); + releaseEndpoint(ep); + isProxyReleased = true; + }; + } + if (prop === "then") { + if (path.length === 0) { + return { then: () => proxy }; + } + const r = requestResponseMessage(ep, { + type: "GET" /* MessageType.GET */, + path: path.map((p) => p.toString()), + }).then(fromWireValue); + return r.then.bind(r); + } + return createProxy(ep, [...path, prop]); + }, + set(_target, prop, rawValue) { + throwIfProxyReleased(isProxyReleased); + // FIXME: ES6 Proxy Handler `set` methods are supposed to return a + // boolean. To show good will, we return true asynchronously ¯\_(ツ)_/¯ + const [value, transferables] = toWireValue(rawValue); + return requestResponseMessage(ep, { + type: "SET" /* MessageType.SET */, + path: [...path, prop].map((p) => p.toString()), + value, + }, transferables).then(fromWireValue); + }, + apply(_target, _thisArg, rawArgumentList) { + throwIfProxyReleased(isProxyReleased); + const last = path[path.length - 1]; + if (last === createEndpoint) { + return requestResponseMessage(ep, { + type: "ENDPOINT" /* MessageType.ENDPOINT */, + }).then(fromWireValue); + } + // We just pretend that `bind()` didn’t happen. + if (last === "bind") { + return createProxy(ep, path.slice(0, -1)); + } + const [argumentList, transferables] = processArguments(rawArgumentList); + return requestResponseMessage(ep, { + type: "APPLY" /* MessageType.APPLY */, + path: path.map((p) => p.toString()), + argumentList, + }, transferables).then(fromWireValue); + }, + construct(_target, rawArgumentList) { + throwIfProxyReleased(isProxyReleased); + const [argumentList, transferables] = processArguments(rawArgumentList); + return requestResponseMessage(ep, { + type: "CONSTRUCT" /* MessageType.CONSTRUCT */, + path: path.map((p) => p.toString()), + argumentList, + }, transferables).then(fromWireValue); + }, + }); + registerProxy(proxy, ep); + return proxy; + } + function myFlat(arr) { + return Array.prototype.concat.apply([], arr); + } + function processArguments(argumentList) { + const processed = argumentList.map(toWireValue); + return [processed.map((v) => v[0]), myFlat(processed.map((v) => v[1]))]; + } + const transferCache = new WeakMap(); + function transfer(obj, transfers) { + transferCache.set(obj, transfers); + return obj; + } + function proxy(obj) { + return Object.assign(obj, { [proxyMarker]: true }); + } + function windowEndpoint(w, context = globalThis, targetOrigin = "*") { + return { + postMessage: (msg, transferables) => w.postMessage(msg, targetOrigin, transferables), + addEventListener: context.addEventListener.bind(context), + removeEventListener: context.removeEventListener.bind(context), + }; + } + function toWireValue(value) { + for (const [name, handler] of transferHandlers) { + if (handler.canHandle(value)) { + const [serializedValue, transferables] = handler.serialize(value); + return [ + { + type: "HANDLER" /* WireValueType.HANDLER */, + name, + value: serializedValue, + }, + transferables, + ]; + } + } + return [ + { + type: "RAW" /* WireValueType.RAW */, + value, + }, + transferCache.get(value) || [], + ]; + } + function fromWireValue(value) { + switch (value.type) { + case "HANDLER" /* WireValueType.HANDLER */: + return transferHandlers.get(value.name).deserialize(value.value); + case "RAW" /* WireValueType.RAW */: + return value.value; + } + } + function requestResponseMessage(ep, msg, transfers) { + return new Promise((resolve) => { + const id = generateUUID(); + ep.addEventListener("message", function l(ev) { + if (!ev.data || !ev.data.id || ev.data.id !== id) { + return; + } + ep.removeEventListener("message", l); + resolve(ev.data); + }); + if (ep.start) { + ep.start(); + } + ep.postMessage(Object.assign({ id }, msg), transfers); + }); + } + function generateUUID() { + return new Array(4) + .fill(0) + .map(() => Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(16)) + .join("-"); + } + + exports.createEndpoint = createEndpoint; + exports.expose = expose; + exports.finalizer = finalizer; + exports.proxy = proxy; + exports.proxyMarker = proxyMarker; + exports.releaseProxy = releaseProxy; + exports.transfer = transfer; + exports.transferHandlers = transferHandlers; + exports.windowEndpoint = windowEndpoint; + exports.wrap = wrap; + +})); + + +},{}]},{},[2]); diff --git a/dist/build/twilio-video-processors.min.js b/dist/build/twilio-video-processors.min.js new file mode 100644 index 0000000..e95f60b --- /dev/null +++ b/dist/build/twilio-video-processors.min.js @@ -0,0 +1,38 @@ +/*! twilio-video-processors.js 3.0.0-preview.1 + +The following license applies to all parts of this software except as +documented below. + + Copyright (C) 2022 Twilio Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + 3. Neither the name of Twilio nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ +(function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i0&&t[t.length-1])&&(op[0]===6||op[0]===2)){_=0;continue}if(op[0]===3&&(!t||op[1]>t[0]&&op[1]0&&t[t.length-1])&&(op[0]===6||op[0]===2)){_=0;continue}if(op[0]===3&&(!t||op[1]>t[0]&&op[1]0&&t[t.length-1])&&(op[0]===6||op[0]===2)){_=0;continue}if(op[0]===3&&(!t||op[1]>t[0]&&op[1]0&&t[t.length-1])&&(op[0]===6||op[0]===2)){_=0;continue}if(op[0]===3&&(!t||op[1]>t[0]&&op[1]0&&t[t.length-1])&&(op[0]===6||op[0]===2)){_=0;continue}if(op[0]===3&&(!t||op[1]>t[0]&&op[1]0&&t[t.length-1])&&(op[0]===6||op[0]===2)){_=0;continue}if(op[0]===3&&(!t||op[1]>t[0]&&op[1]0&&t[t.length-1])&&(op[0]===6||op[0]===2)){_=0;continue}if(op[0]===3&&(!t||op[1]>t[0]&&op[1]viewportHeight||type===types_1.ImageFit.Cover&&viewportHeight>newContentHeight){factor=viewportHeight/newContentHeight;newContentWidth=factor*newContentWidth;newContentHeight=viewportHeight}var x=(viewportWidth-newContentWidth)/2;var y=(viewportHeight-newContentHeight)/2;return{x:x,y:y,w:newContentWidth,h:newContentHeight}};return VirtualBackgroundProcessorPipeline}(BackgroundProcessorPipeline_1.BackgroundProcessorPipeline);exports.VirtualBackgroundProcessorPipeline=VirtualBackgroundProcessorPipeline},{"../../../../types":26,"./BackgroundProcessorPipeline":7}],14:[function(require,module,exports){"use strict";var __extends=this&&this.__extends||function(){var extendStatics=function(d,b){extendStatics=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(d,b){d.__proto__=b}||function(d,b){for(var p in b)if(Object.prototype.hasOwnProperty.call(b,p))d[p]=b[p]};return extendStatics(d,b)};return function(d,b){if(typeof b!=="function"&&b!==null)throw new TypeError("Class extends value "+String(b)+" is not a constructor or null");extendStatics(d,b);function __(){this.constructor=d}d.prototype=b===null?Object.create(b):(__.prototype=b.prototype,new __)}}();var __awaiter=this&&this.__awaiter||function(thisArg,_arguments,P,generator){function adopt(value){return value instanceof P?value:new P(function(resolve){resolve(value)})}return new(P||(P=Promise))(function(resolve,reject){function fulfilled(value){try{step(generator.next(value))}catch(e){reject(e)}}function rejected(value){try{step(generator["throw"](value))}catch(e){reject(e)}}function step(result){result.done?resolve(result.value):adopt(result.value).then(fulfilled,rejected)}step((generator=generator.apply(thisArg,_arguments||[])).next())})};var __generator=this&&this.__generator||function(thisArg,body){var _={label:0,sent:function(){if(t[0]&1)throw t[1];return t[1]},trys:[],ops:[]},f,y,t,g;return g={next:verb(0),throw:verb(1),return:verb(2)},typeof Symbol==="function"&&(g[Symbol.iterator]=function(){return this}),g;function verb(n){return function(v){return step([n,v])}}function step(op){if(f)throw new TypeError("Generator is already executing.");while(g&&(g=0,op[0]&&(_=0)),_)try{if(f=1,y&&(t=op[0]&2?y["return"]:op[0]?y["throw"]||((t=y["return"])&&t.call(y),0):y.next)&&!(t=t.call(y,op[1])).done)return t;if(y=0,t)op=[op[0]&2,t.value];switch(op[0]){case 0:case 1:t=op;break;case 4:_.label++;return{value:op[1],done:false};case 5:_.label++;y=op[1];op=[0];continue;case 7:op=_.ops.pop();_.trys.pop();continue;default:if(!(t=_.trys,t=t.length>0&&t[t.length-1])&&(op[0]===6||op[0]===2)){_=0;continue}if(op[0]===3&&(!t||op[1]>t[0]&&op[1] 0.0 && i <= u_radius; i += u_radius) {\n for (float j = -u_radius; j <= u_radius; j += u_radius * (j == 0.0 ? 2.0 : 1.0)) {\n vec2 shift = vec2(i, j) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n totalAlpha += texture(u_segmentationMask, coord).a;\n totalPixels++;\n }\n }\n\n return totalAlpha / totalPixels;\n }\n\n void main() {\n vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb;\n float outAlpha = texture(u_segmentationMask, v_texCoord).a;\n float averageAlpha = edgePixelsAverageAlpha(outAlpha);\n float totalWeight = 1.0;\n\n if (averageAlpha == 0.0 || averageAlpha == 1.0) {\n outColor = vec4(averageAlpha * centerColor, averageAlpha);\n return;\n }\n\n for (float i = 1.0; i <= u_radius; i += u_step) {\n float x = (1.0 - u_direction) * i;\n float y = u_direction * i;\n vec2 shift = vec2(x, y) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n float spaceWeight = u_spaceWeights[int(i - 1.0)];\n float colorWeight = calculateColorWeight(coord, centerColor);\n float weight = spaceWeight * colorWeight;\n float alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n\n shift = vec2(-x, -y) * u_texelSize;\n coord = vec2(v_texCoord + shift);\n colorWeight = calculateColorWeight(coord, centerColor);\n weight = spaceWeight * colorWeight;\n alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n }\n\n outAlpha /= totalWeight;\n outColor = vec4(outAlpha * centerColor, outAlpha);\n }\n ",glOut:glOut,height:height,textureUnit:outputTextureUnit,type:outputType,width:width,uniformVars:[{name:"u_inputFrame",type:"int",values:[0]},{name:"u_direction",type:"float",values:[direction==="vertical"?1:0]},{name:"u_texelSize",type:"float",values:[1/width,1/height]}]})||this;_this._direction=direction;_this._inputDimensions=inputDimensions;_this.updateSigmaColor(0);_this.updateSigmaSpace(0);return _this}SinglePassBilateralFilterStage.prototype.updateSigmaColor=function(sigmaColor){this._setUniformVars([{name:"u_colorWeights",type:"float:v",values:createColorWeights(sigmaColor)}])};SinglePassBilateralFilterStage.prototype.updateSigmaSpace=function(sigmaSpace){var _a=this._inputDimensions,inputHeight=_a.height,inputWidth=_a.width;var _b=this._outputDimensions,outputHeight=_b.height,outputWidth=_b.width;sigmaSpace*=Math.max(outputWidth/inputWidth,outputHeight/inputHeight);var step=Math.floor(.5*sigmaSpace/Math.log(sigmaSpace));var sigmaTexel=Math.max(1/outputWidth,1/outputHeight)*sigmaSpace;var texelSize=1/(this._direction==="horizontal"?outputWidth:outputHeight);this._setUniformVars([{name:"u_radius",type:"float",values:[sigmaSpace]},{name:"u_spaceWeights",type:"float:v",values:createSpaceWeights(sigmaSpace,sigmaTexel,texelSize)},{name:"u_step",type:"float",values:[step]}])};return SinglePassBilateralFilterStage}(pipelines_1.WebGL2Pipeline.ProcessingStage);exports.SinglePassBilateralFilterStage=SinglePassBilateralFilterStage},{"../../../pipelines":21}],19:[function(require,module,exports){"use strict";var __extends=this&&this.__extends||function(){var extendStatics=function(d,b){extendStatics=Object.setPrototypeOf||{__proto__:[]}instanceof Array&&function(d,b){d.__proto__=b}||function(d,b){for(var p in b)if(Object.prototype.hasOwnProperty.call(b,p))d[p]=b[p]};return extendStatics(d,b)};return function(d,b){if(typeof b!=="function"&&b!==null)throw new TypeError("Class extends value "+String(b)+" is not a constructor or null");extendStatics(d,b);function __(){this.constructor=d}d.prototype=b===null?Object.create(b):(__.prototype=b.prototype,new __)}}();Object.defineProperty(exports,"__esModule",{value:true});exports.PersonMaskUpscalePipeline=void 0;var pipelines_1=require("../../../pipelines");var SinglePassBilateralFilterStage_1=require("./SinglePassBilateralFilterStage");var PersonMaskUpscalePipeline=function(_super){__extends(PersonMaskUpscalePipeline,_super);function PersonMaskUpscalePipeline(inputDimensions,outputCanvas){var _this=_super.call(this)||this;var glOut=outputCanvas.getContext("webgl2");var outputDimensions={height:outputCanvas.height,width:outputCanvas.width};_this.addStage(new pipelines_1.WebGL2Pipeline.InputStage(glOut));_this.addStage(new SinglePassBilateralFilterStage_1.SinglePassBilateralFilterStage(glOut,"horizontal","texture",inputDimensions,outputDimensions,1,2));_this.addStage(new SinglePassBilateralFilterStage_1.SinglePassBilateralFilterStage(glOut,"vertical","canvas",inputDimensions,outputDimensions,2));return _this}PersonMaskUpscalePipeline.prototype.updateBilateralFilterConfig=function(config){var _a=this._stages,bilateralFilterStages=_a.slice(1);var sigmaSpace=config.sigmaSpace;if(typeof sigmaSpace==="number"){bilateralFilterStages.forEach(function(stage){stage.updateSigmaColor(.1);stage.updateSigmaSpace(sigmaSpace)})}};return PersonMaskUpscalePipeline}(pipelines_1.WebGL2Pipeline);exports.PersonMaskUpscalePipeline=PersonMaskUpscalePipeline},{"../../../pipelines":21,"./SinglePassBilateralFilterStage":18}],20:[function(require,module,exports){"use strict";Object.defineProperty(exports,"__esModule",{value:true});exports.Pipeline=void 0;var Pipeline=function(){function Pipeline(){this._stages=[]}Pipeline.prototype.addStage=function(stage){this._stages.push(stage)};Pipeline.prototype.render=function(){var args=[];for(var _i=0;_iBenchmark.cacheSize){timingCache.splice(0,timingCache.length-Benchmark.cacheSize)}};Benchmark.cacheSize=41;return Benchmark}();exports.Benchmark=Benchmark},{}],28:[function(require,module,exports){"use strict";var __awaiter=this&&this.__awaiter||function(thisArg,_arguments,P,generator){function adopt(value){return value instanceof P?value:new P(function(resolve){resolve(value)})}return new(P||(P=Promise))(function(resolve,reject){function fulfilled(value){try{step(generator.next(value))}catch(e){reject(e)}}function rejected(value){try{step(generator["throw"](value))}catch(e){reject(e)}}function step(result){result.done?resolve(result.value):adopt(result.value).then(fulfilled,rejected)}step((generator=generator.apply(thisArg,_arguments||[])).next())})};var __generator=this&&this.__generator||function(thisArg,body){var _={label:0,sent:function(){if(t[0]&1)throw t[1];return t[1]},trys:[],ops:[]},f,y,t,g;return g={next:verb(0),throw:verb(1),return:verb(2)},typeof Symbol==="function"&&(g[Symbol.iterator]=function(){return this}),g;function verb(n){return function(v){return step([n,v])}}function step(op){if(f)throw new TypeError("Generator is already executing.");while(g&&(g=0,op[0]&&(_=0)),_)try{if(f=1,y&&(t=op[0]&2?y["return"]:op[0]?y["throw"]||((t=y["return"])&&t.call(y),0):y.next)&&!(t=t.call(y,op[1])).done)return t;if(y=0,t)op=[op[0]&2,t.value];switch(op[0]){case 0:case 1:t=op;break;case 4:_.label++;return{value:op[1],done:false};case 5:_.label++;y=op[1];op=[0];continue;case 7:op=_.ops.pop();_.trys.pop();continue;default:if(!(t=_.trys,t=t.length>0&&t[t.length-1])&&(op[0]===6||op[0]===2)){_=0;continue}if(op[0]===3&&(!t||op[1]>t[0]&&op[1]0}();exports.isSupported=isBrowserSupported()},{}],30:[function(require,module,exports){"use strict";Object.defineProperty(exports,"__esModule",{value:true});exports.version=void 0;exports.version="3.0.0-preview.1"},{}],31:[function(require,module,exports){(function(global,factory){typeof exports==="object"&&typeof module!=="undefined"?factory(exports):typeof define==="function"&&define.amd?define(["exports"],factory):(global=typeof globalThis!=="undefined"?globalThis:global||self,factory(global.Comlink={}))})(this,function(exports){"use strict";const proxyMarker=Symbol("Comlink.proxy");const createEndpoint=Symbol("Comlink.endpoint");const releaseProxy=Symbol("Comlink.releaseProxy");const finalizer=Symbol("Comlink.finalizer");const throwMarker=Symbol("Comlink.thrown");const isObject=val=>typeof val==="object"&&val!==null||typeof val==="function";const proxyTransferHandler={canHandle:val=>isObject(val)&&val[proxyMarker],serialize(obj){const{port1,port2}=new MessageChannel;expose(obj,port1);return[port2,[port2]]},deserialize(port){port.start();return wrap(port)}};const throwTransferHandler={canHandle:value=>isObject(value)&&throwMarker in value,serialize({value}){let serialized;if(value instanceof Error){serialized={isError:true,value:{message:value.message,name:value.name,stack:value.stack}}}else{serialized={isError:false,value:value}}return[serialized,[]]},deserialize(serialized){if(serialized.isError){throw Object.assign(new Error(serialized.value.message),serialized.value)}throw serialized.value}};const transferHandlers=new Map([["proxy",proxyTransferHandler],["throw",throwTransferHandler]]);function isAllowedOrigin(allowedOrigins,origin){for(const allowedOrigin of allowedOrigins){if(origin===allowedOrigin||allowedOrigin==="*"){return true}if(allowedOrigin instanceof RegExp&&allowedOrigin.test(origin)){return true}}return false}function expose(obj,ep=globalThis,allowedOrigins=["*"]){ep.addEventListener("message",function callback(ev){if(!ev||!ev.data){return}if(!isAllowedOrigin(allowedOrigins,ev.origin)){console.warn(`Invalid origin '${ev.origin}' for comlink proxy`);return}const{id,type,path}=Object.assign({path:[]},ev.data);const argumentList=(ev.data.argumentList||[]).map(fromWireValue);let returnValue;try{const parent=path.slice(0,-1).reduce((obj,prop)=>obj[prop],obj);const rawValue=path.reduce((obj,prop)=>obj[prop],obj);switch(type){case"GET":{returnValue=rawValue}break;case"SET":{parent[path.slice(-1)[0]]=fromWireValue(ev.data.value);returnValue=true}break;case"APPLY":{returnValue=rawValue.apply(parent,argumentList)}break;case"CONSTRUCT":{const value=new rawValue(...argumentList);returnValue=proxy(value)}break;case"ENDPOINT":{const{port1,port2}=new MessageChannel;expose(obj,port2);returnValue=transfer(port1,[port1])}break;case"RELEASE":{returnValue=undefined}break;default:return}}catch(value){returnValue={value:value,[throwMarker]:0}}Promise.resolve(returnValue).catch(value=>{return{value:value,[throwMarker]:0}}).then(returnValue=>{const[wireValue,transferables]=toWireValue(returnValue);ep.postMessage(Object.assign(Object.assign({},wireValue),{id:id}),transferables);if(type==="RELEASE"){ep.removeEventListener("message",callback);closeEndPoint(ep);if(finalizer in obj&&typeof obj[finalizer]==="function"){obj[finalizer]()}}}).catch(error=>{const[wireValue,transferables]=toWireValue({value:new TypeError("Unserializable return value"),[throwMarker]:0});ep.postMessage(Object.assign(Object.assign({},wireValue),{id:id}),transferables)})});if(ep.start){ep.start()}}function isMessagePort(endpoint){return endpoint.constructor.name==="MessagePort"}function closeEndPoint(endpoint){if(isMessagePort(endpoint))endpoint.close()}function wrap(ep,target){return createProxy(ep,[],target)}function throwIfProxyReleased(isReleased){if(isReleased){throw new Error("Proxy has been released and is not useable")}}function releaseEndpoint(ep){return requestResponseMessage(ep,{type:"RELEASE"}).then(()=>{closeEndPoint(ep)})}const proxyCounter=new WeakMap;const proxyFinalizers="FinalizationRegistry"in globalThis&&new FinalizationRegistry(ep=>{const newCount=(proxyCounter.get(ep)||0)-1;proxyCounter.set(ep,newCount);if(newCount===0){releaseEndpoint(ep)}});function registerProxy(proxy,ep){const newCount=(proxyCounter.get(ep)||0)+1;proxyCounter.set(ep,newCount);if(proxyFinalizers){proxyFinalizers.register(proxy,ep,proxy)}}function unregisterProxy(proxy){if(proxyFinalizers){proxyFinalizers.unregister(proxy)}}function createProxy(ep,path=[],target=function(){}){let isProxyReleased=false;const proxy=new Proxy(target,{get(_target,prop){throwIfProxyReleased(isProxyReleased);if(prop===releaseProxy){return()=>{unregisterProxy(proxy);releaseEndpoint(ep);isProxyReleased=true}}if(prop==="then"){if(path.length===0){return{then:()=>proxy}}const r=requestResponseMessage(ep,{type:"GET",path:path.map(p=>p.toString())}).then(fromWireValue);return r.then.bind(r)}return createProxy(ep,[...path,prop])},set(_target,prop,rawValue){throwIfProxyReleased(isProxyReleased);const[value,transferables]=toWireValue(rawValue);return requestResponseMessage(ep,{type:"SET",path:[...path,prop].map(p=>p.toString()),value:value},transferables).then(fromWireValue)},apply(_target,_thisArg,rawArgumentList){throwIfProxyReleased(isProxyReleased);const last=path[path.length-1];if(last===createEndpoint){return requestResponseMessage(ep,{type:"ENDPOINT"}).then(fromWireValue)}if(last==="bind"){return createProxy(ep,path.slice(0,-1))}const[argumentList,transferables]=processArguments(rawArgumentList);return requestResponseMessage(ep,{type:"APPLY",path:path.map(p=>p.toString()),argumentList:argumentList},transferables).then(fromWireValue)},construct(_target,rawArgumentList){throwIfProxyReleased(isProxyReleased);const[argumentList,transferables]=processArguments(rawArgumentList);return requestResponseMessage(ep,{type:"CONSTRUCT",path:path.map(p=>p.toString()),argumentList:argumentList},transferables).then(fromWireValue)}});registerProxy(proxy,ep);return proxy}function myFlat(arr){return Array.prototype.concat.apply([],arr)}function processArguments(argumentList){const processed=argumentList.map(toWireValue);return[processed.map(v=>v[0]),myFlat(processed.map(v=>v[1]))]}const transferCache=new WeakMap;function transfer(obj,transfers){transferCache.set(obj,transfers);return obj}function proxy(obj){return Object.assign(obj,{[proxyMarker]:true})}function windowEndpoint(w,context=globalThis,targetOrigin="*"){return{postMessage:(msg,transferables)=>w.postMessage(msg,targetOrigin,transferables),addEventListener:context.addEventListener.bind(context),removeEventListener:context.removeEventListener.bind(context)}}function toWireValue(value){for(const[name,handler]of transferHandlers){if(handler.canHandle(value)){const[serializedValue,transferables]=handler.serialize(value);return[{type:"HANDLER",name:name,value:serializedValue},transferables]}}return[{type:"RAW",value:value},transferCache.get(value)||[]]}function fromWireValue(value){switch(value.type){case"HANDLER":return transferHandlers.get(value.name).deserialize(value.value);case"RAW":return value.value}}function requestResponseMessage(ep,msg,transfers){return new Promise(resolve=>{const id=generateUUID();ep.addEventListener("message",function l(ev){if(!ev.data||!ev.data.id||ev.data.id!==id){return}ep.removeEventListener("message",l);resolve(ev.data)});if(ep.start){ep.start()}ep.postMessage(Object.assign({id:id},msg),transfers)})}function generateUUID(){return new Array(4).fill(0).map(()=>Math.floor(Math.random()*Number.MAX_SAFE_INTEGER).toString(16)).join("-")}exports.createEndpoint=createEndpoint;exports.expose=expose;exports.finalizer=finalizer;exports.proxy=proxy;exports.proxyMarker=proxyMarker;exports.releaseProxy=releaseProxy;exports.transfer=transfer;exports.transferHandlers=transferHandlers;exports.windowEndpoint=windowEndpoint;exports.wrap=wrap})},{}]},{},[2]); \ No newline at end of file diff --git a/dist/build/twilio-virtual-background-processor-pipeline-worker.js b/dist/build/twilio-virtual-background-processor-pipeline-worker.js new file mode 100644 index 0000000..9bd82e5 --- /dev/null +++ b/dist/build/twilio-virtual-background-processor-pipeline-worker.js @@ -0,0 +1,1814 @@ +/*! twilio-video-processors.js 3.0.0-preview.1 + +The following license applies to all parts of this software except as +documented below. + + Copyright (C) 2022 Twilio Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + 3. Neither the name of Twilio nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +*/ +(function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.BackgroundProcessorPipeline = void 0; +var constants_1 = require("../../../../constants"); +var Benchmark_1 = require("../../../../utils/Benchmark"); +var support_1 = require("../../../../utils/support"); +var TwilioTFLite_1 = require("../../../../utils/TwilioTFLite"); +var pipelines_1 = require("../../../pipelines"); +var InputFrameDownscaleStage_1 = require("./InputFrameDownscaleStage"); +var PostProcessingStage_1 = require("./PostProcessingStage"); +/** + * @private + */ +var BackgroundProcessorPipeline = /** @class */ (function (_super) { + __extends(BackgroundProcessorPipeline, _super); + function BackgroundProcessorPipeline(options, onResizeWebGL2Canvas) { + if (onResizeWebGL2Canvas === void 0) { onResizeWebGL2Canvas = function () { }; } + var _this = _super.call(this) || this; + _this._outputCanvas = new OffscreenCanvas(1, 1); + _this._webgl2Canvas = new OffscreenCanvas(1, 1); + _this._benchmark = new Benchmark_1.Benchmark(); + _this._inferenceInputCanvas = new OffscreenCanvas(constants_1.WASM_INFERENCE_DIMENSIONS.width, constants_1.WASM_INFERENCE_DIMENSIONS.height); + _this._inputFrameDownscaleMode = (0, support_1.isChromiumImageBitmap)() ? 'image-bitmap' : 'canvas'; + var assetsPath = options.assetsPath, deferInputFrameDownscale = options.deferInputFrameDownscale, maskBlurRadius = options.maskBlurRadius; + _this._assetsPath = assetsPath; + _this._deferInputFrameDownscale = deferInputFrameDownscale; + _this._onResizeWebGL2Canvas = onResizeWebGL2Canvas; + _this.addStage(new InputFrameDownscaleStage_1.InputFrameDowscaleStage(_this._inferenceInputCanvas, _this._inputFrameDownscaleMode)); + _this.addStage(new PostProcessingStage_1.PostProcessingStage(constants_1.WASM_INFERENCE_DIMENSIONS, _this._webgl2Canvas, _this._outputCanvas, maskBlurRadius, function (inputFrame) { return _this._setBackground(inputFrame); })); + return _this; + } + BackgroundProcessorPipeline.prototype.loadTwilioTFLite = function () { + return __awaiter(this, void 0, void 0, function () { + var _twilioTFLite; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + _twilioTFLite = BackgroundProcessorPipeline._twilioTFLite; + if (!!_twilioTFLite) return [3 /*break*/, 2]; + _twilioTFLite = new TwilioTFLite_1.TwilioTFLite(); + return [4 /*yield*/, _twilioTFLite.initialize(this._assetsPath, constants_1.MODEL_NAME, constants_1.TFLITE_LOADER_NAME, constants_1.TFLITE_SIMD_LOADER_NAME)]; + case 1: + _a.sent(); + BackgroundProcessorPipeline._twilioTFLite = _twilioTFLite; + _a.label = 2; + case 2: return [2 /*return*/, _twilioTFLite.isSimdEnabled]; + } + }); + }); + }; + BackgroundProcessorPipeline.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var _a, inputFrameDownscaleStage, postProcessingStage, _b, _benchmark, _deferInputFrameDownscale, _c, inferenceInputHeight, inferenceInputWidth, _outputCanvas, _webgl2Canvas, _twilioTFLite, isInputVideoFrame, _d, height, width, didResizeWebGL2Canvas, downscalePromise, personMask; + return __generator(this, function (_e) { + switch (_e.label) { + case 0: + if (!BackgroundProcessorPipeline._twilioTFLite) { + return [2 /*return*/, null]; + } + _a = this._stages, inputFrameDownscaleStage = _a[0], postProcessingStage = _a[1]; + _b = this, _benchmark = _b._benchmark, _deferInputFrameDownscale = _b._deferInputFrameDownscale, _c = _b._inferenceInputCanvas, inferenceInputHeight = _c.height, inferenceInputWidth = _c.width, _outputCanvas = _b._outputCanvas, _webgl2Canvas = _b._webgl2Canvas; + _twilioTFLite = BackgroundProcessorPipeline._twilioTFLite; + isInputVideoFrame = typeof VideoFrame === 'function' + && inputFrame instanceof VideoFrame; + _d = isInputVideoFrame + ? { height: inputFrame.displayHeight, width: inputFrame.displayWidth } + : inputFrame, height = _d.height, width = _d.width; + didResizeWebGL2Canvas = false; + if (_outputCanvas.width !== width) { + _outputCanvas.width = width; + _webgl2Canvas.width = width; + didResizeWebGL2Canvas = true; + } + if (_outputCanvas.height !== height) { + _outputCanvas.height = height; + _webgl2Canvas.height = height; + didResizeWebGL2Canvas = true; + } + if (didResizeWebGL2Canvas) { + postProcessingStage.resetPersonMaskUpscalePipeline(); + this._onResizeWebGL2Canvas(); + } + _benchmark.start('inputImageResizeDelay'); + downscalePromise = inputFrameDownscaleStage.render(inputFrame) + .then(function (downscaledFrameData) { + _twilioTFLite.loadInputBuffer(downscaledFrameData); + }); + if (!!_deferInputFrameDownscale) return [3 /*break*/, 2]; + return [4 /*yield*/, downscalePromise]; + case 1: + _e.sent(); + _e.label = 2; + case 2: + _benchmark.end('inputImageResizeDelay'); + _benchmark.start('segmentationDelay'); + personMask = new ImageData(_twilioTFLite.runInference(), inferenceInputWidth, inferenceInputHeight); + _benchmark.end('segmentationDelay'); + _benchmark.start('imageCompositionDelay'); + postProcessingStage.render(inputFrame, personMask); + _benchmark.end('imageCompositionDelay'); + if (typeof VideoFrame === 'function' + && inputFrame instanceof VideoFrame) { + inputFrame.close(); + } + return [2 /*return*/, this._outputCanvas]; + } + }); + }); + }; + BackgroundProcessorPipeline.prototype.setDeferInputFrameDownscale = function (defer) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + this._deferInputFrameDownscale = defer; + return [2 /*return*/]; + }); + }); + }; + BackgroundProcessorPipeline.prototype.setMaskBlurRadius = function (radius) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + this._stages[1] + .updateMaskBlurRadius(radius); + return [2 /*return*/]; + }); + }); + }; + BackgroundProcessorPipeline._twilioTFLite = null; + return BackgroundProcessorPipeline; +}(pipelines_1.Pipeline)); +exports.BackgroundProcessorPipeline = BackgroundProcessorPipeline; + +},{"../../../../constants":1,"../../../../utils/Benchmark":16,"../../../../utils/TwilioTFLite":17,"../../../../utils/support":18,"../../../pipelines":10,"./InputFrameDownscaleStage":3,"./PostProcessingStage":4}],3:[function(require,module,exports){ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.InputFrameDowscaleStage = void 0; +/** + * @private + */ +var InputFrameDowscaleStage = /** @class */ (function () { + function InputFrameDowscaleStage(outputCanvas, inputFrameDownscaleMode) { + this._inputFrameDownscaleMode = inputFrameDownscaleMode; + this._outputContext = outputCanvas.getContext('2d', { willReadFrequently: true }); + } + InputFrameDowscaleStage.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var _a, _outputContext, _inputFrameDownscaleMode, _b, resizeHeight, resizeWidth, downscaledBitmap, data; + return __generator(this, function (_c) { + switch (_c.label) { + case 0: + _a = this, _outputContext = _a._outputContext, _inputFrameDownscaleMode = _a._inputFrameDownscaleMode; + _b = _outputContext.canvas, resizeHeight = _b.height, resizeWidth = _b.width; + if (!(_inputFrameDownscaleMode === 'image-bitmap')) return [3 /*break*/, 2]; + return [4 /*yield*/, createImageBitmap(inputFrame, { + resizeWidth: resizeWidth, + resizeHeight: resizeHeight, + resizeQuality: 'pixelated' + })]; + case 1: + downscaledBitmap = _c.sent(); + _outputContext.drawImage(downscaledBitmap, 0, 0); + downscaledBitmap.close(); + return [3 /*break*/, 3]; + case 2: + _outputContext.drawImage(inputFrame, 0, 0, resizeWidth, resizeHeight); + _c.label = 3; + case 3: + data = _outputContext.getImageData(0, 0, resizeWidth, resizeHeight).data; + return [2 /*return*/, data]; + } + }); + }); + }; + return InputFrameDowscaleStage; +}()); +exports.InputFrameDowscaleStage = InputFrameDowscaleStage; + +},{}],4:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PostProcessingStage = void 0; +var personmaskupscalepipeline_1 = require("../personmaskupscalepipeline"); +/** + * @private + */ +var PostProcessingStage = /** @class */ (function () { + function PostProcessingStage(inputDimensions, webgl2Canvas, outputCanvas, maskBlurRadius, setBackground) { + this._personMaskUpscalePipeline = null; + this._inputDimensions = inputDimensions; + this._maskBlurRadius = maskBlurRadius; + this._outputContext = outputCanvas.getContext('2d'); + this._webgl2Canvas = webgl2Canvas; + this._setBackground = setBackground; + } + PostProcessingStage.prototype.render = function (inputFrame, personMask) { + var _a = this, _outputContext = _a._outputContext, _setBackground = _a._setBackground, _webgl2Canvas = _a._webgl2Canvas; + if (!this._personMaskUpscalePipeline) { + this.resetPersonMaskUpscalePipeline(); + } + this._personMaskUpscalePipeline.render(inputFrame, personMask); + _outputContext.save(); + _outputContext.globalCompositeOperation = 'copy'; + _outputContext.drawImage(_webgl2Canvas, 0, 0); + _outputContext.globalCompositeOperation = 'destination-over'; + _setBackground(inputFrame); + _outputContext.restore(); + }; + PostProcessingStage.prototype.resetPersonMaskUpscalePipeline = function () { + var _a; + var _b = this, _inputDimensions = _b._inputDimensions, _maskBlurRadius = _b._maskBlurRadius, _webgl2Canvas = _b._webgl2Canvas; + (_a = this._personMaskUpscalePipeline) === null || _a === void 0 ? void 0 : _a.cleanUp(); + this._personMaskUpscalePipeline = new personmaskupscalepipeline_1.PersonMaskUpscalePipeline(_inputDimensions, _webgl2Canvas); + this._personMaskUpscalePipeline.updateBilateralFilterConfig({ + sigmaSpace: _maskBlurRadius + }); + }; + PostProcessingStage.prototype.updateMaskBlurRadius = function (radius) { + var _a; + if (this._maskBlurRadius !== radius) { + this._maskBlurRadius = radius; + (_a = this._personMaskUpscalePipeline) === null || _a === void 0 ? void 0 : _a.updateBilateralFilterConfig({ + sigmaSpace: radius + }); + } + }; + return PostProcessingStage; +}()); +exports.PostProcessingStage = PostProcessingStage; + +},{"../personmaskupscalepipeline":8}],5:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessorPipeline = void 0; +var types_1 = require("../../../../types"); +var BackgroundProcessorPipeline_1 = require("./BackgroundProcessorPipeline"); +/** + * @private + */ +var VirtualBackgroundProcessorPipeline = /** @class */ (function (_super) { + __extends(VirtualBackgroundProcessorPipeline, _super); + function VirtualBackgroundProcessorPipeline(options) { + var _this = _super.call(this, options) || this; + var fitType = options.fitType; + _this._backgroundImage = null; + _this._fitType = fitType; + return _this; + } + VirtualBackgroundProcessorPipeline.prototype.setBackgroundImage = function (backgroundImage) { + var _a; + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_b) { + (_a = this._backgroundImage) === null || _a === void 0 ? void 0 : _a.close(); + this._backgroundImage = backgroundImage; + return [2 /*return*/]; + }); + }); + }; + VirtualBackgroundProcessorPipeline.prototype.setFitType = function (fitType) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + this._fitType = fitType; + return [2 /*return*/]; + }); + }); + }; + VirtualBackgroundProcessorPipeline.prototype._setBackground = function () { + var _a = this, _backgroundImage = _a._backgroundImage, _fitType = _a._fitType, _outputCanvas = _a._outputCanvas; + if (!_backgroundImage) { + return; + } + var ctx = _outputCanvas.getContext('2d'); + var imageWidth = _backgroundImage.width; + var imageHeight = _backgroundImage.height; + var canvasWidth = _outputCanvas.width; + var canvasHeight = _outputCanvas.height; + if (_fitType === types_1.ImageFit.Fill) { + ctx.drawImage(_backgroundImage, 0, 0, imageWidth, imageHeight, 0, 0, canvasWidth, canvasHeight); + } + else if (_fitType === types_1.ImageFit.None) { + ctx.drawImage(_backgroundImage, 0, 0, imageWidth, imageHeight); + } + else { + var _b = this._getFitPosition(imageWidth, imageHeight, canvasWidth, canvasHeight, _fitType), x = _b.x, y = _b.y, w = _b.w, h = _b.h; + ctx.drawImage(_backgroundImage, 0, 0, imageWidth, imageHeight, x, y, w, h); + } + }; + VirtualBackgroundProcessorPipeline.prototype._getFitPosition = function (contentWidth, contentHeight, viewportWidth, viewportHeight, type) { + // Calculate new content width to fit viewport width + var factor = viewportWidth / contentWidth; + var newContentWidth = viewportWidth; + var newContentHeight = factor * contentHeight; + // Scale down the resulting height and width more + // to fit viewport height if the content still exceeds it + if ((type === types_1.ImageFit.Contain && newContentHeight > viewportHeight) + || (type === types_1.ImageFit.Cover && viewportHeight > newContentHeight)) { + factor = viewportHeight / newContentHeight; + newContentWidth = factor * newContentWidth; + newContentHeight = viewportHeight; + } + // Calculate the destination top left corner to center the content + var x = (viewportWidth - newContentWidth) / 2; + var y = (viewportHeight - newContentHeight) / 2; + return { + x: x, + y: y, + w: newContentWidth, + h: newContentHeight, + }; + }; + return VirtualBackgroundProcessorPipeline; +}(BackgroundProcessorPipeline_1.BackgroundProcessorPipeline)); +exports.VirtualBackgroundProcessorPipeline = VirtualBackgroundProcessorPipeline; + +},{"../../../../types":15,"./BackgroundProcessorPipeline":2}],6:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessorPipelineWorker = void 0; +var comlink_1 = require("comlink"); +var VirtualBackgroundProcessorPipeline_1 = require("./VirtualBackgroundProcessorPipeline"); +/** + * @private + */ +var VirtualBackgroundProcessorPipelineWorker = /** @class */ (function (_super) { + __extends(VirtualBackgroundProcessorPipelineWorker, _super); + function VirtualBackgroundProcessorPipelineWorker() { + return _super !== null && _super.apply(this, arguments) || this; + } + VirtualBackgroundProcessorPipelineWorker.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var outputFrame, outputBitmap; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, _super.prototype.render.call(this, inputFrame)]; + case 1: + outputFrame = _a.sent(); + outputBitmap = outputFrame instanceof OffscreenCanvas + ? outputFrame.transferToImageBitmap() + : outputFrame; + return [2 /*return*/, outputBitmap && (0, comlink_1.transfer)(outputBitmap, [outputBitmap])]; + } + }); + }); + }; + return VirtualBackgroundProcessorPipelineWorker; +}(VirtualBackgroundProcessorPipeline_1.VirtualBackgroundProcessorPipeline)); +exports.VirtualBackgroundProcessorPipelineWorker = VirtualBackgroundProcessorPipelineWorker; +(0, comlink_1.expose)(VirtualBackgroundProcessorPipelineWorker); + +},{"./VirtualBackgroundProcessorPipeline":5,"comlink":19}],7:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.SinglePassBilateralFilterStage = void 0; +var pipelines_1 = require("../../../pipelines"); +/** + * @private + */ +function createSpaceWeights(radius, sigma, texelSize) { + return '0'.repeat(radius).split('').map(function (zero, i) { + var x = (i + 1) * texelSize; + return Math.exp(-0.5 * x * x / sigma / sigma); + }); +} +/** + * @private + */ +function createColorWeights(sigma) { + return '0'.repeat(256).split('').map(function (zero, i) { + var x = i / 255; + return Math.exp(-0.5 * x * x / sigma / sigma); + }); +} +/** + * @private + */ +var SinglePassBilateralFilterStage = /** @class */ (function (_super) { + __extends(SinglePassBilateralFilterStage, _super); + function SinglePassBilateralFilterStage(glOut, direction, outputType, inputDimensions, outputDimensions, inputTextureUnit, outputTextureUnit) { + var _this = this; + if (outputTextureUnit === void 0) { outputTextureUnit = inputTextureUnit + 1; } + var height = outputDimensions.height, width = outputDimensions.width; + _this = _super.call(this, { + textureName: 'u_segmentationMask', + textureUnit: inputTextureUnit + }, { + fragmentShaderSource: "#version 300 es\n precision highp float;\n\n uniform sampler2D u_inputFrame;\n uniform sampler2D u_segmentationMask;\n uniform vec2 u_texelSize;\n uniform float u_direction;\n uniform float u_radius;\n uniform float u_step;\n uniform float u_spaceWeights[128];\n uniform float u_colorWeights[256];\n\n in vec2 v_texCoord;\n\n out vec4 outColor;\n\n float calculateColorWeight(vec2 coord, vec3 centerColor) {\n vec3 coordColor = texture(u_inputFrame, coord).rgb;\n float x = distance(centerColor, coordColor);\n return u_colorWeights[int(x * 255.0)];\n }\n\n float edgePixelsAverageAlpha(float outAlpha) {\n float totalAlpha = outAlpha;\n float totalPixels = 1.0;\n\n for (float i = -u_radius; u_radius > 0.0 && i <= u_radius; i += u_radius) {\n for (float j = -u_radius; j <= u_radius; j += u_radius * (j == 0.0 ? 2.0 : 1.0)) {\n vec2 shift = vec2(i, j) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n totalAlpha += texture(u_segmentationMask, coord).a;\n totalPixels++;\n }\n }\n\n return totalAlpha / totalPixels;\n }\n\n void main() {\n vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb;\n float outAlpha = texture(u_segmentationMask, v_texCoord).a;\n float averageAlpha = edgePixelsAverageAlpha(outAlpha);\n float totalWeight = 1.0;\n\n if (averageAlpha == 0.0 || averageAlpha == 1.0) {\n outColor = vec4(averageAlpha * centerColor, averageAlpha);\n return;\n }\n\n for (float i = 1.0; i <= u_radius; i += u_step) {\n float x = (1.0 - u_direction) * i;\n float y = u_direction * i;\n vec2 shift = vec2(x, y) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n float spaceWeight = u_spaceWeights[int(i - 1.0)];\n float colorWeight = calculateColorWeight(coord, centerColor);\n float weight = spaceWeight * colorWeight;\n float alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n\n shift = vec2(-x, -y) * u_texelSize;\n coord = vec2(v_texCoord + shift);\n colorWeight = calculateColorWeight(coord, centerColor);\n weight = spaceWeight * colorWeight;\n alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n }\n\n outAlpha /= totalWeight;\n outColor = vec4(outAlpha * centerColor, outAlpha);\n }\n ", + glOut: glOut, + height: height, + textureUnit: outputTextureUnit, + type: outputType, + width: width, + uniformVars: [ + { + name: 'u_inputFrame', + type: 'int', + values: [0] + }, + { + name: 'u_direction', + type: 'float', + values: [direction === 'vertical' ? 1 : 0] + }, + { + name: 'u_texelSize', + type: 'float', + values: [1 / width, 1 / height] + } + ] + }) || this; + _this._direction = direction; + _this._inputDimensions = inputDimensions; + _this.updateSigmaColor(0); + _this.updateSigmaSpace(0); + return _this; + } + SinglePassBilateralFilterStage.prototype.updateSigmaColor = function (sigmaColor) { + this._setUniformVars([ + { + name: 'u_colorWeights', + type: 'float:v', + values: createColorWeights(sigmaColor) + } + ]); + }; + SinglePassBilateralFilterStage.prototype.updateSigmaSpace = function (sigmaSpace) { + var _a = this._inputDimensions, inputHeight = _a.height, inputWidth = _a.width; + var _b = this._outputDimensions, outputHeight = _b.height, outputWidth = _b.width; + sigmaSpace *= Math.max(outputWidth / inputWidth, outputHeight / inputHeight); + var step = Math.floor(0.5 * sigmaSpace / Math.log(sigmaSpace)); + var sigmaTexel = Math.max(1 / outputWidth, 1 / outputHeight) * sigmaSpace; + var texelSize = 1 / (this._direction === 'horizontal' + ? outputWidth + : outputHeight); + this._setUniformVars([ + { + name: 'u_radius', + type: 'float', + values: [sigmaSpace] + }, + { + name: 'u_spaceWeights', + type: 'float:v', + values: createSpaceWeights(sigmaSpace, sigmaTexel, texelSize) + }, + { + name: 'u_step', + type: 'float', + values: [step] + } + ]); + }; + return SinglePassBilateralFilterStage; +}(pipelines_1.WebGL2Pipeline.ProcessingStage)); +exports.SinglePassBilateralFilterStage = SinglePassBilateralFilterStage; + +},{"../../../pipelines":10}],8:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PersonMaskUpscalePipeline = void 0; +var pipelines_1 = require("../../../pipelines"); +var SinglePassBilateralFilterStage_1 = require("./SinglePassBilateralFilterStage"); +/** + * @private + */ +var PersonMaskUpscalePipeline = /** @class */ (function (_super) { + __extends(PersonMaskUpscalePipeline, _super); + function PersonMaskUpscalePipeline(inputDimensions, outputCanvas) { + var _this = _super.call(this) || this; + var glOut = outputCanvas.getContext('webgl2'); + var outputDimensions = { + height: outputCanvas.height, + width: outputCanvas.width + }; + _this.addStage(new pipelines_1.WebGL2Pipeline.InputStage(glOut)); + _this.addStage(new SinglePassBilateralFilterStage_1.SinglePassBilateralFilterStage(glOut, 'horizontal', 'texture', inputDimensions, outputDimensions, 1, 2)); + _this.addStage(new SinglePassBilateralFilterStage_1.SinglePassBilateralFilterStage(glOut, 'vertical', 'canvas', inputDimensions, outputDimensions, 2)); + return _this; + } + PersonMaskUpscalePipeline.prototype.updateBilateralFilterConfig = function (config) { + var _a = this._stages, bilateralFilterStages = _a.slice(1); + var sigmaSpace = config.sigmaSpace; + if (typeof sigmaSpace === 'number') { + bilateralFilterStages.forEach(function (stage) { + stage.updateSigmaColor(0.1); + stage.updateSigmaSpace(sigmaSpace); + }); + } + }; + return PersonMaskUpscalePipeline; +}(pipelines_1.WebGL2Pipeline)); +exports.PersonMaskUpscalePipeline = PersonMaskUpscalePipeline; + +},{"../../../pipelines":10,"./SinglePassBilateralFilterStage":7}],9:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Pipeline = void 0; +/** + * @private + */ +var Pipeline = /** @class */ (function () { + function Pipeline() { + this._stages = []; + } + Pipeline.prototype.addStage = function (stage) { + this._stages.push(stage); + }; + Pipeline.prototype.render = function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + this._stages.forEach(function (stage) { + stage.render.apply(stage, args); + }); + }; + return Pipeline; +}()); +exports.Pipeline = Pipeline; + +},{}],10:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2Pipeline = exports.Pipeline = void 0; +var Pipeline_1 = require("./Pipeline"); +Object.defineProperty(exports, "Pipeline", { enumerable: true, get: function () { return Pipeline_1.Pipeline; } }); +var webgl2pipeline_1 = require("./webgl2pipeline"); +Object.defineProperty(exports, "WebGL2Pipeline", { enumerable: true, get: function () { return webgl2pipeline_1.WebGL2Pipeline; } }); + +},{"./Pipeline":9,"./webgl2pipeline":13}],11:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2PipelineInputStage = void 0; +var webgl2PipelineHelpers_1 = require("./webgl2PipelineHelpers"); +/** + * @private + */ +var WebGL2PipelineInputStage = /** @class */ (function () { + function WebGL2PipelineInputStage(glOut) { + this._inputTexture = null; + var _a = glOut.canvas, height = _a.height, width = _a.width; + this._glOut = glOut; + this._inputFrameTexture = (0, webgl2PipelineHelpers_1.createTexture)(glOut, glOut.RGBA8, width, height, glOut.NEAREST, glOut.NEAREST); + } + WebGL2PipelineInputStage.prototype.cleanUp = function () { + var _a = this, _glOut = _a._glOut, _inputFrameTexture = _a._inputFrameTexture, _inputTexture = _a._inputTexture; + _glOut.deleteTexture(_inputFrameTexture); + _glOut.deleteTexture(_inputTexture); + }; + WebGL2PipelineInputStage.prototype.render = function (inputFrame, inputTextureData) { + var _a = this, _glOut = _a._glOut, _inputFrameTexture = _a._inputFrameTexture; + var _b = _glOut.canvas, height = _b.height, width = _b.width; + _glOut.viewport(0, 0, width, height); + _glOut.clearColor(0, 0, 0, 0); + _glOut.clear(_glOut.COLOR_BUFFER_BIT); + if (inputFrame) { + _glOut.activeTexture(_glOut.TEXTURE0); + _glOut.bindTexture(_glOut.TEXTURE_2D, _inputFrameTexture); + _glOut.texSubImage2D(_glOut.TEXTURE_2D, 0, 0, 0, width, height, _glOut.RGBA, _glOut.UNSIGNED_BYTE, inputFrame); + } + if (!inputTextureData) { + return; + } + var data = inputTextureData.data, textureHeight = inputTextureData.height, textureWidth = inputTextureData.width; + if (!this._inputTexture) { + this._inputTexture = (0, webgl2PipelineHelpers_1.createTexture)(_glOut, _glOut.RGBA8, textureWidth, textureHeight, _glOut.NEAREST, _glOut.NEAREST); + } + _glOut.viewport(0, 0, textureWidth, textureHeight); + _glOut.activeTexture(_glOut.TEXTURE1); + _glOut.bindTexture(_glOut.TEXTURE_2D, this._inputTexture); + _glOut.texSubImage2D(_glOut.TEXTURE_2D, 0, 0, 0, textureWidth, textureHeight, _glOut.RGBA, _glOut.UNSIGNED_BYTE, data); + }; + return WebGL2PipelineInputStage; +}()); +exports.WebGL2PipelineInputStage = WebGL2PipelineInputStage; + +},{"./webgl2PipelineHelpers":14}],12:[function(require,module,exports){ +"use strict"; +var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) { + if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) { + if (ar || !(i in from)) { + if (!ar) ar = Array.prototype.slice.call(from, 0, i); + ar[i] = from[i]; + } + } + return to.concat(ar || Array.prototype.slice.call(from)); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2PipelineProcessingStage = void 0; +var webgl2PipelineHelpers_1 = require("./webgl2PipelineHelpers"); +/**; + * @private + */ +var WebGL2PipelineProcessingStage = /** @class */ (function () { + function WebGL2PipelineProcessingStage(inputConfig, outputConfig) { + this._outputFramebuffer = null; + this._outputTexture = null; + var textureName = inputConfig.textureName, textureUnit = inputConfig.textureUnit; + var glOut = outputConfig.glOut; + this._glOut = glOut; + var fragmentShaderSource = outputConfig.fragmentShaderSource, _a = outputConfig.height, height = _a === void 0 ? glOut.canvas.height : _a, _b = outputConfig.textureUnit, outputTextureUnit = _b === void 0 ? textureUnit + 1 : _b, outputType = outputConfig.type, _c = outputConfig.uniformVars, uniformVars = _c === void 0 ? [] : _c, _d = outputConfig.vertexShaderSource, vertexShaderSource = _d === void 0 ? "#version 300 es\n in vec2 a_position;\n in vec2 a_texCoord;\n\n out vec2 v_texCoord;\n\n void main() {\n gl_Position = vec4(a_position".concat(outputType === 'canvas' + ? ' * vec2(1.0, -1.0)' + : '', ", 0.0, 1.0);\n v_texCoord = a_texCoord;\n }\n ") : _d, _e = outputConfig.width, width = _e === void 0 ? glOut.canvas.width : _e; + this._outputDimensions = { + height: height, + width: width + }; + this._outputTextureUnit = outputTextureUnit; + this._fragmentShader = (0, webgl2PipelineHelpers_1.compileShader)(glOut, glOut.FRAGMENT_SHADER, fragmentShaderSource); + this._vertexShader = (0, webgl2PipelineHelpers_1.compileShader)(glOut, glOut.VERTEX_SHADER, vertexShaderSource); + this._positionBuffer = (0, webgl2PipelineHelpers_1.initBuffer)(glOut, [ + -1.0, -1.0, + 1.0, -1.0, + -1.0, 1.0, + 1.0, 1.0, + ]); + this._texCoordBuffer = (0, webgl2PipelineHelpers_1.initBuffer)(glOut, [ + 0.0, 0.0, + 1.0, 0.0, + 0.0, 1.0, + 1.0, 1.0, + ]); + if (outputType === 'texture') { + this._outputTexture = (0, webgl2PipelineHelpers_1.createTexture)(glOut, glOut.RGBA8, width, height); + this._outputFramebuffer = glOut.createFramebuffer(); + glOut.bindFramebuffer(glOut.FRAMEBUFFER, this._outputFramebuffer); + glOut.framebufferTexture2D(glOut.FRAMEBUFFER, glOut.COLOR_ATTACHMENT0, glOut.TEXTURE_2D, this._outputTexture, 0); + } + var program = (0, webgl2PipelineHelpers_1.createPipelineStageProgram)(glOut, this._vertexShader, this._fragmentShader, this._positionBuffer, this._texCoordBuffer); + this._program = program; + this._setUniformVars(__spreadArray([ + { + name: textureName, + type: 'int', + values: [textureUnit] + } + ], uniformVars, true)); + } + WebGL2PipelineProcessingStage.prototype.cleanUp = function () { + var _a = this, _fragmentShader = _a._fragmentShader, _glOut = _a._glOut, _positionBuffer = _a._positionBuffer, _program = _a._program, _texCoordBuffer = _a._texCoordBuffer, _vertexShader = _a._vertexShader; + _glOut.deleteProgram(_program); + _glOut.deleteBuffer(_texCoordBuffer); + _glOut.deleteBuffer(_positionBuffer); + _glOut.deleteShader(_vertexShader); + _glOut.deleteShader(_fragmentShader); + }; + WebGL2PipelineProcessingStage.prototype.render = function () { + var _a = this, _glOut = _a._glOut, _b = _a._outputDimensions, height = _b.height, width = _b.width, _outputFramebuffer = _a._outputFramebuffer, _outputTexture = _a._outputTexture, _outputTextureUnit = _a._outputTextureUnit, _program = _a._program; + _glOut.viewport(0, 0, width, height); + _glOut.useProgram(_program); + if (_outputTexture) { + _glOut.activeTexture(_glOut.TEXTURE0 + + _outputTextureUnit); + _glOut.bindTexture(_glOut.TEXTURE_2D, _outputTexture); + } + _glOut.bindFramebuffer(_glOut.FRAMEBUFFER, _outputFramebuffer); + _glOut.drawArrays(_glOut.TRIANGLE_STRIP, 0, 4); + }; + WebGL2PipelineProcessingStage.prototype._setUniformVars = function (uniformVars) { + var _a = this, _glOut = _a._glOut, _program = _a._program; + _glOut.useProgram(_program); + uniformVars.forEach(function (_a) { + var name = _a.name, type = _a.type, values = _a.values; + var uniformVarLocation = _glOut + .getUniformLocation(_program, name); + var isVector = type.split(':')[1] === 'v'; + if (isVector) { + // @ts-ignore + _glOut["uniform1".concat(type[0], "v")](uniformVarLocation, values); + } + else { + // @ts-ignore + _glOut["uniform".concat(values.length).concat(type[0])].apply(_glOut, __spreadArray([uniformVarLocation], values, false)); + } + }); + }; + return WebGL2PipelineProcessingStage; +}()); +exports.WebGL2PipelineProcessingStage = WebGL2PipelineProcessingStage; + +},{"./webgl2PipelineHelpers":14}],13:[function(require,module,exports){ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2Pipeline = void 0; +var Pipeline_1 = require("../Pipeline"); +var WebGL2PipelineInputStage_1 = require("./WebGL2PipelineInputStage"); +var WebGL2PipelineProcessingStage_1 = require("./WebGL2PipelineProcessingStage"); +/** + * @private + */ +var WebGL2Pipeline = /** @class */ (function (_super) { + __extends(WebGL2Pipeline, _super); + function WebGL2Pipeline() { + var _this = _super !== null && _super.apply(this, arguments) || this; + _this._stages = []; + return _this; + } + WebGL2Pipeline.prototype.cleanUp = function () { + this._stages.forEach(function (stage) { return stage.cleanUp(); }); + }; + WebGL2Pipeline.prototype.render = function (inputFrame, inputTextureData) { + var _a = this._stages, inputStage = _a[0], otherStages = _a.slice(1); + inputStage.render(inputFrame, inputTextureData); + otherStages.forEach(function (stage) { return stage + .render(); }); + }; + WebGL2Pipeline.InputStage = WebGL2PipelineInputStage_1.WebGL2PipelineInputStage; + WebGL2Pipeline.ProcessingStage = WebGL2PipelineProcessingStage_1.WebGL2PipelineProcessingStage; + return WebGL2Pipeline; +}(Pipeline_1.Pipeline)); +exports.WebGL2Pipeline = WebGL2Pipeline; + +},{"../Pipeline":9,"./WebGL2PipelineInputStage":11,"./WebGL2PipelineProcessingStage":12}],14:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.initBuffer = exports.createTexture = exports.compileShader = exports.createProgram = exports.createPipelineStageProgram = void 0; +/** + * @private + */ +function createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer) { + var program = createProgram(gl, vertexShader, fragmentShader); + var positionAttributeLocation = gl.getAttribLocation(program, 'a_position'); + gl.enableVertexAttribArray(positionAttributeLocation); + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + gl.vertexAttribPointer(positionAttributeLocation, 2, gl.FLOAT, false, 0, 0); + var texCoordAttributeLocation = gl.getAttribLocation(program, 'a_texCoord'); + gl.enableVertexAttribArray(texCoordAttributeLocation); + gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); + gl.vertexAttribPointer(texCoordAttributeLocation, 2, gl.FLOAT, false, 0, 0); + return program; +} +exports.createPipelineStageProgram = createPipelineStageProgram; +/** + * @private + */ +function createProgram(gl, vertexShader, fragmentShader) { + var program = gl.createProgram(); + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + throw new Error("Could not link WebGL program: ".concat(gl.getProgramInfoLog(program))); + } + return program; +} +exports.createProgram = createProgram; +/** + * @private + */ +function compileShader(gl, shaderType, shaderSource) { + var shader = gl.createShader(shaderType); + gl.shaderSource(shader, shaderSource); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + throw new Error("Could not compile shader: ".concat(gl.getShaderInfoLog(shader))); + } + return shader; +} +exports.compileShader = compileShader; +/** + * @private + */ +function createTexture(gl, internalformat, width, height, minFilter, magFilter) { + if (minFilter === void 0) { minFilter = gl.NEAREST; } + if (magFilter === void 0) { magFilter = gl.NEAREST; } + var texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, minFilter); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, magFilter); + gl.texStorage2D(gl.TEXTURE_2D, 1, internalformat, width, height); + return texture; +} +exports.createTexture = createTexture; +/** + * @private + */ +function initBuffer(gl, data) { + var buffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(data), gl.STATIC_DRAW); + return buffer; +} +exports.initBuffer = initBuffer; + +},{}],15:[function(require,module,exports){ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.ImageFit = void 0; +/** + * ImageFit specifies the positioning of an image inside a viewport. + */ +var ImageFit; +(function (ImageFit) { + /** + * Scale the image up or down to fill the viewport while preserving the aspect ratio. + * The image will be fully visible but will add empty space in the viewport if + * aspect ratios do not match. + */ + ImageFit["Contain"] = "Contain"; + /** + * Scale the image to fill both height and width of the viewport while preserving + * the aspect ratio, but will crop the image if aspect ratios do not match. + */ + ImageFit["Cover"] = "Cover"; + /** + * Stretches the image to fill the viewport regardless of aspect ratio. + */ + ImageFit["Fill"] = "Fill"; + /** + * Ignore height and width and use the original size. + */ + ImageFit["None"] = "None"; +})(ImageFit || (exports.ImageFit = ImageFit = {})); + +},{}],16:[function(require,module,exports){ +"use strict"; +var __assign = (this && this.__assign) || function () { + __assign = Object.assign || function(t) { + for (var s, i = 1, n = arguments.length; i < n; i++) { + s = arguments[i]; + for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) + t[p] = s[p]; + } + return t; + }; + return __assign.apply(this, arguments); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Benchmark = void 0; +/** + * @private + */ +var Benchmark = /** @class */ (function () { + function Benchmark() { + this._timingCache = new Map(); + this._timings = new Map(); + } + Benchmark.prototype.end = function (name) { + var timing = this._timings.get(name); + if (!timing) { + return; + } + timing.end = Date.now(); + timing.delay = timing.end - timing.start; + this._save(name, __assign({}, timing)); + }; + Benchmark.prototype.getAverageDelay = function (name) { + var timingCache = this._timingCache.get(name); + if (!timingCache || !timingCache.length) { + return; + } + return timingCache.map(function (timing) { return timing.delay; }) + .reduce(function (total, value) { return total += value; }, 0) / timingCache.length; + }; + Benchmark.prototype.getNames = function () { + return Array.from(this._timingCache.keys()); + }; + Benchmark.prototype.getRate = function (name) { + var timingCache = this._timingCache.get(name); + if (!timingCache || timingCache.length < 2) { + return; + } + var totalDelay = timingCache[timingCache.length - 1].end - timingCache[0].start; + return (timingCache.length / totalDelay) * 1000; + }; + Benchmark.prototype.merge = function (benchmark) { + var _this = this; + var _timingCache = benchmark._timingCache, _timings = benchmark._timings; + _timingCache.forEach(function (cache, name) { return _this._timingCache.set(name, cache); }); + _timings.forEach(function (timing, name) { return _this._timings.set(name, timing); }); + }; + Benchmark.prototype.start = function (name) { + var timing = this._timings.get(name); + if (!timing) { + timing = {}; + this._timings.set(name, timing); + } + timing.start = Date.now(); + delete timing.end; + delete timing.delay; + }; + Benchmark.prototype._save = function (name, timing) { + var timingCache = this._timingCache.get(name); + if (!timingCache) { + timingCache = []; + this._timingCache.set(name, timingCache); + } + timingCache.push(timing); + if (timingCache.length > Benchmark.cacheSize) { + timingCache.splice(0, timingCache.length - Benchmark.cacheSize); + } + }; + // NOTE (csantos): How many timing information to save per benchmark. + // This is about the amount of timing info generated on a 24fps input. + // Enough samples to calculate fps + Benchmark.cacheSize = 41; + return Benchmark; +}()); +exports.Benchmark = Benchmark; + +},{}],17:[function(require,module,exports){ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.TwilioTFLite = void 0; +var isWebWorker = typeof WorkerGlobalScope !== 'undefined' + && self instanceof WorkerGlobalScope; +var loadedScripts = new Set(); +var model; +/** + * @private + */ +var TwilioTFLite = /** @class */ (function () { + function TwilioTFLite() { + this._inputBuffer = null; + this._isSimdEnabled = null; + this._tflite = null; + } + Object.defineProperty(TwilioTFLite.prototype, "isSimdEnabled", { + get: function () { + return this._isSimdEnabled; + }, + enumerable: false, + configurable: true + }); + TwilioTFLite.prototype.initialize = function (assetsPath, modelName, moduleLoaderName, moduleSimdLoaderName) { + return __awaiter(this, void 0, void 0, function () { + var _a, modelResponse, _b, tflite, modelBufferOffset; + return __generator(this, function (_c) { + switch (_c.label) { + case 0: + if (this._tflite) { + return [2 /*return*/]; + } + if (isWebWorker) { + // NOTE(mmalavalli): In a web worker, paths to other dependencies + // are determined relative to the assets path, so no need to append + // it to the file names of the dependencies. + assetsPath = ''; + } + return [4 /*yield*/, Promise.all([ + this._loadWasmModule(assetsPath, moduleLoaderName, moduleSimdLoaderName), + fetch("".concat(assetsPath).concat(modelName)), + ])]; + case 1: + _a = _c.sent(), modelResponse = _a[1]; + _b = model; + if (_b) return [3 /*break*/, 3]; + return [4 /*yield*/, modelResponse.arrayBuffer()]; + case 2: + _b = (_c.sent()); + _c.label = 3; + case 3: + model = _b; + tflite = this._tflite; + modelBufferOffset = tflite._getModelBufferMemoryOffset(); + tflite.HEAPU8.set(new Uint8Array(model), modelBufferOffset); + tflite._loadModel(model.byteLength); + return [2 /*return*/]; + } + }); + }); + }; + TwilioTFLite.prototype.loadInputBuffer = function (inputBuffer) { + var tflite = this._tflite; + var height = tflite._getInputHeight(); + var width = tflite._getInputWidth(); + var pixels = width * height; + var tfliteInputMemoryOffset = tflite._getInputMemoryOffset() / 4; + for (var i = 0; i < pixels; i++) { + var curTFLiteOffset = tfliteInputMemoryOffset + i * 3; + var curImageBufferOffset = i * 4; + tflite.HEAPF32[curTFLiteOffset] = inputBuffer[curImageBufferOffset] / 255; + tflite.HEAPF32[curTFLiteOffset + 1] = inputBuffer[curImageBufferOffset + 1] / 255; + tflite.HEAPF32[curTFLiteOffset + 2] = inputBuffer[curImageBufferOffset + 2] / 255; + } + this._inputBuffer = inputBuffer; + }; + TwilioTFLite.prototype.runInference = function () { + var tflite = this._tflite; + var height = tflite._getInputHeight(); + var width = tflite._getInputWidth(); + var pixels = width * height; + var tfliteOutputMemoryOffset = tflite._getOutputMemoryOffset() / 4; + tflite._runInference(); + var inputBuffer = this._inputBuffer || new Uint8ClampedArray(pixels * 4); + for (var i = 0; i < pixels; i++) { + inputBuffer[i * 4 + 3] = Math.round(tflite.HEAPF32[tfliteOutputMemoryOffset + i] * 255); + } + return inputBuffer; + }; + TwilioTFLite.prototype._loadScript = function (path) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + if (loadedScripts.has(path)) { + return [2 /*return*/]; + } + if (isWebWorker) { + importScripts(path); + loadedScripts.add(path); + return [2 /*return*/]; + } + return [2 /*return*/, new Promise(function (resolve, reject) { + var script = document.createElement('script'); + script.onload = function () { + loadedScripts.add(path); + resolve(); + }; + script.onerror = function () { + reject(); + }; + document.head.append(script); + script.src = path; + })]; + }); + }); + }; + TwilioTFLite.prototype._loadWasmModule = function (assetsPath, moduleLoaderName, moduleSimdLoaderName) { + return __awaiter(this, void 0, void 0, function () { + var _a, _b, _c; + return __generator(this, function (_d) { + switch (_d.label) { + case 0: + _d.trys.push([0, 3, , 6]); + return [4 /*yield*/, this._loadScript("".concat(assetsPath).concat(moduleSimdLoaderName))]; + case 1: + _d.sent(); + _a = this; + return [4 /*yield*/, createTwilioTFLiteSIMDModule()]; + case 2: + _a._tflite = _d.sent(); + this._isSimdEnabled = true; + return [3 /*break*/, 6]; + case 3: + _b = _d.sent(); + return [4 /*yield*/, this._loadScript("".concat(assetsPath).concat(moduleLoaderName))]; + case 4: + _d.sent(); + _c = this; + return [4 /*yield*/, createTwilioTFLiteModule()]; + case 5: + _c._tflite = _d.sent(); + this._isSimdEnabled = false; + return [3 /*break*/, 6]; + case 6: return [2 /*return*/]; + } + }); + }); + }; + return TwilioTFLite; +}()); +exports.TwilioTFLite = TwilioTFLite; + +},{}],18:[function(require,module,exports){ +"use strict"; +var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) { + if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) { + if (ar || !(i in from)) { + if (!ar) ar = Array.prototype.slice.call(from, 0, i); + ar[i] = from[i]; + } + } + return to.concat(ar || Array.prototype.slice.call(from)); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.isSupported = exports.isCanvasBlurSupported = exports.isChromiumImageBitmap = exports.isBrowserSupported = void 0; +/** + * @private + */ +function getCanvas() { + return typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); +} +/** + * @private + */ +function isBrowserSupported() { + if (typeof window !== 'undefined' && typeof document !== 'undefined') { + return !!(getCanvas().getContext('2d') || getCanvas().getContext('webgl2')); + } + else { + return false; + } +} +exports.isBrowserSupported = isBrowserSupported; +/** + * @private + */ +function isChromiumImageBitmap() { + return /Chrome/.test(navigator.userAgent) + && typeof createImageBitmap === 'function'; +} +exports.isChromiumImageBitmap = isChromiumImageBitmap; +/** + * @private + */ +exports.isCanvasBlurSupported = (function () { + var blackPixel = [0, 0, 0, 255]; + var whitePixel = [255, 255, 255, 255]; + var inputImageData = new ImageData(new Uint8ClampedArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], blackPixel, true), blackPixel, true), blackPixel, true), blackPixel, true), whitePixel, true), blackPixel, true), blackPixel, true), blackPixel, true), blackPixel, true)), 3, 3); + var canvas = getCanvas(); + var context = canvas.getContext('2d'); + canvas.width = 3; + canvas.height = 3; + context.putImageData(inputImageData, 0, 0); + context.filter = 'blur(1px)'; + context.drawImage(canvas, 0, 0); + var data = context.getImageData(0, 0, 3, 3).data; + return data[0] > 0; +})(); +/** + * Check if the current browser is officially supported by twilio-video-procesors.js. + * This is set to `true` for browsers that supports canvas + * [2D](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D) or + * [webgl2](https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext) + * rendering context. + * @example + * ```ts + * import { isSupported } from '@twilio/video-processors'; + * + * if (isSupported) { + * // Initialize the background processors + * } + * ``` + */ +exports.isSupported = isBrowserSupported(); + +},{}],19:[function(require,module,exports){ +(function (global, factory) { + typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports) : + typeof define === 'function' && define.amd ? define(['exports'], factory) : + (global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.Comlink = {})); +})(this, (function (exports) { 'use strict'; + + /** + * @license + * Copyright 2019 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + const proxyMarker = Symbol("Comlink.proxy"); + const createEndpoint = Symbol("Comlink.endpoint"); + const releaseProxy = Symbol("Comlink.releaseProxy"); + const finalizer = Symbol("Comlink.finalizer"); + const throwMarker = Symbol("Comlink.thrown"); + const isObject = (val) => (typeof val === "object" && val !== null) || typeof val === "function"; + /** + * Internal transfer handle to handle objects marked to proxy. + */ + const proxyTransferHandler = { + canHandle: (val) => isObject(val) && val[proxyMarker], + serialize(obj) { + const { port1, port2 } = new MessageChannel(); + expose(obj, port1); + return [port2, [port2]]; + }, + deserialize(port) { + port.start(); + return wrap(port); + }, + }; + /** + * Internal transfer handler to handle thrown exceptions. + */ + const throwTransferHandler = { + canHandle: (value) => isObject(value) && throwMarker in value, + serialize({ value }) { + let serialized; + if (value instanceof Error) { + serialized = { + isError: true, + value: { + message: value.message, + name: value.name, + stack: value.stack, + }, + }; + } + else { + serialized = { isError: false, value }; + } + return [serialized, []]; + }, + deserialize(serialized) { + if (serialized.isError) { + throw Object.assign(new Error(serialized.value.message), serialized.value); + } + throw serialized.value; + }, + }; + /** + * Allows customizing the serialization of certain values. + */ + const transferHandlers = new Map([ + ["proxy", proxyTransferHandler], + ["throw", throwTransferHandler], + ]); + function isAllowedOrigin(allowedOrigins, origin) { + for (const allowedOrigin of allowedOrigins) { + if (origin === allowedOrigin || allowedOrigin === "*") { + return true; + } + if (allowedOrigin instanceof RegExp && allowedOrigin.test(origin)) { + return true; + } + } + return false; + } + function expose(obj, ep = globalThis, allowedOrigins = ["*"]) { + ep.addEventListener("message", function callback(ev) { + if (!ev || !ev.data) { + return; + } + if (!isAllowedOrigin(allowedOrigins, ev.origin)) { + console.warn(`Invalid origin '${ev.origin}' for comlink proxy`); + return; + } + const { id, type, path } = Object.assign({ path: [] }, ev.data); + const argumentList = (ev.data.argumentList || []).map(fromWireValue); + let returnValue; + try { + const parent = path.slice(0, -1).reduce((obj, prop) => obj[prop], obj); + const rawValue = path.reduce((obj, prop) => obj[prop], obj); + switch (type) { + case "GET" /* MessageType.GET */: + { + returnValue = rawValue; + } + break; + case "SET" /* MessageType.SET */: + { + parent[path.slice(-1)[0]] = fromWireValue(ev.data.value); + returnValue = true; + } + break; + case "APPLY" /* MessageType.APPLY */: + { + returnValue = rawValue.apply(parent, argumentList); + } + break; + case "CONSTRUCT" /* MessageType.CONSTRUCT */: + { + const value = new rawValue(...argumentList); + returnValue = proxy(value); + } + break; + case "ENDPOINT" /* MessageType.ENDPOINT */: + { + const { port1, port2 } = new MessageChannel(); + expose(obj, port2); + returnValue = transfer(port1, [port1]); + } + break; + case "RELEASE" /* MessageType.RELEASE */: + { + returnValue = undefined; + } + break; + default: + return; + } + } + catch (value) { + returnValue = { value, [throwMarker]: 0 }; + } + Promise.resolve(returnValue) + .catch((value) => { + return { value, [throwMarker]: 0 }; + }) + .then((returnValue) => { + const [wireValue, transferables] = toWireValue(returnValue); + ep.postMessage(Object.assign(Object.assign({}, wireValue), { id }), transferables); + if (type === "RELEASE" /* MessageType.RELEASE */) { + // detach and deactive after sending release response above. + ep.removeEventListener("message", callback); + closeEndPoint(ep); + if (finalizer in obj && typeof obj[finalizer] === "function") { + obj[finalizer](); + } + } + }) + .catch((error) => { + // Send Serialization Error To Caller + const [wireValue, transferables] = toWireValue({ + value: new TypeError("Unserializable return value"), + [throwMarker]: 0, + }); + ep.postMessage(Object.assign(Object.assign({}, wireValue), { id }), transferables); + }); + }); + if (ep.start) { + ep.start(); + } + } + function isMessagePort(endpoint) { + return endpoint.constructor.name === "MessagePort"; + } + function closeEndPoint(endpoint) { + if (isMessagePort(endpoint)) + endpoint.close(); + } + function wrap(ep, target) { + return createProxy(ep, [], target); + } + function throwIfProxyReleased(isReleased) { + if (isReleased) { + throw new Error("Proxy has been released and is not useable"); + } + } + function releaseEndpoint(ep) { + return requestResponseMessage(ep, { + type: "RELEASE" /* MessageType.RELEASE */, + }).then(() => { + closeEndPoint(ep); + }); + } + const proxyCounter = new WeakMap(); + const proxyFinalizers = "FinalizationRegistry" in globalThis && + new FinalizationRegistry((ep) => { + const newCount = (proxyCounter.get(ep) || 0) - 1; + proxyCounter.set(ep, newCount); + if (newCount === 0) { + releaseEndpoint(ep); + } + }); + function registerProxy(proxy, ep) { + const newCount = (proxyCounter.get(ep) || 0) + 1; + proxyCounter.set(ep, newCount); + if (proxyFinalizers) { + proxyFinalizers.register(proxy, ep, proxy); + } + } + function unregisterProxy(proxy) { + if (proxyFinalizers) { + proxyFinalizers.unregister(proxy); + } + } + function createProxy(ep, path = [], target = function () { }) { + let isProxyReleased = false; + const proxy = new Proxy(target, { + get(_target, prop) { + throwIfProxyReleased(isProxyReleased); + if (prop === releaseProxy) { + return () => { + unregisterProxy(proxy); + releaseEndpoint(ep); + isProxyReleased = true; + }; + } + if (prop === "then") { + if (path.length === 0) { + return { then: () => proxy }; + } + const r = requestResponseMessage(ep, { + type: "GET" /* MessageType.GET */, + path: path.map((p) => p.toString()), + }).then(fromWireValue); + return r.then.bind(r); + } + return createProxy(ep, [...path, prop]); + }, + set(_target, prop, rawValue) { + throwIfProxyReleased(isProxyReleased); + // FIXME: ES6 Proxy Handler `set` methods are supposed to return a + // boolean. To show good will, we return true asynchronously ¯\_(ツ)_/¯ + const [value, transferables] = toWireValue(rawValue); + return requestResponseMessage(ep, { + type: "SET" /* MessageType.SET */, + path: [...path, prop].map((p) => p.toString()), + value, + }, transferables).then(fromWireValue); + }, + apply(_target, _thisArg, rawArgumentList) { + throwIfProxyReleased(isProxyReleased); + const last = path[path.length - 1]; + if (last === createEndpoint) { + return requestResponseMessage(ep, { + type: "ENDPOINT" /* MessageType.ENDPOINT */, + }).then(fromWireValue); + } + // We just pretend that `bind()` didn’t happen. + if (last === "bind") { + return createProxy(ep, path.slice(0, -1)); + } + const [argumentList, transferables] = processArguments(rawArgumentList); + return requestResponseMessage(ep, { + type: "APPLY" /* MessageType.APPLY */, + path: path.map((p) => p.toString()), + argumentList, + }, transferables).then(fromWireValue); + }, + construct(_target, rawArgumentList) { + throwIfProxyReleased(isProxyReleased); + const [argumentList, transferables] = processArguments(rawArgumentList); + return requestResponseMessage(ep, { + type: "CONSTRUCT" /* MessageType.CONSTRUCT */, + path: path.map((p) => p.toString()), + argumentList, + }, transferables).then(fromWireValue); + }, + }); + registerProxy(proxy, ep); + return proxy; + } + function myFlat(arr) { + return Array.prototype.concat.apply([], arr); + } + function processArguments(argumentList) { + const processed = argumentList.map(toWireValue); + return [processed.map((v) => v[0]), myFlat(processed.map((v) => v[1]))]; + } + const transferCache = new WeakMap(); + function transfer(obj, transfers) { + transferCache.set(obj, transfers); + return obj; + } + function proxy(obj) { + return Object.assign(obj, { [proxyMarker]: true }); + } + function windowEndpoint(w, context = globalThis, targetOrigin = "*") { + return { + postMessage: (msg, transferables) => w.postMessage(msg, targetOrigin, transferables), + addEventListener: context.addEventListener.bind(context), + removeEventListener: context.removeEventListener.bind(context), + }; + } + function toWireValue(value) { + for (const [name, handler] of transferHandlers) { + if (handler.canHandle(value)) { + const [serializedValue, transferables] = handler.serialize(value); + return [ + { + type: "HANDLER" /* WireValueType.HANDLER */, + name, + value: serializedValue, + }, + transferables, + ]; + } + } + return [ + { + type: "RAW" /* WireValueType.RAW */, + value, + }, + transferCache.get(value) || [], + ]; + } + function fromWireValue(value) { + switch (value.type) { + case "HANDLER" /* WireValueType.HANDLER */: + return transferHandlers.get(value.name).deserialize(value.value); + case "RAW" /* WireValueType.RAW */: + return value.value; + } + } + function requestResponseMessage(ep, msg, transfers) { + return new Promise((resolve) => { + const id = generateUUID(); + ep.addEventListener("message", function l(ev) { + if (!ev.data || !ev.data.id || ev.data.id !== id) { + return; + } + ep.removeEventListener("message", l); + resolve(ev.data); + }); + if (ep.start) { + ep.start(); + } + ep.postMessage(Object.assign({ id }, msg), transfers); + }); + } + function generateUUID() { + return new Array(4) + .fill(0) + .map(() => Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(16)) + .join("-"); + } + + exports.createEndpoint = createEndpoint; + exports.expose = expose; + exports.finalizer = finalizer; + exports.proxy = proxy; + exports.proxyMarker = proxyMarker; + exports.releaseProxy = releaseProxy; + exports.transfer = transfer; + exports.transferHandlers = transferHandlers; + exports.windowEndpoint = windowEndpoint; + exports.wrap = wrap; + +})); + + +},{}]},{},[6]); diff --git a/dist/docs/.nojekyll b/dist/docs/.nojekyll new file mode 100644 index 0000000..e2ac661 --- /dev/null +++ b/dist/docs/.nojekyll @@ -0,0 +1 @@ +TypeDoc added this file to prevent GitHub Pages from using Jekyll. You can turn off this behavior by setting the `githubPages` option to false. \ No newline at end of file diff --git a/dist/docs/assets/highlight.css b/dist/docs/assets/highlight.css new file mode 100644 index 0000000..47059f9 --- /dev/null +++ b/dist/docs/assets/highlight.css @@ -0,0 +1,127 @@ +:root { + --light-hl-0: #001080; + --dark-hl-0: #9CDCFE; + --light-hl-1: #000000; + --dark-hl-1: #D4D4D4; + --light-hl-2: #AF00DB; + --dark-hl-2: #C586C0; + --light-hl-3: #0000FF; + --dark-hl-3: #569CD6; + --light-hl-4: #A31515; + --dark-hl-4: #CE9178; + --light-hl-5: #800000; + --dark-hl-5: #808080; + --light-hl-6: #800000; + --dark-hl-6: #569CD6; + --light-hl-7: #000000FF; + --dark-hl-7: #D4D4D4; + --light-hl-8: #E50000; + --dark-hl-8: #9CDCFE; + --light-hl-9: #0000FF; + --dark-hl-9: #CE9178; + --light-hl-10: #0070C1; + --dark-hl-10: #4FC1FF; + --light-hl-11: #267F99; + --dark-hl-11: #4EC9B0; + --light-hl-12: #795E26; + --dark-hl-12: #DCDCAA; + --light-hl-13: #008000; + --dark-hl-13: #6A9955; + --light-hl-14: #098658; + --dark-hl-14: #B5CEA8; + --light-code-background: #FFFFFF; + --dark-code-background: #1E1E1E; +} + +@media (prefers-color-scheme: light) { :root { + --hl-0: var(--light-hl-0); + --hl-1: var(--light-hl-1); + --hl-2: var(--light-hl-2); + --hl-3: var(--light-hl-3); + --hl-4: var(--light-hl-4); + --hl-5: var(--light-hl-5); + --hl-6: var(--light-hl-6); + --hl-7: var(--light-hl-7); + --hl-8: var(--light-hl-8); + --hl-9: var(--light-hl-9); + --hl-10: var(--light-hl-10); + --hl-11: var(--light-hl-11); + --hl-12: var(--light-hl-12); + --hl-13: var(--light-hl-13); + --hl-14: var(--light-hl-14); + --code-background: var(--light-code-background); +} } + +@media (prefers-color-scheme: dark) { :root { + --hl-0: var(--dark-hl-0); + --hl-1: var(--dark-hl-1); + --hl-2: var(--dark-hl-2); + --hl-3: var(--dark-hl-3); + --hl-4: var(--dark-hl-4); + --hl-5: var(--dark-hl-5); + --hl-6: var(--dark-hl-6); + --hl-7: var(--dark-hl-7); + --hl-8: var(--dark-hl-8); + --hl-9: var(--dark-hl-9); + --hl-10: var(--dark-hl-10); + --hl-11: var(--dark-hl-11); + --hl-12: var(--dark-hl-12); + --hl-13: var(--dark-hl-13); + --hl-14: var(--dark-hl-14); + --code-background: var(--dark-code-background); +} } + +:root[data-theme='light'] { + --hl-0: var(--light-hl-0); + --hl-1: var(--light-hl-1); + --hl-2: var(--light-hl-2); + --hl-3: var(--light-hl-3); + --hl-4: var(--light-hl-4); + --hl-5: var(--light-hl-5); + --hl-6: var(--light-hl-6); + --hl-7: var(--light-hl-7); + --hl-8: var(--light-hl-8); + --hl-9: var(--light-hl-9); + --hl-10: var(--light-hl-10); + --hl-11: var(--light-hl-11); + --hl-12: var(--light-hl-12); + --hl-13: var(--light-hl-13); + --hl-14: var(--light-hl-14); + --code-background: var(--light-code-background); +} + +:root[data-theme='dark'] { + --hl-0: var(--dark-hl-0); + --hl-1: var(--dark-hl-1); + --hl-2: var(--dark-hl-2); + --hl-3: var(--dark-hl-3); + --hl-4: var(--dark-hl-4); + --hl-5: var(--dark-hl-5); + --hl-6: var(--dark-hl-6); + --hl-7: var(--dark-hl-7); + --hl-8: var(--dark-hl-8); + --hl-9: var(--dark-hl-9); + --hl-10: var(--dark-hl-10); + --hl-11: var(--dark-hl-11); + --hl-12: var(--dark-hl-12); + --hl-13: var(--dark-hl-13); + --hl-14: var(--dark-hl-14); + --code-background: var(--dark-code-background); +} + +.hl-0 { color: var(--hl-0); } +.hl-1 { color: var(--hl-1); } +.hl-2 { color: var(--hl-2); } +.hl-3 { color: var(--hl-3); } +.hl-4 { color: var(--hl-4); } +.hl-5 { color: var(--hl-5); } +.hl-6 { color: var(--hl-6); } +.hl-7 { color: var(--hl-7); } +.hl-8 { color: var(--hl-8); } +.hl-9 { color: var(--hl-9); } +.hl-10 { color: var(--hl-10); } +.hl-11 { color: var(--hl-11); } +.hl-12 { color: var(--hl-12); } +.hl-13 { color: var(--hl-13); } +.hl-14 { color: var(--hl-14); } +pre, code { background: var(--code-background); } diff --git a/dist/docs/assets/main.js b/dist/docs/assets/main.js new file mode 100644 index 0000000..3cee05e --- /dev/null +++ b/dist/docs/assets/main.js @@ -0,0 +1,58 @@ +"use strict"; +"use strict";(()=>{var Se=Object.create;var re=Object.defineProperty;var we=Object.getOwnPropertyDescriptor;var Te=Object.getOwnPropertyNames;var ke=Object.getPrototypeOf,Qe=Object.prototype.hasOwnProperty;var Pe=(t,e)=>()=>(e||t((e={exports:{}}).exports,e),e.exports);var Ie=(t,e,r,n)=>{if(e&&typeof e=="object"||typeof e=="function")for(let i of Te(e))!Qe.call(t,i)&&i!==r&&re(t,i,{get:()=>e[i],enumerable:!(n=we(e,i))||n.enumerable});return t};var Ce=(t,e,r)=>(r=t!=null?Se(ke(t)):{},Ie(e||!t||!t.__esModule?re(r,"default",{value:t,enumerable:!0}):r,t));var ae=Pe((se,oe)=>{(function(){var t=function(e){var r=new t.Builder;return r.pipeline.add(t.trimmer,t.stopWordFilter,t.stemmer),r.searchPipeline.add(t.stemmer),e.call(r,r),r.build()};t.version="2.3.9";t.utils={},t.utils.warn=function(e){return function(r){e.console&&console.warn&&console.warn(r)}}(this),t.utils.asString=function(e){return e==null?"":e.toString()},t.utils.clone=function(e){if(e==null)return e;for(var r=Object.create(null),n=Object.keys(e),i=0;i0){var d=t.utils.clone(r)||{};d.position=[a,u],d.index=s.length,s.push(new t.Token(n.slice(a,o),d))}a=o+1}}return s},t.tokenizer.separator=/[\s\-]+/;t.Pipeline=function(){this._stack=[]},t.Pipeline.registeredFunctions=Object.create(null),t.Pipeline.registerFunction=function(e,r){r in this.registeredFunctions&&t.utils.warn("Overwriting existing registered function: "+r),e.label=r,t.Pipeline.registeredFunctions[e.label]=e},t.Pipeline.warnIfFunctionNotRegistered=function(e){var r=e.label&&e.label in this.registeredFunctions;r||t.utils.warn(`Function is not registered with pipeline. This may cause problems when serialising the index. +`,e)},t.Pipeline.load=function(e){var r=new t.Pipeline;return e.forEach(function(n){var i=t.Pipeline.registeredFunctions[n];if(i)r.add(i);else throw new Error("Cannot load unregistered function: "+n)}),r},t.Pipeline.prototype.add=function(){var e=Array.prototype.slice.call(arguments);e.forEach(function(r){t.Pipeline.warnIfFunctionNotRegistered(r),this._stack.push(r)},this)},t.Pipeline.prototype.after=function(e,r){t.Pipeline.warnIfFunctionNotRegistered(r);var n=this._stack.indexOf(e);if(n==-1)throw new Error("Cannot find existingFn");n=n+1,this._stack.splice(n,0,r)},t.Pipeline.prototype.before=function(e,r){t.Pipeline.warnIfFunctionNotRegistered(r);var n=this._stack.indexOf(e);if(n==-1)throw new Error("Cannot find existingFn");this._stack.splice(n,0,r)},t.Pipeline.prototype.remove=function(e){var r=this._stack.indexOf(e);r!=-1&&this._stack.splice(r,1)},t.Pipeline.prototype.run=function(e){for(var r=this._stack.length,n=0;n1&&(oe&&(n=s),o!=e);)i=n-r,s=r+Math.floor(i/2),o=this.elements[s*2];if(o==e||o>e)return s*2;if(ol?d+=2:a==l&&(r+=n[u+1]*i[d+1],u+=2,d+=2);return r},t.Vector.prototype.similarity=function(e){return this.dot(e)/this.magnitude()||0},t.Vector.prototype.toArray=function(){for(var e=new Array(this.elements.length/2),r=1,n=0;r0){var o=s.str.charAt(0),a;o in s.node.edges?a=s.node.edges[o]:(a=new t.TokenSet,s.node.edges[o]=a),s.str.length==1&&(a.final=!0),i.push({node:a,editsRemaining:s.editsRemaining,str:s.str.slice(1)})}if(s.editsRemaining!=0){if("*"in s.node.edges)var l=s.node.edges["*"];else{var l=new t.TokenSet;s.node.edges["*"]=l}if(s.str.length==0&&(l.final=!0),i.push({node:l,editsRemaining:s.editsRemaining-1,str:s.str}),s.str.length>1&&i.push({node:s.node,editsRemaining:s.editsRemaining-1,str:s.str.slice(1)}),s.str.length==1&&(s.node.final=!0),s.str.length>=1){if("*"in s.node.edges)var u=s.node.edges["*"];else{var u=new t.TokenSet;s.node.edges["*"]=u}s.str.length==1&&(u.final=!0),i.push({node:u,editsRemaining:s.editsRemaining-1,str:s.str.slice(1)})}if(s.str.length>1){var d=s.str.charAt(0),m=s.str.charAt(1),y;m in s.node.edges?y=s.node.edges[m]:(y=new t.TokenSet,s.node.edges[m]=y),s.str.length==1&&(y.final=!0),i.push({node:y,editsRemaining:s.editsRemaining-1,str:d+s.str.slice(2)})}}}return n},t.TokenSet.fromString=function(e){for(var r=new t.TokenSet,n=r,i=0,s=e.length;i=e;r--){var n=this.uncheckedNodes[r],i=n.child.toString();i in this.minimizedNodes?n.parent.edges[n.char]=this.minimizedNodes[i]:(n.child._str=i,this.minimizedNodes[i]=n.child),this.uncheckedNodes.pop()}};t.Index=function(e){this.invertedIndex=e.invertedIndex,this.fieldVectors=e.fieldVectors,this.tokenSet=e.tokenSet,this.fields=e.fields,this.pipeline=e.pipeline},t.Index.prototype.search=function(e){return this.query(function(r){var n=new t.QueryParser(e,r);n.parse()})},t.Index.prototype.query=function(e){for(var r=new t.Query(this.fields),n=Object.create(null),i=Object.create(null),s=Object.create(null),o=Object.create(null),a=Object.create(null),l=0;l1?this._b=1:this._b=e},t.Builder.prototype.k1=function(e){this._k1=e},t.Builder.prototype.add=function(e,r){var n=e[this._ref],i=Object.keys(this._fields);this._documents[n]=r||{},this.documentCount+=1;for(var s=0;s=this.length)return t.QueryLexer.EOS;var e=this.str.charAt(this.pos);return this.pos+=1,e},t.QueryLexer.prototype.width=function(){return this.pos-this.start},t.QueryLexer.prototype.ignore=function(){this.start==this.pos&&(this.pos+=1),this.start=this.pos},t.QueryLexer.prototype.backup=function(){this.pos-=1},t.QueryLexer.prototype.acceptDigitRun=function(){var e,r;do e=this.next(),r=e.charCodeAt(0);while(r>47&&r<58);e!=t.QueryLexer.EOS&&this.backup()},t.QueryLexer.prototype.more=function(){return this.pos1&&(e.backup(),e.emit(t.QueryLexer.TERM)),e.ignore(),e.more())return t.QueryLexer.lexText},t.QueryLexer.lexEditDistance=function(e){return e.ignore(),e.acceptDigitRun(),e.emit(t.QueryLexer.EDIT_DISTANCE),t.QueryLexer.lexText},t.QueryLexer.lexBoost=function(e){return e.ignore(),e.acceptDigitRun(),e.emit(t.QueryLexer.BOOST),t.QueryLexer.lexText},t.QueryLexer.lexEOS=function(e){e.width()>0&&e.emit(t.QueryLexer.TERM)},t.QueryLexer.termSeparator=t.tokenizer.separator,t.QueryLexer.lexText=function(e){for(;;){var r=e.next();if(r==t.QueryLexer.EOS)return t.QueryLexer.lexEOS;if(r.charCodeAt(0)==92){e.escapeCharacter();continue}if(r==":")return t.QueryLexer.lexField;if(r=="~")return e.backup(),e.width()>0&&e.emit(t.QueryLexer.TERM),t.QueryLexer.lexEditDistance;if(r=="^")return e.backup(),e.width()>0&&e.emit(t.QueryLexer.TERM),t.QueryLexer.lexBoost;if(r=="+"&&e.width()===1||r=="-"&&e.width()===1)return e.emit(t.QueryLexer.PRESENCE),t.QueryLexer.lexText;if(r.match(t.QueryLexer.termSeparator))return t.QueryLexer.lexTerm}},t.QueryParser=function(e,r){this.lexer=new t.QueryLexer(e),this.query=r,this.currentClause={},this.lexemeIdx=0},t.QueryParser.prototype.parse=function(){this.lexer.run(),this.lexemes=this.lexer.lexemes;for(var e=t.QueryParser.parseClause;e;)e=e(this);return this.query},t.QueryParser.prototype.peekLexeme=function(){return this.lexemes[this.lexemeIdx]},t.QueryParser.prototype.consumeLexeme=function(){var e=this.peekLexeme();return this.lexemeIdx+=1,e},t.QueryParser.prototype.nextClause=function(){var e=this.currentClause;this.query.clause(e),this.currentClause={}},t.QueryParser.parseClause=function(e){var r=e.peekLexeme();if(r!=null)switch(r.type){case t.QueryLexer.PRESENCE:return t.QueryParser.parsePresence;case t.QueryLexer.FIELD:return t.QueryParser.parseField;case t.QueryLexer.TERM:return t.QueryParser.parseTerm;default:var n="expected either a field or a term, found "+r.type;throw r.str.length>=1&&(n+=" with value '"+r.str+"'"),new t.QueryParseError(n,r.start,r.end)}},t.QueryParser.parsePresence=function(e){var r=e.consumeLexeme();if(r!=null){switch(r.str){case"-":e.currentClause.presence=t.Query.presence.PROHIBITED;break;case"+":e.currentClause.presence=t.Query.presence.REQUIRED;break;default:var n="unrecognised presence operator'"+r.str+"'";throw new t.QueryParseError(n,r.start,r.end)}var i=e.peekLexeme();if(i==null){var n="expecting term or field, found nothing";throw new t.QueryParseError(n,r.start,r.end)}switch(i.type){case t.QueryLexer.FIELD:return t.QueryParser.parseField;case t.QueryLexer.TERM:return t.QueryParser.parseTerm;default:var n="expecting term or field, found '"+i.type+"'";throw new t.QueryParseError(n,i.start,i.end)}}},t.QueryParser.parseField=function(e){var r=e.consumeLexeme();if(r!=null){if(e.query.allFields.indexOf(r.str)==-1){var n=e.query.allFields.map(function(o){return"'"+o+"'"}).join(", "),i="unrecognised field '"+r.str+"', possible fields: "+n;throw new t.QueryParseError(i,r.start,r.end)}e.currentClause.fields=[r.str];var s=e.peekLexeme();if(s==null){var i="expecting term, found nothing";throw new t.QueryParseError(i,r.start,r.end)}switch(s.type){case t.QueryLexer.TERM:return t.QueryParser.parseTerm;default:var i="expecting term, found '"+s.type+"'";throw new t.QueryParseError(i,s.start,s.end)}}},t.QueryParser.parseTerm=function(e){var r=e.consumeLexeme();if(r!=null){e.currentClause.term=r.str.toLowerCase(),r.str.indexOf("*")!=-1&&(e.currentClause.usePipeline=!1);var n=e.peekLexeme();if(n==null){e.nextClause();return}switch(n.type){case t.QueryLexer.TERM:return e.nextClause(),t.QueryParser.parseTerm;case t.QueryLexer.FIELD:return e.nextClause(),t.QueryParser.parseField;case t.QueryLexer.EDIT_DISTANCE:return t.QueryParser.parseEditDistance;case t.QueryLexer.BOOST:return t.QueryParser.parseBoost;case t.QueryLexer.PRESENCE:return e.nextClause(),t.QueryParser.parsePresence;default:var i="Unexpected lexeme type '"+n.type+"'";throw new t.QueryParseError(i,n.start,n.end)}}},t.QueryParser.parseEditDistance=function(e){var r=e.consumeLexeme();if(r!=null){var n=parseInt(r.str,10);if(isNaN(n)){var i="edit distance must be numeric";throw new t.QueryParseError(i,r.start,r.end)}e.currentClause.editDistance=n;var s=e.peekLexeme();if(s==null){e.nextClause();return}switch(s.type){case t.QueryLexer.TERM:return e.nextClause(),t.QueryParser.parseTerm;case t.QueryLexer.FIELD:return e.nextClause(),t.QueryParser.parseField;case t.QueryLexer.EDIT_DISTANCE:return t.QueryParser.parseEditDistance;case t.QueryLexer.BOOST:return t.QueryParser.parseBoost;case t.QueryLexer.PRESENCE:return e.nextClause(),t.QueryParser.parsePresence;default:var i="Unexpected lexeme type '"+s.type+"'";throw new t.QueryParseError(i,s.start,s.end)}}},t.QueryParser.parseBoost=function(e){var r=e.consumeLexeme();if(r!=null){var n=parseInt(r.str,10);if(isNaN(n)){var i="boost must be numeric";throw new t.QueryParseError(i,r.start,r.end)}e.currentClause.boost=n;var s=e.peekLexeme();if(s==null){e.nextClause();return}switch(s.type){case t.QueryLexer.TERM:return e.nextClause(),t.QueryParser.parseTerm;case t.QueryLexer.FIELD:return e.nextClause(),t.QueryParser.parseField;case t.QueryLexer.EDIT_DISTANCE:return t.QueryParser.parseEditDistance;case t.QueryLexer.BOOST:return t.QueryParser.parseBoost;case t.QueryLexer.PRESENCE:return e.nextClause(),t.QueryParser.parsePresence;default:var i="Unexpected lexeme type '"+s.type+"'";throw new t.QueryParseError(i,s.start,s.end)}}},function(e,r){typeof define=="function"&&define.amd?define(r):typeof se=="object"?oe.exports=r():e.lunr=r()}(this,function(){return t})})()});var ne=[];function G(t,e){ne.push({selector:e,constructor:t})}var U=class{constructor(){this.alwaysVisibleMember=null;this.createComponents(document.body),this.ensureActivePageVisible(),this.ensureFocusedElementVisible(),this.listenForCodeCopies(),window.addEventListener("hashchange",()=>this.ensureFocusedElementVisible())}createComponents(e){ne.forEach(r=>{e.querySelectorAll(r.selector).forEach(n=>{n.dataset.hasInstance||(new r.constructor({el:n,app:this}),n.dataset.hasInstance=String(!0))})})}filterChanged(){this.ensureFocusedElementVisible()}ensureActivePageVisible(){let e=document.querySelector(".tsd-navigation .current"),r=e?.parentElement;for(;r&&!r.classList.contains(".tsd-navigation");)r instanceof HTMLDetailsElement&&(r.open=!0),r=r.parentElement;if(e){let n=e.getBoundingClientRect().top-document.documentElement.clientHeight/4;document.querySelector(".site-menu").scrollTop=n}}ensureFocusedElementVisible(){if(this.alwaysVisibleMember&&(this.alwaysVisibleMember.classList.remove("always-visible"),this.alwaysVisibleMember.firstElementChild.remove(),this.alwaysVisibleMember=null),!location.hash)return;let e=document.getElementById(location.hash.substring(1));if(!e)return;let r=e.parentElement;for(;r&&r.tagName!=="SECTION";)r=r.parentElement;if(r&&r.offsetParent==null){this.alwaysVisibleMember=r,r.classList.add("always-visible");let n=document.createElement("p");n.classList.add("warning"),n.textContent="This member is normally hidden due to your filter settings.",r.prepend(n)}}listenForCodeCopies(){document.querySelectorAll("pre > button").forEach(e=>{let r;e.addEventListener("click",()=>{e.previousElementSibling instanceof HTMLElement&&navigator.clipboard.writeText(e.previousElementSibling.innerText.trim()),e.textContent="Copied!",e.classList.add("visible"),clearTimeout(r),r=setTimeout(()=>{e.classList.remove("visible"),r=setTimeout(()=>{e.textContent="Copy"},100)},1e3)})})}};var ie=(t,e=100)=>{let r;return()=>{clearTimeout(r),r=setTimeout(()=>t(),e)}};var ce=Ce(ae());function de(){let t=document.getElementById("tsd-search");if(!t)return;let e=document.getElementById("tsd-search-script");t.classList.add("loading"),e&&(e.addEventListener("error",()=>{t.classList.remove("loading"),t.classList.add("failure")}),e.addEventListener("load",()=>{t.classList.remove("loading"),t.classList.add("ready")}),window.searchData&&t.classList.remove("loading"));let r=document.querySelector("#tsd-search input"),n=document.querySelector("#tsd-search .results");if(!r||!n)throw new Error("The input field or the result list wrapper was not found");let i=!1;n.addEventListener("mousedown",()=>i=!0),n.addEventListener("mouseup",()=>{i=!1,t.classList.remove("has-focus")}),r.addEventListener("focus",()=>t.classList.add("has-focus")),r.addEventListener("blur",()=>{i||(i=!1,t.classList.remove("has-focus"))});let s={base:t.dataset.base+"/"};Oe(t,n,r,s)}function Oe(t,e,r,n){r.addEventListener("input",ie(()=>{Re(t,e,r,n)},200));let i=!1;r.addEventListener("keydown",s=>{i=!0,s.key=="Enter"?Fe(e,r):s.key=="Escape"?r.blur():s.key=="ArrowUp"?ue(e,-1):s.key==="ArrowDown"?ue(e,1):i=!1}),r.addEventListener("keypress",s=>{i&&s.preventDefault()}),document.body.addEventListener("keydown",s=>{s.altKey||s.ctrlKey||s.metaKey||!r.matches(":focus")&&s.key==="/"&&(r.focus(),s.preventDefault())})}function _e(t,e){t.index||window.searchData&&(e.classList.remove("loading"),e.classList.add("ready"),t.data=window.searchData,t.index=ce.Index.load(window.searchData.index))}function Re(t,e,r,n){if(_e(n,t),!n.index||!n.data)return;e.textContent="";let i=r.value.trim(),s=i?n.index.search(`*${i}*`):[];for(let o=0;oa.score-o.score);for(let o=0,a=Math.min(10,s.length);o${le(l.parent,i)}.${u}`);let d=document.createElement("li");d.classList.value=l.classes??"";let m=document.createElement("a");m.href=n.base+l.url,m.innerHTML=u,d.append(m),e.appendChild(d)}}function ue(t,e){let r=t.querySelector(".current");if(!r)r=t.querySelector(e==1?"li:first-child":"li:last-child"),r&&r.classList.add("current");else{let n=r;if(e===1)do n=n.nextElementSibling??void 0;while(n instanceof HTMLElement&&n.offsetParent==null);else do n=n.previousElementSibling??void 0;while(n instanceof HTMLElement&&n.offsetParent==null);n&&(r.classList.remove("current"),n.classList.add("current"))}}function Fe(t,e){let r=t.querySelector(".current");if(r||(r=t.querySelector("li:first-child")),r){let n=r.querySelector("a");n&&(window.location.href=n.href),e.blur()}}function le(t,e){if(e==="")return t;let r=t.toLocaleLowerCase(),n=e.toLocaleLowerCase(),i=[],s=0,o=r.indexOf(n);for(;o!=-1;)i.push(K(t.substring(s,o)),`${K(t.substring(o,o+n.length))}`),s=o+n.length,o=r.indexOf(n,s);return i.push(K(t.substring(s))),i.join("")}var Me={"&":"&","<":"<",">":">","'":"'",'"':"""};function K(t){return t.replace(/[&<>"'"]/g,e=>Me[e])}var P=class{constructor(e){this.el=e.el,this.app=e.app}};var M="mousedown",fe="mousemove",N="mouseup",J={x:0,y:0},he=!1,ee=!1,De=!1,D=!1,pe=/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent);document.documentElement.classList.add(pe?"is-mobile":"not-mobile");pe&&"ontouchstart"in document.documentElement&&(De=!0,M="touchstart",fe="touchmove",N="touchend");document.addEventListener(M,t=>{ee=!0,D=!1;let e=M=="touchstart"?t.targetTouches[0]:t;J.y=e.pageY||0,J.x=e.pageX||0});document.addEventListener(fe,t=>{if(ee&&!D){let e=M=="touchstart"?t.targetTouches[0]:t,r=J.x-(e.pageX||0),n=J.y-(e.pageY||0);D=Math.sqrt(r*r+n*n)>10}});document.addEventListener(N,()=>{ee=!1});document.addEventListener("click",t=>{he&&(t.preventDefault(),t.stopImmediatePropagation(),he=!1)});var X=class extends P{constructor(r){super(r);this.className=this.el.dataset.toggle||"",this.el.addEventListener(N,n=>this.onPointerUp(n)),this.el.addEventListener("click",n=>n.preventDefault()),document.addEventListener(M,n=>this.onDocumentPointerDown(n)),document.addEventListener(N,n=>this.onDocumentPointerUp(n))}setActive(r){if(this.active==r)return;this.active=r,document.documentElement.classList.toggle("has-"+this.className,r),this.el.classList.toggle("active",r);let n=(this.active?"to-has-":"from-has-")+this.className;document.documentElement.classList.add(n),setTimeout(()=>document.documentElement.classList.remove(n),500)}onPointerUp(r){D||(this.setActive(!0),r.preventDefault())}onDocumentPointerDown(r){if(this.active){if(r.target.closest(".col-sidebar, .tsd-filter-group"))return;this.setActive(!1)}}onDocumentPointerUp(r){if(!D&&this.active&&r.target.closest(".col-sidebar")){let n=r.target.closest("a");if(n){let i=window.location.href;i.indexOf("#")!=-1&&(i=i.substring(0,i.indexOf("#"))),n.href.substring(0,i.length)==i&&setTimeout(()=>this.setActive(!1),250)}}}};var te;try{te=localStorage}catch{te={getItem(){return null},setItem(){}}}var Q=te;var me=document.head.appendChild(document.createElement("style"));me.dataset.for="filters";var Y=class extends P{constructor(r){super(r);this.key=`filter-${this.el.name}`,this.value=this.el.checked,this.el.addEventListener("change",()=>{this.setLocalStorage(this.el.checked)}),this.setLocalStorage(this.fromLocalStorage()),me.innerHTML+=`html:not(.${this.key}) .tsd-is-${this.el.name} { display: none; } +`}fromLocalStorage(){let r=Q.getItem(this.key);return r?r==="true":this.el.checked}setLocalStorage(r){Q.setItem(this.key,r.toString()),this.value=r,this.handleValueChange()}handleValueChange(){this.el.checked=this.value,document.documentElement.classList.toggle(this.key,this.value),this.app.filterChanged(),document.querySelectorAll(".tsd-index-section").forEach(r=>{r.style.display="block";let n=Array.from(r.querySelectorAll(".tsd-index-link")).every(i=>i.offsetParent==null);r.style.display=n?"none":"block"})}};var Z=class extends P{constructor(r){super(r);this.summary=this.el.querySelector(".tsd-accordion-summary"),this.icon=this.summary.querySelector("svg"),this.key=`tsd-accordion-${this.summary.dataset.key??this.summary.textContent.trim().replace(/\s+/g,"-").toLowerCase()}`;let n=Q.getItem(this.key);this.el.open=n?n==="true":this.el.open,this.el.addEventListener("toggle",()=>this.update()),this.update()}update(){this.icon.style.transform=`rotate(${this.el.open?0:-90}deg)`,Q.setItem(this.key,this.el.open.toString())}};function ve(t){let e=Q.getItem("tsd-theme")||"os";t.value=e,ye(e),t.addEventListener("change",()=>{Q.setItem("tsd-theme",t.value),ye(t.value)})}function ye(t){document.documentElement.dataset.theme=t}de();G(X,"a[data-toggle]");G(Z,".tsd-index-accordion");G(Y,".tsd-filter-item input[type=checkbox]");var ge=document.getElementById("tsd-theme");ge&&ve(ge);var Ae=new U;Object.defineProperty(window,"app",{value:Ae});document.querySelectorAll("summary a").forEach(t=>{t.addEventListener("click",()=>{location.assign(t.href)})});})(); +/*! Bundled license information: + +lunr/lunr.js: + (** + * lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 2.3.9 + * Copyright (C) 2020 Oliver Nightingale + * @license MIT + *) + (*! + * lunr.utils + * Copyright (C) 2020 Oliver Nightingale + *) + (*! + * lunr.Set + * Copyright (C) 2020 Oliver Nightingale + *) + (*! + * lunr.tokenizer + * Copyright (C) 2020 Oliver Nightingale + *) + (*! + * lunr.Pipeline + * Copyright (C) 2020 Oliver Nightingale + *) + (*! + * lunr.Vector + * Copyright (C) 2020 Oliver Nightingale + *) + (*! + * lunr.stemmer + * Copyright (C) 2020 Oliver Nightingale + * Includes code from - http://tartarus.org/~martin/PorterStemmer/js.txt + *) + (*! + * lunr.stopWordFilter + * Copyright (C) 2020 Oliver Nightingale + *) + (*! + * lunr.trimmer + * Copyright (C) 2020 Oliver Nightingale + *) + (*! + * lunr.TokenSet + * Copyright (C) 2020 Oliver Nightingale + *) + (*! + * lunr.Index + * Copyright (C) 2020 Oliver Nightingale + *) + (*! + * lunr.Builder + * Copyright (C) 2020 Oliver Nightingale + *) +*/ diff --git a/dist/docs/assets/search.js b/dist/docs/assets/search.js new file mode 100644 index 0000000..d1020e0 --- /dev/null +++ b/dist/docs/assets/search.js @@ -0,0 +1 @@ +window.searchData = JSON.parse("{\"rows\":[{\"kind\":128,\"name\":\"GaussianBlurBackgroundProcessor\",\"url\":\"classes/GaussianBlurBackgroundProcessor.html\",\"classes\":\"\"},{\"kind\":512,\"name\":\"constructor\",\"url\":\"classes/GaussianBlurBackgroundProcessor.html#constructor\",\"classes\":\"\",\"parent\":\"GaussianBlurBackgroundProcessor\"},{\"kind\":262144,\"name\":\"blurFilterRadius\",\"url\":\"classes/GaussianBlurBackgroundProcessor.html#blurFilterRadius\",\"classes\":\"\",\"parent\":\"GaussianBlurBackgroundProcessor\"},{\"kind\":262144,\"name\":\"deferInputFrameDownscale\",\"url\":\"classes/GaussianBlurBackgroundProcessor.html#deferInputFrameDownscale\",\"classes\":\"tsd-is-inherited\",\"parent\":\"GaussianBlurBackgroundProcessor\"},{\"kind\":262144,\"name\":\"maskBlurRadius\",\"url\":\"classes/GaussianBlurBackgroundProcessor.html#maskBlurRadius\",\"classes\":\"tsd-is-inherited\",\"parent\":\"GaussianBlurBackgroundProcessor\"},{\"kind\":2048,\"name\":\"loadModel\",\"url\":\"classes/GaussianBlurBackgroundProcessor.html#loadModel\",\"classes\":\"tsd-is-inherited\",\"parent\":\"GaussianBlurBackgroundProcessor\"},{\"kind\":2048,\"name\":\"processFrame\",\"url\":\"classes/GaussianBlurBackgroundProcessor.html#processFrame\",\"classes\":\"tsd-is-inherited\",\"parent\":\"GaussianBlurBackgroundProcessor\"},{\"kind\":256,\"name\":\"GaussianBlurBackgroundProcessorOptions\",\"url\":\"interfaces/GaussianBlurBackgroundProcessorOptions.html\",\"classes\":\"\"},{\"kind\":1024,\"name\":\"blurFilterRadius\",\"url\":\"interfaces/GaussianBlurBackgroundProcessorOptions.html#blurFilterRadius\",\"classes\":\"\",\"parent\":\"GaussianBlurBackgroundProcessorOptions\"},{\"kind\":1024,\"name\":\"assetsPath\",\"url\":\"interfaces/GaussianBlurBackgroundProcessorOptions.html#assetsPath\",\"classes\":\"tsd-is-inherited\",\"parent\":\"GaussianBlurBackgroundProcessorOptions\"},{\"kind\":1024,\"name\":\"deferInputFrameDownscale\",\"url\":\"interfaces/GaussianBlurBackgroundProcessorOptions.html#deferInputFrameDownscale\",\"classes\":\"tsd-is-inherited\",\"parent\":\"GaussianBlurBackgroundProcessorOptions\"},{\"kind\":1024,\"name\":\"maskBlurRadius\",\"url\":\"interfaces/GaussianBlurBackgroundProcessorOptions.html#maskBlurRadius\",\"classes\":\"tsd-is-inherited\",\"parent\":\"GaussianBlurBackgroundProcessorOptions\"},{\"kind\":1024,\"name\":\"useWebWorker\",\"url\":\"interfaces/GaussianBlurBackgroundProcessorOptions.html#useWebWorker\",\"classes\":\"tsd-is-inherited\",\"parent\":\"GaussianBlurBackgroundProcessorOptions\"},{\"kind\":8,\"name\":\"ImageFit\",\"url\":\"enums/ImageFit.html\",\"classes\":\"\"},{\"kind\":16,\"name\":\"Contain\",\"url\":\"enums/ImageFit.html#Contain\",\"classes\":\"\",\"parent\":\"ImageFit\"},{\"kind\":16,\"name\":\"Cover\",\"url\":\"enums/ImageFit.html#Cover\",\"classes\":\"\",\"parent\":\"ImageFit\"},{\"kind\":16,\"name\":\"Fill\",\"url\":\"enums/ImageFit.html#Fill\",\"classes\":\"\",\"parent\":\"ImageFit\"},{\"kind\":16,\"name\":\"None\",\"url\":\"enums/ImageFit.html#None\",\"classes\":\"\",\"parent\":\"ImageFit\"},{\"kind\":32,\"name\":\"isSupported\",\"url\":\"variables/isSupported.html\",\"classes\":\"\"},{\"kind\":32,\"name\":\"version\",\"url\":\"variables/version.html\",\"classes\":\"\"},{\"kind\":128,\"name\":\"VirtualBackgroundProcessor\",\"url\":\"classes/VirtualBackgroundProcessor.html\",\"classes\":\"\"},{\"kind\":512,\"name\":\"constructor\",\"url\":\"classes/VirtualBackgroundProcessor.html#constructor\",\"classes\":\"\",\"parent\":\"VirtualBackgroundProcessor\"},{\"kind\":262144,\"name\":\"backgroundImage\",\"url\":\"classes/VirtualBackgroundProcessor.html#backgroundImage\",\"classes\":\"\",\"parent\":\"VirtualBackgroundProcessor\"},{\"kind\":262144,\"name\":\"fitType\",\"url\":\"classes/VirtualBackgroundProcessor.html#fitType\",\"classes\":\"\",\"parent\":\"VirtualBackgroundProcessor\"},{\"kind\":262144,\"name\":\"deferInputFrameDownscale\",\"url\":\"classes/VirtualBackgroundProcessor.html#deferInputFrameDownscale\",\"classes\":\"tsd-is-inherited\",\"parent\":\"VirtualBackgroundProcessor\"},{\"kind\":262144,\"name\":\"maskBlurRadius\",\"url\":\"classes/VirtualBackgroundProcessor.html#maskBlurRadius\",\"classes\":\"tsd-is-inherited\",\"parent\":\"VirtualBackgroundProcessor\"},{\"kind\":2048,\"name\":\"loadModel\",\"url\":\"classes/VirtualBackgroundProcessor.html#loadModel\",\"classes\":\"tsd-is-inherited\",\"parent\":\"VirtualBackgroundProcessor\"},{\"kind\":2048,\"name\":\"processFrame\",\"url\":\"classes/VirtualBackgroundProcessor.html#processFrame\",\"classes\":\"tsd-is-inherited\",\"parent\":\"VirtualBackgroundProcessor\"},{\"kind\":256,\"name\":\"VirtualBackgroundProcessorOptions\",\"url\":\"interfaces/VirtualBackgroundProcessorOptions.html\",\"classes\":\"\"},{\"kind\":1024,\"name\":\"backgroundImage\",\"url\":\"interfaces/VirtualBackgroundProcessorOptions.html#backgroundImage\",\"classes\":\"\",\"parent\":\"VirtualBackgroundProcessorOptions\"},{\"kind\":1024,\"name\":\"fitType\",\"url\":\"interfaces/VirtualBackgroundProcessorOptions.html#fitType\",\"classes\":\"\",\"parent\":\"VirtualBackgroundProcessorOptions\"},{\"kind\":1024,\"name\":\"assetsPath\",\"url\":\"interfaces/VirtualBackgroundProcessorOptions.html#assetsPath\",\"classes\":\"tsd-is-inherited\",\"parent\":\"VirtualBackgroundProcessorOptions\"},{\"kind\":1024,\"name\":\"deferInputFrameDownscale\",\"url\":\"interfaces/VirtualBackgroundProcessorOptions.html#deferInputFrameDownscale\",\"classes\":\"tsd-is-inherited\",\"parent\":\"VirtualBackgroundProcessorOptions\"},{\"kind\":1024,\"name\":\"maskBlurRadius\",\"url\":\"interfaces/VirtualBackgroundProcessorOptions.html#maskBlurRadius\",\"classes\":\"tsd-is-inherited\",\"parent\":\"VirtualBackgroundProcessorOptions\"},{\"kind\":1024,\"name\":\"useWebWorker\",\"url\":\"interfaces/VirtualBackgroundProcessorOptions.html#useWebWorker\",\"classes\":\"tsd-is-inherited\",\"parent\":\"VirtualBackgroundProcessorOptions\"}],\"index\":{\"version\":\"2.3.9\",\"fields\":[\"name\",\"comment\"],\"fieldVectors\":[[\"name/0\",[0,31.781]],[\"comment/0\",[]],[\"name/1\",[1,26.672]],[\"comment/1\",[]],[\"name/2\",[2,26.672]],[\"comment/2\",[]],[\"name/3\",[3,20.794]],[\"comment/3\",[]],[\"name/4\",[4,20.794]],[\"comment/4\",[]],[\"name/5\",[5,26.672]],[\"comment/5\",[]],[\"name/6\",[6,26.672]],[\"comment/6\",[]],[\"name/7\",[7,31.781]],[\"comment/7\",[]],[\"name/8\",[2,26.672]],[\"comment/8\",[]],[\"name/9\",[8,26.672]],[\"comment/9\",[]],[\"name/10\",[3,20.794]],[\"comment/10\",[]],[\"name/11\",[4,20.794]],[\"comment/11\",[]],[\"name/12\",[9,26.672]],[\"comment/12\",[]],[\"name/13\",[10,31.781]],[\"comment/13\",[]],[\"name/14\",[11,31.781]],[\"comment/14\",[]],[\"name/15\",[12,31.781]],[\"comment/15\",[]],[\"name/16\",[13,31.781]],[\"comment/16\",[]],[\"name/17\",[14,31.781]],[\"comment/17\",[]],[\"name/18\",[15,31.781]],[\"comment/18\",[]],[\"name/19\",[16,31.781]],[\"comment/19\",[]],[\"name/20\",[17,31.781]],[\"comment/20\",[]],[\"name/21\",[1,26.672]],[\"comment/21\",[]],[\"name/22\",[18,26.672]],[\"comment/22\",[]],[\"name/23\",[19,26.672]],[\"comment/23\",[]],[\"name/24\",[3,20.794]],[\"comment/24\",[]],[\"name/25\",[4,20.794]],[\"comment/25\",[]],[\"name/26\",[5,26.672]],[\"comment/26\",[]],[\"name/27\",[6,26.672]],[\"comment/27\",[]],[\"name/28\",[20,31.781]],[\"comment/28\",[]],[\"name/29\",[18,26.672]],[\"comment/29\",[]],[\"name/30\",[19,26.672]],[\"comment/30\",[]],[\"name/31\",[8,26.672]],[\"comment/31\",[]],[\"name/32\",[3,20.794]],[\"comment/32\",[]],[\"name/33\",[4,20.794]],[\"comment/33\",[]],[\"name/34\",[9,26.672]],[\"comment/34\",[]]],\"invertedIndex\":[[\"assetspath\",{\"_index\":8,\"name\":{\"9\":{},\"31\":{}},\"comment\":{}}],[\"backgroundimage\",{\"_index\":18,\"name\":{\"22\":{},\"29\":{}},\"comment\":{}}],[\"blurfilterradius\",{\"_index\":2,\"name\":{\"2\":{},\"8\":{}},\"comment\":{}}],[\"constructor\",{\"_index\":1,\"name\":{\"1\":{},\"21\":{}},\"comment\":{}}],[\"contain\",{\"_index\":11,\"name\":{\"14\":{}},\"comment\":{}}],[\"cover\",{\"_index\":12,\"name\":{\"15\":{}},\"comment\":{}}],[\"deferinputframedownscale\",{\"_index\":3,\"name\":{\"3\":{},\"10\":{},\"24\":{},\"32\":{}},\"comment\":{}}],[\"fill\",{\"_index\":13,\"name\":{\"16\":{}},\"comment\":{}}],[\"fittype\",{\"_index\":19,\"name\":{\"23\":{},\"30\":{}},\"comment\":{}}],[\"gaussianblurbackgroundprocessor\",{\"_index\":0,\"name\":{\"0\":{}},\"comment\":{}}],[\"gaussianblurbackgroundprocessoroptions\",{\"_index\":7,\"name\":{\"7\":{}},\"comment\":{}}],[\"imagefit\",{\"_index\":10,\"name\":{\"13\":{}},\"comment\":{}}],[\"issupported\",{\"_index\":15,\"name\":{\"18\":{}},\"comment\":{}}],[\"loadmodel\",{\"_index\":5,\"name\":{\"5\":{},\"26\":{}},\"comment\":{}}],[\"maskblurradius\",{\"_index\":4,\"name\":{\"4\":{},\"11\":{},\"25\":{},\"33\":{}},\"comment\":{}}],[\"none\",{\"_index\":14,\"name\":{\"17\":{}},\"comment\":{}}],[\"processframe\",{\"_index\":6,\"name\":{\"6\":{},\"27\":{}},\"comment\":{}}],[\"usewebworker\",{\"_index\":9,\"name\":{\"12\":{},\"34\":{}},\"comment\":{}}],[\"version\",{\"_index\":16,\"name\":{\"19\":{}},\"comment\":{}}],[\"virtualbackgroundprocessor\",{\"_index\":17,\"name\":{\"20\":{}},\"comment\":{}}],[\"virtualbackgroundprocessoroptions\",{\"_index\":20,\"name\":{\"28\":{}},\"comment\":{}}]],\"pipeline\":[]}}"); \ No newline at end of file diff --git a/dist/docs/assets/style.css b/dist/docs/assets/style.css new file mode 100644 index 0000000..258146f --- /dev/null +++ b/dist/docs/assets/style.css @@ -0,0 +1,1379 @@ +:root { + /* Light */ + --light-color-background: #f2f4f8; + --light-color-background-secondary: #eff0f1; + --light-color-warning-text: #222; + --light-color-background-warning: #e6e600; + --light-color-icon-background: var(--light-color-background); + --light-color-accent: #c5c7c9; + --light-color-active-menu-item: var(--light-color-accent); + --light-color-text: #222; + --light-color-text-aside: #6e6e6e; + --light-color-link: #1f70c2; + + --light-color-ts-project: #b111c9; + --light-color-ts-module: var(--light-color-ts-project); + --light-color-ts-namespace: var(--light-color-ts-project); + --light-color-ts-enum: #7e6f15; + --light-color-ts-enum-member: var(--light-color-ts-enum); + --light-color-ts-variable: #4760ec; + --light-color-ts-function: #572be7; + --light-color-ts-class: #1f70c2; + --light-color-ts-interface: #108024; + --light-color-ts-constructor: var(--light-color-ts-class); + --light-color-ts-property: var(--light-color-ts-variable); + --light-color-ts-method: var(--light-color-ts-function); + --light-color-ts-call-signature: var(--light-color-ts-method); + --light-color-ts-index-signature: var(--light-color-ts-property); + --light-color-ts-constructor-signature: var(--light-color-ts-constructor); + --light-color-ts-parameter: var(--light-color-ts-variable); + /* type literal not included as links will never be generated to it */ + --light-color-ts-type-parameter: var(--light-color-ts-type-alias); + --light-color-ts-accessor: var(--light-color-ts-property); + --light-color-ts-get-signature: var(--light-color-ts-accessor); + --light-color-ts-set-signature: var(--light-color-ts-accessor); + --light-color-ts-type-alias: #d51270; + /* reference not included as links will be colored with the kind that it points to */ + + --light-external-icon: url("data:image/svg+xml;utf8,"); + --light-color-scheme: light; + + /* Dark */ + --dark-color-background: #2b2e33; + --dark-color-background-secondary: #1e2024; + --dark-color-background-warning: #bebe00; + --dark-color-warning-text: #222; + --dark-color-icon-background: var(--dark-color-background-secondary); + --dark-color-accent: #9096a2; + --dark-color-active-menu-item: #5d5d6a; + --dark-color-text: #f5f5f5; + --dark-color-text-aside: #dddddd; + --dark-color-link: #00aff4; + + --dark-color-ts-project: #e358ff; + --dark-color-ts-module: var(--dark-color-ts-project); + --dark-color-ts-namespace: var(--dark-color-ts-project); + --dark-color-ts-enum: #f4d93e; + --dark-color-ts-enum-member: var(--dark-color-ts-enum); + --dark-color-ts-variable: #798dff; + --dark-color-ts-function: #a280ff; + --dark-color-ts-class: #8ac4ff; + --dark-color-ts-interface: #6cff87; + --dark-color-ts-constructor: var(--dark-color-ts-class); + --dark-color-ts-property: var(--dark-color-ts-variable); + --dark-color-ts-method: var(--dark-color-ts-function); + --dark-color-ts-call-signature: var(--dark-color-ts-method); + --dark-color-ts-index-signature: var(--dark-color-ts-property); + --dark-color-ts-constructor-signature: var(--dark-color-ts-constructor); + --dark-color-ts-parameter: var(--dark-color-ts-variable); + /* type literal not included as links will never be generated to it */ + --dark-color-ts-type-parameter: var(--dark-color-ts-type-alias); + --dark-color-ts-accessor: var(--dark-color-ts-property); + --dark-color-ts-get-signature: var(--dark-color-ts-accessor); + --dark-color-ts-set-signature: var(--dark-color-ts-accessor); + --dark-color-ts-type-alias: #ff6492; + /* reference not included as links will be colored with the kind that it points to */ + + --dark-external-icon: url("data:image/svg+xml;utf8,"); + --dark-color-scheme: dark; +} + +@media (prefers-color-scheme: light) { + :root { + --color-background: var(--light-color-background); + --color-background-secondary: var(--light-color-background-secondary); + --color-background-warning: var(--light-color-background-warning); + --color-warning-text: var(--light-color-warning-text); + --color-icon-background: var(--light-color-icon-background); + --color-accent: var(--light-color-accent); + --color-active-menu-item: var(--light-color-active-menu-item); + --color-text: var(--light-color-text); + --color-text-aside: var(--light-color-text-aside); + --color-link: var(--light-color-link); + + --color-ts-module: var(--light-color-ts-module); + --color-ts-namespace: var(--light-color-ts-namespace); + --color-ts-enum: var(--light-color-ts-enum); + --color-ts-enum-member: var(--light-color-ts-enum-member); + --color-ts-variable: var(--light-color-ts-variable); + --color-ts-function: var(--light-color-ts-function); + --color-ts-class: var(--light-color-ts-class); + --color-ts-interface: var(--light-color-ts-interface); + --color-ts-constructor: var(--light-color-ts-constructor); + --color-ts-property: var(--light-color-ts-property); + --color-ts-method: var(--light-color-ts-method); + --color-ts-call-signature: var(--light-color-ts-call-signature); + --color-ts-index-signature: var(--light-color-ts-index-signature); + --color-ts-constructor-signature: var( + --light-color-ts-constructor-signature + ); + --color-ts-parameter: var(--light-color-ts-parameter); + --color-ts-type-parameter: var(--light-color-ts-type-parameter); + --color-ts-accessor: var(--light-color-ts-accessor); + --color-ts-get-signature: var(--light-color-ts-get-signature); + --color-ts-set-signature: var(--light-color-ts-set-signature); + --color-ts-type-alias: var(--light-color-ts-type-alias); + + --external-icon: var(--light-external-icon); + --color-scheme: var(--light-color-scheme); + } +} + +@media (prefers-color-scheme: dark) { + :root { + --color-background: var(--dark-color-background); + --color-background-secondary: var(--dark-color-background-secondary); + --color-background-warning: var(--dark-color-background-warning); + --color-warning-text: var(--dark-color-warning-text); + --color-icon-background: var(--dark-color-icon-background); + --color-accent: var(--dark-color-accent); + --color-active-menu-item: var(--dark-color-active-menu-item); + --color-text: var(--dark-color-text); + --color-text-aside: var(--dark-color-text-aside); + --color-link: var(--dark-color-link); + + --color-ts-module: var(--dark-color-ts-module); + --color-ts-namespace: var(--dark-color-ts-namespace); + --color-ts-enum: var(--dark-color-ts-enum); + --color-ts-enum-member: var(--dark-color-ts-enum-member); + --color-ts-variable: var(--dark-color-ts-variable); + --color-ts-function: var(--dark-color-ts-function); + --color-ts-class: var(--dark-color-ts-class); + --color-ts-interface: var(--dark-color-ts-interface); + --color-ts-constructor: var(--dark-color-ts-constructor); + --color-ts-property: var(--dark-color-ts-property); + --color-ts-method: var(--dark-color-ts-method); + --color-ts-call-signature: var(--dark-color-ts-call-signature); + --color-ts-index-signature: var(--dark-color-ts-index-signature); + --color-ts-constructor-signature: var( + --dark-color-ts-constructor-signature + ); + --color-ts-parameter: var(--dark-color-ts-parameter); + --color-ts-type-parameter: var(--dark-color-ts-type-parameter); + --color-ts-accessor: var(--dark-color-ts-accessor); + --color-ts-get-signature: var(--dark-color-ts-get-signature); + --color-ts-set-signature: var(--dark-color-ts-set-signature); + --color-ts-type-alias: var(--dark-color-ts-type-alias); + + --external-icon: var(--dark-external-icon); + --color-scheme: var(--dark-color-scheme); + } +} + +html { + color-scheme: var(--color-scheme); +} + +body { + margin: 0; +} + +:root[data-theme="light"] { + --color-background: var(--light-color-background); + --color-background-secondary: var(--light-color-background-secondary); + --color-background-warning: var(--light-color-background-warning); + --color-warning-text: var(--light-color-warning-text); + --color-icon-background: var(--light-color-icon-background); + --color-accent: var(--light-color-accent); + --color-active-menu-item: var(--light-color-active-menu-item); + --color-text: var(--light-color-text); + --color-text-aside: var(--light-color-text-aside); + --color-link: var(--light-color-link); + + --color-ts-module: var(--light-color-ts-module); + --color-ts-namespace: var(--light-color-ts-namespace); + --color-ts-enum: var(--light-color-ts-enum); + --color-ts-enum-member: var(--light-color-ts-enum-member); + --color-ts-variable: var(--light-color-ts-variable); + --color-ts-function: var(--light-color-ts-function); + --color-ts-class: var(--light-color-ts-class); + --color-ts-interface: var(--light-color-ts-interface); + --color-ts-constructor: var(--light-color-ts-constructor); + --color-ts-property: var(--light-color-ts-property); + --color-ts-method: var(--light-color-ts-method); + --color-ts-call-signature: var(--light-color-ts-call-signature); + --color-ts-index-signature: var(--light-color-ts-index-signature); + --color-ts-constructor-signature: var( + --light-color-ts-constructor-signature + ); + --color-ts-parameter: var(--light-color-ts-parameter); + --color-ts-type-parameter: var(--light-color-ts-type-parameter); + --color-ts-accessor: var(--light-color-ts-accessor); + --color-ts-get-signature: var(--light-color-ts-get-signature); + --color-ts-set-signature: var(--light-color-ts-set-signature); + --color-ts-type-alias: var(--light-color-ts-type-alias); + + --external-icon: var(--light-external-icon); + --color-scheme: var(--light-color-scheme); +} + +:root[data-theme="dark"] { + --color-background: var(--dark-color-background); + --color-background-secondary: var(--dark-color-background-secondary); + --color-background-warning: var(--dark-color-background-warning); + --color-warning-text: var(--dark-color-warning-text); + --color-icon-background: var(--dark-color-icon-background); + --color-accent: var(--dark-color-accent); + --color-active-menu-item: var(--dark-color-active-menu-item); + --color-text: var(--dark-color-text); + --color-text-aside: var(--dark-color-text-aside); + --color-link: var(--dark-color-link); + + --color-ts-module: var(--dark-color-ts-module); + --color-ts-namespace: var(--dark-color-ts-namespace); + --color-ts-enum: var(--dark-color-ts-enum); + --color-ts-enum-member: var(--dark-color-ts-enum-member); + --color-ts-variable: var(--dark-color-ts-variable); + --color-ts-function: var(--dark-color-ts-function); + --color-ts-class: var(--dark-color-ts-class); + --color-ts-interface: var(--dark-color-ts-interface); + --color-ts-constructor: var(--dark-color-ts-constructor); + --color-ts-property: var(--dark-color-ts-property); + --color-ts-method: var(--dark-color-ts-method); + --color-ts-call-signature: var(--dark-color-ts-call-signature); + --color-ts-index-signature: var(--dark-color-ts-index-signature); + --color-ts-constructor-signature: var( + --dark-color-ts-constructor-signature + ); + --color-ts-parameter: var(--dark-color-ts-parameter); + --color-ts-type-parameter: var(--dark-color-ts-type-parameter); + --color-ts-accessor: var(--dark-color-ts-accessor); + --color-ts-get-signature: var(--dark-color-ts-get-signature); + --color-ts-set-signature: var(--dark-color-ts-set-signature); + --color-ts-type-alias: var(--dark-color-ts-type-alias); + + --external-icon: var(--dark-external-icon); + --color-scheme: var(--dark-color-scheme); +} + +.always-visible, +.always-visible .tsd-signatures { + display: inherit !important; +} + +h1, +h2, +h3, +h4, +h5, +h6 { + line-height: 1.2; +} + +h1 > a, +h2 > a, +h3 > a, +h4 > a, +h5 > a, +h6 > a { + text-decoration: none; + color: var(--color-text); +} + +h1 { + font-size: 1.875rem; + margin: 0.67rem 0; +} + +h2 { + font-size: 1.5rem; + margin: 0.83rem 0; +} + +h3 { + font-size: 1.25rem; + margin: 1rem 0; +} + +h4 { + font-size: 1.05rem; + margin: 1.33rem 0; +} + +h5 { + font-size: 1rem; + margin: 1.5rem 0; +} + +h6 { + font-size: 0.875rem; + margin: 2.33rem 0; +} + +.uppercase { + text-transform: uppercase; +} + +dl, +menu, +ol, +ul { + margin: 1em 0; +} + +dd { + margin: 0 0 0 40px; +} + +.container { + max-width: 1700px; + padding: 0 2rem; +} + +/* Footer */ +.tsd-generator { + border-top: 1px solid var(--color-accent); + padding-top: 1rem; + padding-bottom: 1rem; + max-height: 3.5rem; +} + +.tsd-generator > p { + margin-top: 0; + margin-bottom: 0; + padding: 0 1rem; +} + +.container-main { + margin: 0 auto; + /* toolbar, footer, margin */ + min-height: calc(100vh - 41px - 56px - 4rem); +} + +@keyframes fade-in { + from { + opacity: 0; + } + to { + opacity: 1; + } +} +@keyframes fade-out { + from { + opacity: 1; + visibility: visible; + } + to { + opacity: 0; + } +} +@keyframes fade-in-delayed { + 0% { + opacity: 0; + } + 33% { + opacity: 0; + } + 100% { + opacity: 1; + } +} +@keyframes fade-out-delayed { + 0% { + opacity: 1; + visibility: visible; + } + 66% { + opacity: 0; + } + 100% { + opacity: 0; + } +} +@keyframes pop-in-from-right { + from { + transform: translate(100%, 0); + } + to { + transform: translate(0, 0); + } +} +@keyframes pop-out-to-right { + from { + transform: translate(0, 0); + visibility: visible; + } + to { + transform: translate(100%, 0); + } +} +body { + background: var(--color-background); + font-family: "Segoe UI", sans-serif; + font-size: 16px; + color: var(--color-text); +} + +a { + color: var(--color-link); + text-decoration: none; +} +a:hover { + text-decoration: underline; +} +a.external[target="_blank"] { + background-image: var(--external-icon); + background-position: top 3px right; + background-repeat: no-repeat; + padding-right: 13px; +} + +code, +pre { + font-family: Menlo, Monaco, Consolas, "Courier New", monospace; + padding: 0.2em; + margin: 0; + font-size: 0.875rem; + border-radius: 0.8em; +} + +pre { + position: relative; + white-space: pre; + white-space: pre-wrap; + word-wrap: break-word; + padding: 10px; + border: 1px solid var(--color-accent); +} +pre code { + padding: 0; + font-size: 100%; +} +pre > button { + position: absolute; + top: 10px; + right: 10px; + opacity: 0; + transition: opacity 0.1s; + box-sizing: border-box; +} +pre:hover > button, +pre > button.visible { + opacity: 1; +} + +blockquote { + margin: 1em 0; + padding-left: 1em; + border-left: 4px solid gray; +} + +.tsd-typography { + line-height: 1.333em; +} +.tsd-typography ul { + list-style: square; + padding: 0 0 0 20px; + margin: 0; +} +.tsd-typography .tsd-index-panel h3, +.tsd-index-panel .tsd-typography h3, +.tsd-typography h4, +.tsd-typography h5, +.tsd-typography h6 { + font-size: 1em; +} +.tsd-typography h5, +.tsd-typography h6 { + font-weight: normal; +} +.tsd-typography p, +.tsd-typography ul, +.tsd-typography ol { + margin: 1em 0; +} +.tsd-typography table { + border-collapse: collapse; + border: none; +} +.tsd-typography td, +.tsd-typography th { + padding: 6px 13px; + border: 1px solid var(--color-accent); +} +.tsd-typography thead, +.tsd-typography tr:nth-child(even) { + background-color: var(--color-background-secondary); +} + +.tsd-breadcrumb { + margin: 0; + padding: 0; + color: var(--color-text-aside); +} +.tsd-breadcrumb a { + color: var(--color-text-aside); + text-decoration: none; +} +.tsd-breadcrumb a:hover { + text-decoration: underline; +} +.tsd-breadcrumb li { + display: inline; +} +.tsd-breadcrumb li:after { + content: " / "; +} + +.tsd-comment-tags { + display: flex; + flex-direction: column; +} +dl.tsd-comment-tag-group { + display: flex; + align-items: center; + overflow: hidden; + margin: 0.5em 0; +} +dl.tsd-comment-tag-group dt { + display: flex; + margin-right: 0.5em; + font-size: 0.875em; + font-weight: normal; +} +dl.tsd-comment-tag-group dd { + margin: 0; +} +code.tsd-tag { + padding: 0.25em 0.4em; + border: 0.1em solid var(--color-accent); + margin-right: 0.25em; + font-size: 70%; +} +h1 code.tsd-tag:first-of-type { + margin-left: 0.25em; +} + +dl.tsd-comment-tag-group dd:before, +dl.tsd-comment-tag-group dd:after { + content: " "; +} +dl.tsd-comment-tag-group dd pre, +dl.tsd-comment-tag-group dd:after { + clear: both; +} +dl.tsd-comment-tag-group p { + margin: 0; +} + +.tsd-panel.tsd-comment .lead { + font-size: 1.1em; + line-height: 1.333em; + margin-bottom: 2em; +} +.tsd-panel.tsd-comment .lead:last-child { + margin-bottom: 0; +} + +.tsd-filter-visibility h4 { + font-size: 1rem; + padding-top: 0.75rem; + padding-bottom: 0.5rem; + margin: 0; +} +.tsd-filter-item:not(:last-child) { + margin-bottom: 0.5rem; +} +.tsd-filter-input { + display: flex; + width: fit-content; + width: -moz-fit-content; + align-items: center; + user-select: none; + -webkit-user-select: none; + -moz-user-select: none; + -ms-user-select: none; + cursor: pointer; +} +.tsd-filter-input input[type="checkbox"] { + cursor: pointer; + position: absolute; + width: 1.5em; + height: 1.5em; + opacity: 0; +} +.tsd-filter-input input[type="checkbox"]:disabled { + pointer-events: none; +} +.tsd-filter-input svg { + cursor: pointer; + width: 1.5em; + height: 1.5em; + margin-right: 0.5em; + border-radius: 0.33em; + /* Leaving this at full opacity breaks event listeners on Firefox. + Don't remove unless you know what you're doing. */ + opacity: 0.99; +} +.tsd-filter-input input[type="checkbox"]:focus + svg { + transform: scale(0.95); +} +.tsd-filter-input input[type="checkbox"]:focus:not(:focus-visible) + svg { + transform: scale(1); +} +.tsd-checkbox-background { + fill: var(--color-accent); +} +input[type="checkbox"]:checked ~ svg .tsd-checkbox-checkmark { + stroke: var(--color-text); +} +.tsd-filter-input input:disabled ~ svg > .tsd-checkbox-background { + fill: var(--color-background); + stroke: var(--color-accent); + stroke-width: 0.25rem; +} +.tsd-filter-input input:disabled ~ svg > .tsd-checkbox-checkmark { + stroke: var(--color-accent); +} + +.tsd-theme-toggle { + padding-top: 0.75rem; +} +.tsd-theme-toggle > h4 { + display: inline; + vertical-align: middle; + margin-right: 0.75rem; +} + +.tsd-hierarchy { + list-style: square; + margin: 0; +} +.tsd-hierarchy .target { + font-weight: bold; +} + +.tsd-panel-group.tsd-index-group { + margin-bottom: 0; +} +.tsd-index-panel .tsd-index-list { + list-style: none; + line-height: 1.333em; + margin: 0; + padding: 0.25rem 0 0 0; + overflow: hidden; + display: grid; + grid-template-columns: repeat(3, 1fr); + column-gap: 1rem; + grid-template-rows: auto; +} +@media (max-width: 1024px) { + .tsd-index-panel .tsd-index-list { + grid-template-columns: repeat(2, 1fr); + } +} +@media (max-width: 768px) { + .tsd-index-panel .tsd-index-list { + grid-template-columns: repeat(1, 1fr); + } +} +.tsd-index-panel .tsd-index-list li { + -webkit-page-break-inside: avoid; + -moz-page-break-inside: avoid; + -ms-page-break-inside: avoid; + -o-page-break-inside: avoid; + page-break-inside: avoid; +} + +.tsd-flag { + display: inline-block; + padding: 0.25em 0.4em; + border-radius: 4px; + color: var(--color-comment-tag-text); + background-color: var(--color-comment-tag); + text-indent: 0; + font-size: 75%; + line-height: 1; + font-weight: normal; +} + +.tsd-anchor { + position: relative; + top: -100px; +} + +.tsd-member { + position: relative; +} +.tsd-member .tsd-anchor + h3 { + display: flex; + align-items: center; + margin-top: 0; + margin-bottom: 0; + border-bottom: none; +} + +.tsd-navigation.settings { + margin: 1rem 0; +} +.tsd-navigation > a, +.tsd-navigation .tsd-accordion-summary { + width: calc(100% - 0.5rem); +} +.tsd-navigation a, +.tsd-navigation summary > span, +.tsd-page-navigation a { + display: inline-flex; + align-items: center; + padding: 0.25rem; + color: var(--color-text); + text-decoration: none; + box-sizing: border-box; +} +.tsd-navigation a.current, +.tsd-page-navigation a.current { + background: var(--color-active-menu-item); +} +.tsd-navigation a:hover, +.tsd-page-navigation a:hover { + text-decoration: underline; +} +.tsd-navigation ul, +.tsd-page-navigation ul { + margin-top: 0; + margin-bottom: 0; + padding: 0; + list-style: none; +} +.tsd-navigation li, +.tsd-page-navigation li { + padding: 0; + max-width: 100%; +} +.tsd-nested-navigation { + margin-left: 3rem; +} +.tsd-nested-navigation > li > details { + margin-left: -1.5rem; +} +.tsd-small-nested-navigation { + margin-left: 1.5rem; +} +.tsd-small-nested-navigation > li > details { + margin-left: -1.5rem; +} + +.tsd-nested-navigation > li > a, +.tsd-nested-navigation > li > span { + width: calc(100% - 1.75rem - 0.5rem); +} + +.tsd-page-navigation ul { + padding-left: 1.75rem; +} + +#tsd-sidebar-links a { + margin-top: 0; + margin-bottom: 0.5rem; + line-height: 1.25rem; +} +#tsd-sidebar-links a:last-of-type { + margin-bottom: 0; +} + +a.tsd-index-link { + padding: 0.25rem 0 !important; + font-size: 1rem; + line-height: 1.25rem; + display: inline-flex; + align-items: center; + color: var(--color-text); +} +.tsd-accordion-summary { + list-style-type: none; /* hide marker on non-safari */ + outline: none; /* broken on safari, so just hide it */ +} +.tsd-accordion-summary::-webkit-details-marker { + display: none; /* hide marker on safari */ +} +.tsd-accordion-summary, +.tsd-accordion-summary a { + user-select: none; + -moz-user-select: none; + -webkit-user-select: none; + -ms-user-select: none; + + cursor: pointer; +} +.tsd-accordion-summary a { + width: calc(100% - 1.5rem); +} +.tsd-accordion-summary > * { + margin-top: 0; + margin-bottom: 0; + padding-top: 0; + padding-bottom: 0; +} +.tsd-index-accordion .tsd-accordion-summary > svg { + margin-left: 0.25rem; +} +.tsd-index-content > :not(:first-child) { + margin-top: 0.75rem; +} +.tsd-index-heading { + margin-top: 1.5rem; + margin-bottom: 0.75rem; +} + +.tsd-kind-icon { + margin-right: 0.5rem; + width: 1.25rem; + height: 1.25rem; + min-width: 1.25rem; + min-height: 1.25rem; +} +.tsd-kind-icon path { + transform-origin: center; + transform: scale(1.1); +} +.tsd-signature > .tsd-kind-icon { + margin-right: 0.8rem; +} + +.tsd-panel { + margin-bottom: 2.5rem; +} +.tsd-panel.tsd-member { + margin-bottom: 4rem; +} +.tsd-panel:empty { + display: none; +} +.tsd-panel > h1, +.tsd-panel > h2, +.tsd-panel > h3 { + margin: 1.5rem -1.5rem 0.75rem -1.5rem; + padding: 0 1.5rem 0.75rem 1.5rem; +} +.tsd-panel > h1.tsd-before-signature, +.tsd-panel > h2.tsd-before-signature, +.tsd-panel > h3.tsd-before-signature { + margin-bottom: 0; + border-bottom: none; +} + +.tsd-panel-group { + margin: 4rem 0; +} +.tsd-panel-group.tsd-index-group { + margin: 2rem 0; +} +.tsd-panel-group.tsd-index-group details { + margin: 2rem 0; +} + +#tsd-search { + transition: background-color 0.2s; +} +#tsd-search .title { + position: relative; + z-index: 2; +} +#tsd-search .field { + position: absolute; + left: 0; + top: 0; + right: 2.5rem; + height: 100%; +} +#tsd-search .field input { + box-sizing: border-box; + position: relative; + top: -50px; + z-index: 1; + width: 100%; + padding: 0 10px; + opacity: 0; + outline: 0; + border: 0; + background: transparent; + color: var(--color-text); +} +#tsd-search .field label { + position: absolute; + overflow: hidden; + right: -40px; +} +#tsd-search .field input, +#tsd-search .title, +#tsd-toolbar-links a { + transition: opacity 0.2s; +} +#tsd-search .results { + position: absolute; + visibility: hidden; + top: 40px; + width: 100%; + margin: 0; + padding: 0; + list-style: none; + box-shadow: 0 0 4px rgba(0, 0, 0, 0.25); +} +#tsd-search .results li { + padding: 0 10px; + background-color: var(--color-background); +} +#tsd-search .results li:nth-child(even) { + background-color: var(--color-background-secondary); +} +#tsd-search .results li.state { + display: none; +} +#tsd-search .results li.current:not(.no-results), +#tsd-search .results li:hover:not(.no-results) { + background-color: var(--color-accent); +} +#tsd-search .results a { + display: block; +} +#tsd-search .results a:before { + top: 10px; +} +#tsd-search .results span.parent { + color: var(--color-text-aside); + font-weight: normal; +} +#tsd-search.has-focus { + background-color: var(--color-accent); +} +#tsd-search.has-focus .field input { + top: 0; + opacity: 1; +} +#tsd-search.has-focus .title, +#tsd-search.has-focus #tsd-toolbar-links a { + z-index: 0; + opacity: 0; +} +#tsd-search.has-focus .results { + visibility: visible; +} +#tsd-search.loading .results li.state.loading { + display: block; +} +#tsd-search.failure .results li.state.failure { + display: block; +} + +#tsd-toolbar-links { + position: absolute; + top: 0; + right: 2rem; + height: 100%; + display: flex; + align-items: center; + justify-content: flex-end; +} +#tsd-toolbar-links a { + margin-left: 1.5rem; +} +#tsd-toolbar-links a:hover { + text-decoration: underline; +} + +.tsd-signature { + margin: 0 0 1rem 0; + padding: 1rem 0.5rem; + border: 1px solid var(--color-accent); + font-family: Menlo, Monaco, Consolas, "Courier New", monospace; + font-size: 14px; + overflow-x: auto; +} + +.tsd-signature-symbol { + color: var(--color-text-aside); + font-weight: normal; +} + +.tsd-signature-type { + font-style: italic; + font-weight: normal; +} + +.tsd-signatures { + padding: 0; + margin: 0 0 1em 0; + list-style-type: none; +} +.tsd-signatures .tsd-signature { + margin: 0; + border-color: var(--color-accent); + border-width: 1px 0; + transition: background-color 0.1s; +} +.tsd-description .tsd-signatures .tsd-signature { + border-width: 1px; +} + +ul.tsd-parameter-list, +ul.tsd-type-parameter-list { + list-style: square; + margin: 0; + padding-left: 20px; +} +ul.tsd-parameter-list > li.tsd-parameter-signature, +ul.tsd-type-parameter-list > li.tsd-parameter-signature { + list-style: none; + margin-left: -20px; +} +ul.tsd-parameter-list h5, +ul.tsd-type-parameter-list h5 { + font-size: 16px; + margin: 1em 0 0.5em 0; +} +.tsd-sources { + margin-top: 1rem; + font-size: 0.875em; +} +.tsd-sources a { + color: var(--color-text-aside); + text-decoration: underline; +} +.tsd-sources ul { + list-style: none; + padding: 0; +} + +.tsd-page-toolbar { + position: sticky; + z-index: 1; + top: 0; + left: 0; + width: 100%; + color: var(--color-text); + background: var(--color-background-secondary); + border-bottom: 1px var(--color-accent) solid; + transition: transform 0.3s ease-in-out; +} +.tsd-page-toolbar a { + color: var(--color-text); + text-decoration: none; +} +.tsd-page-toolbar a.title { + font-weight: bold; +} +.tsd-page-toolbar a.title:hover { + text-decoration: underline; +} +.tsd-page-toolbar .tsd-toolbar-contents { + display: flex; + justify-content: space-between; + height: 2.5rem; + margin: 0 auto; +} +.tsd-page-toolbar .table-cell { + position: relative; + white-space: nowrap; + line-height: 40px; +} +.tsd-page-toolbar .table-cell:first-child { + width: 100%; +} +.tsd-page-toolbar .tsd-toolbar-icon { + box-sizing: border-box; + line-height: 0; + padding: 12px 0; +} + +.tsd-widget { + display: inline-block; + overflow: hidden; + opacity: 0.8; + height: 40px; + transition: + opacity 0.1s, + background-color 0.2s; + vertical-align: bottom; + cursor: pointer; +} +.tsd-widget:hover { + opacity: 0.9; +} +.tsd-widget.active { + opacity: 1; + background-color: var(--color-accent); +} +.tsd-widget.no-caption { + width: 40px; +} +.tsd-widget.no-caption:before { + margin: 0; +} + +.tsd-widget.options, +.tsd-widget.menu { + display: none; +} +input[type="checkbox"] + .tsd-widget:before { + background-position: -120px 0; +} +input[type="checkbox"]:checked + .tsd-widget:before { + background-position: -160px 0; +} + +img { + max-width: 100%; +} + +.tsd-anchor-icon { + display: inline-flex; + align-items: center; + margin-left: 0.5rem; + vertical-align: middle; + color: var(--color-text); +} + +.tsd-anchor-icon svg { + width: 1em; + height: 1em; + visibility: hidden; +} + +.tsd-anchor-link:hover > .tsd-anchor-icon svg { + visibility: visible; +} + +.deprecated { + text-decoration: line-through; +} + +.warning { + padding: 1rem; + color: var(--color-warning-text); + background: var(--color-background-warning); +} + +.tsd-kind-project { + color: var(--color-ts-project); +} +.tsd-kind-module { + color: var(--color-ts-module); +} +.tsd-kind-namespace { + color: var(--color-ts-namespace); +} +.tsd-kind-enum { + color: var(--color-ts-enum); +} +.tsd-kind-enum-member { + color: var(--color-ts-enum-member); +} +.tsd-kind-variable { + color: var(--color-ts-variable); +} +.tsd-kind-function { + color: var(--color-ts-function); +} +.tsd-kind-class { + color: var(--color-ts-class); +} +.tsd-kind-interface { + color: var(--color-ts-interface); +} +.tsd-kind-constructor { + color: var(--color-ts-constructor); +} +.tsd-kind-property { + color: var(--color-ts-property); +} +.tsd-kind-method { + color: var(--color-ts-method); +} +.tsd-kind-call-signature { + color: var(--color-ts-call-signature); +} +.tsd-kind-index-signature { + color: var(--color-ts-index-signature); +} +.tsd-kind-constructor-signature { + color: var(--color-ts-constructor-signature); +} +.tsd-kind-parameter { + color: var(--color-ts-parameter); +} +.tsd-kind-type-literal { + color: var(--color-ts-type-literal); +} +.tsd-kind-type-parameter { + color: var(--color-ts-type-parameter); +} +.tsd-kind-accessor { + color: var(--color-ts-accessor); +} +.tsd-kind-get-signature { + color: var(--color-ts-get-signature); +} +.tsd-kind-set-signature { + color: var(--color-ts-set-signature); +} +.tsd-kind-type-alias { + color: var(--color-ts-type-alias); +} + +/* if we have a kind icon, don't color the text by kind */ +.tsd-kind-icon ~ span { + color: var(--color-text); +} + +* { + scrollbar-width: thin; + scrollbar-color: var(--color-accent) var(--color-icon-background); +} + +*::-webkit-scrollbar { + width: 0.75rem; +} + +*::-webkit-scrollbar-track { + background: var(--color-icon-background); +} + +*::-webkit-scrollbar-thumb { + background-color: var(--color-accent); + border-radius: 999rem; + border: 0.25rem solid var(--color-icon-background); +} + +/* mobile */ +@media (max-width: 769px) { + .tsd-widget.options, + .tsd-widget.menu { + display: inline-block; + } + + .container-main { + display: flex; + } + html .col-content { + float: none; + max-width: 100%; + width: 100%; + } + html .col-sidebar { + position: fixed !important; + overflow-y: auto; + -webkit-overflow-scrolling: touch; + z-index: 1024; + top: 0 !important; + bottom: 0 !important; + left: auto !important; + right: 0 !important; + padding: 1.5rem 1.5rem 0 0; + width: 75vw; + visibility: hidden; + background-color: var(--color-background); + transform: translate(100%, 0); + } + html .col-sidebar > *:last-child { + padding-bottom: 20px; + } + html .overlay { + content: ""; + display: block; + position: fixed; + z-index: 1023; + top: 0; + left: 0; + right: 0; + bottom: 0; + background-color: rgba(0, 0, 0, 0.75); + visibility: hidden; + } + + .to-has-menu .overlay { + animation: fade-in 0.4s; + } + + .to-has-menu .col-sidebar { + animation: pop-in-from-right 0.4s; + } + + .from-has-menu .overlay { + animation: fade-out 0.4s; + } + + .from-has-menu .col-sidebar { + animation: pop-out-to-right 0.4s; + } + + .has-menu body { + overflow: hidden; + } + .has-menu .overlay { + visibility: visible; + } + .has-menu .col-sidebar { + visibility: visible; + transform: translate(0, 0); + display: flex; + flex-direction: column; + gap: 1.5rem; + max-height: 100vh; + padding: 1rem 2rem; + } + .has-menu .tsd-navigation { + max-height: 100%; + } +} + +/* one sidebar */ +@media (min-width: 770px) { + .container-main { + display: grid; + grid-template-columns: minmax(0, 1fr) minmax(0, 2fr); + grid-template-areas: "sidebar content"; + margin: 2rem auto; + } + + .col-sidebar { + grid-area: sidebar; + } + .col-content { + grid-area: content; + padding: 0 1rem; + } +} +@media (min-width: 770px) and (max-width: 1399px) { + .col-sidebar { + max-height: calc(100vh - 2rem - 42px); + overflow: auto; + position: sticky; + top: 42px; + padding-top: 1rem; + } + .site-menu { + margin-top: 1rem; + } +} + +/* two sidebars */ +@media (min-width: 1200px) { + .container-main { + grid-template-columns: minmax(0, 1fr) minmax(0, 2.5fr) minmax(0, 20rem); + grid-template-areas: "sidebar content toc"; + } + + .col-sidebar { + display: contents; + } + + .page-menu { + grid-area: toc; + padding-left: 1rem; + } + .site-menu { + grid-area: sidebar; + } + + .site-menu { + margin-top: 1rem 0; + } + + .page-menu, + .site-menu { + max-height: calc(100vh - 2rem - 42px); + overflow: auto; + position: sticky; + top: 42px; + } +} diff --git a/dist/docs/classes/GaussianBlurBackgroundProcessor.html b/dist/docs/classes/GaussianBlurBackgroundProcessor.html new file mode 100644 index 0000000..e1ded9a --- /dev/null +++ b/dist/docs/classes/GaussianBlurBackgroundProcessor.html @@ -0,0 +1,231 @@ +GaussianBlurBackgroundProcessor | @twilio/video-processors
+
+ +
+
+
+
+ +

Class GaussianBlurBackgroundProcessor

+
+

The GaussianBlurBackgroundProcessor, when added to a VideoTrack, +applies a gaussian blur filter on the background in each video frame +and leaves the foreground (person(s)) untouched. Each instance of +GaussianBlurBackgroundProcessor should be added to only one VideoTrack +at a time to prevent overlapping of image data from multiple VideoTracks.

+
+
+

Example

import { createLocalVideoTrack } from 'twilio-video';
import { GaussianBlurBackgroundProcessor } from '@twilio/video-processors';

let blurBackground: GaussianBlurBackgroundProcessor;

(async() => {
  blurBackground = new GaussianBlurBackgroundProcessor({
    assetsPath: 'https://my-server-path/assets'
  });
  await blurBackground.loadModel();

  const track = await createLocalVideoTrack({
    // Increasing the capture resolution decreases the output FPS
    // especially on browsers that do not support SIMD
    // such as desktop Safari and iOS browsers, or on Chrome
    // with capture resolutions above 640x480 for webgl2.
    width: 640,
    height: 480,

    // Any frame rate above 24 fps on desktop browsers increase CPU
    // usage without noticeable increase in quality.
    frameRate: 24
  });
  track.addProcessor(virtualBackground, {
    inputFrameBufferType: 'videoframe',
    outputFrameBufferContextType: 'bitmaprenderer'
  });
})();
+
+
+
+

Hierarchy

+
    +
  • BackgroundProcessor +
      +
    • GaussianBlurBackgroundProcessor
+
+
+
+ +
+
+

Constructors

+
constructor +
+
+

Accessors

+
blurFilterRadius +deferInputFrameDownscale +maskBlurRadius +
+
+

Methods

+
loadModel +processFrame +
+
+

Constructors

+
+

constructor

+
+
+

Accessors

+
+

blurFilterRadius

+
    +
  • get blurFilterRadius(): number
    • +
  • +

    The current background blur filter radius in pixels.

    +
    +

    Returns number

    +
    • +
  • set blurFilterRadius(radius): void
    • +
  • +

    Set a new background blur filter radius in pixels.

    +
    +
    +

    Parameters

    +
      +
    • +
      radius: number
    +

    Returns void

    +
+
+

deferInputFrameDownscale

+
    +
  • get deferInputFrameDownscale(): boolean
    • +
  • +

    Whether the pipeline is calculating the person mask without +waiting for the current input frame to be downscaled (Chrome only).

    +
    +

    Returns boolean

    +
    • +
  • set deferInputFrameDownscale(defer): void
    • +
  • +

    Toggle whether the pipeline should calculate the person mask +without waiting for the current input frame to be downscaled +(Chrome only).

    +
    +
    +

    Parameters

    +
      +
    • +
      defer: boolean
    +

    Returns void

    +
+
+

maskBlurRadius

+
    +
  • get maskBlurRadius(): number
    • +
  • +

    The current blur radius when smoothing out the edges of the person's mask.

    +
    +

    Returns number

    +
    • +
  • set maskBlurRadius(radius): void
    • +
  • +

    Set a new blur radius to be used when smoothing out the edges of the person's mask.

    +
    +
    +

    Parameters

    +
      +
    • +
      radius: number
    +

    Returns void

    +
+
+

Methods

+
+

loadModel

+
    + +
  • +

    Load the segmentation model. +Call this method before attaching the processor to ensure +video frames are processed correctly.

    +
    +

    Returns Promise<void>

    +
+
+

processFrame

+
    + +
  • +

    Apply a transform to the background of an input video frame and leaving +the foreground (person(s)) untouched. Any exception detected will +result in the frame being dropped.

    +
    +
    +

    Parameters

    +
      +
    • +
      inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | VideoFrame | HTMLVideoElement
      +

      The source of the input frame to process. +
      +
      +OffscreenCanvas - Good for canvas-related processing +that can be rendered off screen. +
      +
      +HTMLCanvasElement - This is recommended on browsers +that doesn't support OffscreenCanvas, or if you need to render the frame on the screen. +
      +
      +HTMLVideoElement +
      +
      +VideoFrame - Recommended on browsers that support the +Insertable Streams API. +

      +
      • +
    • +
      outputFrameBuffer: HTMLCanvasElement
      +

      The output frame buffer to use to draw the processed frame.

      +
    +

    Returns Promise<void>

    +
+
+
+
+
+

Settings

 +
+
+

Member Visibility

+
    +
    • +
+
+

Theme

+
+

On This Page

 +
+
+
+
+
+

Generated using TypeDoc

+
\ No newline at end of file diff --git a/dist/docs/classes/VirtualBackgroundProcessor.html b/dist/docs/classes/VirtualBackgroundProcessor.html new file mode 100644 index 0000000..4e38595 --- /dev/null +++ b/dist/docs/classes/VirtualBackgroundProcessor.html @@ -0,0 +1,256 @@ +VirtualBackgroundProcessor | @twilio/video-processors
+
+ +
+
+
+
+ +

Class VirtualBackgroundProcessor

+
+

The VirtualBackgroundProcessor, when added to a VideoTrack, +replaces the background in each video frame with a given image, +and leaves the foreground (person(s)) untouched. Each instance of +VirtualBackgroundProcessor should be added to only one VideoTrack +at a time to prevent overlapping of image data from multiple VideoTracks.

+
+
+

Example

import { createLocalVideoTrack } from 'twilio-video';
import { VirtualBackgroundProcessor } from '@twilio/video-processors';

let virtualBackground: VirtualBackgroundProcessor;
const img = new Image();

img.onload = async () => {
  virtualBackground = new VirtualBackgroundProcessor({
    assetsPath: 'https://my-server-path/assets',
    backgroundImage: img
  });
  await virtualBackground.loadModel();

  const track = await createLocalVideoTrack({
    // Increasing the capture resolution decreases the output FPS
    // especially on browsers that do not support SIMD
    // such as desktop Safari and iOS browsers, or on Chrome
    // with capture resolutions above 640x480 for webgl2.
    width: 640,
    height: 480,

    // Any frame rate above 24 fps on desktop browsers increase CPU
    // usage without noticeable increase in quality.
    frameRate: 24
  });
  track.addProcessor(virtualBackground, {
    inputFrameBufferType: 'videoframe',
    outputFrameBufferContextType: 'bitmaprenderer'
  });
};

img.src = '/background.jpg';
+
+
+
+

Hierarchy

+
    +
  • BackgroundProcessor +
      +
    • VirtualBackgroundProcessor
+
+
+
+ +
+
+

Constructors

+
constructor +
+
+

Accessors

+
backgroundImage +deferInputFrameDownscale +fitType +maskBlurRadius +
+
+

Methods

+
loadModel +processFrame +
+
+

Constructors

+
+

constructor

+
+
+

Accessors

+
+

backgroundImage

+
    +
  • get backgroundImage(): HTMLImageElement
    • +
  • +

    The HTMLImageElement representing the current background image.

    +
    +

    Returns HTMLImageElement

    +
    • +
  • set backgroundImage(image): void
    • +
  • +

    Set an HTMLImageElement as the new background image. +An error will be raised if the image hasn't been fully loaded yet. Additionally, the image must follow +security guidelines +when loading the image from a different origin. Failing to do so will result to an empty output frame.

    +
    +
    +

    Parameters

    +
      +
    • +
      image: HTMLImageElement
    +

    Returns void

    +
+
+

deferInputFrameDownscale

+
    +
  • get deferInputFrameDownscale(): boolean
    • +
  • +

    Whether the pipeline is calculating the person mask without +waiting for the current input frame to be downscaled (Chrome only).

    +
    +

    Returns boolean

    +
    • +
  • set deferInputFrameDownscale(defer): void
    • +
  • +

    Toggle whether the pipeline should calculate the person mask +without waiting for the current input frame to be downscaled +(Chrome only).

    +
    +
    +

    Parameters

    +
      +
    • +
      defer: boolean
    +

    Returns void

    +
+
+

fitType

+
    +
  • get fitType(): ImageFit
    • +
  • +

    The current [[ImageFit]] for positioning of the background image in the viewport.

    +
    +

    Returns ImageFit

    +
    • +
  • set fitType(fitType): void
    • +
  • +

    Set a new [[ImageFit]] to be used for positioning the background image in the viewport.

    +
    +
    +

    Parameters

    +
    +

    Returns void

    +
+
+

maskBlurRadius

+
    +
  • get maskBlurRadius(): number
    • +
  • +

    The current blur radius when smoothing out the edges of the person's mask.

    +
    +

    Returns number

    +
    • +
  • set maskBlurRadius(radius): void
    • +
  • +

    Set a new blur radius to be used when smoothing out the edges of the person's mask.

    +
    +
    +

    Parameters

    +
      +
    • +
      radius: number
    +

    Returns void

    +
+
+

Methods

+
+

loadModel

+
    + +
  • +

    Load the segmentation model. +Call this method before attaching the processor to ensure +video frames are processed correctly.

    +
    +

    Returns Promise<void>

    +
+
+

processFrame

+
    + +
  • +

    Apply a transform to the background of an input video frame and leaving +the foreground (person(s)) untouched. Any exception detected will +result in the frame being dropped.

    +
    +
    +

    Parameters

    +
      +
    • +
      inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | VideoFrame | HTMLVideoElement
      +

      The source of the input frame to process. +
      +
      +OffscreenCanvas - Good for canvas-related processing +that can be rendered off screen. +
      +
      +HTMLCanvasElement - This is recommended on browsers +that doesn't support OffscreenCanvas, or if you need to render the frame on the screen. +
      +
      +HTMLVideoElement +
      +
      +VideoFrame - Recommended on browsers that support the +Insertable Streams API. +

      +
      • +
    • +
      outputFrameBuffer: HTMLCanvasElement
      +

      The output frame buffer to use to draw the processed frame.

      +
    +

    Returns Promise<void>

    +
+
+
+
+
+

Settings

 +
+
+

Member Visibility

+
    +
    • +
+
+

Theme

+
+

On This Page

 +
+
+
+
+
+

Generated using TypeDoc

+
\ No newline at end of file diff --git a/dist/docs/enums/ImageFit.html b/dist/docs/enums/ImageFit.html new file mode 100644 index 0000000..d770772 --- /dev/null +++ b/dist/docs/enums/ImageFit.html @@ -0,0 +1,96 @@ +ImageFit | @twilio/video-processors
+
+ +
+
+
+
+ +

Enumeration ImageFit

+
+

ImageFit specifies the positioning of an image inside a viewport.

+
+
+
+
+
+ +
+
+

Enumeration Members

+
Contain +Cover +Fill +None +
+
+

Enumeration Members

+
+

Contain

+
Contain: "Contain"
+

Scale the image up or down to fill the viewport while preserving the aspect ratio. +The image will be fully visible but will add empty space in the viewport if +aspect ratios do not match.

+
+
+
+

Cover

+
Cover: "Cover"
+

Scale the image to fill both height and width of the viewport while preserving +the aspect ratio, but will crop the image if aspect ratios do not match.

+
+
+
+

Fill

+
Fill: "Fill"
+

Stretches the image to fill the viewport regardless of aspect ratio.

+
+
+
+

None

+
None: "None"
+

Ignore height and width and use the original size.

+
+
+
+
+
+
+

Settings

 +
+
+

Member Visibility

+
    +
    • +
+
+

Theme

+
+

On This Page

 +
+
+
+
+
+

Generated using TypeDoc

+
\ No newline at end of file diff --git a/dist/docs/index.html b/dist/docs/index.html new file mode 100644 index 0000000..9272848 --- /dev/null +++ b/dist/docs/index.html @@ -0,0 +1,98 @@ +@twilio/video-processors
+
+ +
+
+
+
+

@twilio/video-processors

+

Twilio Video Processors

+

[!WARNING]
We are no longer allowing new customers to onboard to Twilio Video. Effective December 5th, 2026, Twilio Video will End of Life (EOL) and will cease to function for all customers. Customers may transition to any video provider they choose, however, we are recommending customers migrate to the Zoom Video SDK and we have prepared a Migration Guide. Additional information on this EOL is available in our Help Center here.

+
+

Twilio Video Processors is a collection of video processing tools which can be used with Twilio Video JavaScript SDK to apply transformations and filters to a VideoTrack.

+

   See it live here!

+

Features

The following Video Processors are provided to apply transformations and filters to a person's background. You can also use them as a reference for creating your own Video Processors that can be used with Twilio Video JavaScript SDK.

+ +

Prerequisites

+

Note

The Node.js and NPM requirements do not apply if the goal is to use this library as a dependency of your project. They only apply if you want to check the source code out and build the artifacts and/or run tests.

+

Installation

NPM

You can install directly from npm.

+
npm install @twilio/video-processors --save
+
+

Using this method, you can import twilio-video-processors like so:

+
import * as VideoProcessors from '@twilio/video-processors';
+
+

Script tag

You can also copy twilio-video-processors.js from the dist/build folder and include it directly in your web app using a <script> tag.

+
<script src="https://my-server-path/twilio-video-processors.js"></script>
+
+

Using this method, twilio-video-processors.js will set a browser global:

+
const VideoProcessors = Twilio.VideoProcessors;
+
+

Assets

In order to achieve the best performance, the VideoProcessors use WebAssembly to run TensorFlow Lite for person segmentation. You need to serve the tflite model and binaries so they can be loaded properly. These files can be downloaded from the dist/build folder. Check the API docs for details and the examples folder for reference.

+

Usage

These processors run TensorFlow Lite using MediaPipe Selfie Segmentation Landscape Model and requires WebAssembly SIMD support in order to achieve the best performance. We recommend that, when calling Video.createLocalVideoTrack, the video capture constraints be set to 24 fps frame rate with 640x480 capture dimensions. Higher resolutions can still be used for increased accuracy, but may degrade performance, resulting in a lower output frame rate on low powered devices.

+

Best Practice

Please check out the following pages for best practice.

+ +
+
+
+
+
+

Settings

 +
+
+

Member Visibility

+
    +
    • +
+
+

Theme

+
+

On This Page

 +
+
+
+
+
+

Generated using TypeDoc

+
\ No newline at end of file diff --git a/dist/docs/interfaces/GaussianBlurBackgroundProcessorOptions.html b/dist/docs/interfaces/GaussianBlurBackgroundProcessorOptions.html new file mode 100644 index 0000000..ac6b314 --- /dev/null +++ b/dist/docs/interfaces/GaussianBlurBackgroundProcessorOptions.html @@ -0,0 +1,143 @@ +GaussianBlurBackgroundProcessorOptions | @twilio/video-processors
+
+ +
+
+
+
+ +

Interface GaussianBlurBackgroundProcessorOptions

+
+

Options passed to [[GaussianBlurBackgroundProcessor]] constructor.

+
+
+
+

Hierarchy

+
    +
  • BackgroundProcessorOptions +
      +
    • GaussianBlurBackgroundProcessorOptions
+
+
+
+ +
+
+

Properties

+
assetsPath +blurFilterRadius? +deferInputFrameDownscale? +maskBlurRadius? +useWebWorker? +
+
+

Properties

+
+

assetsPath

+
assetsPath: string
+

The VideoProcessors load assets dynamically depending on certain browser features. +You need to serve all the assets and provide the root path so they can be referenced properly. +These assets can be copied from the dist/build folder which you can add as part of your deployment process.

+
+
+

Example


+
+For virtual background: +
+ +
const virtualBackground = new VirtualBackgroundProcessor({
  assetsPath: 'https://my-server-path/assets',
  backgroundImage: img,
});
await virtualBackground.loadModel();
+
+
+For blur background: +
+ +
const blurBackground = new GaussianBlurBackgroundProcessor({
  assetsPath: 'https://my-server-path/assets'
});
await blurBackground.loadModel();
+
+
+
+

Optional blurFilterRadius

+
blurFilterRadius?: number
+

The background blur filter radius to use in pixels.

+
+
+

Default

15
+
+
+
+

Optional deferInputFrameDownscale

+
deferInputFrameDownscale?: boolean
+

Whether the pipeline should calculate the person mask without +waiting for the current input frame to be downscaled. Setting +this to true will potentially increase the output frame rate at +the expense of a slight trailing effect around the person mask +(Chrome only).

+
+
+

Default

false
+
+
+
+

Optional maskBlurRadius

+
maskBlurRadius?: number
+

The blur radius to use when smoothing out the edges of the person's mask.

+
+
+

Default

8
+
+
+
+

Optional useWebWorker

+
useWebWorker?: boolean
+

Whether to use a web worker (Chrome only).

+
+
+

Default

true
+
+
+
+
+
+
+

Settings

 +
+
+

Member Visibility

+
    +
    • +
+
+

Theme

+
+

On This Page

 +
+
+
+
+
+

Generated using TypeDoc

+
\ No newline at end of file diff --git a/dist/docs/interfaces/VirtualBackgroundProcessorOptions.html b/dist/docs/interfaces/VirtualBackgroundProcessorOptions.html new file mode 100644 index 0000000..16fd7d0 --- /dev/null +++ b/dist/docs/interfaces/VirtualBackgroundProcessorOptions.html @@ -0,0 +1,155 @@ +VirtualBackgroundProcessorOptions | @twilio/video-processors
+
+ +
+
+
+
+ +

Interface VirtualBackgroundProcessorOptions

+
+

Options passed to [[VirtualBackgroundProcessor]] constructor.

+
+
+
+

Hierarchy

+
    +
  • BackgroundProcessorOptions +
      +
    • VirtualBackgroundProcessorOptions
+
+
+
+ +
+
+

Properties

+
assetsPath +backgroundImage +deferInputFrameDownscale? +fitType? +maskBlurRadius? +useWebWorker? +
+
+

Properties

+
+

assetsPath

+
assetsPath: string
+

The VideoProcessors load assets dynamically depending on certain browser features. +You need to serve all the assets and provide the root path so they can be referenced properly. +These assets can be copied from the dist/build folder which you can add as part of your deployment process.

+
+
+

Example


+
+For virtual background: +
+ +
const virtualBackground = new VirtualBackgroundProcessor({
  assetsPath: 'https://my-server-path/assets',
  backgroundImage: img,
});
await virtualBackground.loadModel();
+
+
+For blur background: +
+ +
const blurBackground = new GaussianBlurBackgroundProcessor({
  assetsPath: 'https://my-server-path/assets'
});
await blurBackground.loadModel();
+
+
+
+

backgroundImage

+
backgroundImage: HTMLImageElement
+

The HTMLImageElement to use for background replacement. +An error will be raised if the image hasn't been fully loaded yet. Additionally, the image must follow +security guidelines +when loading the image from a different origin. Failing to do so will result to an empty output frame.

+
+
+
+

Optional deferInputFrameDownscale

+
deferInputFrameDownscale?: boolean
+

Whether the pipeline should calculate the person mask without +waiting for the current input frame to be downscaled. Setting +this to true will potentially increase the output frame rate at +the expense of a slight trailing effect around the person mask +(Chrome only).

+
+
+

Default

false
+
+
+
+

Optional fitType

+
fitType?: ImageFit
+

The [[ImageFit]] to use for positioning of the background image in the viewport. Only the Canvas2D [[Pipeline]] +supports this option. WebGL2 ignores this option and falls back to Cover.

+
+
+

Default

'Fill'
+
+
+
+

Optional maskBlurRadius

+
maskBlurRadius?: number
+

The blur radius to use when smoothing out the edges of the person's mask.

+
+
+

Default

8
+
+
+
+

Optional useWebWorker

+
useWebWorker?: boolean
+

Whether to use a web worker (Chrome only).

+
+
+

Default

true
+
+
+
+
+
+
+

Settings

 +
+
+

Member Visibility

+
    +
    • +
+
+

Theme

+
+

On This Page

 +
+
+
+
+
+

Generated using TypeDoc

+
\ No newline at end of file diff --git a/dist/docs/modules.html b/dist/docs/modules.html new file mode 100644 index 0000000..4a5808d --- /dev/null +++ b/dist/docs/modules.html @@ -0,0 +1,62 @@ +@twilio/video-processors
+
+ +
+
+
+
+

@twilio/video-processors

+
+
+

Index

+
+

Enumerations

+
ImageFit +
+
+

Classes

+
GaussianBlurBackgroundProcessor +VirtualBackgroundProcessor +
+
+

Interfaces

+
GaussianBlurBackgroundProcessorOptions +VirtualBackgroundProcessorOptions +
+
+

Variables

+
isSupported +version +
+
+
+
+
+

Settings

 +
+
+

Member Visibility

+
    +
    • +
+
+

Theme

+
+
+
+

Generated using TypeDoc

+
\ No newline at end of file diff --git a/dist/docs/variables/isSupported.html b/dist/docs/variables/isSupported.html new file mode 100644 index 0000000..3480733 --- /dev/null +++ b/dist/docs/variables/isSupported.html @@ -0,0 +1,54 @@ +isSupported | @twilio/video-processors
+
+ +
+
+
+
+ +

Variable isSupportedConst

+
isSupported: boolean = ...
+

Check if the current browser is officially supported by twilio-video-procesors.js. +This is set to true for browsers that supports canvas +2D or +webgl2 +rendering context.

+
+
+

Example

import { isSupported } from '@twilio/video-processors';

if (isSupported) {
  // Initialize the background processors
}
+
+
+
+
+
+
+

Settings

 +
+
+

Member Visibility

+
    +
    • +
+
+

Theme

+
+
+
+

Generated using TypeDoc

+
\ No newline at end of file diff --git a/dist/docs/variables/version.html b/dist/docs/variables/version.html new file mode 100644 index 0000000..47b9d82 --- /dev/null +++ b/dist/docs/variables/version.html @@ -0,0 +1,47 @@ +version | @twilio/video-processors
+
+ +
+
+
+
+ +

Variable versionConst

+
version: string = '3.0.0-preview.1'
+

The current version of the library.

+
+
+
+
+
+
+

Settings

 +
+
+

Member Visibility

+
    +
    • +
+
+

Theme

+
+
+
+

Generated using TypeDoc

+
\ No newline at end of file diff --git a/es5/constants.d.ts b/es5/constants.d.ts new file mode 100644 index 0000000..bab1da0 --- /dev/null +++ b/es5/constants.d.ts @@ -0,0 +1,9 @@ +import { Dimensions } from './types'; +export declare const BLUR_FILTER_RADIUS = 15; +export declare const MASK_BLUR_RADIUS = 8; +export declare const MODEL_NAME = "selfie_segmentation_landscape.tflite"; +export declare const TFLITE_LOADER_NAME = "tflite-1-0-0.js"; +export declare const TFLITE_SIMD_LOADER_NAME = "tflite-simd-1-0-0.js"; +export declare const TWILIO_GAUSSIAN_BLUR_BACKGROUND_PROCESSOR_PIPELINE_WORKER = "twilio-gaussian-blur-background-processor-pipeline-worker.js"; +export declare const TWILIO_VIRTUAL_BACKGROUND_PROCESSOR_PIPELINE_WORKER = "twilio-virtual-background-processor-pipeline-worker.js"; +export declare const WASM_INFERENCE_DIMENSIONS: Dimensions; diff --git a/es5/constants.js b/es5/constants.js new file mode 100644 index 0000000..5252c1b --- /dev/null +++ b/es5/constants.js @@ -0,0 +1,15 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WASM_INFERENCE_DIMENSIONS = exports.TWILIO_VIRTUAL_BACKGROUND_PROCESSOR_PIPELINE_WORKER = exports.TWILIO_GAUSSIAN_BLUR_BACKGROUND_PROCESSOR_PIPELINE_WORKER = exports.TFLITE_SIMD_LOADER_NAME = exports.TFLITE_LOADER_NAME = exports.MODEL_NAME = exports.MASK_BLUR_RADIUS = exports.BLUR_FILTER_RADIUS = void 0; +exports.BLUR_FILTER_RADIUS = 15; +exports.MASK_BLUR_RADIUS = 8; +exports.MODEL_NAME = 'selfie_segmentation_landscape.tflite'; +exports.TFLITE_LOADER_NAME = 'tflite-1-0-0.js'; +exports.TFLITE_SIMD_LOADER_NAME = 'tflite-simd-1-0-0.js'; +exports.TWILIO_GAUSSIAN_BLUR_BACKGROUND_PROCESSOR_PIPELINE_WORKER = 'twilio-gaussian-blur-background-processor-pipeline-worker.js'; +exports.TWILIO_VIRTUAL_BACKGROUND_PROCESSOR_PIPELINE_WORKER = 'twilio-virtual-background-processor-pipeline-worker.js'; +exports.WASM_INFERENCE_DIMENSIONS = { + width: 256, + height: 144, +}; +//# sourceMappingURL=constants.js.map \ No newline at end of file diff --git a/es5/constants.js.map b/es5/constants.js.map new file mode 100644 index 0000000..0f97395 --- /dev/null +++ b/es5/constants.js.map @@ -0,0 +1 @@ +{"version":3,"file":"constants.js","sourceRoot":"","sources":["../lib/constants.ts"],"names":[],"mappings":";;;AAEa,QAAA,kBAAkB,GAAG,EAAE,CAAC;AACxB,QAAA,gBAAgB,GAAG,CAAC,CAAC;AACrB,QAAA,UAAU,GAAG,sCAAsC,CAAC;AACpD,QAAA,kBAAkB,GAAG,iBAAiB,CAAC;AACvC,QAAA,uBAAuB,GAAG,sBAAsB,CAAC;AACjD,QAAA,yDAAyD,GAAG,8DAA8D,CAAC;AAC3H,QAAA,mDAAmD,GAAG,wDAAwD,CAAC;AAG/G,QAAA,yBAAyB,GAAe;IACnD,KAAK,EAAE,GAAG;IACV,MAAM,EAAE,GAAG;CACZ,CAAC","sourcesContent":["import { Dimensions } from './types';\n\nexport const BLUR_FILTER_RADIUS = 15;\nexport const MASK_BLUR_RADIUS = 8;\nexport const MODEL_NAME = 'selfie_segmentation_landscape.tflite';\nexport const TFLITE_LOADER_NAME = 'tflite-1-0-0.js';\nexport const TFLITE_SIMD_LOADER_NAME = 'tflite-simd-1-0-0.js';\nexport const TWILIO_GAUSSIAN_BLUR_BACKGROUND_PROCESSOR_PIPELINE_WORKER = 'twilio-gaussian-blur-background-processor-pipeline-worker.js';\nexport const TWILIO_VIRTUAL_BACKGROUND_PROCESSOR_PIPELINE_WORKER = 'twilio-virtual-background-processor-pipeline-worker.js';\n\n\nexport const WASM_INFERENCE_DIMENSIONS: Dimensions = {\n width: 256,\n height: 144,\n};\n"]} \ No newline at end of file diff --git a/es5/index.d.ts b/es5/index.d.ts new file mode 100644 index 0000000..bde98a9 --- /dev/null +++ b/es5/index.d.ts @@ -0,0 +1,6 @@ +import { GaussianBlurBackgroundProcessor, GaussianBlurBackgroundProcessorOptions } from './processors/background/GaussianBlurBackgroundProcessor'; +import { VirtualBackgroundProcessor, VirtualBackgroundProcessorOptions } from './processors/background/VirtualBackgroundProcessor'; +import { ImageFit } from './types'; +import { isSupported } from './utils/support'; +import { version } from './utils/version'; +export { GaussianBlurBackgroundProcessor, GaussianBlurBackgroundProcessorOptions, ImageFit, isSupported, version, VirtualBackgroundProcessor, VirtualBackgroundProcessorOptions, }; diff --git a/es5/index.js b/es5/index.js new file mode 100644 index 0000000..7e061ca --- /dev/null +++ b/es5/index.js @@ -0,0 +1,29 @@ +"use strict"; +var __assign = (this && this.__assign) || function () { + __assign = Object.assign || function(t) { + for (var s, i = 1, n = arguments.length; i < n; i++) { + s = arguments[i]; + for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) + t[p] = s[p]; + } + return t; + }; + return __assign.apply(this, arguments); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessor = exports.version = exports.isSupported = exports.ImageFit = exports.GaussianBlurBackgroundProcessor = void 0; +var GaussianBlurBackgroundProcessor_1 = require("./processors/background/GaussianBlurBackgroundProcessor"); +Object.defineProperty(exports, "GaussianBlurBackgroundProcessor", { enumerable: true, get: function () { return GaussianBlurBackgroundProcessor_1.GaussianBlurBackgroundProcessor; } }); +var VirtualBackgroundProcessor_1 = require("./processors/background/VirtualBackgroundProcessor"); +Object.defineProperty(exports, "VirtualBackgroundProcessor", { enumerable: true, get: function () { return VirtualBackgroundProcessor_1.VirtualBackgroundProcessor; } }); +var types_1 = require("./types"); +Object.defineProperty(exports, "ImageFit", { enumerable: true, get: function () { return types_1.ImageFit; } }); +var support_1 = require("./utils/support"); +Object.defineProperty(exports, "isSupported", { enumerable: true, get: function () { return support_1.isSupported; } }); +var version_1 = require("./utils/version"); +Object.defineProperty(exports, "version", { enumerable: true, get: function () { return version_1.version; } }); +if (typeof window !== 'undefined') { + window.Twilio = window.Twilio || {}; + window.Twilio.VideoProcessors = __assign(__assign({}, window.Twilio.VideoProcessors), { GaussianBlurBackgroundProcessor: GaussianBlurBackgroundProcessor_1.GaussianBlurBackgroundProcessor, ImageFit: types_1.ImageFit, isSupported: support_1.isSupported, version: version_1.version, VirtualBackgroundProcessor: VirtualBackgroundProcessor_1.VirtualBackgroundProcessor }); +} +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/es5/index.js.map b/es5/index.js.map new file mode 100644 index 0000000..c4216fd --- /dev/null +++ b/es5/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../lib/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAAA,2GAAkJ;AAmBhJ,gHAnBO,iEAA+B,OAmBP;AAlBjC,iGAAmI;AAuBjI,2GAvBO,uDAA0B,OAuBP;AAtB5B,iCAAmC;AAmBjC,yFAnBO,gBAAQ,OAmBP;AAlBV,2CAA8C;AAmB5C,4FAnBO,qBAAW,OAmBP;AAlBb,2CAA0C;AAmBxC,wFAnBO,iBAAO,OAmBP;AAjBT,IAAI,OAAO,MAAM,KAAK,WAAW,EAAE;IACjC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;IACpC,MAAM,CAAC,MAAM,CAAC,eAAe,yBACxB,MAAM,CAAC,MAAM,CAAC,eAAe,KAChC,+BAA+B,mEAAA,EAC/B,QAAQ,kBAAA,EACR,WAAW,uBAAA,EACX,OAAO,mBAAA,EACP,0BAA0B,yDAAA,GAC3B,CAAC;CACH","sourcesContent":["import { GaussianBlurBackgroundProcessor, GaussianBlurBackgroundProcessorOptions } from './processors/background/GaussianBlurBackgroundProcessor';\nimport { VirtualBackgroundProcessor, VirtualBackgroundProcessorOptions } from './processors/background/VirtualBackgroundProcessor';\nimport { ImageFit } from './types';\nimport { isSupported } from './utils/support';\nimport { version } from './utils/version';\n\nif (typeof window !== 'undefined') {\n window.Twilio = window.Twilio || {};\n window.Twilio.VideoProcessors = {\n ...window.Twilio.VideoProcessors,\n GaussianBlurBackgroundProcessor,\n ImageFit,\n isSupported,\n version,\n VirtualBackgroundProcessor,\n };\n}\n\nexport {\n GaussianBlurBackgroundProcessor,\n GaussianBlurBackgroundProcessorOptions,\n ImageFit,\n isSupported,\n version,\n VirtualBackgroundProcessor,\n VirtualBackgroundProcessorOptions,\n};\n"]} \ No newline at end of file diff --git a/es5/processors/Processor.d.ts b/es5/processors/Processor.d.ts new file mode 100644 index 0000000..9a77b13 --- /dev/null +++ b/es5/processors/Processor.d.ts @@ -0,0 +1,13 @@ +/** + * @private + * The [[Processor]] is an abstract class for building your own custom processors. + */ +export declare abstract class Processor { + /** + * Applies a transform to an input frame and draw the results to an output frame buffer. + * The frame will be dropped if this method raises an exception. + * @param inputFrameBuffer - The source of the input frame to process. + * @param outputFrameBuffer - The output frame buffer to use to draw the processed frame. + */ + abstract processFrame(inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement | VideoFrame, outputFrameBuffer: HTMLCanvasElement): Promise | void; +} diff --git a/es5/processors/Processor.js b/es5/processors/Processor.js new file mode 100644 index 0000000..3bf6156 --- /dev/null +++ b/es5/processors/Processor.js @@ -0,0 +1,14 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Processor = void 0; +/** + * @private + * The [[Processor]] is an abstract class for building your own custom processors. + */ +var Processor = /** @class */ (function () { + function Processor() { + } + return Processor; +}()); +exports.Processor = Processor; +//# sourceMappingURL=Processor.js.map \ No newline at end of file diff --git a/es5/processors/Processor.js.map b/es5/processors/Processor.js.map new file mode 100644 index 0000000..641361d --- /dev/null +++ b/es5/processors/Processor.js.map @@ -0,0 +1 @@ +{"version":3,"file":"Processor.js","sourceRoot":"","sources":["../../lib/processors/Processor.ts"],"names":[],"mappings":";;;AAAA;;;GAGG;AACH;IAAA;IAWA,CAAC;IAAD,gBAAC;AAAD,CAAC,AAXD,IAWC;AAXqB,8BAAS","sourcesContent":["/**\n * @private\n * The [[Processor]] is an abstract class for building your own custom processors.\n */\nexport abstract class Processor {\n\n /**\n * Applies a transform to an input frame and draw the results to an output frame buffer.\n * The frame will be dropped if this method raises an exception.\n * @param inputFrameBuffer - The source of the input frame to process.\n * @param outputFrameBuffer - The output frame buffer to use to draw the processed frame.\n */\n abstract processFrame(\n inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement | VideoFrame,\n outputFrameBuffer: HTMLCanvasElement): Promise | void;\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/BackgroundProcessor.d.ts b/es5/processors/background/BackgroundProcessor.d.ts new file mode 100644 index 0000000..ebe84b4 --- /dev/null +++ b/es5/processors/background/BackgroundProcessor.d.ts @@ -0,0 +1,130 @@ +import { Processor } from '../Processor'; +import { BackgroundProcessorPipeline, BackgroundProcessorPipelineProxy } from './pipelines/backgroundprocessorpipeline'; +/** + * @private + */ +export interface BackgroundProcessorOptions { + /** + * The VideoProcessors load assets dynamically depending on certain browser features. + * You need to serve all the assets and provide the root path so they can be referenced properly. + * These assets can be copied from the `dist/build` folder which you can add as part of your deployment process. + * @example + *
+ *
+ * For virtual background: + *
+ * + * ```ts + * const virtualBackground = new VirtualBackgroundProcessor({ + * assetsPath: 'https://my-server-path/assets', + * backgroundImage: img, + * }); + * await virtualBackground.loadModel(); + * ``` + * + *
+ * For blur background: + *
+ * + * ```ts + * const blurBackground = new GaussianBlurBackgroundProcessor({ + * assetsPath: 'https://my-server-path/assets' + * }); + * await blurBackground.loadModel(); + * ``` + */ + assetsPath: string; + /** + * Whether the pipeline should calculate the person mask without + * waiting for the current input frame to be downscaled. Setting + * this to true will potentially increase the output frame rate at + * the expense of a slight trailing effect around the person mask + * (Chrome only). + * @default + * ```html + * false + * ``` + */ + deferInputFrameDownscale?: boolean; + /** + * The blur radius to use when smoothing out the edges of the person's mask. + * @default + * ```html + * 8 + * ``` + */ + maskBlurRadius?: number; + /** + * Whether to use a web worker (Chrome only). + * @default + * ```html + * true + * ``` + */ + useWebWorker?: boolean; +} +/** + * @private + */ +export declare class BackgroundProcessor extends Processor { + protected readonly _assetsPath: string; + protected readonly _backgroundProcessorPipeline: BackgroundProcessorPipeline | BackgroundProcessorPipelineProxy; + private readonly _benchmark; + private _deferInputFrameDownscale; + private readonly _inputFrameCanvas; + private readonly _inputFrameContext; + private _isSimdEnabled; + private _maskBlurRadius; + private _outputFrameBuffer; + private _outputFrameBufferContext; + protected constructor(backgroundProcessorPipeline: BackgroundProcessorPipeline | BackgroundProcessorPipelineProxy, options: BackgroundProcessorOptions); + /** + * Whether the pipeline is calculating the person mask without + * waiting for the current input frame to be downscaled (Chrome only). + */ + get deferInputFrameDownscale(): boolean; + /** + * Toggle whether the pipeline should calculate the person mask + * without waiting for the current input frame to be downscaled + * (Chrome only). + */ + set deferInputFrameDownscale(defer: boolean); + /** + * The current blur radius when smoothing out the edges of the person's mask. + */ + get maskBlurRadius(): number; + /** + * Set a new blur radius to be used when smoothing out the edges of the person's mask. + */ + set maskBlurRadius(radius: number); + /** + * Load the segmentation model. + * Call this method before attaching the processor to ensure + * video frames are processed correctly. + */ + loadModel(): Promise; + /** + * Apply a transform to the background of an input video frame and leaving + * the foreground (person(s)) untouched. Any exception detected will + * result in the frame being dropped. + * @param inputFrameBuffer - The source of the input frame to process. + *
+ *
+ * [OffscreenCanvas](https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas) - Good for canvas-related processing + * that can be rendered off screen. + *
+ *
+ * [HTMLCanvasElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement) - This is recommended on browsers + * that doesn't support `OffscreenCanvas`, or if you need to render the frame on the screen. + *
+ *
+ * [HTMLVideoElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement) + *
+ *
+ * [VideoFrame](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame) - Recommended on browsers that support the + * [Insertable Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Insertable_Streams_for_MediaStreamTrack_API). + *
+ * @param outputFrameBuffer - The output frame buffer to use to draw the processed frame. + */ + processFrame(inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement | VideoFrame, outputFrameBuffer: HTMLCanvasElement): Promise; +} diff --git a/es5/processors/background/BackgroundProcessor.js b/es5/processors/background/BackgroundProcessor.js new file mode 100644 index 0000000..ca9c804 --- /dev/null +++ b/es5/processors/background/BackgroundProcessor.js @@ -0,0 +1,253 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.BackgroundProcessor = void 0; +var constants_1 = require("../../constants"); +var Processor_1 = require("../Processor"); +var backgroundprocessorpipeline_1 = require("./pipelines/backgroundprocessorpipeline"); +/** + * @private + */ +var BackgroundProcessor = /** @class */ (function (_super) { + __extends(BackgroundProcessor, _super); + function BackgroundProcessor(backgroundProcessorPipeline, options) { + var _this = _super.call(this) || this; + _this._deferInputFrameDownscale = false; + _this._inputFrameCanvas = new OffscreenCanvas(1, 1); + _this._inputFrameContext = _this._inputFrameCanvas.getContext('2d', { willReadFrequently: true }); + _this._isSimdEnabled = null; + _this._maskBlurRadius = constants_1.MASK_BLUR_RADIUS; + _this._outputFrameBuffer = null; + _this._outputFrameBufferContext = null; + var assetsPath = options.assetsPath, _a = options.deferInputFrameDownscale, deferInputFrameDownscale = _a === void 0 ? _this._deferInputFrameDownscale : _a, _b = options.maskBlurRadius, maskBlurRadius = _b === void 0 ? _this._maskBlurRadius : _b; + if (typeof assetsPath !== 'string') { + throw new Error('assetsPath parameter must be a string'); + } + _this._assetsPath = assetsPath.replace(/([^/])$/, '$1/'); + _this._backgroundProcessorPipeline = backgroundProcessorPipeline; + // @ts-ignore + _this._benchmark = _this._backgroundProcessorPipeline._benchmark; + _this.deferInputFrameDownscale = deferInputFrameDownscale; + _this.maskBlurRadius = maskBlurRadius; + return _this; + } + Object.defineProperty(BackgroundProcessor.prototype, "deferInputFrameDownscale", { + /** + * Whether the pipeline is calculating the person mask without + * waiting for the current input frame to be downscaled (Chrome only). + */ + get: function () { + return this._deferInputFrameDownscale; + }, + /** + * Toggle whether the pipeline should calculate the person mask + * without waiting for the current input frame to be downscaled + * (Chrome only). + */ + set: function (defer) { + if (typeof defer !== 'boolean') { + console.warn('Provided deferInputFrameDownscale is not a boolean.'); + defer = this._deferInputFrameDownscale; + } + if (this._deferInputFrameDownscale !== defer) { + this._deferInputFrameDownscale = defer; + this._backgroundProcessorPipeline.setDeferInputFrameDownscale(this._deferInputFrameDownscale).catch(function () { + /* noop */ + }); + } + }, + enumerable: false, + configurable: true + }); + Object.defineProperty(BackgroundProcessor.prototype, "maskBlurRadius", { + /** + * The current blur radius when smoothing out the edges of the person's mask. + */ + get: function () { + return this._maskBlurRadius; + }, + /** + * Set a new blur radius to be used when smoothing out the edges of the person's mask. + */ + set: function (radius) { + if (typeof radius !== 'number' || radius < 0) { + console.warn("Valid mask blur radius not found. Using ".concat(constants_1.MASK_BLUR_RADIUS, " as default.")); + radius = constants_1.MASK_BLUR_RADIUS; + } + if (this._maskBlurRadius !== radius) { + this._maskBlurRadius = radius; + this._backgroundProcessorPipeline + .setMaskBlurRadius(this._maskBlurRadius) + .catch(function () { + /* noop */ + }); + } + }, + enumerable: false, + configurable: true + }); + /** + * Load the segmentation model. + * Call this method before attaching the processor to ensure + * video frames are processed correctly. + */ + BackgroundProcessor.prototype.loadModel = function () { + return __awaiter(this, void 0, void 0, function () { + var _a; + return __generator(this, function (_b) { + switch (_b.label) { + case 0: + _a = this; + return [4 /*yield*/, this + ._backgroundProcessorPipeline + .loadTwilioTFLite()]; + case 1: + _a._isSimdEnabled = _b.sent(); + return [2 /*return*/]; + } + }); + }); + }; + /** + * Apply a transform to the background of an input video frame and leaving + * the foreground (person(s)) untouched. Any exception detected will + * result in the frame being dropped. + * @param inputFrameBuffer - The source of the input frame to process. + *
+ *
+ * [OffscreenCanvas](https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas) - Good for canvas-related processing + * that can be rendered off screen. + *
+ *
+ * [HTMLCanvasElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement) - This is recommended on browsers + * that doesn't support `OffscreenCanvas`, or if you need to render the frame on the screen. + *
+ *
+ * [HTMLVideoElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement) + *
+ *
+ * [VideoFrame](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame) - Recommended on browsers that support the + * [Insertable Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Insertable_Streams_for_MediaStreamTrack_API). + *
+ * @param outputFrameBuffer - The output frame buffer to use to draw the processed frame. + */ + BackgroundProcessor.prototype.processFrame = function (inputFrameBuffer, outputFrameBuffer) { + return __awaiter(this, void 0, void 0, function () { + var _a, _backgroundProcessorPipeline, _benchmark, _outputFrameBufferContext, _b, captureWidth, captureHeight, inputFrame, outputFrame, _c, outputBitmap; + return __generator(this, function (_d) { + switch (_d.label) { + case 0: + if (!inputFrameBuffer || !outputFrameBuffer) { + throw new Error('Missing input or output frame buffer'); + } + _a = this, _backgroundProcessorPipeline = _a._backgroundProcessorPipeline, _benchmark = _a._benchmark, _outputFrameBufferContext = _a._outputFrameBufferContext; + _benchmark.end('captureFrameDelay'); + _benchmark.end('totalProcessingDelay'); + _benchmark.start('totalProcessingDelay'); + _benchmark.start('processFrameDelay'); + _b = inputFrameBuffer instanceof HTMLVideoElement + ? { width: inputFrameBuffer.videoWidth, height: inputFrameBuffer.videoHeight } + : typeof VideoFrame === 'function' && inputFrameBuffer instanceof VideoFrame + ? { width: inputFrameBuffer.displayWidth, height: inputFrameBuffer.displayHeight } + : inputFrameBuffer, captureWidth = _b.width, captureHeight = _b.height; + if (this._outputFrameBuffer !== outputFrameBuffer) { + this._outputFrameBuffer = outputFrameBuffer; + this._outputFrameBufferContext = outputFrameBuffer.getContext('2d') + || outputFrameBuffer.getContext('bitmaprenderer'); + } + if (this._inputFrameCanvas.width !== captureWidth) { + this._inputFrameCanvas.width = captureWidth; + } + if (this._inputFrameCanvas.height !== captureHeight) { + this._inputFrameCanvas.height = captureHeight; + } + if (inputFrameBuffer instanceof HTMLVideoElement) { + this._inputFrameContext.drawImage(inputFrameBuffer, 0, 0); + inputFrame = this._inputFrameCanvas; + } + else { + inputFrame = inputFrameBuffer; + } + if (!(_backgroundProcessorPipeline instanceof backgroundprocessorpipeline_1.BackgroundProcessorPipeline)) return [3 /*break*/, 2]; + return [4 /*yield*/, _backgroundProcessorPipeline + .render(inputFrame)]; + case 1: + _c = _d.sent(); + return [3 /*break*/, 4]; + case 2: return [4 /*yield*/, _backgroundProcessorPipeline + .render(inputFrame)]; + case 3: + _c = _d.sent(); + _d.label = 4; + case 4: + outputFrame = _c; + if (_outputFrameBufferContext instanceof ImageBitmapRenderingContext) { + outputBitmap = outputFrame instanceof OffscreenCanvas + ? outputFrame.transferToImageBitmap() + : outputFrame; + _outputFrameBufferContext.transferFromImageBitmap(outputBitmap); + } + else if (_outputFrameBufferContext instanceof CanvasRenderingContext2D && outputFrame) { + _outputFrameBufferContext.drawImage(outputFrame, 0, 0); + } + _benchmark.end('processFrameDelay'); + _benchmark.start('captureFrameDelay'); + return [2 /*return*/]; + } + }); + }); + }; + return BackgroundProcessor; +}(Processor_1.Processor)); +exports.BackgroundProcessor = BackgroundProcessor; +//# sourceMappingURL=BackgroundProcessor.js.map \ No newline at end of file diff --git a/es5/processors/background/BackgroundProcessor.js.map b/es5/processors/background/BackgroundProcessor.js.map new file mode 100644 index 0000000..ffd94af --- /dev/null +++ b/es5/processors/background/BackgroundProcessor.js.map @@ -0,0 +1 @@ +{"version":3,"file":"BackgroundProcessor.js","sourceRoot":"","sources":["../../../lib/processors/background/BackgroundProcessor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAAmD;AAGnD,0CAAyC;AACzC,uFAAwH;AAqExH;;GAEG;AACH;IAAyC,uCAAS;IAahD,6BACE,2BAA2F,EAC3F,OAAmC;QAFrC,YAIE,iBAAO,SAkBR;QA9BO,+BAAyB,GAAY,KAAK,CAAC;QAClC,uBAAiB,GAAoB,IAAI,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/D,wBAAkB,GAAsC,KAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAE,CAAC;QACxI,oBAAc,GAAmB,IAAI,CAAC;QACtC,qBAAe,GAAW,4BAAgB,CAAC;QAC3C,wBAAkB,GAA6B,IAAI,CAAC;QACpD,+BAAyB,GAAkE,IAAI,CAAC;QASpG,IAAA,UAAU,GAGR,OAAO,WAHC,EACV,KAEE,OAAO,yBAFgD,EAAzD,wBAAwB,mBAAG,KAAI,CAAC,yBAAyB,KAAA,EACzD,KACE,OAAO,eAD4B,EAArC,cAAc,mBAAG,KAAI,CAAC,eAAe,KAAA,CAC3B;QAEZ,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE;YAClC,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;SAC1D;QAED,KAAI,CAAC,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACxD,KAAI,CAAC,4BAA4B,GAAG,2BAA2B,CAAC;QAChE,aAAa;QACb,KAAI,CAAC,UAAU,GAAG,KAAI,CAAC,4BAA4B,CAAC,UAAU,CAAC;QAC/D,KAAI,CAAC,wBAAwB,GAAG,wBAAwB,CAAC;QACzD,KAAI,CAAC,cAAc,GAAG,cAAc,CAAC;;IACvC,CAAC;IAMD,sBAAI,yDAAwB;QAJ5B;;;WAGG;aACH;YACE,OAAO,IAAI,CAAC,yBAAyB,CAAC;QACxC,CAAC;QAED;;;;WAIG;aACH,UAA6B,KAAc;YACzC,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;gBAC9B,OAAO,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;gBACpE,KAAK,GAAG,IAAI,CAAC,yBAAyB,CAAC;aACxC;YACD,IAAI,IAAI,CAAC,yBAAyB,KAAK,KAAK,EAAE;gBAC5C,IAAI,CAAC,yBAAyB,GAAG,KAAK,CAAC;gBACvC,IAAI,CAAC,4BAA4B,CAAC,2BAA2B,CAC3D,IAAI,CAAC,yBAAyB,CAC/B,CAAC,KAAK,CAAC;oBACN,UAAU;gBACZ,CAAC,CAAC,CAAC;aACJ;QACH,CAAC;;;OApBA;IAyBD,sBAAI,+CAAc;QAHlB;;WAEG;aACH;YACE,OAAO,IAAI,CAAC,eAAe,CAAC;QAC9B,CAAC;QAED;;WAEG;aACH,UAAmB,MAAc;YAC/B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,GAAG,CAAC,EAAE;gBAC5C,OAAO,CAAC,IAAI,CAAC,kDAA2C,4BAAgB,iBAAc,CAAC,CAAC;gBACxF,MAAM,GAAG,4BAAgB,CAAC;aAC3B;YACD,IAAI,IAAI,CAAC,eAAe,KAAK,MAAM,EAAE;gBACnC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;gBAC9B,IAAI,CAAC,4BAA4B;qBAC9B,iBAAiB,CAAC,IAAI,CAAC,eAAe,CAAC;qBACvC,KAAK,CAAC;oBACL,UAAU;gBACZ,CAAC,CAAC,CAAC;aACN;QACH,CAAC;;;OAlBA;IAoBD;;;;OAIG;IACG,uCAAS,GAAf;;;;;;wBACE,KAAA,IAAI,CAAA;wBAAkB,qBAAM,IAAI;iCAC7B,4BAA4B;iCAC5B,gBAAgB,EAAE,EAAA;;wBAFrB,GAAK,cAAc,GAAG,SAED,CAAC;;;;;KACvB;IAED;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACG,0CAAY,GAAlB,UACE,gBAAqF,EACrF,iBAAoC;;;;;;wBAEpC,IAAI,CAAC,gBAAgB,IAAI,CAAC,iBAAiB,EAAE;4BAC3C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;yBACzD;wBAEK,KAIF,IAAI,EAHN,4BAA4B,kCAAA,EAC5B,UAAU,gBAAA,EACV,yBAAyB,+BAAA,CAClB;wBAET,UAAU,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;wBACpC,UAAU,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;wBACvC,UAAU,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;wBACzC,UAAU,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;wBAEhC,KAGF,gBAAgB,YAAY,gBAAgB;4BAC9C,CAAC,CAAC,EAAE,KAAK,EAAE,gBAAgB,CAAC,UAAU,EAAE,MAAM,EAAE,gBAAgB,CAAC,WAAW,EAAE;4BAC9E,CAAC,CAAC,OAAO,UAAU,KAAK,UAAU,IAAI,gBAAgB,YAAY,UAAU;gCAC1E,CAAC,CAAC,EAAE,KAAK,EAAE,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,gBAAgB,CAAC,aAAa,EAAE;gCAClF,CAAC,CAAC,gBAAyD,EANtD,YAAY,WAAA,EACX,aAAa,YAAA,CAKyC;wBAEhE,IAAI,IAAI,CAAC,kBAAkB,KAAK,iBAAiB,EAAE;4BACjD,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC;4BAC5C,IAAI,CAAC,yBAAyB,GAAG,iBAAiB,CAAC,UAAU,CAAC,IAAI,CAAC;mCAC9D,iBAAiB,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;yBACrD;wBACD,IAAI,IAAI,CAAC,iBAAiB,CAAC,KAAK,KAAK,YAAY,EAAE;4BACjD,IAAI,CAAC,iBAAiB,CAAC,KAAK,GAAG,YAAY,CAAC;yBAC7C;wBACD,IAAI,IAAI,CAAC,iBAAiB,CAAC,MAAM,KAAK,aAAa,EAAE;4BACnD,IAAI,CAAC,iBAAiB,CAAC,MAAM,GAAG,aAAa,CAAC;yBAC/C;wBAGD,IAAI,gBAAgB,YAAY,gBAAgB,EAAE;4BAChD,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAC/B,gBAAgB,EAChB,CAAC,EACD,CAAC,CACF,CAAC;4BACF,UAAU,GAAG,IAAI,CAAC,iBAAiB,CAAC;yBACrC;6BAAM;4BACL,UAAU,GAAG,gBAAgB,CAAC;yBAC/B;6BAEmB,CAAA,4BAA4B,YAAY,yDAA2B,CAAA,EAAnE,wBAAmE;wBACnF,qBAAO,4BAA4D;iCAChE,MAAM,CACL,UAAU,CACX,EAAA;;wBAHH,KAAA,SAGG,CAAA;;4BACH,qBAAO,4BAAiE;6BACrE,MAAM,CACL,UAAwB,CACzB,EAAA;;wBAHH,KAAA,SAGG,CAAA;;;wBARD,WAAW,KAQV;wBAEP,IAAI,yBAAyB,YAAY,2BAA2B,EAAE;4BAC9D,YAAY,GAAG,WAAW,YAAY,eAAe;gCACzD,CAAC,CAAC,WAAW,CAAC,qBAAqB,EAAE;gCACrC,CAAC,CAAC,WAAW,CAAC;4BAChB,yBAAyB,CAAC,uBAAuB,CAAC,YAAY,CAAC,CAAC;yBACjE;6BAAM,IAAI,yBAAyB,YAAY,wBAAwB,IAAI,WAAW,EAAE;4BACvF,yBAAyB,CAAC,SAAS,CACjC,WAAW,EACX,CAAC,EACD,CAAC,CACF,CAAC;yBACH;wBAED,UAAU,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;wBACpC,UAAU,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;;;;;KACvC;IACH,0BAAC;AAAD,CAAC,AA1MD,CAAyC,qBAAS,GA0MjD;AA1MY,kDAAmB","sourcesContent":["import { MASK_BLUR_RADIUS } from '../../constants';\nimport { Benchmark } from '../../utils/Benchmark';\nimport { InputFrame } from '../../types';\nimport { Processor } from '../Processor';\nimport { BackgroundProcessorPipeline, BackgroundProcessorPipelineProxy } from './pipelines/backgroundprocessorpipeline';\n\n/**\n * @private\n */\nexport interface BackgroundProcessorOptions {\n /**\n * The VideoProcessors load assets dynamically depending on certain browser features.\n * You need to serve all the assets and provide the root path so they can be referenced properly.\n * These assets can be copied from the `dist/build` folder which you can add as part of your deployment process.\n * @example\n *
\n *
\n * For virtual background:\n *
\n *\n * ```ts\n * const virtualBackground = new VirtualBackgroundProcessor({\n * assetsPath: 'https://my-server-path/assets',\n * backgroundImage: img,\n * });\n * await virtualBackground.loadModel();\n * ```\n *\n *
\n * For blur background:\n *
\n *\n * ```ts\n * const blurBackground = new GaussianBlurBackgroundProcessor({\n * assetsPath: 'https://my-server-path/assets'\n * });\n * await blurBackground.loadModel();\n * ```\n */\n assetsPath: string;\n\n /**\n * Whether the pipeline should calculate the person mask without\n * waiting for the current input frame to be downscaled. Setting\n * this to true will potentially increase the output frame rate at\n * the expense of a slight trailing effect around the person mask\n * (Chrome only).\n * @default\n * ```html\n * false\n * ```\n */\n deferInputFrameDownscale?: boolean;\n\n /**\n * The blur radius to use when smoothing out the edges of the person's mask.\n * @default\n * ```html\n * 8\n * ```\n */\n maskBlurRadius?: number;\n\n /**\n * Whether to use a web worker (Chrome only).\n * @default\n * ```html\n * true\n * ```\n */\n useWebWorker?: boolean;\n}\n\n/**\n * @private\n */\nexport class BackgroundProcessor extends Processor {\n protected readonly _assetsPath: string;\n protected readonly _backgroundProcessorPipeline: BackgroundProcessorPipeline | BackgroundProcessorPipelineProxy;\n\n private readonly _benchmark: Benchmark;\n private _deferInputFrameDownscale: boolean = false;\n private readonly _inputFrameCanvas: OffscreenCanvas = new OffscreenCanvas(1, 1);\n private readonly _inputFrameContext: OffscreenCanvasRenderingContext2D = this._inputFrameCanvas.getContext('2d', { willReadFrequently: true })!;\n private _isSimdEnabled: boolean | null = null;\n private _maskBlurRadius: number = MASK_BLUR_RADIUS;\n private _outputFrameBuffer: HTMLCanvasElement | null = null;\n private _outputFrameBufferContext: CanvasRenderingContext2D | ImageBitmapRenderingContext | null = null;\n\n protected constructor(\n backgroundProcessorPipeline: BackgroundProcessorPipeline | BackgroundProcessorPipelineProxy,\n options: BackgroundProcessorOptions\n ) {\n super();\n\n const {\n assetsPath,\n deferInputFrameDownscale = this._deferInputFrameDownscale,\n maskBlurRadius = this._maskBlurRadius\n } = options;\n\n if (typeof assetsPath !== 'string') {\n throw new Error('assetsPath parameter must be a string');\n }\n\n this._assetsPath = assetsPath.replace(/([^/])$/, '$1/');\n this._backgroundProcessorPipeline = backgroundProcessorPipeline;\n // @ts-ignore\n this._benchmark = this._backgroundProcessorPipeline._benchmark;\n this.deferInputFrameDownscale = deferInputFrameDownscale;\n this.maskBlurRadius = maskBlurRadius;\n }\n\n /**\n * Whether the pipeline is calculating the person mask without\n * waiting for the current input frame to be downscaled (Chrome only).\n */\n get deferInputFrameDownscale(): boolean {\n return this._deferInputFrameDownscale;\n }\n\n /**\n * Toggle whether the pipeline should calculate the person mask\n * without waiting for the current input frame to be downscaled\n * (Chrome only).\n */\n set deferInputFrameDownscale(defer: boolean) {\n if (typeof defer !== 'boolean') {\n console.warn('Provided deferInputFrameDownscale is not a boolean.');\n defer = this._deferInputFrameDownscale;\n }\n if (this._deferInputFrameDownscale !== defer) {\n this._deferInputFrameDownscale = defer;\n this._backgroundProcessorPipeline.setDeferInputFrameDownscale(\n this._deferInputFrameDownscale\n ).catch(() => {\n /* noop */\n });\n }\n }\n\n /**\n * The current blur radius when smoothing out the edges of the person's mask.\n */\n get maskBlurRadius(): number {\n return this._maskBlurRadius;\n }\n\n /**\n * Set a new blur radius to be used when smoothing out the edges of the person's mask.\n */\n set maskBlurRadius(radius: number) {\n if (typeof radius !== 'number' || radius < 0) {\n console.warn(`Valid mask blur radius not found. Using ${MASK_BLUR_RADIUS} as default.`);\n radius = MASK_BLUR_RADIUS;\n }\n if (this._maskBlurRadius !== radius) {\n this._maskBlurRadius = radius;\n this._backgroundProcessorPipeline\n .setMaskBlurRadius(this._maskBlurRadius)\n .catch(() => {\n /* noop */\n });\n }\n }\n\n /**\n * Load the segmentation model.\n * Call this method before attaching the processor to ensure\n * video frames are processed correctly.\n */\n async loadModel(): Promise {\n this._isSimdEnabled = await this\n ._backgroundProcessorPipeline\n .loadTwilioTFLite();\n }\n\n /**\n * Apply a transform to the background of an input video frame and leaving\n * the foreground (person(s)) untouched. Any exception detected will\n * result in the frame being dropped.\n * @param inputFrameBuffer - The source of the input frame to process.\n *
\n *
\n * [OffscreenCanvas](https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas) - Good for canvas-related processing\n * that can be rendered off screen.\n *
\n *
\n * [HTMLCanvasElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLCanvasElement) - This is recommended on browsers\n * that doesn't support `OffscreenCanvas`, or if you need to render the frame on the screen.\n *
\n *
\n * [HTMLVideoElement](https://developer.mozilla.org/en-US/docs/Web/API/HTMLVideoElement)\n *
\n *
\n * [VideoFrame](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame) - Recommended on browsers that support the\n * [Insertable Streams API](https://developer.mozilla.org/en-US/docs/Web/API/Insertable_Streams_for_MediaStreamTrack_API).\n *
\n * @param outputFrameBuffer - The output frame buffer to use to draw the processed frame.\n */\n async processFrame(\n inputFrameBuffer: OffscreenCanvas | HTMLCanvasElement | HTMLVideoElement | VideoFrame,\n outputFrameBuffer: HTMLCanvasElement\n ): Promise {\n if (!inputFrameBuffer || !outputFrameBuffer) {\n throw new Error('Missing input or output frame buffer');\n }\n\n const {\n _backgroundProcessorPipeline,\n _benchmark,\n _outputFrameBufferContext\n } = this;\n\n _benchmark.end('captureFrameDelay');\n _benchmark.end('totalProcessingDelay');\n _benchmark.start('totalProcessingDelay');\n _benchmark.start('processFrameDelay');\n\n const {\n width: captureWidth,\n height: captureHeight\n } = inputFrameBuffer instanceof HTMLVideoElement\n ? { width: inputFrameBuffer.videoWidth, height: inputFrameBuffer.videoHeight }\n : typeof VideoFrame === 'function' && inputFrameBuffer instanceof VideoFrame\n ? { width: inputFrameBuffer.displayWidth, height: inputFrameBuffer.displayHeight }\n : inputFrameBuffer as (OffscreenCanvas | HTMLCanvasElement);\n\n if (this._outputFrameBuffer !== outputFrameBuffer) {\n this._outputFrameBuffer = outputFrameBuffer;\n this._outputFrameBufferContext = outputFrameBuffer.getContext('2d')\n || outputFrameBuffer.getContext('bitmaprenderer');\n }\n if (this._inputFrameCanvas.width !== captureWidth) {\n this._inputFrameCanvas.width = captureWidth;\n }\n if (this._inputFrameCanvas.height !== captureHeight) {\n this._inputFrameCanvas.height = captureHeight;\n }\n\n let inputFrame: InputFrame;\n if (inputFrameBuffer instanceof HTMLVideoElement) {\n this._inputFrameContext.drawImage(\n inputFrameBuffer,\n 0,\n 0\n );\n inputFrame = this._inputFrameCanvas;\n } else {\n inputFrame = inputFrameBuffer;\n }\n\n const outputFrame = _backgroundProcessorPipeline instanceof BackgroundProcessorPipeline\n ? await (_backgroundProcessorPipeline as BackgroundProcessorPipeline)\n .render(\n inputFrame\n )\n : await (_backgroundProcessorPipeline as BackgroundProcessorPipelineProxy)\n .render(\n inputFrame as VideoFrame\n );\n\n if (_outputFrameBufferContext instanceof ImageBitmapRenderingContext) {\n const outputBitmap = outputFrame instanceof OffscreenCanvas\n ? outputFrame.transferToImageBitmap()\n : outputFrame;\n _outputFrameBufferContext.transferFromImageBitmap(outputBitmap);\n } else if (_outputFrameBufferContext instanceof CanvasRenderingContext2D && outputFrame) {\n _outputFrameBufferContext.drawImage(\n outputFrame,\n 0,\n 0\n );\n }\n\n _benchmark.end('processFrameDelay');\n _benchmark.start('captureFrameDelay');\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/GaussianBlurBackgroundProcessor.d.ts b/es5/processors/background/GaussianBlurBackgroundProcessor.d.ts new file mode 100644 index 0000000..003b7cf --- /dev/null +++ b/es5/processors/background/GaussianBlurBackgroundProcessor.d.ts @@ -0,0 +1,72 @@ +import { BackgroundProcessor, BackgroundProcessorOptions } from './BackgroundProcessor'; +/** + * Options passed to [[GaussianBlurBackgroundProcessor]] constructor. + */ +export interface GaussianBlurBackgroundProcessorOptions extends BackgroundProcessorOptions { + /** + * The background blur filter radius to use in pixels. + * @default + * ```html + * 15 + * ``` + */ + blurFilterRadius?: number; +} +/** + * The GaussianBlurBackgroundProcessor, when added to a VideoTrack, + * applies a gaussian blur filter on the background in each video frame + * and leaves the foreground (person(s)) untouched. Each instance of + * GaussianBlurBackgroundProcessor should be added to only one VideoTrack + * at a time to prevent overlapping of image data from multiple VideoTracks. + * + * @example + * + * ```ts + * import { createLocalVideoTrack } from 'twilio-video'; + * import { GaussianBlurBackgroundProcessor } from '@twilio/video-processors'; + * + * let blurBackground: GaussianBlurBackgroundProcessor; + * + * (async() => { + * blurBackground = new GaussianBlurBackgroundProcessor({ + * assetsPath: 'https://my-server-path/assets' + * }); + * await blurBackground.loadModel(); + * + * const track = await createLocalVideoTrack({ + * // Increasing the capture resolution decreases the output FPS + * // especially on browsers that do not support SIMD + * // such as desktop Safari and iOS browsers, or on Chrome + * // with capture resolutions above 640x480 for webgl2. + * width: 640, + * height: 480, + * + * // Any frame rate above 24 fps on desktop browsers increase CPU + * // usage without noticeable increase in quality. + * frameRate: 24 + * }); + * track.addProcessor(virtualBackground, { + * inputFrameBufferType: 'videoframe', + * outputFrameBufferContextType: 'bitmaprenderer' + * }); + * })(); + * ``` + */ +export declare class GaussianBlurBackgroundProcessor extends BackgroundProcessor { + private _blurFilterRadius; + private readonly _name; + /** + * Construct a GaussianBlurBackgroundProcessor. Default values will be used for + * any missing properties in [[GaussianBlurBackgroundProcessorOptions]], and + * invalid properties will be ignored. + */ + constructor(options: GaussianBlurBackgroundProcessorOptions); + /** + * The current background blur filter radius in pixels. + */ + get blurFilterRadius(): number; + /** + * Set a new background blur filter radius in pixels. + */ + set blurFilterRadius(radius: number); +} diff --git a/es5/processors/background/GaussianBlurBackgroundProcessor.js b/es5/processors/background/GaussianBlurBackgroundProcessor.js new file mode 100644 index 0000000..519fe05 --- /dev/null +++ b/es5/processors/background/GaussianBlurBackgroundProcessor.js @@ -0,0 +1,120 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurBackgroundProcessor = void 0; +var constants_1 = require("../../constants"); +var support_1 = require("../../utils/support"); +var BackgroundProcessor_1 = require("./BackgroundProcessor"); +var backgroundprocessorpipeline_1 = require("./pipelines/backgroundprocessorpipeline"); +/** + * The GaussianBlurBackgroundProcessor, when added to a VideoTrack, + * applies a gaussian blur filter on the background in each video frame + * and leaves the foreground (person(s)) untouched. Each instance of + * GaussianBlurBackgroundProcessor should be added to only one VideoTrack + * at a time to prevent overlapping of image data from multiple VideoTracks. + * + * @example + * + * ```ts + * import { createLocalVideoTrack } from 'twilio-video'; + * import { GaussianBlurBackgroundProcessor } from '@twilio/video-processors'; + * + * let blurBackground: GaussianBlurBackgroundProcessor; + * + * (async() => { + * blurBackground = new GaussianBlurBackgroundProcessor({ + * assetsPath: 'https://my-server-path/assets' + * }); + * await blurBackground.loadModel(); + * + * const track = await createLocalVideoTrack({ + * // Increasing the capture resolution decreases the output FPS + * // especially on browsers that do not support SIMD + * // such as desktop Safari and iOS browsers, or on Chrome + * // with capture resolutions above 640x480 for webgl2. + * width: 640, + * height: 480, + * + * // Any frame rate above 24 fps on desktop browsers increase CPU + * // usage without noticeable increase in quality. + * frameRate: 24 + * }); + * track.addProcessor(virtualBackground, { + * inputFrameBufferType: 'videoframe', + * outputFrameBufferContextType: 'bitmaprenderer' + * }); + * })(); + * ``` + */ +var GaussianBlurBackgroundProcessor = /** @class */ (function (_super) { + __extends(GaussianBlurBackgroundProcessor, _super); + /** + * Construct a GaussianBlurBackgroundProcessor. Default values will be used for + * any missing properties in [[GaussianBlurBackgroundProcessorOptions]], and + * invalid properties will be ignored. + */ + function GaussianBlurBackgroundProcessor(options) { + var _this = this; + var _a = options.blurFilterRadius, blurFilterRadius = _a === void 0 ? constants_1.BLUR_FILTER_RADIUS : _a, _b = options.deferInputFrameDownscale, deferInputFrameDownscale = _b === void 0 ? false : _b, _c = options.maskBlurRadius, maskBlurRadius = _c === void 0 ? constants_1.MASK_BLUR_RADIUS : _c, _d = options.useWebWorker, useWebWorker = _d === void 0 ? true : _d; + var assetsPath = options + .assetsPath + .replace(/([^/])$/, '$1/'); + var BackgroundProcessorPipelineOrProxy = useWebWorker && (0, support_1.isChromiumImageBitmap)() + ? backgroundprocessorpipeline_1.GaussianBlurBackgroundProcessorPipelineProxy + : backgroundprocessorpipeline_1.GaussianBlurBackgroundProcessorPipeline; + var backgroundProcessorPipeline = new BackgroundProcessorPipelineOrProxy({ + assetsPath: assetsPath, + blurFilterRadius: blurFilterRadius, + deferInputFrameDownscale: deferInputFrameDownscale, + maskBlurRadius: maskBlurRadius + }); + _this = _super.call(this, backgroundProcessorPipeline, options) || this; + _this._blurFilterRadius = constants_1.BLUR_FILTER_RADIUS; + // tslint:disable-next-line no-unused-variable + _this._name = 'GaussianBlurBackgroundProcessor'; + _this.blurFilterRadius = options.blurFilterRadius; + return _this; + } + Object.defineProperty(GaussianBlurBackgroundProcessor.prototype, "blurFilterRadius", { + /** + * The current background blur filter radius in pixels. + */ + get: function () { + return this._blurFilterRadius; + }, + /** + * Set a new background blur filter radius in pixels. + */ + set: function (radius) { + if (!radius) { + console.warn("Valid blur filter radius not found. Using ".concat(constants_1.BLUR_FILTER_RADIUS, " as default.")); + radius = constants_1.BLUR_FILTER_RADIUS; + } + this._blurFilterRadius = radius; + this._backgroundProcessorPipeline + .setBlurFilterRadius(this._blurFilterRadius) + .catch(function () { + /* noop */ + }); + }, + enumerable: false, + configurable: true + }); + return GaussianBlurBackgroundProcessor; +}(BackgroundProcessor_1.BackgroundProcessor)); +exports.GaussianBlurBackgroundProcessor = GaussianBlurBackgroundProcessor; +//# sourceMappingURL=GaussianBlurBackgroundProcessor.js.map \ No newline at end of file diff --git a/es5/processors/background/GaussianBlurBackgroundProcessor.js.map b/es5/processors/background/GaussianBlurBackgroundProcessor.js.map new file mode 100644 index 0000000..ced1782 --- /dev/null +++ b/es5/processors/background/GaussianBlurBackgroundProcessor.js.map @@ -0,0 +1 @@ +{"version":3,"file":"GaussianBlurBackgroundProcessor.js","sourceRoot":"","sources":["../../../lib/processors/background/GaussianBlurBackgroundProcessor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AAAA,6CAAuE;AACvE,+CAA4D;AAC5D,6DAAwF;AACxF,uFAAgJ;AAgBhJ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuCG;AACH;IAAqD,mDAAmB;IAKtE;;;;OAIG;IACH,yCAAY,OAA+C;QAA3D,iBA6BC;QA3BG,IAAA,KAIE,OAAO,iBAJ4B,EAArC,gBAAgB,mBAAG,8BAAkB,KAAA,EACrC,KAGE,OAAO,yBAHuB,EAAhC,wBAAwB,mBAAG,KAAK,KAAA,EAChC,KAEE,OAAO,eAFwB,EAAjC,cAAc,mBAAG,4BAAgB,KAAA,EACjC,KACE,OAAO,aADU,EAAnB,YAAY,mBAAG,IAAI,KAAA,CACT;QAEZ,IAAM,UAAU,GAAG,OAAO;aACvB,UAAU;aACV,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAE7B,IAAM,kCAAkC,GAAG,YAAY,IAAI,IAAA,+BAAqB,GAAE;YAChF,CAAC,CAAC,0EAA4C;YAC9C,CAAC,CAAC,qEAAuC,CAAC;QAE5C,IAAM,2BAA2B,GAAG,IAAI,kCAAkC,CAAC;YACzE,UAAU,YAAA;YACV,gBAAgB,kBAAA;YAChB,wBAAwB,0BAAA;YACxB,cAAc,gBAAA;SACf,CAAC,CAAC;gBAEH,kBACE,2BAA2B,EAC3B,OAAO,CACR;QAnCK,uBAAiB,GAAW,8BAAkB,CAAC;QACvD,8CAA8C;QAC7B,WAAK,GAAW,iCAAiC,CAAC;QAmCjE,KAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAiB,CAAC;;IACpD,CAAC;IAKD,sBAAI,6DAAgB;QAHpB;;WAEG;aACH;YACE,OAAO,IAAI,CAAC,iBAAiB,CAAC;QAChC,CAAC;QAED;;WAEG;aACH,UAAqB,MAAc;YACjC,IAAI,CAAC,MAAM,EAAE;gBACX,OAAO,CAAC,IAAI,CAAC,oDAA6C,8BAAkB,iBAAc,CAAC,CAAC;gBAC5F,MAAM,GAAG,8BAAkB,CAAC;aAC7B;YACD,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC;YAC/B,IAAI,CAAC,4BAAuH;iBAC1H,mBAAmB,CAAC,IAAI,CAAC,iBAAiB,CAAC;iBAC3C,KAAK,CAAC;gBACL,UAAU;YACZ,CAAC,CAAC,CAAC;QACP,CAAC;;;OAhBA;IAiBH,sCAAC;AAAD,CAAC,AA/DD,CAAqD,yCAAmB,GA+DvE;AA/DY,0EAA+B","sourcesContent":["import { BLUR_FILTER_RADIUS, MASK_BLUR_RADIUS } from '../../constants';\nimport { isChromiumImageBitmap } from '../../utils/support';\nimport { BackgroundProcessor, BackgroundProcessorOptions } from './BackgroundProcessor';\nimport { GaussianBlurBackgroundProcessorPipeline, GaussianBlurBackgroundProcessorPipelineProxy } from './pipelines/backgroundprocessorpipeline';\n\n/**\n * Options passed to [[GaussianBlurBackgroundProcessor]] constructor.\n */\nexport interface GaussianBlurBackgroundProcessorOptions extends BackgroundProcessorOptions {\n /**\n * The background blur filter radius to use in pixels.\n * @default\n * ```html\n * 15\n * ```\n */\n blurFilterRadius?: number;\n}\n\n/**\n * The GaussianBlurBackgroundProcessor, when added to a VideoTrack,\n * applies a gaussian blur filter on the background in each video frame\n * and leaves the foreground (person(s)) untouched. Each instance of\n * GaussianBlurBackgroundProcessor should be added to only one VideoTrack\n * at a time to prevent overlapping of image data from multiple VideoTracks.\n *\n * @example\n *\n * ```ts\n * import { createLocalVideoTrack } from 'twilio-video';\n * import { GaussianBlurBackgroundProcessor } from '@twilio/video-processors';\n *\n * let blurBackground: GaussianBlurBackgroundProcessor;\n *\n * (async() => {\n * blurBackground = new GaussianBlurBackgroundProcessor({\n * assetsPath: 'https://my-server-path/assets'\n * });\n * await blurBackground.loadModel();\n *\n * const track = await createLocalVideoTrack({\n * // Increasing the capture resolution decreases the output FPS\n * // especially on browsers that do not support SIMD\n * // such as desktop Safari and iOS browsers, or on Chrome\n * // with capture resolutions above 640x480 for webgl2.\n * width: 640,\n * height: 480,\n *\n * // Any frame rate above 24 fps on desktop browsers increase CPU\n * // usage without noticeable increase in quality.\n * frameRate: 24\n * });\n * track.addProcessor(virtualBackground, {\n * inputFrameBufferType: 'videoframe',\n * outputFrameBufferContextType: 'bitmaprenderer'\n * });\n * })();\n * ```\n */\nexport class GaussianBlurBackgroundProcessor extends BackgroundProcessor {\n private _blurFilterRadius: number = BLUR_FILTER_RADIUS;\n // tslint:disable-next-line no-unused-variable\n private readonly _name: string = 'GaussianBlurBackgroundProcessor';\n\n /**\n * Construct a GaussianBlurBackgroundProcessor. Default values will be used for\n * any missing properties in [[GaussianBlurBackgroundProcessorOptions]], and\n * invalid properties will be ignored.\n */\n constructor(options: GaussianBlurBackgroundProcessorOptions) {\n const {\n blurFilterRadius = BLUR_FILTER_RADIUS,\n deferInputFrameDownscale = false,\n maskBlurRadius = MASK_BLUR_RADIUS,\n useWebWorker = true\n } = options;\n\n const assetsPath = options\n .assetsPath\n .replace(/([^/])$/, '$1/');\n\n const BackgroundProcessorPipelineOrProxy = useWebWorker && isChromiumImageBitmap()\n ? GaussianBlurBackgroundProcessorPipelineProxy\n : GaussianBlurBackgroundProcessorPipeline;\n\n const backgroundProcessorPipeline = new BackgroundProcessorPipelineOrProxy({\n assetsPath,\n blurFilterRadius,\n deferInputFrameDownscale,\n maskBlurRadius\n });\n\n super(\n backgroundProcessorPipeline,\n options\n );\n\n this.blurFilterRadius = options.blurFilterRadius!;\n }\n\n /**\n * The current background blur filter radius in pixels.\n */\n get blurFilterRadius(): number {\n return this._blurFilterRadius;\n }\n\n /**\n * Set a new background blur filter radius in pixels.\n */\n set blurFilterRadius(radius: number) {\n if (!radius) {\n console.warn(`Valid blur filter radius not found. Using ${BLUR_FILTER_RADIUS} as default.`);\n radius = BLUR_FILTER_RADIUS;\n }\n this._blurFilterRadius = radius;\n (this._backgroundProcessorPipeline as GaussianBlurBackgroundProcessorPipeline | GaussianBlurBackgroundProcessorPipelineProxy)\n .setBlurFilterRadius(this._blurFilterRadius)\n .catch(() => {\n /* noop */\n });\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/VirtualBackgroundProcessor.d.ts b/es5/processors/background/VirtualBackgroundProcessor.d.ts new file mode 100644 index 0000000..b2d1233 --- /dev/null +++ b/es5/processors/background/VirtualBackgroundProcessor.d.ts @@ -0,0 +1,97 @@ +import { ImageFit } from '../../types'; +import { BackgroundProcessor, BackgroundProcessorOptions } from './BackgroundProcessor'; +/** + * Options passed to [[VirtualBackgroundProcessor]] constructor. + */ +export interface VirtualBackgroundProcessorOptions extends BackgroundProcessorOptions { + /** + * The HTMLImageElement to use for background replacement. + * An error will be raised if the image hasn't been fully loaded yet. Additionally, the image must follow + * [security guidelines](https://developer.mozilla.org/en-US/docs/Web/HTML/CORS_enabled_image) + * when loading the image from a different origin. Failing to do so will result to an empty output frame. + */ + backgroundImage: HTMLImageElement; + /** + * The [[ImageFit]] to use for positioning of the background image in the viewport. Only the Canvas2D [[Pipeline]] + * supports this option. WebGL2 ignores this option and falls back to Cover. + * @default + * ```html + * 'Fill' + * ``` + */ + fitType?: ImageFit; +} +/** + * The VirtualBackgroundProcessor, when added to a VideoTrack, + * replaces the background in each video frame with a given image, + * and leaves the foreground (person(s)) untouched. Each instance of + * VirtualBackgroundProcessor should be added to only one VideoTrack + * at a time to prevent overlapping of image data from multiple VideoTracks. + * + * @example + * + * ```ts + * import { createLocalVideoTrack } from 'twilio-video'; + * import { VirtualBackgroundProcessor } from '@twilio/video-processors'; + * + * let virtualBackground: VirtualBackgroundProcessor; + * const img = new Image(); + * + * img.onload = async () => { + * virtualBackground = new VirtualBackgroundProcessor({ + * assetsPath: 'https://my-server-path/assets', + * backgroundImage: img + * }); + * await virtualBackground.loadModel(); + * + * const track = await createLocalVideoTrack({ + * // Increasing the capture resolution decreases the output FPS + * // especially on browsers that do not support SIMD + * // such as desktop Safari and iOS browsers, or on Chrome + * // with capture resolutions above 640x480 for webgl2. + * width: 640, + * height: 480, + * + * // Any frame rate above 24 fps on desktop browsers increase CPU + * // usage without noticeable increase in quality. + * frameRate: 24 + * }); + * track.addProcessor(virtualBackground, { + * inputFrameBufferType: 'videoframe', + * outputFrameBufferContextType: 'bitmaprenderer' + * }); + * }; + * + * img.src = '/background.jpg'; + * ``` + */ +export declare class VirtualBackgroundProcessor extends BackgroundProcessor { + private _backgroundImage; + private _fitType; + private readonly _name; + /** + * Construct a VirtualBackgroundProcessor. Default values will be used for + * any missing optional properties in [[VirtualBackgroundProcessorOptions]], + * and invalid properties will be ignored. + */ + constructor(options: VirtualBackgroundProcessorOptions); + /** + * The HTMLImageElement representing the current background image. + */ + get backgroundImage(): HTMLImageElement; + /** + * Set an HTMLImageElement as the new background image. + * An error will be raised if the image hasn't been fully loaded yet. Additionally, the image must follow + * [security guidelines](https://developer.mozilla.org/en-US/docs/Web/HTML/CORS_enabled_image) + * when loading the image from a different origin. Failing to do so will result to an empty output frame. + */ + set backgroundImage(image: HTMLImageElement); + /** + * The current [[ImageFit]] for positioning of the background image in the viewport. + */ + get fitType(): ImageFit; + /** + * Set a new [[ImageFit]] to be used for positioning the background image in the viewport. + */ + set fitType(fitType: ImageFit); +} diff --git a/es5/processors/background/VirtualBackgroundProcessor.js b/es5/processors/background/VirtualBackgroundProcessor.js new file mode 100644 index 0000000..c3e1a29 --- /dev/null +++ b/es5/processors/background/VirtualBackgroundProcessor.js @@ -0,0 +1,153 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessor = void 0; +var types_1 = require("../../types"); +var constants_1 = require("../../constants"); +var support_1 = require("../../utils/support"); +var BackgroundProcessor_1 = require("./BackgroundProcessor"); +var backgroundprocessorpipeline_1 = require("./pipelines/backgroundprocessorpipeline"); +/** + * The VirtualBackgroundProcessor, when added to a VideoTrack, + * replaces the background in each video frame with a given image, + * and leaves the foreground (person(s)) untouched. Each instance of + * VirtualBackgroundProcessor should be added to only one VideoTrack + * at a time to prevent overlapping of image data from multiple VideoTracks. + * + * @example + * + * ```ts + * import { createLocalVideoTrack } from 'twilio-video'; + * import { VirtualBackgroundProcessor } from '@twilio/video-processors'; + * + * let virtualBackground: VirtualBackgroundProcessor; + * const img = new Image(); + * + * img.onload = async () => { + * virtualBackground = new VirtualBackgroundProcessor({ + * assetsPath: 'https://my-server-path/assets', + * backgroundImage: img + * }); + * await virtualBackground.loadModel(); + * + * const track = await createLocalVideoTrack({ + * // Increasing the capture resolution decreases the output FPS + * // especially on browsers that do not support SIMD + * // such as desktop Safari and iOS browsers, or on Chrome + * // with capture resolutions above 640x480 for webgl2. + * width: 640, + * height: 480, + * + * // Any frame rate above 24 fps on desktop browsers increase CPU + * // usage without noticeable increase in quality. + * frameRate: 24 + * }); + * track.addProcessor(virtualBackground, { + * inputFrameBufferType: 'videoframe', + * outputFrameBufferContextType: 'bitmaprenderer' + * }); + * }; + * + * img.src = '/background.jpg'; + * ``` + */ +var VirtualBackgroundProcessor = /** @class */ (function (_super) { + __extends(VirtualBackgroundProcessor, _super); + /** + * Construct a VirtualBackgroundProcessor. Default values will be used for + * any missing optional properties in [[VirtualBackgroundProcessorOptions]], + * and invalid properties will be ignored. + */ + function VirtualBackgroundProcessor(options) { + var _this = this; + var backgroundImage = options.backgroundImage, _a = options.deferInputFrameDownscale, deferInputFrameDownscale = _a === void 0 ? false : _a, _b = options.fitType, fitType = _b === void 0 ? types_1.ImageFit.Fill : _b, _c = options.maskBlurRadius, maskBlurRadius = _c === void 0 ? constants_1.MASK_BLUR_RADIUS : _c, _d = options.useWebWorker, useWebWorker = _d === void 0 ? true : _d; + var assetsPath = options + .assetsPath + .replace(/([^/])$/, '$1/'); + var VirtualBackgroundProcessorPipelineOrProxy = useWebWorker && (0, support_1.isChromiumImageBitmap)() + ? backgroundprocessorpipeline_1.VirtualBackgroundProcessorPipelineProxy + : backgroundprocessorpipeline_1.VirtualBackgroundProcessorPipeline; + var backgroundProcessorPipeline = new VirtualBackgroundProcessorPipelineOrProxy({ + assetsPath: assetsPath, + deferInputFrameDownscale: deferInputFrameDownscale, + fitType: fitType, + maskBlurRadius: maskBlurRadius + }); + _this = _super.call(this, backgroundProcessorPipeline, options) || this; + // tslint:disable-next-line no-unused-variable + _this._name = 'VirtualBackgroundProcessor'; + _this.backgroundImage = backgroundImage; + _this.fitType = fitType; + return _this; + } + Object.defineProperty(VirtualBackgroundProcessor.prototype, "backgroundImage", { + /** + * The HTMLImageElement representing the current background image. + */ + get: function () { + return this._backgroundImage; + }, + /** + * Set an HTMLImageElement as the new background image. + * An error will be raised if the image hasn't been fully loaded yet. Additionally, the image must follow + * [security guidelines](https://developer.mozilla.org/en-US/docs/Web/HTML/CORS_enabled_image) + * when loading the image from a different origin. Failing to do so will result to an empty output frame. + */ + set: function (image) { + var _this = this; + if (!image || !image.complete || !image.naturalHeight) { + throw new Error('Invalid image. Make sure that the image is an HTMLImageElement and has been successfully loaded'); + } + this._backgroundImage = image; + createImageBitmap(this._backgroundImage).then(function (imageBitmap) { return _this._backgroundProcessorPipeline + .setBackgroundImage(imageBitmap); }).catch(function () { + /* noop */ + }); + }, + enumerable: false, + configurable: true + }); + Object.defineProperty(VirtualBackgroundProcessor.prototype, "fitType", { + /** + * The current [[ImageFit]] for positioning of the background image in the viewport. + */ + get: function () { + return this._fitType; + }, + /** + * Set a new [[ImageFit]] to be used for positioning the background image in the viewport. + */ + set: function (fitType) { + var validTypes = Object.keys(types_1.ImageFit); + if (!validTypes.includes(fitType)) { + console.warn("Valid fitType not found. Using '".concat(types_1.ImageFit.Fill, "' as default.")); + fitType = types_1.ImageFit.Fill; + } + this._fitType = fitType; + this._backgroundProcessorPipeline + .setFitType(this._fitType) + .catch(function () { + /* noop */ + }); + }, + enumerable: false, + configurable: true + }); + return VirtualBackgroundProcessor; +}(BackgroundProcessor_1.BackgroundProcessor)); +exports.VirtualBackgroundProcessor = VirtualBackgroundProcessor; +//# sourceMappingURL=VirtualBackgroundProcessor.js.map \ No newline at end of file diff --git a/es5/processors/background/VirtualBackgroundProcessor.js.map b/es5/processors/background/VirtualBackgroundProcessor.js.map new file mode 100644 index 0000000..7f94efc --- /dev/null +++ b/es5/processors/background/VirtualBackgroundProcessor.js.map @@ -0,0 +1 @@ +{"version":3,"file":"VirtualBackgroundProcessor.js","sourceRoot":"","sources":["../../../lib/processors/background/VirtualBackgroundProcessor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AAAA,qCAAuC;AACvC,6CAAmD;AACnD,+CAA4D;AAC5D,6DAAwF;AACxF,uFAAsI;AAyBtI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH;IAAgD,8CAAmB;IAMjE;;;;OAIG;IACH,oCAAY,OAA0C;QAAtD,iBA+BC;QA7BG,IAAA,eAAe,GAKb,OAAO,gBALM,EACf,KAIE,OAAO,yBAJuB,EAAhC,wBAAwB,mBAAG,KAAK,KAAA,EAChC,KAGE,OAAO,QAHc,EAAvB,OAAO,mBAAG,gBAAQ,CAAC,IAAI,KAAA,EACvB,KAEE,OAAO,eAFwB,EAAjC,cAAc,mBAAG,4BAAgB,KAAA,EACjC,KACE,OAAO,aADU,EAAnB,YAAY,mBAAG,IAAI,KAAA,CACT;QAEZ,IAAM,UAAU,GAAG,OAAO;aACvB,UAAU;aACV,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAE7B,IAAM,yCAAyC,GAAG,YAAY,IAAI,IAAA,+BAAqB,GAAE;YACvF,CAAC,CAAC,qEAAuC;YACzC,CAAC,CAAC,gEAAkC,CAAC;QAEvC,IAAM,2BAA2B,GAAG,IAAI,yCAAyC,CAAC;YAChF,UAAU,YAAA;YACV,wBAAwB,0BAAA;YACxB,OAAO,SAAA;YACP,cAAc,gBAAA;SACf,CAAC,CAAC;gBAEH,kBACE,2BAA2B,EAC3B,OAAO,CACR;QAnCH,8CAA8C;QAC7B,WAAK,GAAW,4BAA4B,CAAC;QAoC5D,KAAI,CAAC,eAAe,GAAG,eAAe,CAAC;QACvC,KAAI,CAAC,OAAO,GAAG,OAAO,CAAC;;IACzB,CAAC;IAKD,sBAAI,uDAAe;QAHnB;;WAEG;aACH;YACE,OAAO,IAAI,CAAC,gBAAgB,CAAC;QAC/B,CAAC;QAED;;;;;WAKG;aACH,UAAoB,KAAuB;YAA3C,iBAWC;YAVC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,aAAa,EAAE;gBACrD,MAAM,IAAI,KAAK,CAAC,iGAAiG,CAAC,CAAC;aACpH;YACD,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;YAC9B,iBAAiB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAC3C,UAAC,WAAW,IAAK,OAAC,KAAI,CAAC,4BAA6G;iBACjI,kBAAkB,CAAC,WAAW,CAAC,EADjB,CACiB,CACnC,CAAC,KAAK,CAAC;gBACN,UAAU;YACZ,CAAC,CAAC,CAAC;QACL,CAAC;;;OAnBA;IAwBD,sBAAI,+CAAO;QAHX;;WAEG;aACH;YACE,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QAED;;WAEG;aACH,UAAY,OAAiB;YAC3B,IAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,gBAAQ,CAAC,CAAC;YACzC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAiB,CAAC,EAAE;gBAC3C,OAAO,CAAC,IAAI,CAAC,0CAAmC,gBAAQ,CAAC,IAAI,kBAAe,CAAC,CAAC;gBAC9E,OAAO,GAAG,gBAAQ,CAAC,IAAI,CAAC;aACzB;YACD,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;YACvB,IAAI,CAAC,4BAA6G;iBAChH,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC;iBACzB,KAAK,CAAC;gBACL,UAAU;YACZ,CAAC,CAAC,CAAC;QACP,CAAC;;;OAjBA;IAkBH,iCAAC;AAAD,CAAC,AA7FD,CAAgD,yCAAmB,GA6FlE;AA7FY,gEAA0B","sourcesContent":["import { ImageFit } from '../../types';\nimport { MASK_BLUR_RADIUS } from '../../constants';\nimport { isChromiumImageBitmap } from '../../utils/support';\nimport { BackgroundProcessor, BackgroundProcessorOptions } from './BackgroundProcessor';\nimport { VirtualBackgroundProcessorPipeline, VirtualBackgroundProcessorPipelineProxy } from './pipelines/backgroundprocessorpipeline';\n\n/**\n * Options passed to [[VirtualBackgroundProcessor]] constructor.\n */\nexport interface VirtualBackgroundProcessorOptions extends BackgroundProcessorOptions {\n /**\n * The HTMLImageElement to use for background replacement.\n * An error will be raised if the image hasn't been fully loaded yet. Additionally, the image must follow\n * [security guidelines](https://developer.mozilla.org/en-US/docs/Web/HTML/CORS_enabled_image)\n * when loading the image from a different origin. Failing to do so will result to an empty output frame.\n */\n backgroundImage: HTMLImageElement;\n\n /**\n * The [[ImageFit]] to use for positioning of the background image in the viewport. Only the Canvas2D [[Pipeline]]\n * supports this option. WebGL2 ignores this option and falls back to Cover.\n * @default\n * ```html\n * 'Fill'\n * ```\n */\n fitType?: ImageFit;\n}\n\n/**\n * The VirtualBackgroundProcessor, when added to a VideoTrack,\n * replaces the background in each video frame with a given image,\n * and leaves the foreground (person(s)) untouched. Each instance of\n * VirtualBackgroundProcessor should be added to only one VideoTrack\n * at a time to prevent overlapping of image data from multiple VideoTracks.\n *\n * @example\n *\n * ```ts\n * import { createLocalVideoTrack } from 'twilio-video';\n * import { VirtualBackgroundProcessor } from '@twilio/video-processors';\n *\n * let virtualBackground: VirtualBackgroundProcessor;\n * const img = new Image();\n *\n * img.onload = async () => {\n * virtualBackground = new VirtualBackgroundProcessor({\n * assetsPath: 'https://my-server-path/assets',\n * backgroundImage: img\n * });\n * await virtualBackground.loadModel();\n *\n * const track = await createLocalVideoTrack({\n * // Increasing the capture resolution decreases the output FPS\n * // especially on browsers that do not support SIMD\n * // such as desktop Safari and iOS browsers, or on Chrome\n * // with capture resolutions above 640x480 for webgl2.\n * width: 640,\n * height: 480,\n *\n * // Any frame rate above 24 fps on desktop browsers increase CPU\n * // usage without noticeable increase in quality.\n * frameRate: 24\n * });\n * track.addProcessor(virtualBackground, {\n * inputFrameBufferType: 'videoframe',\n * outputFrameBufferContextType: 'bitmaprenderer'\n * });\n * };\n *\n * img.src = '/background.jpg';\n * ```\n */\nexport class VirtualBackgroundProcessor extends BackgroundProcessor {\n private _backgroundImage!: HTMLImageElement;\n private _fitType!: ImageFit;\n // tslint:disable-next-line no-unused-variable\n private readonly _name: string = 'VirtualBackgroundProcessor';\n\n /**\n * Construct a VirtualBackgroundProcessor. Default values will be used for\n * any missing optional properties in [[VirtualBackgroundProcessorOptions]],\n * and invalid properties will be ignored.\n */\n constructor(options: VirtualBackgroundProcessorOptions) {\n const {\n backgroundImage,\n deferInputFrameDownscale = false,\n fitType = ImageFit.Fill,\n maskBlurRadius = MASK_BLUR_RADIUS,\n useWebWorker = true\n } = options;\n\n const assetsPath = options\n .assetsPath\n .replace(/([^/])$/, '$1/');\n\n const VirtualBackgroundProcessorPipelineOrProxy = useWebWorker && isChromiumImageBitmap()\n ? VirtualBackgroundProcessorPipelineProxy\n : VirtualBackgroundProcessorPipeline;\n\n const backgroundProcessorPipeline = new VirtualBackgroundProcessorPipelineOrProxy({\n assetsPath,\n deferInputFrameDownscale,\n fitType,\n maskBlurRadius\n });\n\n super(\n backgroundProcessorPipeline,\n options\n );\n\n this.backgroundImage = backgroundImage;\n this.fitType = fitType;\n }\n\n /**\n * The HTMLImageElement representing the current background image.\n */\n get backgroundImage(): HTMLImageElement {\n return this._backgroundImage;\n }\n\n /**\n * Set an HTMLImageElement as the new background image.\n * An error will be raised if the image hasn't been fully loaded yet. Additionally, the image must follow\n * [security guidelines](https://developer.mozilla.org/en-US/docs/Web/HTML/CORS_enabled_image)\n * when loading the image from a different origin. Failing to do so will result to an empty output frame.\n */\n set backgroundImage(image: HTMLImageElement) {\n if (!image || !image.complete || !image.naturalHeight) {\n throw new Error('Invalid image. Make sure that the image is an HTMLImageElement and has been successfully loaded');\n }\n this._backgroundImage = image;\n createImageBitmap(this._backgroundImage).then(\n (imageBitmap) => (this._backgroundProcessorPipeline as VirtualBackgroundProcessorPipeline | VirtualBackgroundProcessorPipelineProxy)\n .setBackgroundImage(imageBitmap)\n ).catch(() => {\n /* noop */\n });\n }\n\n /**\n * The current [[ImageFit]] for positioning of the background image in the viewport.\n */\n get fitType(): ImageFit {\n return this._fitType;\n }\n\n /**\n * Set a new [[ImageFit]] to be used for positioning the background image in the viewport.\n */\n set fitType(fitType: ImageFit) {\n const validTypes = Object.keys(ImageFit);\n if (!validTypes.includes(fitType as string)) {\n console.warn(`Valid fitType not found. Using '${ImageFit.Fill}' as default.`);\n fitType = ImageFit.Fill;\n }\n this._fitType = fitType;\n (this._backgroundProcessorPipeline as VirtualBackgroundProcessorPipeline | VirtualBackgroundProcessorPipelineProxy)\n .setFitType(this._fitType)\n .catch(() => {\n /* noop */\n });\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.d.ts b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.d.ts new file mode 100644 index 0000000..dafaa41 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.d.ts @@ -0,0 +1,30 @@ +import { InputFrame } from '../../../../types'; +import { Pipeline } from '../../../pipelines'; +/** + * @private + */ +export interface BackgroundProcessorPipelineOptions { + assetsPath: string; + deferInputFrameDownscale: boolean; + maskBlurRadius: number; +} +/** + * @private + */ +export declare abstract class BackgroundProcessorPipeline extends Pipeline { + private static _twilioTFLite; + protected readonly _outputCanvas: OffscreenCanvas; + protected readonly _webgl2Canvas: OffscreenCanvas; + private readonly _assetsPath; + private readonly _benchmark; + private _deferInputFrameDownscale; + private readonly _inferenceInputCanvas; + private readonly _inputFrameDownscaleMode; + private readonly _onResizeWebGL2Canvas; + protected constructor(options: BackgroundProcessorPipelineOptions, onResizeWebGL2Canvas?: () => void); + loadTwilioTFLite(): Promise; + render(inputFrame: InputFrame): Promise; + setDeferInputFrameDownscale(defer: boolean): Promise; + setMaskBlurRadius(radius: number): Promise; + protected abstract _setBackground(inputFrame?: InputFrame): void; +} diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.js b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.js new file mode 100644 index 0000000..20fc8b0 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.js @@ -0,0 +1,182 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.BackgroundProcessorPipeline = void 0; +var constants_1 = require("../../../../constants"); +var Benchmark_1 = require("../../../../utils/Benchmark"); +var support_1 = require("../../../../utils/support"); +var TwilioTFLite_1 = require("../../../../utils/TwilioTFLite"); +var pipelines_1 = require("../../../pipelines"); +var InputFrameDownscaleStage_1 = require("./InputFrameDownscaleStage"); +var PostProcessingStage_1 = require("./PostProcessingStage"); +/** + * @private + */ +var BackgroundProcessorPipeline = /** @class */ (function (_super) { + __extends(BackgroundProcessorPipeline, _super); + function BackgroundProcessorPipeline(options, onResizeWebGL2Canvas) { + if (onResizeWebGL2Canvas === void 0) { onResizeWebGL2Canvas = function () { }; } + var _this = _super.call(this) || this; + _this._outputCanvas = new OffscreenCanvas(1, 1); + _this._webgl2Canvas = new OffscreenCanvas(1, 1); + _this._benchmark = new Benchmark_1.Benchmark(); + _this._inferenceInputCanvas = new OffscreenCanvas(constants_1.WASM_INFERENCE_DIMENSIONS.width, constants_1.WASM_INFERENCE_DIMENSIONS.height); + _this._inputFrameDownscaleMode = (0, support_1.isChromiumImageBitmap)() ? 'image-bitmap' : 'canvas'; + var assetsPath = options.assetsPath, deferInputFrameDownscale = options.deferInputFrameDownscale, maskBlurRadius = options.maskBlurRadius; + _this._assetsPath = assetsPath; + _this._deferInputFrameDownscale = deferInputFrameDownscale; + _this._onResizeWebGL2Canvas = onResizeWebGL2Canvas; + _this.addStage(new InputFrameDownscaleStage_1.InputFrameDowscaleStage(_this._inferenceInputCanvas, _this._inputFrameDownscaleMode)); + _this.addStage(new PostProcessingStage_1.PostProcessingStage(constants_1.WASM_INFERENCE_DIMENSIONS, _this._webgl2Canvas, _this._outputCanvas, maskBlurRadius, function (inputFrame) { return _this._setBackground(inputFrame); })); + return _this; + } + BackgroundProcessorPipeline.prototype.loadTwilioTFLite = function () { + return __awaiter(this, void 0, void 0, function () { + var _twilioTFLite; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + _twilioTFLite = BackgroundProcessorPipeline._twilioTFLite; + if (!!_twilioTFLite) return [3 /*break*/, 2]; + _twilioTFLite = new TwilioTFLite_1.TwilioTFLite(); + return [4 /*yield*/, _twilioTFLite.initialize(this._assetsPath, constants_1.MODEL_NAME, constants_1.TFLITE_LOADER_NAME, constants_1.TFLITE_SIMD_LOADER_NAME)]; + case 1: + _a.sent(); + BackgroundProcessorPipeline._twilioTFLite = _twilioTFLite; + _a.label = 2; + case 2: return [2 /*return*/, _twilioTFLite.isSimdEnabled]; + } + }); + }); + }; + BackgroundProcessorPipeline.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var _a, inputFrameDownscaleStage, postProcessingStage, _b, _benchmark, _deferInputFrameDownscale, _c, inferenceInputHeight, inferenceInputWidth, _outputCanvas, _webgl2Canvas, _twilioTFLite, isInputVideoFrame, _d, height, width, didResizeWebGL2Canvas, downscalePromise, personMask; + return __generator(this, function (_e) { + switch (_e.label) { + case 0: + if (!BackgroundProcessorPipeline._twilioTFLite) { + return [2 /*return*/, null]; + } + _a = this._stages, inputFrameDownscaleStage = _a[0], postProcessingStage = _a[1]; + _b = this, _benchmark = _b._benchmark, _deferInputFrameDownscale = _b._deferInputFrameDownscale, _c = _b._inferenceInputCanvas, inferenceInputHeight = _c.height, inferenceInputWidth = _c.width, _outputCanvas = _b._outputCanvas, _webgl2Canvas = _b._webgl2Canvas; + _twilioTFLite = BackgroundProcessorPipeline._twilioTFLite; + isInputVideoFrame = typeof VideoFrame === 'function' + && inputFrame instanceof VideoFrame; + _d = isInputVideoFrame + ? { height: inputFrame.displayHeight, width: inputFrame.displayWidth } + : inputFrame, height = _d.height, width = _d.width; + didResizeWebGL2Canvas = false; + if (_outputCanvas.width !== width) { + _outputCanvas.width = width; + _webgl2Canvas.width = width; + didResizeWebGL2Canvas = true; + } + if (_outputCanvas.height !== height) { + _outputCanvas.height = height; + _webgl2Canvas.height = height; + didResizeWebGL2Canvas = true; + } + if (didResizeWebGL2Canvas) { + postProcessingStage.resetPersonMaskUpscalePipeline(); + this._onResizeWebGL2Canvas(); + } + _benchmark.start('inputImageResizeDelay'); + downscalePromise = inputFrameDownscaleStage.render(inputFrame) + .then(function (downscaledFrameData) { + _twilioTFLite.loadInputBuffer(downscaledFrameData); + }); + if (!!_deferInputFrameDownscale) return [3 /*break*/, 2]; + return [4 /*yield*/, downscalePromise]; + case 1: + _e.sent(); + _e.label = 2; + case 2: + _benchmark.end('inputImageResizeDelay'); + _benchmark.start('segmentationDelay'); + personMask = new ImageData(_twilioTFLite.runInference(), inferenceInputWidth, inferenceInputHeight); + _benchmark.end('segmentationDelay'); + _benchmark.start('imageCompositionDelay'); + postProcessingStage.render(inputFrame, personMask); + _benchmark.end('imageCompositionDelay'); + if (typeof VideoFrame === 'function' + && inputFrame instanceof VideoFrame) { + inputFrame.close(); + } + return [2 /*return*/, this._outputCanvas]; + } + }); + }); + }; + BackgroundProcessorPipeline.prototype.setDeferInputFrameDownscale = function (defer) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + this._deferInputFrameDownscale = defer; + return [2 /*return*/]; + }); + }); + }; + BackgroundProcessorPipeline.prototype.setMaskBlurRadius = function (radius) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + this._stages[1] + .updateMaskBlurRadius(radius); + return [2 /*return*/]; + }); + }); + }; + BackgroundProcessorPipeline._twilioTFLite = null; + return BackgroundProcessorPipeline; +}(pipelines_1.Pipeline)); +exports.BackgroundProcessorPipeline = BackgroundProcessorPipeline; +//# sourceMappingURL=BackgroundProcessorPipeline.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.js.map b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.js.map new file mode 100644 index 0000000..c1e86d7 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.js.map @@ -0,0 +1 @@ +{"version":3,"file":"BackgroundProcessorPipeline.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mDAA2H;AAC3H,yDAAwD;AACxD,qDAAkE;AAClE,+DAA8D;AAE9D,gDAA8C;AAC9C,uEAAqE;AACrE,6DAA4D;AAW5D;;GAEG;AACH;IAA0D,+CAAQ;IAYhE,qCACE,OAA2C,EAC3C,oBAA+B;QAA/B,qCAAA,EAAA,qCAA8B,CAAC;QAFjC,YAIE,iBAAO,SAwBR;QArCkB,mBAAa,GAAG,IAAI,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC1C,mBAAa,GAAG,IAAI,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAE5C,gBAAU,GAAG,IAAI,qBAAS,EAAE,CAAC;QAE7B,2BAAqB,GAAG,IAAI,eAAe,CAAC,qCAAyB,CAAC,KAAK,EAAE,qCAAyB,CAAC,MAAM,CAAC,CAAC;QAC/G,8BAAwB,GAAG,IAAA,+BAAqB,GAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC;QAU5F,IAAA,UAAU,GAGR,OAAO,WAHC,EACV,wBAAwB,GAEtB,OAAO,yBAFe,EACxB,cAAc,GACZ,OAAO,eADK,CACJ;QAEZ,KAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,KAAI,CAAC,yBAAyB,GAAG,wBAAwB,CAAC;QAC1D,KAAI,CAAC,qBAAqB,GAAG,oBAAoB,CAAC;QAElD,KAAI,CAAC,QAAQ,CAAC,IAAI,kDAAuB,CACvC,KAAI,CAAC,qBAAqB,EAC1B,KAAI,CAAC,wBAAwB,CAC9B,CAAC,CAAC;QAEH,KAAI,CAAC,QAAQ,CAAC,IAAI,yCAAmB,CACnC,qCAAyB,EACzB,KAAI,CAAC,aAAa,EAClB,KAAI,CAAC,aAAa,EAClB,cAAc,EACd,UAAC,UAAuB,IAAW,OAAA,KAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAA/B,CAA+B,CACnE,CAAC,CAAC;;IACL,CAAC;IAEK,sDAAgB,GAAtB;;;;;;wBACQ,aAAa,GAAK,2BAA2B,cAAhC,CAAiC;6BAChD,CAAC,aAAa,EAAd,wBAAc;wBAChB,aAAa,GAAG,IAAI,2BAAY,EAAE,CAAC;wBACnC,qBAAM,aAAa,CAAC,UAAU,CAC5B,IAAI,CAAC,WAAW,EAChB,sBAAU,EACV,8BAAkB,EAClB,mCAAuB,CACxB,EAAA;;wBALD,SAKC,CAAC;wBACF,2BAA2B,CAAC,aAAa,GAAG,aAAa,CAAC;;4BAE5D,sBAAO,aAAa,CAAC,aAAc,EAAC;;;;KACrC;IAEK,4CAAM,GAAZ,UAAa,UAAsB;;;;;;wBACjC,IAAI,CAAC,2BAA2B,CAAC,aAAa,EAAE;4BAC9C,sBAAO,IAAI,EAAC;yBACb;wBAEK,KAGF,IAAI,CAAC,OAGR,EALC,wBAAwB,QAAA,EACxB,mBAAmB,QAAA,CAInB;wBAEI,KASF,IAAI,EARN,UAAU,gBAAA,EACV,yBAAyB,+BAAA,EACzB,6BAGC,EAFS,oBAAoB,YAAA,EACrB,mBAAmB,WAAA,EAE5B,aAAa,mBAAA,EACb,aAAa,mBAAA,CACN;wBAGP,aAAa,GACX,2BAA2B,cADhB,CACiB;wBAE1B,iBAAiB,GAAG,OAAO,UAAU,KAAK,UAAU;+BACrD,UAAU,YAAY,UAAU,CAAC;wBAEhC,KAAoB,iBAAiB;4BACzC,CAAC,CAAC,EAAE,MAAM,EAAE,UAAU,CAAC,aAAa,EAAE,KAAK,EAAE,UAAU,CAAC,YAAY,EAAE;4BACtE,CAAC,CAAC,UAAmD,EAF/C,MAAM,YAAA,EAAE,KAAK,WAAA,CAEmC;wBAEpD,qBAAqB,GAAG,KAAK,CAAC;wBAClC,IAAI,aAAa,CAAC,KAAK,KAAK,KAAK,EAAE;4BACjC,aAAa,CAAC,KAAK,GAAG,KAAK,CAAC;4BAC5B,aAAa,CAAC,KAAK,GAAG,KAAK,CAAC;4BAC5B,qBAAqB,GAAG,IAAI,CAAC;yBAC9B;wBACD,IAAI,aAAa,CAAC,MAAM,KAAK,MAAM,EAAE;4BACnC,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC;4BAC9B,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC;4BAC9B,qBAAqB,GAAG,IAAI,CAAC;yBAC9B;wBACD,IAAI,qBAAqB,EAAE;4BACzB,mBAAmB,CAAC,8BAA8B,EAAE,CAAC;4BACrD,IAAI,CAAC,qBAAqB,EAAE,CAAC;yBAC9B;wBAED,UAAU,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;wBACpC,gBAAgB,GAAG,wBAAwB,CAAC,MAAM,CAAC,UAAU,CAAC;6BACjE,IAAI,CAAC,UAAC,mBAAmB;4BACxB,aAAc,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;wBACtD,CAAC,CAAC,CAAC;6BAED,CAAC,yBAAyB,EAA1B,wBAA0B;wBAC5B,qBAAM,gBAAgB,EAAA;;wBAAtB,SAAsB,CAAC;;;wBAEzB,UAAU,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;wBAExC,UAAU,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;wBAChC,UAAU,GAAG,IAAI,SAAS,CAC9B,aAAc,CAAC,YAAY,EAAE,EAC7B,mBAAmB,EACnB,oBAAoB,CACrB,CAAC;wBACF,UAAU,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;wBAEpC,UAAU,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;wBAC1C,mBAAmB,CAAC,MAAM,CACxB,UAAU,EACV,UAAU,CACX,CAAC;wBACF,UAAU,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;wBAExC,IAAI,OAAO,UAAU,KAAK,UAAU;+BAC/B,UAAU,YAAY,UAAU,EAAE;4BACrC,UAAU,CAAC,KAAK,EAAE,CAAC;yBACpB;wBAED,sBAAO,IAAI,CAAC,aAAa,EAAC;;;;KAC3B;IAEK,iEAA2B,GAAjC,UAAkC,KAAc;;;gBAC9C,IAAI,CAAC,yBAAyB,GAAG,KAAK,CAAC;;;;KACxC;IAEK,uDAAiB,GAAvB,UAAwB,MAAc;;;gBACnC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAyB;qBACrC,oBAAoB,CAAC,MAAM,CAAC,CAAC;;;;KACjC;IApJc,yCAAa,GAAwB,IAAI,AAA5B,CAA6B;IAuJ3D,kCAAC;CAAA,AAxJD,CAA0D,oBAAQ,GAwJjE;AAxJqB,kEAA2B","sourcesContent":["import { MODEL_NAME, TFLITE_LOADER_NAME, TFLITE_SIMD_LOADER_NAME, WASM_INFERENCE_DIMENSIONS } from '../../../../constants';\nimport { Benchmark } from '../../../../utils/Benchmark';\nimport { isChromiumImageBitmap } from '../../../../utils/support';\nimport { TwilioTFLite } from '../../../../utils/TwilioTFLite';\nimport { InputFrame } from '../../../../types';\nimport { Pipeline } from '../../../pipelines';\nimport { InputFrameDowscaleStage } from './InputFrameDownscaleStage';\nimport { PostProcessingStage } from './PostProcessingStage';\n\n/**\n * @private\n */\nexport interface BackgroundProcessorPipelineOptions {\n assetsPath: string;\n deferInputFrameDownscale: boolean;\n maskBlurRadius: number;\n}\n\n/**\n * @private\n */\nexport abstract class BackgroundProcessorPipeline extends Pipeline {\n private static _twilioTFLite: TwilioTFLite | null = null;\n\n protected readonly _outputCanvas = new OffscreenCanvas(1, 1);\n protected readonly _webgl2Canvas = new OffscreenCanvas(1, 1);\n private readonly _assetsPath: string;\n private readonly _benchmark = new Benchmark();\n private _deferInputFrameDownscale: boolean;\n private readonly _inferenceInputCanvas = new OffscreenCanvas(WASM_INFERENCE_DIMENSIONS.width, WASM_INFERENCE_DIMENSIONS.height);\n private readonly _inputFrameDownscaleMode = isChromiumImageBitmap() ? 'image-bitmap' : 'canvas';\n private readonly _onResizeWebGL2Canvas: () => void;\n\n protected constructor(\n options: BackgroundProcessorPipelineOptions,\n onResizeWebGL2Canvas = () => {}\n ) {\n super();\n\n const {\n assetsPath,\n deferInputFrameDownscale,\n maskBlurRadius\n } = options;\n\n this._assetsPath = assetsPath;\n this._deferInputFrameDownscale = deferInputFrameDownscale;\n this._onResizeWebGL2Canvas = onResizeWebGL2Canvas;\n\n this.addStage(new InputFrameDowscaleStage(\n this._inferenceInputCanvas,\n this._inputFrameDownscaleMode\n ));\n\n this.addStage(new PostProcessingStage(\n WASM_INFERENCE_DIMENSIONS,\n this._webgl2Canvas,\n this._outputCanvas,\n maskBlurRadius,\n (inputFrame?: InputFrame): void => this._setBackground(inputFrame)\n ));\n }\n\n async loadTwilioTFLite(): Promise {\n let { _twilioTFLite } = BackgroundProcessorPipeline;\n if (!_twilioTFLite) {\n _twilioTFLite = new TwilioTFLite();\n await _twilioTFLite.initialize(\n this._assetsPath,\n MODEL_NAME,\n TFLITE_LOADER_NAME,\n TFLITE_SIMD_LOADER_NAME\n );\n BackgroundProcessorPipeline._twilioTFLite = _twilioTFLite;\n }\n return _twilioTFLite.isSimdEnabled!;\n }\n\n async render(inputFrame: InputFrame): Promise {\n if (!BackgroundProcessorPipeline._twilioTFLite) {\n return null;\n }\n\n const [\n inputFrameDownscaleStage,\n postProcessingStage\n ] = this._stages as [\n InputFrameDowscaleStage,\n PostProcessingStage\n ];\n\n const {\n _benchmark,\n _deferInputFrameDownscale,\n _inferenceInputCanvas: {\n height: inferenceInputHeight,\n width: inferenceInputWidth\n },\n _outputCanvas,\n _webgl2Canvas\n } = this;\n\n const {\n _twilioTFLite\n } = BackgroundProcessorPipeline;\n\n const isInputVideoFrame = typeof VideoFrame === 'function'\n && inputFrame instanceof VideoFrame;\n\n const { height, width } = isInputVideoFrame\n ? { height: inputFrame.displayHeight, width: inputFrame.displayWidth }\n : inputFrame as (OffscreenCanvas | HTMLCanvasElement);\n\n let didResizeWebGL2Canvas = false;\n if (_outputCanvas.width !== width) {\n _outputCanvas.width = width;\n _webgl2Canvas.width = width;\n didResizeWebGL2Canvas = true;\n }\n if (_outputCanvas.height !== height) {\n _outputCanvas.height = height;\n _webgl2Canvas.height = height;\n didResizeWebGL2Canvas = true;\n }\n if (didResizeWebGL2Canvas) {\n postProcessingStage.resetPersonMaskUpscalePipeline();\n this._onResizeWebGL2Canvas();\n }\n\n _benchmark.start('inputImageResizeDelay');\n const downscalePromise = inputFrameDownscaleStage.render(inputFrame)\n .then((downscaledFrameData) => {\n _twilioTFLite!.loadInputBuffer(downscaledFrameData);\n });\n\n if (!_deferInputFrameDownscale) {\n await downscalePromise;\n }\n _benchmark.end('inputImageResizeDelay');\n\n _benchmark.start('segmentationDelay');\n const personMask = new ImageData(\n _twilioTFLite!.runInference(),\n inferenceInputWidth,\n inferenceInputHeight\n );\n _benchmark.end('segmentationDelay');\n\n _benchmark.start('imageCompositionDelay');\n postProcessingStage.render(\n inputFrame,\n personMask\n );\n _benchmark.end('imageCompositionDelay');\n\n if (typeof VideoFrame === 'function'\n && inputFrame instanceof VideoFrame) {\n inputFrame.close();\n }\n\n return this._outputCanvas;\n }\n\n async setDeferInputFrameDownscale(defer: boolean): Promise {\n this._deferInputFrameDownscale = defer;\n }\n\n async setMaskBlurRadius(radius: number): Promise {\n (this._stages[1] as PostProcessingStage)\n .updateMaskBlurRadius(radius);\n }\n\n protected abstract _setBackground(inputFrame?: InputFrame): void;\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.proxy.d.ts b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.proxy.d.ts new file mode 100644 index 0000000..6e26ec7 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.proxy.d.ts @@ -0,0 +1,14 @@ +import { Remote } from 'comlink'; +import { BackgroundProcessorPipeline } from './BackgroundProcessorPipeline'; +/** + * @private + */ +export declare class BackgroundProcessorPipelineProxy { + protected readonly _pipelineWorkerPromise: Promise>; + private readonly _benchmark; + protected constructor(pipelineWorkerPromise: Promise>); + loadTwilioTFLite(): Promise; + render(inputFrame: VideoFrame): Promise; + setDeferInputFrameDownscale(defer: boolean): Promise; + setMaskBlurRadius(radius: number): Promise; +} diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.proxy.js b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.proxy.js new file mode 100644 index 0000000..1757372 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.proxy.js @@ -0,0 +1,114 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.BackgroundProcessorPipelineProxy = void 0; +var comlink_1 = require("comlink"); +var Benchmark_1 = require("../../../../utils/Benchmark"); +/** + * @private + */ +var BackgroundProcessorPipelineProxy = /** @class */ (function () { + function BackgroundProcessorPipelineProxy(pipelineWorkerPromise) { + this._benchmark = new Benchmark_1.Benchmark(); + this._pipelineWorkerPromise = pipelineWorkerPromise; + } + BackgroundProcessorPipelineProxy.prototype.loadTwilioTFLite = function () { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.loadTwilioTFLite()]; + } + }); + }); + }; + BackgroundProcessorPipelineProxy.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker, outputFrame, _a, _b; + return __generator(this, function (_c) { + switch (_c.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _c.sent(); + return [4 /*yield*/, pipelineWorker.render((0, comlink_1.transfer)(inputFrame, [inputFrame]))]; + case 2: + outputFrame = _c.sent(); + // @ts-ignore + _b = (_a = this._benchmark).merge; + return [4 /*yield*/, pipelineWorker._benchmark]; + case 3: + // @ts-ignore + _b.apply(_a, [_c.sent()]); + return [2 /*return*/, outputFrame]; + } + }); + }); + }; + BackgroundProcessorPipelineProxy.prototype.setDeferInputFrameDownscale = function (defer) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.setDeferInputFrameDownscale(defer)]; + } + }); + }); + }; + BackgroundProcessorPipelineProxy.prototype.setMaskBlurRadius = function (radius) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.setMaskBlurRadius(radius)]; + } + }); + }); + }; + return BackgroundProcessorPipelineProxy; +}()); +exports.BackgroundProcessorPipelineProxy = BackgroundProcessorPipelineProxy; +//# sourceMappingURL=BackgroundProcessorPipeline.proxy.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.proxy.js.map b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.proxy.js.map new file mode 100644 index 0000000..ac381af --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.proxy.js.map @@ -0,0 +1 @@ +{"version":3,"file":"BackgroundProcessorPipeline.proxy.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/backgroundprocessorpipeline/BackgroundProcessorPipeline.proxy.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mCAA2C;AAC3C,yDAAwD;AAGxD;;GAEG;AACH;IAIE,0CACE,qBAAmE;QAHpD,eAAU,GAAc,IAAI,qBAAS,EAAE,CAAC;QAKvD,IAAI,CAAC,sBAAsB,GAAG,qBAAqB,CAAC;IACtD,CAAC;IAEK,2DAAgB,GAAtB;;;;;4BACyB,qBAAM,IAAI,CAAC,sBAAsB,EAAA;;wBAAlD,cAAc,GAAG,SAAiC;wBACxD,sBAAO,cAAc,CAAC,gBAAgB,EAAE,EAAC;;;;KAC1C;IAEK,iDAAM,GAAZ,UAAa,UAAsB;;;;;4BACV,qBAAM,IAAI,CAAC,sBAAsB,EAAA;;wBAAlD,cAAc,GAAG,SAAiC;wBACpC,qBAAM,cAAc,CAAC,MAAM,CAC7C,IAAA,kBAAQ,EAAC,UAAU,EAAE,CAAC,UAAU,CAAC,CAAC,CACnC,EAAA;;wBAFK,WAAW,GAAG,SAEnB;wBACD,aAAa;wBACb,KAAA,CAAA,KAAA,IAAI,CAAC,UAAU,CAAA,CAAC,KAAK,CAAA;wBAAC,qBAAM,cAAc,CAAC,UAAU,EAAA;;wBADrD,aAAa;wBACb,cAAsB,SAA+B,EAAC,CAAC;wBACvD,sBAAO,WAAiC,EAAC;;;;KAC1C;IAEK,sEAA2B,GAAjC,UAAkC,KAAc;;;;;4BACvB,qBAAM,IAAI,CAAC,sBAAsB,EAAA;;wBAAlD,cAAc,GAAG,SAAiC;wBACxD,sBAAO,cAAc,CAAC,2BAA2B,CAAC,KAAK,CAAC,EAAC;;;;KAC1D;IAEK,4DAAiB,GAAvB,UAAwB,MAAc;;;;;4BACb,qBAAM,IAAI,CAAC,sBAAsB,EAAA;;wBAAlD,cAAc,GAAG,SAAiC;wBACxD,sBAAO,cAAc,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAC;;;;KACjD;IACH,uCAAC;AAAD,CAAC,AAlCD,IAkCC;AAlCY,4EAAgC","sourcesContent":["import { Remote, transfer } from 'comlink';\nimport { Benchmark } from '../../../../utils/Benchmark';\nimport { BackgroundProcessorPipeline } from './BackgroundProcessorPipeline';\n\n/**\n * @private\n */\nexport class BackgroundProcessorPipelineProxy {\n protected readonly _pipelineWorkerPromise: Promise>;\n private readonly _benchmark: Benchmark = new Benchmark();\n\n protected constructor(\n pipelineWorkerPromise: Promise>\n ) {\n this._pipelineWorkerPromise = pipelineWorkerPromise;\n }\n\n async loadTwilioTFLite(): Promise {\n const pipelineWorker = await this._pipelineWorkerPromise;\n return pipelineWorker.loadTwilioTFLite();\n }\n\n async render(inputFrame: VideoFrame): Promise {\n const pipelineWorker = await this._pipelineWorkerPromise;\n const outputFrame = await pipelineWorker.render(\n transfer(inputFrame, [inputFrame])\n );\n // @ts-ignore\n this._benchmark.merge(await pipelineWorker._benchmark);\n return outputFrame as ImageBitmap | null;\n }\n\n async setDeferInputFrameDownscale(defer: boolean): Promise {\n const pipelineWorker = await this._pipelineWorkerPromise;\n return pipelineWorker.setDeferInputFrameDownscale(defer);\n }\n\n async setMaskBlurRadius(radius: number): Promise {\n const pipelineWorker = await this._pipelineWorkerPromise;\n return pipelineWorker.setMaskBlurRadius(radius);\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.d.ts b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.d.ts new file mode 100644 index 0000000..8e1d608 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.d.ts @@ -0,0 +1,19 @@ +import { InputFrame } from '../../../../types'; +import { BackgroundProcessorPipeline, BackgroundProcessorPipelineOptions } from './BackgroundProcessorPipeline'; +/** + * @private + */ +export interface GaussianBlurBackgroundProcessorPipelineOptions extends BackgroundProcessorPipelineOptions { + blurFilterRadius: number; +} +/** + * @private + */ +export declare class GaussianBlurBackgroundProcessorPipeline extends BackgroundProcessorPipeline { + private _blurFilterRadius; + private _gaussianBlurFilterPipeline; + constructor(options: GaussianBlurBackgroundProcessorPipelineOptions); + setBlurFilterRadius(radius: number): Promise; + protected _setBackground(inputFrame: InputFrame): void; + private _resetGaussianBlurFilterPipeline; +} diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.js b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.js new file mode 100644 index 0000000..fc508cf --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.js @@ -0,0 +1,104 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurBackgroundProcessorPipeline = void 0; +var support_1 = require("../../../../utils/support"); +var gaussianblurfilterpipeline_1 = require("../gaussianblurfilterpipeline"); +var BackgroundProcessorPipeline_1 = require("./BackgroundProcessorPipeline"); +/** + * @private + */ +var GaussianBlurBackgroundProcessorPipeline = /** @class */ (function (_super) { + __extends(GaussianBlurBackgroundProcessorPipeline, _super); + function GaussianBlurBackgroundProcessorPipeline(options) { + var _this = _super.call(this, options, function () { return _this._resetGaussianBlurFilterPipeline(); }) || this; + _this._gaussianBlurFilterPipeline = null; + var blurFilterRadius = options.blurFilterRadius; + _this._blurFilterRadius = blurFilterRadius; + return _this; + } + GaussianBlurBackgroundProcessorPipeline.prototype.setBlurFilterRadius = function (radius) { + var _a; + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_b) { + this._blurFilterRadius = radius; + return [2 /*return*/, (_a = this._gaussianBlurFilterPipeline) === null || _a === void 0 ? void 0 : _a.updateRadius(this._blurFilterRadius)]; + }); + }); + }; + GaussianBlurBackgroundProcessorPipeline.prototype._setBackground = function (inputFrame) { + var _a = this, _blurFilterRadius = _a._blurFilterRadius, _outputCanvas = _a._outputCanvas, _webgl2Canvas = _a._webgl2Canvas; + var ctx = _outputCanvas.getContext('2d'); + if (support_1.isCanvasBlurSupported) { + ctx.filter = "blur(".concat(_blurFilterRadius, "px)"); + ctx.drawImage(inputFrame, 0, 0); + ctx.filter = 'none'; + return; + } + if (!this._gaussianBlurFilterPipeline) { + this._resetGaussianBlurFilterPipeline(); + } + this._gaussianBlurFilterPipeline.render(); + ctx.drawImage(_webgl2Canvas, 0, 0); + }; + GaussianBlurBackgroundProcessorPipeline.prototype._resetGaussianBlurFilterPipeline = function () { + var _a; + var _b = this, _blurFilterRadius = _b._blurFilterRadius, _webgl2Canvas = _b._webgl2Canvas; + (_a = this._gaussianBlurFilterPipeline) === null || _a === void 0 ? void 0 : _a.cleanUp(); + this._gaussianBlurFilterPipeline = new gaussianblurfilterpipeline_1.GaussianBlurFilterPipeline(_webgl2Canvas); + this._gaussianBlurFilterPipeline.updateRadius(_blurFilterRadius); + }; + return GaussianBlurBackgroundProcessorPipeline; +}(BackgroundProcessorPipeline_1.BackgroundProcessorPipeline)); +exports.GaussianBlurBackgroundProcessorPipeline = GaussianBlurBackgroundProcessorPipeline; +//# sourceMappingURL=GaussianBlurBackgroundProcessorPipeline.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.js.map b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.js.map new file mode 100644 index 0000000..4ac1299 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.js.map @@ -0,0 +1 @@ +{"version":3,"file":"GaussianBlurBackgroundProcessorPipeline.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,qDAAkE;AAClE,4EAA2E;AAC3E,6EAAgH;AAShH;;GAEG;AACH;IAA6D,2DAA2B;IAItF,iDAAY,OAAuD;QAAnE,YACE,kBACE,OAAO,EACP,cAAM,OAAA,KAAI,CAAC,gCAAgC,EAAE,EAAvC,CAAuC,CAC9C,SAOF;QAbO,iCAA2B,GAAsC,IAAI,CAAC;QAS1E,IAAA,gBAAgB,GACd,OAAO,iBADO,CACN;QAEZ,KAAI,CAAC,iBAAiB,GAAG,gBAAgB,CAAC;;IAC5C,CAAC;IAEK,qEAAmB,GAAzB,UAA0B,MAAc;;;;gBACtC,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC;gBAChC,sBAAO,MAAA,IAAI,CAAC,2BAA2B,0CAAE,YAAY,CACnD,IAAI,CAAC,iBAAiB,CACvB,EAAC;;;KACH;IAES,gEAAc,GAAxB,UAAyB,UAAsB;QACvC,IAAA,KAIF,IAAI,EAHN,iBAAiB,uBAAA,EACjB,aAAa,mBAAA,EACb,aAAa,mBACP,CAAC;QAET,IAAM,GAAG,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,CAAE,CAAC;QAC5C,IAAI,+BAAqB,EAAE;YACzB,GAAG,CAAC,MAAM,GAAG,eAAQ,iBAAiB,QAAK,CAAC;YAC5C,GAAG,CAAC,SAAS,CACX,UAAU,EACV,CAAC,EACD,CAAC,CACF,CAAC;YACF,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;YACpB,OAAO;SACR;QACD,IAAI,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACrC,IAAI,CAAC,gCAAgC,EAAE,CAAC;SACzC;QACD,IAAI,CAAC,2BAA4B,CAAC,MAAM,EAAE,CAAC;QAC3C,GAAG,CAAC,SAAS,CACX,aAAa,EACb,CAAC,EACD,CAAC,CACF,CAAC;IACJ,CAAC;IAEO,kFAAgC,GAAxC;;QACQ,IAAA,KAGF,IAAI,EAFN,iBAAiB,uBAAA,EACjB,aAAa,mBACP,CAAC;QACT,MAAA,IAAI,CAAC,2BAA2B,0CAAE,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC,2BAA2B,GAAG,IAAI,uDAA0B,CAC/D,aAAa,CACd,CAAC;QACF,IAAI,CAAC,2BAA4B,CAAC,YAAY,CAC5C,iBAAiB,CAClB,CAAC;IACJ,CAAC;IACH,8CAAC;AAAD,CAAC,AAlED,CAA6D,yDAA2B,GAkEvF;AAlEY,0FAAuC","sourcesContent":["import { InputFrame } from '../../../../types';\nimport { isCanvasBlurSupported } from '../../../../utils/support';\nimport { GaussianBlurFilterPipeline } from '../gaussianblurfilterpipeline';\nimport { BackgroundProcessorPipeline, BackgroundProcessorPipelineOptions } from './BackgroundProcessorPipeline';\n\n/**\n * @private\n */\nexport interface GaussianBlurBackgroundProcessorPipelineOptions extends BackgroundProcessorPipelineOptions {\n blurFilterRadius: number;\n}\n\n/**\n * @private\n */\nexport class GaussianBlurBackgroundProcessorPipeline extends BackgroundProcessorPipeline {\n private _blurFilterRadius: number;\n private _gaussianBlurFilterPipeline: GaussianBlurFilterPipeline | null = null;\n\n constructor(options: GaussianBlurBackgroundProcessorPipelineOptions) {\n super(\n options,\n () => this._resetGaussianBlurFilterPipeline()\n );\n\n const {\n blurFilterRadius\n } = options;\n\n this._blurFilterRadius = blurFilterRadius;\n }\n\n async setBlurFilterRadius(radius: number): Promise {\n this._blurFilterRadius = radius;\n return this._gaussianBlurFilterPipeline?.updateRadius(\n this._blurFilterRadius\n );\n }\n\n protected _setBackground(inputFrame: InputFrame): void {\n const {\n _blurFilterRadius,\n _outputCanvas,\n _webgl2Canvas\n } = this;\n\n const ctx = _outputCanvas.getContext('2d')!;\n if (isCanvasBlurSupported) {\n ctx.filter = `blur(${_blurFilterRadius}px)`;\n ctx.drawImage(\n inputFrame,\n 0,\n 0\n );\n ctx.filter = 'none';\n return;\n }\n if (!this._gaussianBlurFilterPipeline) {\n this._resetGaussianBlurFilterPipeline();\n }\n this._gaussianBlurFilterPipeline!.render();\n ctx.drawImage(\n _webgl2Canvas,\n 0,\n 0\n );\n }\n\n private _resetGaussianBlurFilterPipeline(): void {\n const {\n _blurFilterRadius,\n _webgl2Canvas\n } = this;\n this._gaussianBlurFilterPipeline?.cleanUp();\n this._gaussianBlurFilterPipeline = new GaussianBlurFilterPipeline(\n _webgl2Canvas\n );\n this._gaussianBlurFilterPipeline!.updateRadius(\n _blurFilterRadius\n );\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.proxy.d.ts b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.proxy.d.ts new file mode 100644 index 0000000..15fe6dd --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.proxy.d.ts @@ -0,0 +1,11 @@ +import { Remote } from 'comlink'; +import { BackgroundProcessorPipelineProxy } from './BackgroundProcessorPipeline.proxy'; +import { GaussianBlurBackgroundProcessorPipeline, GaussianBlurBackgroundProcessorPipelineOptions } from './GaussianBlurBackgroundProcessorPipeline'; +/** + * @private + */ +export declare class GaussianBlurBackgroundProcessorPipelineProxy extends BackgroundProcessorPipelineProxy { + protected readonly _pipelineWorkerPromise: Promise>; + constructor(options: GaussianBlurBackgroundProcessorPipelineOptions); + setBlurFilterRadius(radius: number): Promise; +} diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.proxy.js b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.proxy.js new file mode 100644 index 0000000..f17a5c8 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.proxy.js @@ -0,0 +1,88 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurBackgroundProcessorPipelineProxy = void 0; +var comlink_1 = require("comlink"); +var constants_1 = require("../../../../constants"); +var BackgroundProcessorPipeline_proxy_1 = require("./BackgroundProcessorPipeline.proxy"); +var GaussianBlurBackgroundProcessorPipelineWorker; +/** + * @private + */ +var GaussianBlurBackgroundProcessorPipelineProxy = /** @class */ (function (_super) { + __extends(GaussianBlurBackgroundProcessorPipelineProxy, _super); + function GaussianBlurBackgroundProcessorPipelineProxy(options) { + var _this = this; + GaussianBlurBackgroundProcessorPipelineWorker || (GaussianBlurBackgroundProcessorPipelineWorker = (0, comlink_1.wrap)(new Worker("".concat(options.assetsPath).concat(constants_1.TWILIO_GAUSSIAN_BLUR_BACKGROUND_PROCESSOR_PIPELINE_WORKER)))); + var pipelineWorkerPromise = new GaussianBlurBackgroundProcessorPipelineWorker(options); + _this = _super.call(this, pipelineWorkerPromise) || this; + _this._pipelineWorkerPromise = pipelineWorkerPromise; + return _this; + } + GaussianBlurBackgroundProcessorPipelineProxy.prototype.setBlurFilterRadius = function (radius) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.setBlurFilterRadius(radius)]; + } + }); + }); + }; + return GaussianBlurBackgroundProcessorPipelineProxy; +}(BackgroundProcessorPipeline_proxy_1.BackgroundProcessorPipelineProxy)); +exports.GaussianBlurBackgroundProcessorPipelineProxy = GaussianBlurBackgroundProcessorPipelineProxy; +//# sourceMappingURL=GaussianBlurBackgroundProcessorPipeline.proxy.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.proxy.js.map b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.proxy.js.map new file mode 100644 index 0000000..73296bf --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.proxy.js.map @@ -0,0 +1 @@ +{"version":3,"file":"GaussianBlurBackgroundProcessorPipeline.proxy.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.proxy.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mCAAuC;AACvC,mDAAkG;AAClG,yFAAuF;AAGvF,IAAI,6CAAqG,CAAC;AAE1G;;GAEG;AACH;IAAkE,gEAAgC;IAGhG,sDACE,OAAuD;QADzD,iBASC;QANC,6CAA6C,KAA7C,6CAA6C,GAAK,IAAA,cAAI,EACpD,IAAI,MAAM,CAAC,UAAG,OAAO,CAAC,UAAU,SAAG,qEAAyD,CAAE,CAAC,CAChG,EAAC;QACF,IAAM,qBAAqB,GAAG,IAAI,6CAA6C,CAAC,OAAO,CAAC,CAAC;gBACzF,kBAAM,qBAAqB,CAAC;QAC5B,KAAI,CAAC,sBAAsB,GAAG,qBAAqB,CAAC;;IACtD,CAAC;IAEK,0EAAmB,GAAzB,UAA0B,MAAc;;;;;4BACf,qBAAM,IAAI,CAAC,sBAAsB,EAAA;;wBAAlD,cAAc,GAAG,SAAiC;wBACxD,sBAAO,cAAc,CAAC,mBAAmB,CAAC,MAAM,CAAC,EAAC;;;;KACnD;IACH,mDAAC;AAAD,CAAC,AAlBD,CAAkE,oEAAgC,GAkBjG;AAlBY,oGAA4C","sourcesContent":["import { Remote, wrap } from 'comlink';\nimport { TWILIO_GAUSSIAN_BLUR_BACKGROUND_PROCESSOR_PIPELINE_WORKER } from '../../../../constants';\nimport { BackgroundProcessorPipelineProxy } from './BackgroundProcessorPipeline.proxy';\nimport { GaussianBlurBackgroundProcessorPipeline, GaussianBlurBackgroundProcessorPipelineOptions } from './GaussianBlurBackgroundProcessorPipeline';\n\nlet GaussianBlurBackgroundProcessorPipelineWorker: Remote;\n\n/**\n * @private\n */\nexport class GaussianBlurBackgroundProcessorPipelineProxy extends BackgroundProcessorPipelineProxy {\n protected readonly _pipelineWorkerPromise: Promise>;\n\n constructor(\n options: GaussianBlurBackgroundProcessorPipelineOptions\n ) {\n GaussianBlurBackgroundProcessorPipelineWorker ||= wrap(\n new Worker(`${options.assetsPath}${TWILIO_GAUSSIAN_BLUR_BACKGROUND_PROCESSOR_PIPELINE_WORKER}`)\n );\n const pipelineWorkerPromise = new GaussianBlurBackgroundProcessorPipelineWorker(options);\n super(pipelineWorkerPromise);\n this._pipelineWorkerPromise = pipelineWorkerPromise;\n }\n\n async setBlurFilterRadius(radius: number): Promise {\n const pipelineWorker = await this._pipelineWorkerPromise;\n return pipelineWorker.setBlurFilterRadius(radius);\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.worker.d.ts b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.worker.d.ts new file mode 100644 index 0000000..13d01dc --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.worker.d.ts @@ -0,0 +1,7 @@ +import { GaussianBlurBackgroundProcessorPipeline } from './GaussianBlurBackgroundProcessorPipeline'; +/** + * @private + */ +export declare class GaussianBlurBackgroundProcessorPipelineWorker extends GaussianBlurBackgroundProcessorPipeline { + render(inputFrame: VideoFrame): Promise; +} diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.worker.js b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.worker.js new file mode 100644 index 0000000..886d98a --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.worker.js @@ -0,0 +1,85 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurBackgroundProcessorPipelineWorker = void 0; +var comlink_1 = require("comlink"); +var GaussianBlurBackgroundProcessorPipeline_1 = require("./GaussianBlurBackgroundProcessorPipeline"); +/** + * @private + */ +var GaussianBlurBackgroundProcessorPipelineWorker = /** @class */ (function (_super) { + __extends(GaussianBlurBackgroundProcessorPipelineWorker, _super); + function GaussianBlurBackgroundProcessorPipelineWorker() { + return _super !== null && _super.apply(this, arguments) || this; + } + GaussianBlurBackgroundProcessorPipelineWorker.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var outputFrame, outputBitmap; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, _super.prototype.render.call(this, inputFrame)]; + case 1: + outputFrame = _a.sent(); + outputBitmap = outputFrame instanceof OffscreenCanvas + ? outputFrame.transferToImageBitmap() + : outputFrame; + return [2 /*return*/, outputBitmap && (0, comlink_1.transfer)(outputBitmap, [outputBitmap])]; + } + }); + }); + }; + return GaussianBlurBackgroundProcessorPipelineWorker; +}(GaussianBlurBackgroundProcessorPipeline_1.GaussianBlurBackgroundProcessorPipeline)); +exports.GaussianBlurBackgroundProcessorPipelineWorker = GaussianBlurBackgroundProcessorPipelineWorker; +(0, comlink_1.expose)(GaussianBlurBackgroundProcessorPipelineWorker); +//# sourceMappingURL=GaussianBlurBackgroundProcessorPipeline.worker.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.worker.js.map b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.worker.js.map new file mode 100644 index 0000000..24444fe --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.worker.js.map @@ -0,0 +1 @@ +{"version":3,"file":"GaussianBlurBackgroundProcessorPipeline.worker.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/backgroundprocessorpipeline/GaussianBlurBackgroundProcessorPipeline.worker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mCAA2C;AAC3C,qGAAoG;AAEpG;;GAEG;AACH;IAAmE,iEAAuC;IAA1G;;IAaA,CAAC;IAZO,8DAAM,GAAZ,UAAa,UAAsB;;;;;4BACb,qBAAM,iBAAM,MAAM,YAAC,UAAU,CAAC,EAAA;;wBAA5C,WAAW,GAAG,SAA8B;wBAE5C,YAAY,GAAG,WAAW,YAAY,eAAe;4BACzD,CAAC,CAAC,WAAW,CAAC,qBAAqB,EAAE;4BACrC,CAAC,CAAC,WAAW,CAAC;wBAEhB,sBAAO,YAAY,IAAI,IAAA,kBAAQ,EAC7B,YAAY,EACZ,CAAC,YAAY,CAAC,CACf,EAAC;;;;KACH;IACH,oDAAC;AAAD,CAAC,AAbD,CAAmE,iFAAuC,GAazG;AAbY,sGAA6C;AAe1D,IAAA,gBAAM,EAAC,6CAA6C,CAAC,CAAC","sourcesContent":["import { expose, transfer } from 'comlink';\nimport { GaussianBlurBackgroundProcessorPipeline } from './GaussianBlurBackgroundProcessorPipeline';\n\n/**\n * @private\n */\nexport class GaussianBlurBackgroundProcessorPipelineWorker extends GaussianBlurBackgroundProcessorPipeline {\n async render(inputFrame: VideoFrame): Promise {\n const outputFrame = await super.render(inputFrame);\n\n const outputBitmap = outputFrame instanceof OffscreenCanvas\n ? outputFrame.transferToImageBitmap()\n : outputFrame;\n\n return outputBitmap && transfer(\n outputBitmap,\n [outputBitmap]\n );\n }\n}\n\nexpose(GaussianBlurBackgroundProcessorPipelineWorker);\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.d.ts b/es5/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.d.ts new file mode 100644 index 0000000..fd98c93 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.d.ts @@ -0,0 +1,11 @@ +import { InputFrame } from '../../../../types'; +import { Pipeline } from '../../../pipelines'; +/** + * @private + */ +export declare class InputFrameDowscaleStage implements Pipeline.Stage { + private readonly _inputFrameDownscaleMode; + private readonly _outputContext; + constructor(outputCanvas: OffscreenCanvas, inputFrameDownscaleMode: 'canvas' | 'image-bitmap'); + render(inputFrame: InputFrame): Promise; +} diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.js b/es5/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.js new file mode 100644 index 0000000..f31a025 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.js @@ -0,0 +1,80 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.InputFrameDowscaleStage = void 0; +/** + * @private + */ +var InputFrameDowscaleStage = /** @class */ (function () { + function InputFrameDowscaleStage(outputCanvas, inputFrameDownscaleMode) { + this._inputFrameDownscaleMode = inputFrameDownscaleMode; + this._outputContext = outputCanvas.getContext('2d', { willReadFrequently: true }); + } + InputFrameDowscaleStage.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var _a, _outputContext, _inputFrameDownscaleMode, _b, resizeHeight, resizeWidth, downscaledBitmap, data; + return __generator(this, function (_c) { + switch (_c.label) { + case 0: + _a = this, _outputContext = _a._outputContext, _inputFrameDownscaleMode = _a._inputFrameDownscaleMode; + _b = _outputContext.canvas, resizeHeight = _b.height, resizeWidth = _b.width; + if (!(_inputFrameDownscaleMode === 'image-bitmap')) return [3 /*break*/, 2]; + return [4 /*yield*/, createImageBitmap(inputFrame, { + resizeWidth: resizeWidth, + resizeHeight: resizeHeight, + resizeQuality: 'pixelated' + })]; + case 1: + downscaledBitmap = _c.sent(); + _outputContext.drawImage(downscaledBitmap, 0, 0); + downscaledBitmap.close(); + return [3 /*break*/, 3]; + case 2: + _outputContext.drawImage(inputFrame, 0, 0, resizeWidth, resizeHeight); + _c.label = 3; + case 3: + data = _outputContext.getImageData(0, 0, resizeWidth, resizeHeight).data; + return [2 /*return*/, data]; + } + }); + }); + }; + return InputFrameDowscaleStage; +}()); +exports.InputFrameDowscaleStage = InputFrameDowscaleStage; +//# sourceMappingURL=InputFrameDownscaleStage.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.js.map b/es5/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.js.map new file mode 100644 index 0000000..da9c87e --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.js.map @@ -0,0 +1 @@ +{"version":3,"file":"InputFrameDownscaleStage.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/backgroundprocessorpipeline/InputFrameDownscaleStage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGA;;GAEG;AACH;IAIE,iCACE,YAA6B,EAC7B,uBAAkD;QAGlD,IAAI,CAAC,wBAAwB,GAAG,uBAAuB,CAAC;QACxD,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAE,CAAC;IACrF,CAAC;IAEK,wCAAM,GAAZ,UAAa,UAAsB;;;;;;wBAC3B,KAGF,IAAI,EAFN,cAAc,oBAAA,EACd,wBAAwB,8BAAA,CACjB;wBAGP,KAIE,cAAc,OADf,EAFS,YAAY,YAAA,EACb,WAAW,WAAA,CAEH;6BAEf,CAAA,wBAAwB,KAAK,cAAc,CAAA,EAA3C,wBAA2C;wBACpB,qBAAM,iBAAiB,CAC9C,UAAU,EACV;gCACE,WAAW,aAAA;gCACX,YAAY,cAAA;gCACZ,aAAa,EAAE,WAAW;6BAC3B,CACF,EAAA;;wBAPK,gBAAgB,GAAG,SAOxB;wBACD,cAAc,CAAC,SAAS,CACtB,gBAAgB,EAChB,CAAC,EACD,CAAC,CACF,CAAC;wBACF,gBAAgB,CAAC,KAAK,EAAE,CAAC;;;wBAEzB,cAAc,CAAC,SAAS,CACtB,UAAU,EACV,CAAC,EACD,CAAC,EACD,WAAW,EACX,YAAY,CACb,CAAC;;;wBAGI,IAAI,GAAK,cAAc,CAAC,YAAY,CAC1C,CAAC,EACD,CAAC,EACD,WAAW,EACX,YAAY,CACb,KALW,CAKV;wBAEF,sBAAO,IAAI,EAAC;;;;KACb;IACH,8BAAC;AAAD,CAAC,AA5DD,IA4DC;AA5DY,0DAAuB","sourcesContent":["import { InputFrame } from '../../../../types';\nimport { Pipeline } from '../../../pipelines';\n\n/**\n * @private\n */\nexport class InputFrameDowscaleStage implements Pipeline.Stage {\n private readonly _inputFrameDownscaleMode: 'canvas' | 'image-bitmap';\n private readonly _outputContext: OffscreenCanvasRenderingContext2D;\n\n constructor(\n outputCanvas: OffscreenCanvas,\n inputFrameDownscaleMode: 'canvas' | 'image-bitmap'\n\n) {\n this._inputFrameDownscaleMode = inputFrameDownscaleMode;\n this._outputContext = outputCanvas.getContext('2d', { willReadFrequently: true })!;\n }\n\n async render(inputFrame: InputFrame): Promise {\n const {\n _outputContext,\n _inputFrameDownscaleMode,\n } = this;\n\n const {\n canvas: {\n height: resizeHeight,\n width: resizeWidth\n }\n } = _outputContext;\n\n if (_inputFrameDownscaleMode === 'image-bitmap') {\n const downscaledBitmap = await createImageBitmap(\n inputFrame,\n {\n resizeWidth,\n resizeHeight,\n resizeQuality: 'pixelated'\n }\n );\n _outputContext.drawImage(\n downscaledBitmap,\n 0,\n 0\n );\n downscaledBitmap.close();\n } else {\n _outputContext.drawImage(\n inputFrame,\n 0,\n 0,\n resizeWidth,\n resizeHeight\n );\n }\n\n const { data } = _outputContext.getImageData(\n 0,\n 0,\n resizeWidth,\n resizeHeight\n );\n\n return data;\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.d.ts b/es5/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.d.ts new file mode 100644 index 0000000..b56b995 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.d.ts @@ -0,0 +1,17 @@ +import { Dimensions, InputFrame } from '../../../../types'; +import { Pipeline } from '../../../pipelines'; +/** + * @private + */ +export declare class PostProcessingStage implements Pipeline.Stage { + private readonly _inputDimensions; + private _maskBlurRadius; + private readonly _outputContext; + private _personMaskUpscalePipeline; + private readonly _setBackground; + private readonly _webgl2Canvas; + constructor(inputDimensions: Dimensions, webgl2Canvas: OffscreenCanvas, outputCanvas: OffscreenCanvas, maskBlurRadius: number, setBackground: (inputFrame?: InputFrame) => void); + render(inputFrame: InputFrame, personMask: ImageData): void; + resetPersonMaskUpscalePipeline(): void; + updateMaskBlurRadius(radius: number): void; +} diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.js b/es5/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.js new file mode 100644 index 0000000..9720922 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.js @@ -0,0 +1,51 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PostProcessingStage = void 0; +var personmaskupscalepipeline_1 = require("../personmaskupscalepipeline"); +/** + * @private + */ +var PostProcessingStage = /** @class */ (function () { + function PostProcessingStage(inputDimensions, webgl2Canvas, outputCanvas, maskBlurRadius, setBackground) { + this._personMaskUpscalePipeline = null; + this._inputDimensions = inputDimensions; + this._maskBlurRadius = maskBlurRadius; + this._outputContext = outputCanvas.getContext('2d'); + this._webgl2Canvas = webgl2Canvas; + this._setBackground = setBackground; + } + PostProcessingStage.prototype.render = function (inputFrame, personMask) { + var _a = this, _outputContext = _a._outputContext, _setBackground = _a._setBackground, _webgl2Canvas = _a._webgl2Canvas; + if (!this._personMaskUpscalePipeline) { + this.resetPersonMaskUpscalePipeline(); + } + this._personMaskUpscalePipeline.render(inputFrame, personMask); + _outputContext.save(); + _outputContext.globalCompositeOperation = 'copy'; + _outputContext.drawImage(_webgl2Canvas, 0, 0); + _outputContext.globalCompositeOperation = 'destination-over'; + _setBackground(inputFrame); + _outputContext.restore(); + }; + PostProcessingStage.prototype.resetPersonMaskUpscalePipeline = function () { + var _a; + var _b = this, _inputDimensions = _b._inputDimensions, _maskBlurRadius = _b._maskBlurRadius, _webgl2Canvas = _b._webgl2Canvas; + (_a = this._personMaskUpscalePipeline) === null || _a === void 0 ? void 0 : _a.cleanUp(); + this._personMaskUpscalePipeline = new personmaskupscalepipeline_1.PersonMaskUpscalePipeline(_inputDimensions, _webgl2Canvas); + this._personMaskUpscalePipeline.updateBilateralFilterConfig({ + sigmaSpace: _maskBlurRadius + }); + }; + PostProcessingStage.prototype.updateMaskBlurRadius = function (radius) { + var _a; + if (this._maskBlurRadius !== radius) { + this._maskBlurRadius = radius; + (_a = this._personMaskUpscalePipeline) === null || _a === void 0 ? void 0 : _a.updateBilateralFilterConfig({ + sigmaSpace: radius + }); + } + }; + return PostProcessingStage; +}()); +exports.PostProcessingStage = PostProcessingStage; +//# sourceMappingURL=PostProcessingStage.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.js.map b/es5/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.js.map new file mode 100644 index 0000000..b166440 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.js.map @@ -0,0 +1 @@ +{"version":3,"file":"PostProcessingStage.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/backgroundprocessorpipeline/PostProcessingStage.ts"],"names":[],"mappings":";;;AAEA,0EAAyE;AAEzE;;GAEG;AACH;IAQE,6BACE,eAA2B,EAC3B,YAA6B,EAC7B,YAA6B,EAC7B,cAAsB,EACtB,aAAgD;QAT1C,+BAA0B,GAAqC,IAAI,CAAC;QAW1E,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,IAAI,CAAC,eAAe,GAAG,cAAc,CAAC;QACtC,IAAI,CAAC,cAAc,GAAG,YAAY,CAAC,UAAU,CAAC,IAAI,CAAE,CAAC;QACrD,IAAI,CAAC,aAAa,GAAG,YAAY,CAAC;QAClC,IAAI,CAAC,cAAc,GAAG,aAAa,CAAC;IACtC,CAAC;IAED,oCAAM,GAAN,UACE,UAAsB,EACtB,UAAqB;QAEf,IAAA,KAIF,IAAI,EAHN,cAAc,oBAAA,EACd,cAAc,oBAAA,EACd,aAAa,mBACP,CAAC;QACT,IAAI,CAAC,IAAI,CAAC,0BAA0B,EAAE;YACpC,IAAI,CAAC,8BAA8B,EAAE,CAAC;SACvC;QACD,IAAI,CAAC,0BAA2B,CAAC,MAAM,CACrC,UAAU,EACV,UAAU,CACX,CAAC;QACF,cAAc,CAAC,IAAI,EAAE,CAAC;QACtB,cAAc,CAAC,wBAAwB,GAAG,MAAM,CAAC;QACjD,cAAc,CAAC,SAAS,CACtB,aAAa,EACb,CAAC,EACD,CAAC,CACF,CAAC;QACF,cAAc,CAAC,wBAAwB,GAAG,kBAAkB,CAAC;QAC7D,cAAc,CAAC,UAAU,CAAC,CAAC;QAC3B,cAAc,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;IAED,4DAA8B,GAA9B;;QACQ,IAAA,KAIF,IAAI,EAHN,gBAAgB,sBAAA,EAChB,eAAe,qBAAA,EACf,aAAa,mBACP,CAAC;QACT,MAAA,IAAI,CAAC,0BAA0B,0CAAE,OAAO,EAAE,CAAC;QAC3C,IAAI,CAAC,0BAA0B,GAAG,IAAI,qDAAyB,CAC7D,gBAAgB,EAChB,aAAa,CACd,CAAC;QACF,IAAI,CAAC,0BAA0B,CAAC,2BAA2B,CAAC;YAC1D,UAAU,EAAE,eAAe;SAC5B,CAAC,CAAC;IACL,CAAC;IAED,kDAAoB,GAApB,UAAqB,MAAc;;QACjC,IAAI,IAAI,CAAC,eAAe,KAAK,MAAM,EAAE;YACnC,IAAI,CAAC,eAAe,GAAG,MAAM,CAAC;YAC9B,MAAA,IAAI,CAAC,0BAA0B,0CAC3B,2BAA2B,CAAC;gBAC5B,UAAU,EAAE,MAAM;aACnB,CAAC,CAAC;SACN;IACH,CAAC;IACH,0BAAC;AAAD,CAAC,AA3ED,IA2EC;AA3EY,kDAAmB","sourcesContent":["import { Dimensions, InputFrame } from '../../../../types';\nimport { Pipeline } from '../../../pipelines';\nimport { PersonMaskUpscalePipeline } from '../personmaskupscalepipeline';\n\n/**\n * @private\n */\nexport class PostProcessingStage implements Pipeline.Stage {\n private readonly _inputDimensions: Dimensions;\n private _maskBlurRadius: number;\n private readonly _outputContext: OffscreenCanvasRenderingContext2D;\n private _personMaskUpscalePipeline: PersonMaskUpscalePipeline | null = null;\n private readonly _setBackground: (inputFrame?: InputFrame) => void;\n private readonly _webgl2Canvas: OffscreenCanvas;\n\n constructor(\n inputDimensions: Dimensions,\n webgl2Canvas: OffscreenCanvas,\n outputCanvas: OffscreenCanvas,\n maskBlurRadius: number,\n setBackground: (inputFrame?: InputFrame) => void\n ) {\n this._inputDimensions = inputDimensions;\n this._maskBlurRadius = maskBlurRadius;\n this._outputContext = outputCanvas.getContext('2d')!;\n this._webgl2Canvas = webgl2Canvas;\n this._setBackground = setBackground;\n }\n\n render(\n inputFrame: InputFrame,\n personMask: ImageData\n ): void {\n const {\n _outputContext,\n _setBackground,\n _webgl2Canvas\n } = this;\n if (!this._personMaskUpscalePipeline) {\n this.resetPersonMaskUpscalePipeline();\n }\n this._personMaskUpscalePipeline!.render(\n inputFrame,\n personMask\n );\n _outputContext.save();\n _outputContext.globalCompositeOperation = 'copy';\n _outputContext.drawImage(\n _webgl2Canvas,\n 0,\n 0\n );\n _outputContext.globalCompositeOperation = 'destination-over';\n _setBackground(inputFrame);\n _outputContext.restore();\n }\n\n resetPersonMaskUpscalePipeline(): void {\n const {\n _inputDimensions,\n _maskBlurRadius,\n _webgl2Canvas\n } = this;\n this._personMaskUpscalePipeline?.cleanUp();\n this._personMaskUpscalePipeline = new PersonMaskUpscalePipeline(\n _inputDimensions,\n _webgl2Canvas\n );\n this._personMaskUpscalePipeline.updateBilateralFilterConfig({\n sigmaSpace: _maskBlurRadius\n });\n }\n\n updateMaskBlurRadius(radius: number): void {\n if (this._maskBlurRadius !== radius) {\n this._maskBlurRadius = radius;\n this._personMaskUpscalePipeline\n ?.updateBilateralFilterConfig({\n sigmaSpace: radius\n });\n }\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.d.ts b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.d.ts new file mode 100644 index 0000000..3b57a52 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.d.ts @@ -0,0 +1,20 @@ +import { ImageFit } from '../../../../types'; +import { BackgroundProcessorPipeline, BackgroundProcessorPipelineOptions } from './BackgroundProcessorPipeline'; +/** + * @private + */ +export interface VirtualBackgroundProcessorPipelineOptions extends BackgroundProcessorPipelineOptions { + fitType: ImageFit; +} +/** + * @private + */ +export declare class VirtualBackgroundProcessorPipeline extends BackgroundProcessorPipeline { + private _backgroundImage; + private _fitType; + constructor(options: VirtualBackgroundProcessorPipelineOptions); + setBackgroundImage(backgroundImage: ImageBitmap): Promise; + setFitType(fitType: ImageFit): Promise; + protected _setBackground(): void; + private _getFitPosition; +} diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.js b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.js new file mode 100644 index 0000000..1f1b1b5 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.js @@ -0,0 +1,134 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessorPipeline = void 0; +var types_1 = require("../../../../types"); +var BackgroundProcessorPipeline_1 = require("./BackgroundProcessorPipeline"); +/** + * @private + */ +var VirtualBackgroundProcessorPipeline = /** @class */ (function (_super) { + __extends(VirtualBackgroundProcessorPipeline, _super); + function VirtualBackgroundProcessorPipeline(options) { + var _this = _super.call(this, options) || this; + var fitType = options.fitType; + _this._backgroundImage = null; + _this._fitType = fitType; + return _this; + } + VirtualBackgroundProcessorPipeline.prototype.setBackgroundImage = function (backgroundImage) { + var _a; + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_b) { + (_a = this._backgroundImage) === null || _a === void 0 ? void 0 : _a.close(); + this._backgroundImage = backgroundImage; + return [2 /*return*/]; + }); + }); + }; + VirtualBackgroundProcessorPipeline.prototype.setFitType = function (fitType) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + this._fitType = fitType; + return [2 /*return*/]; + }); + }); + }; + VirtualBackgroundProcessorPipeline.prototype._setBackground = function () { + var _a = this, _backgroundImage = _a._backgroundImage, _fitType = _a._fitType, _outputCanvas = _a._outputCanvas; + if (!_backgroundImage) { + return; + } + var ctx = _outputCanvas.getContext('2d'); + var imageWidth = _backgroundImage.width; + var imageHeight = _backgroundImage.height; + var canvasWidth = _outputCanvas.width; + var canvasHeight = _outputCanvas.height; + if (_fitType === types_1.ImageFit.Fill) { + ctx.drawImage(_backgroundImage, 0, 0, imageWidth, imageHeight, 0, 0, canvasWidth, canvasHeight); + } + else if (_fitType === types_1.ImageFit.None) { + ctx.drawImage(_backgroundImage, 0, 0, imageWidth, imageHeight); + } + else { + var _b = this._getFitPosition(imageWidth, imageHeight, canvasWidth, canvasHeight, _fitType), x = _b.x, y = _b.y, w = _b.w, h = _b.h; + ctx.drawImage(_backgroundImage, 0, 0, imageWidth, imageHeight, x, y, w, h); + } + }; + VirtualBackgroundProcessorPipeline.prototype._getFitPosition = function (contentWidth, contentHeight, viewportWidth, viewportHeight, type) { + // Calculate new content width to fit viewport width + var factor = viewportWidth / contentWidth; + var newContentWidth = viewportWidth; + var newContentHeight = factor * contentHeight; + // Scale down the resulting height and width more + // to fit viewport height if the content still exceeds it + if ((type === types_1.ImageFit.Contain && newContentHeight > viewportHeight) + || (type === types_1.ImageFit.Cover && viewportHeight > newContentHeight)) { + factor = viewportHeight / newContentHeight; + newContentWidth = factor * newContentWidth; + newContentHeight = viewportHeight; + } + // Calculate the destination top left corner to center the content + var x = (viewportWidth - newContentWidth) / 2; + var y = (viewportHeight - newContentHeight) / 2; + return { + x: x, + y: y, + w: newContentWidth, + h: newContentHeight, + }; + }; + return VirtualBackgroundProcessorPipeline; +}(BackgroundProcessorPipeline_1.BackgroundProcessorPipeline)); +exports.VirtualBackgroundProcessorPipeline = VirtualBackgroundProcessorPipeline; +//# sourceMappingURL=VirtualBackgroundProcessorPipeline.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.js.map b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.js.map new file mode 100644 index 0000000..fdab301 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.js.map @@ -0,0 +1 @@ +{"version":3,"file":"VirtualBackgroundProcessorPipeline.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6C;AAC7C,6EAAgH;AAShH;;GAEG;AACH;IAAwD,sDAA2B;IAIjF,4CAAY,OAAkD;QAA9D,YACE,kBAAM,OAAO,CAAC,SAQf;QALG,IAAA,OAAO,GACL,OAAO,QADF,CACG;QAEZ,KAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,KAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;;IAC1B,CAAC;IAEK,+DAAkB,GAAxB,UACE,eAA4B;;;;gBAE5B,MAAA,IAAI,CAAC,gBAAgB,0CAAE,KAAK,EAAE,CAAC;gBAC/B,IAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;;;;KACzC;IAEK,uDAAU,GAAhB,UAAiB,OAAiB;;;gBAChC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;;;;KACzB;IAES,2DAAc,GAAxB;QACQ,IAAA,KAIF,IAAI,EAHN,gBAAgB,sBAAA,EAChB,QAAQ,cAAA,EACR,aAAa,mBACP,CAAC;QAET,IAAI,CAAC,gBAAgB,EAAE;YACrB,OAAO;SACR;QAED,IAAM,GAAG,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,CAAE,CAAC;QAC5C,IAAM,UAAU,GAAG,gBAAgB,CAAC,KAAK,CAAC;QAC1C,IAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC;QAC5C,IAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC;QACxC,IAAM,YAAY,GAAG,aAAa,CAAC,MAAM,CAAC;QAE1C,IAAI,QAAQ,KAAK,gBAAQ,CAAC,IAAI,EAAE;YAC9B,GAAG,CAAC,SAAS,CACX,gBAAgB,EAChB,CAAC,EACD,CAAC,EACD,UAAU,EACV,WAAW,EACX,CAAC,EACD,CAAC,EACD,WAAW,EACX,YAAY,CACb,CAAC;SACH;aAAM,IAAI,QAAQ,KAAK,gBAAQ,CAAC,IAAI,EAAE;YACrC,GAAG,CAAC,SAAS,CACX,gBAAgB,EAChB,CAAC,EACD,CAAC,EACD,UAAU,EACV,WAAW,CACZ,CAAC;SACH;aAAM;YACC,IAAA,KAAiB,IAAI,CAAC,eAAe,CACzC,UAAU,EACV,WAAW,EACX,WAAW,EACX,YAAY,EACZ,QAAQ,CACT,EANO,CAAC,OAAA,EAAE,CAAC,OAAA,EAAE,CAAC,OAAA,EAAE,CAAC,OAMjB,CAAC;YACF,GAAG,CAAC,SAAS,CACX,gBAAgB,EAChB,CAAC,EACD,CAAC,EACD,UAAU,EACV,WAAW,EACX,CAAC,EACD,CAAC,EACD,CAAC,EACD,CAAC,CACF,CAAC;SACH;IACH,CAAC;IAEO,4DAAe,GAAvB,UACE,YAAoB,EACpB,aAAqB,EACrB,aAAqB,EACrB,cAAsB,EACtB,IAAc;QAQd,oDAAoD;QACpD,IAAI,MAAM,GAAG,aAAa,GAAG,YAAY,CAAC;QAC1C,IAAI,eAAe,GAAG,aAAa,CAAC;QACpC,IAAI,gBAAgB,GAAG,MAAM,GAAG,aAAa,CAAC;QAE9C,iDAAiD;QACjD,yDAAyD;QACzD,IAAI,CAAC,IAAI,KAAK,gBAAQ,CAAC,OAAO,IAAI,gBAAgB,GAAG,cAAc,CAAC;eAC/D,CAAC,IAAI,KAAK,gBAAQ,CAAC,KAAK,IAAI,cAAc,GAAG,gBAAgB,CAAC,EAAE;YACnE,MAAM,GAAG,cAAc,GAAG,gBAAgB,CAAC;YAC3C,eAAe,GAAG,MAAM,GAAG,eAAe,CAAC;YAC3C,gBAAgB,GAAG,cAAc,CAAC;SACnC;QAED,kEAAkE;QAClE,IAAM,CAAC,GAAG,CAAC,aAAa,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;QAChD,IAAM,CAAC,GAAG,CAAC,cAAc,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAElD,OAAO;YACL,CAAC,GAAA;YACD,CAAC,GAAA;YACD,CAAC,EAAE,eAAe;YAClB,CAAC,EAAE,gBAAgB;SACpB,CAAC;IACJ,CAAC;IACH,yCAAC;AAAD,CAAC,AA3HD,CAAwD,yDAA2B,GA2HlF;AA3HY,gFAAkC","sourcesContent":["import { ImageFit } from '../../../../types';\nimport { BackgroundProcessorPipeline, BackgroundProcessorPipelineOptions } from './BackgroundProcessorPipeline';\n\n/**\n * @private\n */\nexport interface VirtualBackgroundProcessorPipelineOptions extends BackgroundProcessorPipelineOptions {\n fitType: ImageFit;\n}\n\n/**\n * @private\n */\nexport class VirtualBackgroundProcessorPipeline extends BackgroundProcessorPipeline {\n private _backgroundImage: ImageBitmap | null;\n private _fitType: ImageFit;\n\n constructor(options: VirtualBackgroundProcessorPipelineOptions) {\n super(options);\n\n const {\n fitType\n } = options;\n\n this._backgroundImage = null;\n this._fitType = fitType;\n }\n\n async setBackgroundImage(\n backgroundImage: ImageBitmap\n ): Promise {\n this._backgroundImage?.close();\n this._backgroundImage = backgroundImage;\n }\n\n async setFitType(fitType: ImageFit): Promise {\n this._fitType = fitType;\n }\n\n protected _setBackground(): void {\n const {\n _backgroundImage,\n _fitType,\n _outputCanvas\n } = this;\n\n if (!_backgroundImage) {\n return;\n }\n\n const ctx = _outputCanvas.getContext('2d')!;\n const imageWidth = _backgroundImage.width;\n const imageHeight = _backgroundImage.height;\n const canvasWidth = _outputCanvas.width;\n const canvasHeight = _outputCanvas.height;\n\n if (_fitType === ImageFit.Fill) {\n ctx.drawImage(\n _backgroundImage,\n 0,\n 0,\n imageWidth,\n imageHeight,\n 0,\n 0,\n canvasWidth,\n canvasHeight\n );\n } else if (_fitType === ImageFit.None) {\n ctx.drawImage(\n _backgroundImage,\n 0,\n 0,\n imageWidth,\n imageHeight\n );\n } else {\n const { x, y, w, h } = this._getFitPosition(\n imageWidth,\n imageHeight,\n canvasWidth,\n canvasHeight,\n _fitType\n );\n ctx.drawImage(\n _backgroundImage,\n 0,\n 0,\n imageWidth,\n imageHeight,\n x,\n y,\n w,\n h\n );\n }\n }\n\n private _getFitPosition(\n contentWidth: number,\n contentHeight: number,\n viewportWidth: number,\n viewportHeight: number,\n type: ImageFit\n ): {\n h: number,\n w: number,\n x: number,\n y: number\n } {\n\n // Calculate new content width to fit viewport width\n let factor = viewportWidth / contentWidth;\n let newContentWidth = viewportWidth;\n let newContentHeight = factor * contentHeight;\n\n // Scale down the resulting height and width more\n // to fit viewport height if the content still exceeds it\n if ((type === ImageFit.Contain && newContentHeight > viewportHeight)\n || (type === ImageFit.Cover && viewportHeight > newContentHeight)) {\n factor = viewportHeight / newContentHeight;\n newContentWidth = factor * newContentWidth;\n newContentHeight = viewportHeight;\n }\n\n // Calculate the destination top left corner to center the content\n const x = (viewportWidth - newContentWidth) / 2;\n const y = (viewportHeight - newContentHeight) / 2;\n\n return {\n x,\n y,\n w: newContentWidth,\n h: newContentHeight,\n };\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.proxy.d.ts b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.proxy.d.ts new file mode 100644 index 0000000..b4860ea --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.proxy.d.ts @@ -0,0 +1,13 @@ +import { Remote } from 'comlink'; +import { ImageFit } from '../../../../types'; +import { BackgroundProcessorPipelineProxy } from './BackgroundProcessorPipeline.proxy'; +import { VirtualBackgroundProcessorPipeline, VirtualBackgroundProcessorPipelineOptions } from './VirtualBackgroundProcessorPipeline'; +/** + * @private + */ +export declare class VirtualBackgroundProcessorPipelineProxy extends BackgroundProcessorPipelineProxy { + protected readonly _pipelineWorkerPromise: Promise>; + constructor(options: VirtualBackgroundProcessorPipelineOptions); + setBackgroundImage(backgroundImage: ImageBitmap): Promise; + setFitType(fitType: ImageFit): Promise; +} diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.proxy.js b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.proxy.js new file mode 100644 index 0000000..04546ef --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.proxy.js @@ -0,0 +1,101 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessorPipelineProxy = void 0; +var comlink_1 = require("comlink"); +var constants_1 = require("../../../../constants"); +var BackgroundProcessorPipeline_proxy_1 = require("./BackgroundProcessorPipeline.proxy"); +var VirtualBackgroundProcessorPipelineWorker; +/** + * @private + */ +var VirtualBackgroundProcessorPipelineProxy = /** @class */ (function (_super) { + __extends(VirtualBackgroundProcessorPipelineProxy, _super); + function VirtualBackgroundProcessorPipelineProxy(options) { + var _this = this; + VirtualBackgroundProcessorPipelineWorker || (VirtualBackgroundProcessorPipelineWorker = (0, comlink_1.wrap)(new Worker("".concat(options.assetsPath).concat(constants_1.TWILIO_VIRTUAL_BACKGROUND_PROCESSOR_PIPELINE_WORKER)))); + var pipelineWorkerPromise = new VirtualBackgroundProcessorPipelineWorker(options); + _this = _super.call(this, pipelineWorkerPromise) || this; + _this._pipelineWorkerPromise = pipelineWorkerPromise; + return _this; + } + VirtualBackgroundProcessorPipelineProxy.prototype.setBackgroundImage = function (backgroundImage) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.setBackgroundImage((0, comlink_1.transfer)(backgroundImage, [backgroundImage]))]; + } + }); + }); + }; + VirtualBackgroundProcessorPipelineProxy.prototype.setFitType = function (fitType) { + return __awaiter(this, void 0, void 0, function () { + var pipelineWorker; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, this._pipelineWorkerPromise]; + case 1: + pipelineWorker = _a.sent(); + return [2 /*return*/, pipelineWorker.setFitType(fitType)]; + } + }); + }); + }; + return VirtualBackgroundProcessorPipelineProxy; +}(BackgroundProcessorPipeline_proxy_1.BackgroundProcessorPipelineProxy)); +exports.VirtualBackgroundProcessorPipelineProxy = VirtualBackgroundProcessorPipelineProxy; +//# sourceMappingURL=VirtualBackgroundProcessorPipeline.proxy.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.proxy.js.map b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.proxy.js.map new file mode 100644 index 0000000..4733760 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.proxy.js.map @@ -0,0 +1 @@ +{"version":3,"file":"VirtualBackgroundProcessorPipeline.proxy.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.proxy.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mCAAiD;AACjD,mDAA4F;AAE5F,yFAAuF;AAGvF,IAAI,wCAA2F,CAAC;AAEhG;;GAEG;AACH;IAA6D,2DAAgC;IAG3F,iDACE,OAAkD;QADpD,iBASC;QANC,wCAAwC,KAAxC,wCAAwC,GAAK,IAAA,cAAI,EAC/C,IAAI,MAAM,CAAC,UAAG,OAAO,CAAC,UAAU,SAAG,+DAAmD,CAAE,CAAC,CAC1F,EAAC;QACF,IAAM,qBAAqB,GAAG,IAAI,wCAAwC,CAAC,OAAO,CAAC,CAAC;gBACpF,kBAAM,qBAAqB,CAAC;QAC5B,KAAI,CAAC,sBAAsB,GAAG,qBAAqB,CAAC;;IACtD,CAAC;IAEK,oEAAkB,GAAxB,UAAyB,eAA4B;;;;;4BAC5B,qBAAM,IAAI,CAAC,sBAAsB,EAAA;;wBAAlD,cAAc,GAAG,SAAiC;wBACxD,sBAAO,cAAc,CAAC,kBAAkB,CACtC,IAAA,kBAAQ,EAAC,eAAe,EAAE,CAAC,eAAe,CAAC,CAAC,CAC7C,EAAC;;;;KACH;IAEK,4DAAU,GAAhB,UAAiB,OAAiB;;;;;4BACT,qBAAM,IAAI,CAAC,sBAAsB,EAAA;;wBAAlD,cAAc,GAAG,SAAiC;wBACxD,sBAAO,cAAc,CAAC,UAAU,CAAC,OAAO,CAAC,EAAC;;;;KAC3C;IACH,8CAAC;AAAD,CAAC,AAzBD,CAA6D,oEAAgC,GAyB5F;AAzBY,0FAAuC","sourcesContent":["import { Remote, transfer, wrap } from 'comlink';\nimport { TWILIO_VIRTUAL_BACKGROUND_PROCESSOR_PIPELINE_WORKER } from '../../../../constants';\nimport { ImageFit } from '../../../../types';\nimport { BackgroundProcessorPipelineProxy } from './BackgroundProcessorPipeline.proxy';\nimport { VirtualBackgroundProcessorPipeline, VirtualBackgroundProcessorPipelineOptions } from './VirtualBackgroundProcessorPipeline';\n\nlet VirtualBackgroundProcessorPipelineWorker: Remote;\n\n/**\n * @private\n */\nexport class VirtualBackgroundProcessorPipelineProxy extends BackgroundProcessorPipelineProxy {\n protected readonly _pipelineWorkerPromise: Promise>;\n\n constructor(\n options: VirtualBackgroundProcessorPipelineOptions\n ) {\n VirtualBackgroundProcessorPipelineWorker ||= wrap(\n new Worker(`${options.assetsPath}${TWILIO_VIRTUAL_BACKGROUND_PROCESSOR_PIPELINE_WORKER}`)\n );\n const pipelineWorkerPromise = new VirtualBackgroundProcessorPipelineWorker(options);\n super(pipelineWorkerPromise);\n this._pipelineWorkerPromise = pipelineWorkerPromise;\n }\n\n async setBackgroundImage(backgroundImage: ImageBitmap): Promise {\n const pipelineWorker = await this._pipelineWorkerPromise;\n return pipelineWorker.setBackgroundImage(\n transfer(backgroundImage, [backgroundImage])\n );\n }\n\n async setFitType(fitType: ImageFit): Promise {\n const pipelineWorker = await this._pipelineWorkerPromise;\n return pipelineWorker.setFitType(fitType);\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.worker.d.ts b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.worker.d.ts new file mode 100644 index 0000000..717b48a --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.worker.d.ts @@ -0,0 +1,7 @@ +import { VirtualBackgroundProcessorPipeline } from './VirtualBackgroundProcessorPipeline'; +/** + * @private + */ +export declare class VirtualBackgroundProcessorPipelineWorker extends VirtualBackgroundProcessorPipeline { + render(inputFrame: VideoFrame): Promise; +} diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.worker.js b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.worker.js new file mode 100644 index 0000000..e05b8a1 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.worker.js @@ -0,0 +1,85 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessorPipelineWorker = void 0; +var comlink_1 = require("comlink"); +var VirtualBackgroundProcessorPipeline_1 = require("./VirtualBackgroundProcessorPipeline"); +/** + * @private + */ +var VirtualBackgroundProcessorPipelineWorker = /** @class */ (function (_super) { + __extends(VirtualBackgroundProcessorPipelineWorker, _super); + function VirtualBackgroundProcessorPipelineWorker() { + return _super !== null && _super.apply(this, arguments) || this; + } + VirtualBackgroundProcessorPipelineWorker.prototype.render = function (inputFrame) { + return __awaiter(this, void 0, void 0, function () { + var outputFrame, outputBitmap; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, _super.prototype.render.call(this, inputFrame)]; + case 1: + outputFrame = _a.sent(); + outputBitmap = outputFrame instanceof OffscreenCanvas + ? outputFrame.transferToImageBitmap() + : outputFrame; + return [2 /*return*/, outputBitmap && (0, comlink_1.transfer)(outputBitmap, [outputBitmap])]; + } + }); + }); + }; + return VirtualBackgroundProcessorPipelineWorker; +}(VirtualBackgroundProcessorPipeline_1.VirtualBackgroundProcessorPipeline)); +exports.VirtualBackgroundProcessorPipelineWorker = VirtualBackgroundProcessorPipelineWorker; +(0, comlink_1.expose)(VirtualBackgroundProcessorPipelineWorker); +//# sourceMappingURL=VirtualBackgroundProcessorPipeline.worker.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.worker.js.map b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.worker.js.map new file mode 100644 index 0000000..0a38d41 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.worker.js.map @@ -0,0 +1 @@ +{"version":3,"file":"VirtualBackgroundProcessorPipeline.worker.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/backgroundprocessorpipeline/VirtualBackgroundProcessorPipeline.worker.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,mCAA2C;AAC3C,2FAA0F;AAE1F;;GAEG;AACH;IAA8D,4DAAkC;IAAhG;;IAaA,CAAC;IAZO,yDAAM,GAAZ,UAAa,UAAsB;;;;;4BACb,qBAAM,iBAAM,MAAM,YAAC,UAAU,CAAC,EAAA;;wBAA5C,WAAW,GAAG,SAA8B;wBAE5C,YAAY,GAAG,WAAW,YAAY,eAAe;4BACzD,CAAC,CAAC,WAAW,CAAC,qBAAqB,EAAE;4BACrC,CAAC,CAAC,WAAW,CAAC;wBAEhB,sBAAO,YAAY,IAAI,IAAA,kBAAQ,EAC7B,YAAY,EACZ,CAAC,YAAY,CAAC,CACf,EAAC;;;;KACH;IACH,+CAAC;AAAD,CAAC,AAbD,CAA8D,uEAAkC,GAa/F;AAbY,4FAAwC;AAerD,IAAA,gBAAM,EAAC,wCAAwC,CAAC,CAAC","sourcesContent":["import { expose, transfer } from 'comlink';\nimport { VirtualBackgroundProcessorPipeline } from './VirtualBackgroundProcessorPipeline';\n\n/**\n * @private\n */\nexport class VirtualBackgroundProcessorPipelineWorker extends VirtualBackgroundProcessorPipeline {\n async render(inputFrame: VideoFrame): Promise {\n const outputFrame = await super.render(inputFrame);\n\n const outputBitmap = outputFrame instanceof OffscreenCanvas\n ? outputFrame.transferToImageBitmap()\n : outputFrame;\n\n return outputBitmap && transfer(\n outputBitmap,\n [outputBitmap]\n );\n }\n}\n\nexpose(VirtualBackgroundProcessorPipelineWorker);\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/index.d.ts b/es5/processors/background/pipelines/backgroundprocessorpipeline/index.d.ts new file mode 100644 index 0000000..648e18f --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/index.d.ts @@ -0,0 +1,6 @@ +export { BackgroundProcessorPipeline } from './BackgroundProcessorPipeline'; +export { BackgroundProcessorPipelineProxy } from './BackgroundProcessorPipeline.proxy'; +export { GaussianBlurBackgroundProcessorPipeline } from './GaussianBlurBackgroundProcessorPipeline'; +export { GaussianBlurBackgroundProcessorPipelineProxy } from './GaussianBlurBackgroundProcessorPipeline.proxy'; +export { VirtualBackgroundProcessorPipeline } from './VirtualBackgroundProcessorPipeline'; +export { VirtualBackgroundProcessorPipelineProxy } from './VirtualBackgroundProcessorPipeline.proxy'; diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/index.js b/es5/processors/background/pipelines/backgroundprocessorpipeline/index.js new file mode 100644 index 0000000..ebe91fd --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/index.js @@ -0,0 +1,16 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.VirtualBackgroundProcessorPipelineProxy = exports.VirtualBackgroundProcessorPipeline = exports.GaussianBlurBackgroundProcessorPipelineProxy = exports.GaussianBlurBackgroundProcessorPipeline = exports.BackgroundProcessorPipelineProxy = exports.BackgroundProcessorPipeline = void 0; +var BackgroundProcessorPipeline_1 = require("./BackgroundProcessorPipeline"); +Object.defineProperty(exports, "BackgroundProcessorPipeline", { enumerable: true, get: function () { return BackgroundProcessorPipeline_1.BackgroundProcessorPipeline; } }); +var BackgroundProcessorPipeline_proxy_1 = require("./BackgroundProcessorPipeline.proxy"); +Object.defineProperty(exports, "BackgroundProcessorPipelineProxy", { enumerable: true, get: function () { return BackgroundProcessorPipeline_proxy_1.BackgroundProcessorPipelineProxy; } }); +var GaussianBlurBackgroundProcessorPipeline_1 = require("./GaussianBlurBackgroundProcessorPipeline"); +Object.defineProperty(exports, "GaussianBlurBackgroundProcessorPipeline", { enumerable: true, get: function () { return GaussianBlurBackgroundProcessorPipeline_1.GaussianBlurBackgroundProcessorPipeline; } }); +var GaussianBlurBackgroundProcessorPipeline_proxy_1 = require("./GaussianBlurBackgroundProcessorPipeline.proxy"); +Object.defineProperty(exports, "GaussianBlurBackgroundProcessorPipelineProxy", { enumerable: true, get: function () { return GaussianBlurBackgroundProcessorPipeline_proxy_1.GaussianBlurBackgroundProcessorPipelineProxy; } }); +var VirtualBackgroundProcessorPipeline_1 = require("./VirtualBackgroundProcessorPipeline"); +Object.defineProperty(exports, "VirtualBackgroundProcessorPipeline", { enumerable: true, get: function () { return VirtualBackgroundProcessorPipeline_1.VirtualBackgroundProcessorPipeline; } }); +var VirtualBackgroundProcessorPipeline_proxy_1 = require("./VirtualBackgroundProcessorPipeline.proxy"); +Object.defineProperty(exports, "VirtualBackgroundProcessorPipelineProxy", { enumerable: true, get: function () { return VirtualBackgroundProcessorPipeline_proxy_1.VirtualBackgroundProcessorPipelineProxy; } }); +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/backgroundprocessorpipeline/index.js.map b/es5/processors/background/pipelines/backgroundprocessorpipeline/index.js.map new file mode 100644 index 0000000..50feca3 --- /dev/null +++ b/es5/processors/background/pipelines/backgroundprocessorpipeline/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/backgroundprocessorpipeline/index.ts"],"names":[],"mappings":";;;AAAA,6EAA4E;AAAnE,0IAAA,2BAA2B,OAAA;AACpC,yFAAuF;AAA9E,qJAAA,gCAAgC,OAAA;AACzC,qGAAoG;AAA3F,kKAAA,uCAAuC,OAAA;AAChD,iHAA+G;AAAtG,6KAAA,4CAA4C,OAAA;AACrD,2FAA0F;AAAjF,wJAAA,kCAAkC,OAAA;AAC3C,uGAAqG;AAA5F,mKAAA,uCAAuC,OAAA","sourcesContent":["export { BackgroundProcessorPipeline } from './BackgroundProcessorPipeline';\nexport { BackgroundProcessorPipelineProxy } from './BackgroundProcessorPipeline.proxy';\nexport { GaussianBlurBackgroundProcessorPipeline } from './GaussianBlurBackgroundProcessorPipeline';\nexport { GaussianBlurBackgroundProcessorPipelineProxy } from './GaussianBlurBackgroundProcessorPipeline.proxy';\nexport { VirtualBackgroundProcessorPipeline } from './VirtualBackgroundProcessorPipeline';\nexport { VirtualBackgroundProcessorPipelineProxy } from './VirtualBackgroundProcessorPipeline.proxy';\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.d.ts b/es5/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.d.ts new file mode 100644 index 0000000..2122f4e --- /dev/null +++ b/es5/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.d.ts @@ -0,0 +1,8 @@ +import { WebGL2Pipeline } from '../../../pipelines'; +/** + * @private + */ +export declare class SinglePassGaussianBlurFilterStage extends WebGL2Pipeline.ProcessingStage { + constructor(glOut: WebGL2RenderingContext, direction: 'horizontal' | 'vertical', outputType: 'canvas' | 'texture', inputTextureUnit: number, outputTextureUnit?: number); + updateRadius(radius: number): void; +} diff --git a/es5/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.js b/es5/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.js new file mode 100644 index 0000000..72e0da8 --- /dev/null +++ b/es5/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.js @@ -0,0 +1,81 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.SinglePassGaussianBlurFilterStage = void 0; +var pipelines_1 = require("../../../pipelines"); +/** + * @private + */ +function createGaussianBlurWeights(radius) { + var coeff = 1.0 / Math.sqrt(2.0 * Math.PI) / radius; + return '0'.repeat(radius + 1).split('').map(function (zero, x) { + return coeff * Math.exp(-0.5 * x * x / radius / radius); + }); +} +/** + * @private + */ +var SinglePassGaussianBlurFilterStage = /** @class */ (function (_super) { + __extends(SinglePassGaussianBlurFilterStage, _super); + function SinglePassGaussianBlurFilterStage(glOut, direction, outputType, inputTextureUnit, outputTextureUnit) { + var _this = this; + if (outputTextureUnit === void 0) { outputTextureUnit = inputTextureUnit + 1; } + var _a = glOut.canvas, height = _a.height, width = _a.width; + _this = _super.call(this, { + textureName: 'u_inputTexture', + textureUnit: inputTextureUnit + }, { + fragmentShaderSource: "#version 300 es\n precision highp float;\n\n uniform sampler2D u_inputTexture;\n uniform vec2 u_texelSize;\n uniform float u_direction;\n uniform float u_radius;\n uniform float u_gaussianBlurWeights[128];\n\n in vec2 v_texCoord;\n\n out vec4 outColor;\n\n void main() {\n float totalWeight = u_gaussianBlurWeights[0];\n vec3 newColor = totalWeight * texture(u_inputTexture, v_texCoord).rgb;\n\n for (float i = 1.0; i <= u_radius; i += 1.0) {\n float x = (1.0 - u_direction) * i;\n float y = u_direction * i;\n\n vec2 shift = vec2(x, y) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n float weight = u_gaussianBlurWeights[int(i)];\n newColor += weight * texture(u_inputTexture, coord).rgb;\n totalWeight += weight;\n\n shift = vec2(-x, -y) * u_texelSize;\n coord = vec2(v_texCoord + shift);\n newColor += weight * texture(u_inputTexture, coord).rgb;\n totalWeight += weight;\n }\n\n newColor /= totalWeight;\n outColor = vec4(newColor, 1.0);\n }\n ", + glOut: glOut, + height: height, + textureUnit: outputTextureUnit, + type: outputType, + width: width, + uniformVars: [ + { + name: 'u_direction', + type: 'float', + values: [direction === 'vertical' ? 1 : 0] + }, + { + name: 'u_texelSize', + type: 'float', + values: [1 / width, 1 / height] + } + ] + }) || this; + _this.updateRadius(0); + return _this; + } + SinglePassGaussianBlurFilterStage.prototype.updateRadius = function (radius) { + this._setUniformVars([ + { + name: 'u_radius', + type: 'float', + values: [radius] + }, + { + name: 'u_gaussianBlurWeights', + type: 'float:v', + values: createGaussianBlurWeights(radius) + } + ]); + }; + return SinglePassGaussianBlurFilterStage; +}(pipelines_1.WebGL2Pipeline.ProcessingStage)); +exports.SinglePassGaussianBlurFilterStage = SinglePassGaussianBlurFilterStage; +//# sourceMappingURL=SinglePassGaussianBlurFilterStage.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.js.map b/es5/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.js.map new file mode 100644 index 0000000..497744b --- /dev/null +++ b/es5/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.js.map @@ -0,0 +1 @@ +{"version":3,"file":"SinglePassGaussianBlurFilterStage.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/gaussianblurfilterpipeline/SinglePassGaussianBlurFilterStage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AAAA,gDAAoD;AAEpD;;GAEG;AACH,SAAS,yBAAyB,CAAC,MAAc;IAC/C,IAAM,KAAK,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;IACtD,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,UAAC,IAAI,EAAE,CAAC;QAClD,OAAO,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,MAAM,GAAG,MAAM,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH;IAAuD,qDAA8B;IACnF,2CACE,KAA6B,EAC7B,SAAoC,EACpC,UAAgC,EAChC,gBAAwB,EACxB,iBAAwC;QAL1C,iBA4EC;QAvEC,kCAAA,EAAA,oBAAoB,gBAAgB,GAAG,CAAC;QAElC,IAAA,KAGF,KAAK,CAAC,MAAM,EAFd,MAAM,YAAA,EACN,KAAK,WACS,CAAC;gBAEjB,kBACE;YACE,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,gBAAgB;SAC9B,EACD;YACE,oBAAoB,EAAE,kvCAoCrB;YACD,KAAK,OAAA;YACL,MAAM,QAAA;YACN,WAAW,EAAE,iBAAiB;YAC9B,IAAI,EAAE,UAAU;YAChB,KAAK,OAAA;YACL,WAAW,EAAE;gBACX;oBACE,IAAI,EAAE,aAAa;oBACnB,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC3C;gBACD;oBACE,IAAI,EAAE,aAAa;oBACnB,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,MAAM,CAAC;iBAChC;aACF;SACF,CACF;QAED,KAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;;IACvB,CAAC;IAED,wDAAY,GAAZ,UAAa,MAAc;QACzB,IAAI,CAAC,eAAe,CAAC;YACnB;gBACE,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,CAAC,MAAM,CAAC;aACjB;YACD;gBACE,IAAI,EAAE,uBAAuB;gBAC7B,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,yBAAyB,CAAC,MAAM,CAAC;aAC1C;SACF,CAAC,CAAC;IACL,CAAC;IACH,wCAAC;AAAD,CAAC,AA7FD,CAAuD,0BAAc,CAAC,eAAe,GA6FpF;AA7FY,8EAAiC","sourcesContent":["import { WebGL2Pipeline } from '../../../pipelines';\n\n/**\n * @private\n */\nfunction createGaussianBlurWeights(radius: number): number[] {\n const coeff = 1.0 / Math.sqrt(2.0 * Math.PI) / radius;\n return '0'.repeat(radius + 1).split('').map((zero, x) => {\n return coeff * Math.exp(-0.5 * x * x / radius / radius);\n });\n}\n\n/**\n * @private\n */\nexport class SinglePassGaussianBlurFilterStage extends WebGL2Pipeline.ProcessingStage {\n constructor(\n glOut: WebGL2RenderingContext,\n direction: 'horizontal' | 'vertical',\n outputType: 'canvas' | 'texture',\n inputTextureUnit: number,\n outputTextureUnit = inputTextureUnit + 1\n ) {\n const {\n height,\n width\n } = glOut.canvas;\n\n super(\n {\n textureName: 'u_inputTexture',\n textureUnit: inputTextureUnit\n },\n {\n fragmentShaderSource: `#version 300 es\n precision highp float;\n\n uniform sampler2D u_inputTexture;\n uniform vec2 u_texelSize;\n uniform float u_direction;\n uniform float u_radius;\n uniform float u_gaussianBlurWeights[128];\n\n in vec2 v_texCoord;\n\n out vec4 outColor;\n\n void main() {\n float totalWeight = u_gaussianBlurWeights[0];\n vec3 newColor = totalWeight * texture(u_inputTexture, v_texCoord).rgb;\n\n for (float i = 1.0; i <= u_radius; i += 1.0) {\n float x = (1.0 - u_direction) * i;\n float y = u_direction * i;\n\n vec2 shift = vec2(x, y) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n float weight = u_gaussianBlurWeights[int(i)];\n newColor += weight * texture(u_inputTexture, coord).rgb;\n totalWeight += weight;\n\n shift = vec2(-x, -y) * u_texelSize;\n coord = vec2(v_texCoord + shift);\n newColor += weight * texture(u_inputTexture, coord).rgb;\n totalWeight += weight;\n }\n\n newColor /= totalWeight;\n outColor = vec4(newColor, 1.0);\n }\n `,\n glOut,\n height,\n textureUnit: outputTextureUnit,\n type: outputType,\n width,\n uniformVars: [\n {\n name: 'u_direction',\n type: 'float',\n values: [direction === 'vertical' ? 1 : 0]\n },\n {\n name: 'u_texelSize',\n type: 'float',\n values: [1 / width, 1 / height]\n }\n ]\n }\n );\n\n this.updateRadius(0);\n }\n\n updateRadius(radius: number): void {\n this._setUniformVars([\n {\n name: 'u_radius',\n type: 'float',\n values: [radius]\n },\n {\n name: 'u_gaussianBlurWeights',\n type: 'float:v',\n values: createGaussianBlurWeights(radius)\n }\n ]);\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/gaussianblurfilterpipeline/index.d.ts b/es5/processors/background/pipelines/gaussianblurfilterpipeline/index.d.ts new file mode 100644 index 0000000..971bb67 --- /dev/null +++ b/es5/processors/background/pipelines/gaussianblurfilterpipeline/index.d.ts @@ -0,0 +1,8 @@ +import { WebGL2Pipeline } from '../../../pipelines'; +/** + * @private + */ +export declare class GaussianBlurFilterPipeline extends WebGL2Pipeline { + constructor(outputCanvas: OffscreenCanvas | HTMLCanvasElement); + updateRadius(radius: number): void; +} diff --git a/es5/processors/background/pipelines/gaussianblurfilterpipeline/index.js b/es5/processors/background/pipelines/gaussianblurfilterpipeline/index.js new file mode 100644 index 0000000..61dcb65 --- /dev/null +++ b/es5/processors/background/pipelines/gaussianblurfilterpipeline/index.js @@ -0,0 +1,40 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.GaussianBlurFilterPipeline = void 0; +var pipelines_1 = require("../../../pipelines"); +var SinglePassGaussianBlurFilterStage_1 = require("./SinglePassGaussianBlurFilterStage"); +/** + * @private + */ +var GaussianBlurFilterPipeline = /** @class */ (function (_super) { + __extends(GaussianBlurFilterPipeline, _super); + function GaussianBlurFilterPipeline(outputCanvas) { + var _this = _super.call(this) || this; + var glOut = outputCanvas.getContext('webgl2'); + _this.addStage(new SinglePassGaussianBlurFilterStage_1.SinglePassGaussianBlurFilterStage(glOut, 'horizontal', 'texture', 0, 2)); + _this.addStage(new SinglePassGaussianBlurFilterStage_1.SinglePassGaussianBlurFilterStage(glOut, 'vertical', 'canvas', 2)); + return _this; + } + GaussianBlurFilterPipeline.prototype.updateRadius = function (radius) { + this._stages.forEach(function (stage) { return stage + .updateRadius(radius); }); + }; + return GaussianBlurFilterPipeline; +}(pipelines_1.WebGL2Pipeline)); +exports.GaussianBlurFilterPipeline = GaussianBlurFilterPipeline; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/gaussianblurfilterpipeline/index.js.map b/es5/processors/background/pipelines/gaussianblurfilterpipeline/index.js.map new file mode 100644 index 0000000..d89909c --- /dev/null +++ b/es5/processors/background/pipelines/gaussianblurfilterpipeline/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/gaussianblurfilterpipeline/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AAAA,gDAAoD;AACpD,yFAAwF;AAExF;;GAEG;AACH;IAAgD,8CAAc;IAC5D,oCAAY,YAAiD;QAA7D,YACE,iBAAO,SAkBR;QAhBC,IAAM,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC,QAAQ,CAA4B,CAAC;QAE3E,KAAI,CAAC,QAAQ,CAAC,IAAI,qEAAiC,CACjD,KAAK,EACL,YAAY,EACZ,SAAS,EACT,CAAC,EACD,CAAC,CACF,CAAC,CAAC;QAEH,KAAI,CAAC,QAAQ,CAAC,IAAI,qEAAiC,CACjD,KAAK,EACL,UAAU,EACV,QAAQ,EACR,CAAC,CACF,CAAC,CAAC;;IACL,CAAC;IAED,iDAAY,GAAZ,UAAa,MAAc;QACzB,IAAI,CAAC,OAAO,CAAC,OAAO,CAClB,UAAC,KAAK,IAAK,OAAC,KAA2C;aACpD,YAAY,CAAC,MAAM,CAAC,EADZ,CACY,CACxB,CAAC;IACJ,CAAC;IACH,iCAAC;AAAD,CAAC,AA5BD,CAAgD,0BAAc,GA4B7D;AA5BY,gEAA0B","sourcesContent":["import { WebGL2Pipeline } from '../../../pipelines';\nimport { SinglePassGaussianBlurFilterStage } from './SinglePassGaussianBlurFilterStage';\n\n/**\n * @private\n */\nexport class GaussianBlurFilterPipeline extends WebGL2Pipeline {\n constructor(outputCanvas: OffscreenCanvas | HTMLCanvasElement) {\n super();\n\n const glOut = outputCanvas.getContext('webgl2')! as WebGL2RenderingContext;\n\n this.addStage(new SinglePassGaussianBlurFilterStage(\n glOut,\n 'horizontal',\n 'texture',\n 0,\n 2\n ));\n\n this.addStage(new SinglePassGaussianBlurFilterStage(\n glOut,\n 'vertical',\n 'canvas',\n 2\n ));\n }\n\n updateRadius(radius: number): void {\n this._stages.forEach(\n (stage) => (stage as SinglePassGaussianBlurFilterStage)\n .updateRadius(radius)\n );\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/index.d.ts b/es5/processors/background/pipelines/index.d.ts new file mode 100644 index 0000000..274342f --- /dev/null +++ b/es5/processors/background/pipelines/index.d.ts @@ -0,0 +1,2 @@ +export { GaussianBlurFilterPipeline } from './gaussianblurfilterpipeline'; +export { PersonMaskUpscalePipeline } from './personmaskupscalepipeline'; diff --git a/es5/processors/background/pipelines/index.js b/es5/processors/background/pipelines/index.js new file mode 100644 index 0000000..07ca138 --- /dev/null +++ b/es5/processors/background/pipelines/index.js @@ -0,0 +1,8 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PersonMaskUpscalePipeline = exports.GaussianBlurFilterPipeline = void 0; +var gaussianblurfilterpipeline_1 = require("./gaussianblurfilterpipeline"); +Object.defineProperty(exports, "GaussianBlurFilterPipeline", { enumerable: true, get: function () { return gaussianblurfilterpipeline_1.GaussianBlurFilterPipeline; } }); +var personmaskupscalepipeline_1 = require("./personmaskupscalepipeline"); +Object.defineProperty(exports, "PersonMaskUpscalePipeline", { enumerable: true, get: function () { return personmaskupscalepipeline_1.PersonMaskUpscalePipeline; } }); +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/index.js.map b/es5/processors/background/pipelines/index.js.map new file mode 100644 index 0000000..2b69661 --- /dev/null +++ b/es5/processors/background/pipelines/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../lib/processors/background/pipelines/index.ts"],"names":[],"mappings":";;;AAAA,2EAA0E;AAAjE,wIAAA,0BAA0B,OAAA;AACnC,yEAAwE;AAA/D,sIAAA,yBAAyB,OAAA","sourcesContent":["export { GaussianBlurFilterPipeline } from './gaussianblurfilterpipeline';\nexport { PersonMaskUpscalePipeline } from './personmaskupscalepipeline';\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.d.ts b/es5/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.d.ts new file mode 100644 index 0000000..27df8a4 --- /dev/null +++ b/es5/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.d.ts @@ -0,0 +1,12 @@ +import { Dimensions } from '../../../../types'; +import { WebGL2Pipeline } from '../../../pipelines'; +/** + * @private + */ +export declare class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingStage { + private readonly _direction; + private readonly _inputDimensions; + constructor(glOut: WebGL2RenderingContext, direction: 'horizontal' | 'vertical', outputType: 'canvas' | 'texture', inputDimensions: Dimensions, outputDimensions: Dimensions, inputTextureUnit: number, outputTextureUnit?: number); + updateSigmaColor(sigmaColor: number): void; + updateSigmaSpace(sigmaSpace: number): void; +} diff --git a/es5/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.js b/es5/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.js new file mode 100644 index 0000000..521a0bb --- /dev/null +++ b/es5/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.js @@ -0,0 +1,120 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.SinglePassBilateralFilterStage = void 0; +var pipelines_1 = require("../../../pipelines"); +/** + * @private + */ +function createSpaceWeights(radius, sigma, texelSize) { + return '0'.repeat(radius).split('').map(function (zero, i) { + var x = (i + 1) * texelSize; + return Math.exp(-0.5 * x * x / sigma / sigma); + }); +} +/** + * @private + */ +function createColorWeights(sigma) { + return '0'.repeat(256).split('').map(function (zero, i) { + var x = i / 255; + return Math.exp(-0.5 * x * x / sigma / sigma); + }); +} +/** + * @private + */ +var SinglePassBilateralFilterStage = /** @class */ (function (_super) { + __extends(SinglePassBilateralFilterStage, _super); + function SinglePassBilateralFilterStage(glOut, direction, outputType, inputDimensions, outputDimensions, inputTextureUnit, outputTextureUnit) { + var _this = this; + if (outputTextureUnit === void 0) { outputTextureUnit = inputTextureUnit + 1; } + var height = outputDimensions.height, width = outputDimensions.width; + _this = _super.call(this, { + textureName: 'u_segmentationMask', + textureUnit: inputTextureUnit + }, { + fragmentShaderSource: "#version 300 es\n precision highp float;\n\n uniform sampler2D u_inputFrame;\n uniform sampler2D u_segmentationMask;\n uniform vec2 u_texelSize;\n uniform float u_direction;\n uniform float u_radius;\n uniform float u_step;\n uniform float u_spaceWeights[128];\n uniform float u_colorWeights[256];\n\n in vec2 v_texCoord;\n\n out vec4 outColor;\n\n float calculateColorWeight(vec2 coord, vec3 centerColor) {\n vec3 coordColor = texture(u_inputFrame, coord).rgb;\n float x = distance(centerColor, coordColor);\n return u_colorWeights[int(x * 255.0)];\n }\n\n float edgePixelsAverageAlpha(float outAlpha) {\n float totalAlpha = outAlpha;\n float totalPixels = 1.0;\n\n for (float i = -u_radius; u_radius > 0.0 && i <= u_radius; i += u_radius) {\n for (float j = -u_radius; j <= u_radius; j += u_radius * (j == 0.0 ? 2.0 : 1.0)) {\n vec2 shift = vec2(i, j) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n totalAlpha += texture(u_segmentationMask, coord).a;\n totalPixels++;\n }\n }\n\n return totalAlpha / totalPixels;\n }\n\n void main() {\n vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb;\n float outAlpha = texture(u_segmentationMask, v_texCoord).a;\n float averageAlpha = edgePixelsAverageAlpha(outAlpha);\n float totalWeight = 1.0;\n\n if (averageAlpha == 0.0 || averageAlpha == 1.0) {\n outColor = vec4(averageAlpha * centerColor, averageAlpha);\n return;\n }\n\n for (float i = 1.0; i <= u_radius; i += u_step) {\n float x = (1.0 - u_direction) * i;\n float y = u_direction * i;\n vec2 shift = vec2(x, y) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n float spaceWeight = u_spaceWeights[int(i - 1.0)];\n float colorWeight = calculateColorWeight(coord, centerColor);\n float weight = spaceWeight * colorWeight;\n float alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n\n shift = vec2(-x, -y) * u_texelSize;\n coord = vec2(v_texCoord + shift);\n colorWeight = calculateColorWeight(coord, centerColor);\n weight = spaceWeight * colorWeight;\n alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n }\n\n outAlpha /= totalWeight;\n outColor = vec4(outAlpha * centerColor, outAlpha);\n }\n ", + glOut: glOut, + height: height, + textureUnit: outputTextureUnit, + type: outputType, + width: width, + uniformVars: [ + { + name: 'u_inputFrame', + type: 'int', + values: [0] + }, + { + name: 'u_direction', + type: 'float', + values: [direction === 'vertical' ? 1 : 0] + }, + { + name: 'u_texelSize', + type: 'float', + values: [1 / width, 1 / height] + } + ] + }) || this; + _this._direction = direction; + _this._inputDimensions = inputDimensions; + _this.updateSigmaColor(0); + _this.updateSigmaSpace(0); + return _this; + } + SinglePassBilateralFilterStage.prototype.updateSigmaColor = function (sigmaColor) { + this._setUniformVars([ + { + name: 'u_colorWeights', + type: 'float:v', + values: createColorWeights(sigmaColor) + } + ]); + }; + SinglePassBilateralFilterStage.prototype.updateSigmaSpace = function (sigmaSpace) { + var _a = this._inputDimensions, inputHeight = _a.height, inputWidth = _a.width; + var _b = this._outputDimensions, outputHeight = _b.height, outputWidth = _b.width; + sigmaSpace *= Math.max(outputWidth / inputWidth, outputHeight / inputHeight); + var step = Math.floor(0.5 * sigmaSpace / Math.log(sigmaSpace)); + var sigmaTexel = Math.max(1 / outputWidth, 1 / outputHeight) * sigmaSpace; + var texelSize = 1 / (this._direction === 'horizontal' + ? outputWidth + : outputHeight); + this._setUniformVars([ + { + name: 'u_radius', + type: 'float', + values: [sigmaSpace] + }, + { + name: 'u_spaceWeights', + type: 'float:v', + values: createSpaceWeights(sigmaSpace, sigmaTexel, texelSize) + }, + { + name: 'u_step', + type: 'float', + values: [step] + } + ]); + }; + return SinglePassBilateralFilterStage; +}(pipelines_1.WebGL2Pipeline.ProcessingStage)); +exports.SinglePassBilateralFilterStage = SinglePassBilateralFilterStage; +//# sourceMappingURL=SinglePassBilateralFilterStage.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.js.map b/es5/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.js.map new file mode 100644 index 0000000..3708624 --- /dev/null +++ b/es5/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.js.map @@ -0,0 +1 @@ +{"version":3,"file":"SinglePassBilateralFilterStage.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/personmaskupscalepipeline/SinglePassBilateralFilterStage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AACA,gDAAoD;AAEpD;;GAEG;AACH,SAAS,kBAAkB,CACzB,MAAc,EACd,KAAa,EACb,SAAiB;IAEjB,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,UAAC,IAAI,EAAE,CAAC;QAC9C,IAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC;QAC9B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CACzB,KAAa;IAEb,OAAO,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,UAAC,IAAI,EAAE,CAAC;QAC3C,IAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;QAClB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH;IAAoD,kDAA8B;IAIhF,wCACE,KAA6B,EAC7B,SAAoC,EACpC,UAAgC,EAChC,eAA2B,EAC3B,gBAA4B,EAC5B,gBAAwB,EACxB,iBAAwC;QAP1C,iBA2HC;QApHC,kCAAA,EAAA,oBAAoB,gBAAgB,GAAG,CAAC;QAGtC,IAAA,MAAM,GAEJ,gBAAgB,OAFZ,EACN,KAAK,GACH,gBAAgB,MADb,CACc;gBAErB,kBACE;YACE,WAAW,EAAE,oBAAoB;YACjC,WAAW,EAAE,gBAAgB;SAC9B,EACD;YACE,oBAAoB,EAAE,k2FAyErB;YACD,KAAK,OAAA;YACL,MAAM,QAAA;YACN,WAAW,EAAE,iBAAiB;YAC9B,IAAI,EAAE,UAAU;YAChB,KAAK,OAAA;YACL,WAAW,EAAE;gBACX;oBACE,IAAI,EAAE,cAAc;oBACpB,IAAI,EAAE,KAAK;oBACX,MAAM,EAAE,CAAC,CAAC,CAAC;iBACZ;gBACD;oBACE,IAAI,EAAE,aAAa;oBACnB,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,CAAC,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC3C;gBACD;oBACE,IAAI,EAAE,aAAa;oBACnB,IAAI,EAAE,OAAO;oBACb,MAAM,EAAE,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,MAAM,CAAC;iBAChC;aACF;SACF,CACF;QAED,KAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,KAAI,CAAC,gBAAgB,GAAG,eAAe,CAAC;QACxC,KAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;QACzB,KAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;;IAC3B,CAAC;IAED,yDAAgB,GAAhB,UAAiB,UAAkB;QACjC,IAAI,CAAC,eAAe,CAAC;YACnB;gBACE,IAAI,EAAE,gBAAgB;gBACtB,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,kBAAkB,CACxB,UAAU,CACX;aACF;SACF,CAAC,CAAC;IACL,CAAC;IAED,yDAAgB,GAAhB,UAAiB,UAAkB;QAC3B,IAAA,KAGF,IAAI,CAAC,gBAAgB,EAFf,WAAW,YAAA,EACZ,UAAU,WACM,CAAC;QAEpB,IAAA,KAGF,IAAI,CAAC,iBAAiB,EAFhB,YAAY,YAAA,EACb,WAAW,WACM,CAAC;QAE3B,UAAU,IAAI,IAAI,CAAC,GAAG,CACpB,WAAW,GAAG,UAAU,EACxB,YAAY,GAAG,WAAW,CAC3B,CAAC;QAEF,IAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CACrB,GAAG,GAAG,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CACxC,CAAC;QAEF,IAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CACzB,CAAC,GAAG,WAAW,EACf,CAAC,GAAG,YAAY,CACjB,GAAG,UAAU,CAAC;QAEf,IAAM,SAAS,GAAG,CAAC,GAAG,CACpB,IAAI,CAAC,UAAU,KAAK,YAAY;YAC9B,CAAC,CAAC,WAAW;YACb,CAAC,CAAC,YAAY,CACjB,CAAC;QAEF,IAAI,CAAC,eAAe,CAAC;YACnB;gBACE,IAAI,EAAE,UAAU;gBAChB,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,CAAC,UAAU,CAAC;aACrB;YACD;gBACE,IAAI,EAAE,gBAAgB;gBACtB,IAAI,EAAE,SAAS;gBACf,MAAM,EAAE,kBAAkB,CACxB,UAAU,EACV,UAAU,EACV,SAAS,CACV;aACF;YACD;gBACE,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,CAAC,IAAI,CAAC;aACf;SACF,CAAC,CAAC;IACL,CAAC;IACH,qCAAC;AAAD,CAAC,AAlMD,CAAoD,0BAAc,CAAC,eAAe,GAkMjF;AAlMY,wEAA8B","sourcesContent":["import { Dimensions } from '../../../../types';\nimport { WebGL2Pipeline } from '../../../pipelines';\n\n/**\n * @private\n */\nfunction createSpaceWeights(\n radius: number,\n sigma: number,\n texelSize: number\n): number[] {\n return '0'.repeat(radius).split('').map((zero, i) => {\n const x = (i + 1) * texelSize;\n return Math.exp(-0.5 * x * x / sigma / sigma);\n });\n}\n\n/**\n * @private\n */\nfunction createColorWeights(\n sigma: number\n): number[] {\n return '0'.repeat(256).split('').map((zero, i) => {\n const x = i / 255;\n return Math.exp(-0.5 * x * x / sigma / sigma);\n });\n}\n\n/**\n * @private\n */\nexport class SinglePassBilateralFilterStage extends WebGL2Pipeline.ProcessingStage {\n private readonly _direction: 'horizontal' | 'vertical';\n private readonly _inputDimensions: Dimensions;\n\n constructor(\n glOut: WebGL2RenderingContext,\n direction: 'horizontal' | 'vertical',\n outputType: 'canvas' | 'texture',\n inputDimensions: Dimensions,\n outputDimensions: Dimensions,\n inputTextureUnit: number,\n outputTextureUnit = inputTextureUnit + 1\n ) {\n const {\n height,\n width\n } = outputDimensions;\n\n super(\n {\n textureName: 'u_segmentationMask',\n textureUnit: inputTextureUnit\n },\n {\n fragmentShaderSource: `#version 300 es\n precision highp float;\n\n uniform sampler2D u_inputFrame;\n uniform sampler2D u_segmentationMask;\n uniform vec2 u_texelSize;\n uniform float u_direction;\n uniform float u_radius;\n uniform float u_step;\n uniform float u_spaceWeights[128];\n uniform float u_colorWeights[256];\n\n in vec2 v_texCoord;\n\n out vec4 outColor;\n\n float calculateColorWeight(vec2 coord, vec3 centerColor) {\n vec3 coordColor = texture(u_inputFrame, coord).rgb;\n float x = distance(centerColor, coordColor);\n return u_colorWeights[int(x * 255.0)];\n }\n\n float edgePixelsAverageAlpha(float outAlpha) {\n float totalAlpha = outAlpha;\n float totalPixels = 1.0;\n\n for (float i = -u_radius; u_radius > 0.0 && i <= u_radius; i += u_radius) {\n for (float j = -u_radius; j <= u_radius; j += u_radius * (j == 0.0 ? 2.0 : 1.0)) {\n vec2 shift = vec2(i, j) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n totalAlpha += texture(u_segmentationMask, coord).a;\n totalPixels++;\n }\n }\n\n return totalAlpha / totalPixels;\n }\n\n void main() {\n vec3 centerColor = texture(u_inputFrame, v_texCoord).rgb;\n float outAlpha = texture(u_segmentationMask, v_texCoord).a;\n float averageAlpha = edgePixelsAverageAlpha(outAlpha);\n float totalWeight = 1.0;\n\n if (averageAlpha == 0.0 || averageAlpha == 1.0) {\n outColor = vec4(averageAlpha * centerColor, averageAlpha);\n return;\n }\n\n for (float i = 1.0; i <= u_radius; i += u_step) {\n float x = (1.0 - u_direction) * i;\n float y = u_direction * i;\n vec2 shift = vec2(x, y) * u_texelSize;\n vec2 coord = vec2(v_texCoord + shift);\n float spaceWeight = u_spaceWeights[int(i - 1.0)];\n float colorWeight = calculateColorWeight(coord, centerColor);\n float weight = spaceWeight * colorWeight;\n float alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n\n shift = vec2(-x, -y) * u_texelSize;\n coord = vec2(v_texCoord + shift);\n colorWeight = calculateColorWeight(coord, centerColor);\n weight = spaceWeight * colorWeight;\n alpha = texture(u_segmentationMask, coord).a;\n totalWeight += weight;\n outAlpha += weight * alpha;\n }\n\n outAlpha /= totalWeight;\n outColor = vec4(outAlpha * centerColor, outAlpha);\n }\n `,\n glOut,\n height,\n textureUnit: outputTextureUnit,\n type: outputType,\n width,\n uniformVars: [\n {\n name: 'u_inputFrame',\n type: 'int',\n values: [0]\n },\n {\n name: 'u_direction',\n type: 'float',\n values: [direction === 'vertical' ? 1 : 0]\n },\n {\n name: 'u_texelSize',\n type: 'float',\n values: [1 / width, 1 / height]\n }\n ]\n }\n );\n\n this._direction = direction;\n this._inputDimensions = inputDimensions;\n this.updateSigmaColor(0);\n this.updateSigmaSpace(0);\n }\n\n updateSigmaColor(sigmaColor: number): void {\n this._setUniformVars([\n {\n name: 'u_colorWeights',\n type: 'float:v',\n values: createColorWeights(\n sigmaColor\n )\n }\n ]);\n }\n\n updateSigmaSpace(sigmaSpace: number): void {\n const {\n height: inputHeight,\n width: inputWidth\n } = this._inputDimensions;\n\n const {\n height: outputHeight,\n width: outputWidth\n } = this._outputDimensions;\n\n sigmaSpace *= Math.max(\n outputWidth / inputWidth,\n outputHeight / inputHeight\n );\n\n const step = Math.floor(\n 0.5 * sigmaSpace / Math.log(sigmaSpace)\n );\n\n const sigmaTexel = Math.max(\n 1 / outputWidth,\n 1 / outputHeight\n ) * sigmaSpace;\n\n const texelSize = 1 / (\n this._direction === 'horizontal'\n ? outputWidth\n : outputHeight\n );\n\n this._setUniformVars([\n {\n name: 'u_radius',\n type: 'float',\n values: [sigmaSpace]\n },\n {\n name: 'u_spaceWeights',\n type: 'float:v',\n values: createSpaceWeights(\n sigmaSpace,\n sigmaTexel,\n texelSize\n )\n },\n {\n name: 'u_step',\n type: 'float',\n values: [step]\n }\n ]);\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/background/pipelines/personmaskupscalepipeline/index.d.ts b/es5/processors/background/pipelines/personmaskupscalepipeline/index.d.ts new file mode 100644 index 0000000..51ef17b --- /dev/null +++ b/es5/processors/background/pipelines/personmaskupscalepipeline/index.d.ts @@ -0,0 +1,9 @@ +import { BilateralFilterConfig, Dimensions } from '../../../../types'; +import { WebGL2Pipeline } from '../../../pipelines'; +/** + * @private + */ +export declare class PersonMaskUpscalePipeline extends WebGL2Pipeline { + constructor(inputDimensions: Dimensions, outputCanvas: OffscreenCanvas | HTMLCanvasElement); + updateBilateralFilterConfig(config: BilateralFilterConfig): void; +} diff --git a/es5/processors/background/pipelines/personmaskupscalepipeline/index.js b/es5/processors/background/pipelines/personmaskupscalepipeline/index.js new file mode 100644 index 0000000..cadd2ed --- /dev/null +++ b/es5/processors/background/pipelines/personmaskupscalepipeline/index.js @@ -0,0 +1,51 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PersonMaskUpscalePipeline = void 0; +var pipelines_1 = require("../../../pipelines"); +var SinglePassBilateralFilterStage_1 = require("./SinglePassBilateralFilterStage"); +/** + * @private + */ +var PersonMaskUpscalePipeline = /** @class */ (function (_super) { + __extends(PersonMaskUpscalePipeline, _super); + function PersonMaskUpscalePipeline(inputDimensions, outputCanvas) { + var _this = _super.call(this) || this; + var glOut = outputCanvas.getContext('webgl2'); + var outputDimensions = { + height: outputCanvas.height, + width: outputCanvas.width + }; + _this.addStage(new pipelines_1.WebGL2Pipeline.InputStage(glOut)); + _this.addStage(new SinglePassBilateralFilterStage_1.SinglePassBilateralFilterStage(glOut, 'horizontal', 'texture', inputDimensions, outputDimensions, 1, 2)); + _this.addStage(new SinglePassBilateralFilterStage_1.SinglePassBilateralFilterStage(glOut, 'vertical', 'canvas', inputDimensions, outputDimensions, 2)); + return _this; + } + PersonMaskUpscalePipeline.prototype.updateBilateralFilterConfig = function (config) { + var _a = this._stages, bilateralFilterStages = _a.slice(1); + var sigmaSpace = config.sigmaSpace; + if (typeof sigmaSpace === 'number') { + bilateralFilterStages.forEach(function (stage) { + stage.updateSigmaColor(0.1); + stage.updateSigmaSpace(sigmaSpace); + }); + } + }; + return PersonMaskUpscalePipeline; +}(pipelines_1.WebGL2Pipeline)); +exports.PersonMaskUpscalePipeline = PersonMaskUpscalePipeline; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/es5/processors/background/pipelines/personmaskupscalepipeline/index.js.map b/es5/processors/background/pipelines/personmaskupscalepipeline/index.js.map new file mode 100644 index 0000000..a68d33e --- /dev/null +++ b/es5/processors/background/pipelines/personmaskupscalepipeline/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../lib/processors/background/pipelines/personmaskupscalepipeline/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AACA,gDAAoD;AACpD,mFAAkF;AAElF;;GAEG;AACH;IAA+C,6CAAc;IAC3D,mCACE,eAA2B,EAC3B,YAAiD;QAFnD,YAIE,iBAAO,SA4BR;QA3BC,IAAM,KAAK,GAAG,YAAY,CAAC,UAAU,CAAC,QAAQ,CAA4B,CAAC;QAE3E,IAAM,gBAAgB,GAAG;YACvB,MAAM,EAAE,YAAY,CAAC,MAAM;YAC3B,KAAK,EAAE,YAAY,CAAC,KAAK;SAC1B,CAAC;QAEF,KAAI,CAAC,QAAQ,CAAC,IAAI,0BAAc,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;QAEpD,KAAI,CAAC,QAAQ,CAAC,IAAI,+DAA8B,CAC9C,KAAK,EACL,YAAY,EACZ,SAAS,EACT,eAAe,EACf,gBAAgB,EAChB,CAAC,EACD,CAAC,CACF,CAAC,CAAC;QAEH,KAAI,CAAC,QAAQ,CAAC,IAAI,+DAA8B,CAC9C,KAAK,EACL,UAAU,EACV,QAAQ,EACR,eAAe,EACf,gBAAgB,EAChB,CAAC,CACF,CAAC,CAAC;;IACL,CAAC;IAED,+DAA2B,GAA3B,UAA4B,MAA6B;QACjD,IAAA,KAGF,IAAI,CAAC,OAAO,EADX,qBAAqB,cACV,CAAC;QACT,IAAA,UAAU,GAAK,MAAM,WAAX,CAAY;QAE9B,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE;YACjC,qBAA0D,CAAC,OAAO,CACjE,UAAC,KAAK;gBACJ,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;gBAC5B,KAAK,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;YACrC,CAAC,CACF,CAAC;SACH;IACH,CAAC;IACH,gCAAC;AAAD,CAAC,AAnDD,CAA+C,0BAAc,GAmD5D;AAnDY,8DAAyB","sourcesContent":["import { BilateralFilterConfig, Dimensions } from '../../../../types';\nimport { WebGL2Pipeline } from '../../../pipelines';\nimport { SinglePassBilateralFilterStage } from './SinglePassBilateralFilterStage';\n\n/**\n * @private\n */\nexport class PersonMaskUpscalePipeline extends WebGL2Pipeline {\n constructor(\n inputDimensions: Dimensions,\n outputCanvas: OffscreenCanvas | HTMLCanvasElement\n ) {\n super();\n const glOut = outputCanvas.getContext('webgl2')! as WebGL2RenderingContext;\n\n const outputDimensions = {\n height: outputCanvas.height,\n width: outputCanvas.width\n };\n\n this.addStage(new WebGL2Pipeline.InputStage(glOut));\n\n this.addStage(new SinglePassBilateralFilterStage(\n glOut,\n 'horizontal',\n 'texture',\n inputDimensions,\n outputDimensions,\n 1,\n 2\n ));\n\n this.addStage(new SinglePassBilateralFilterStage(\n glOut,\n 'vertical',\n 'canvas',\n inputDimensions,\n outputDimensions,\n 2\n ));\n }\n\n updateBilateralFilterConfig(config: BilateralFilterConfig) {\n const [\n /* inputStage */,\n ...bilateralFilterStages\n ] = this._stages;\n const { sigmaSpace } = config;\n\n if (typeof sigmaSpace === 'number') {\n (bilateralFilterStages as SinglePassBilateralFilterStage[]).forEach(\n (stage) => {\n stage.updateSigmaColor(0.1);\n stage.updateSigmaSpace(sigmaSpace);\n }\n );\n }\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/pipelines/Pipeline.d.ts b/es5/processors/pipelines/Pipeline.d.ts new file mode 100644 index 0000000..52c8977 --- /dev/null +++ b/es5/processors/pipelines/Pipeline.d.ts @@ -0,0 +1,13 @@ +/** + * @private + */ +export declare class Pipeline implements Pipeline.Stage { + protected readonly _stages: Pipeline.Stage[]; + addStage(stage: Pipeline.Stage): void; + render(...args: any[]): void; +} +export declare namespace Pipeline { + interface Stage { + render(...args: any[]): void; + } +} diff --git a/es5/processors/pipelines/Pipeline.js b/es5/processors/pipelines/Pipeline.js new file mode 100644 index 0000000..c6645f5 --- /dev/null +++ b/es5/processors/pipelines/Pipeline.js @@ -0,0 +1,26 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Pipeline = void 0; +/** + * @private + */ +var Pipeline = /** @class */ (function () { + function Pipeline() { + this._stages = []; + } + Pipeline.prototype.addStage = function (stage) { + this._stages.push(stage); + }; + Pipeline.prototype.render = function () { + var args = []; + for (var _i = 0; _i < arguments.length; _i++) { + args[_i] = arguments[_i]; + } + this._stages.forEach(function (stage) { + stage.render.apply(stage, args); + }); + }; + return Pipeline; +}()); +exports.Pipeline = Pipeline; +//# sourceMappingURL=Pipeline.js.map \ No newline at end of file diff --git a/es5/processors/pipelines/Pipeline.js.map b/es5/processors/pipelines/Pipeline.js.map new file mode 100644 index 0000000..2fbc3c0 --- /dev/null +++ b/es5/processors/pipelines/Pipeline.js.map @@ -0,0 +1 @@ +{"version":3,"file":"Pipeline.js","sourceRoot":"","sources":["../../../lib/processors/pipelines/Pipeline.ts"],"names":[],"mappings":";;;AAAA;;GAEG;AACH;IAAA;QACqB,YAAO,GAAqB,EAAE,CAAC;IAWpD,CAAC;IATC,2BAAQ,GAAR,UAAS,KAAqB;QAC5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,yBAAM,GAAN;QAAO,cAAc;aAAd,UAAc,EAAd,qBAAc,EAAd,IAAc;YAAd,yBAAc;;QACnB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,UAAC,KAAK;YACzB,KAAK,CAAC,MAAM,OAAZ,KAAK,EAAW,IAAI,EAAE;QACxB,CAAC,CAAC,CAAC;IACL,CAAC;IACH,eAAC;AAAD,CAAC,AAZD,IAYC;AAZY,4BAAQ","sourcesContent":["/**\n * @private\n */\nexport class Pipeline implements Pipeline.Stage {\n protected readonly _stages: Pipeline.Stage[] = [];\n\n addStage(stage: Pipeline.Stage): void {\n this._stages.push(stage);\n }\n\n render(...args: any[]): void {\n this._stages.forEach((stage) => {\n stage.render(...args);\n });\n }\n}\n\nexport namespace Pipeline {\n export interface Stage {\n render(...args: any[]): void;\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/pipelines/index.d.ts b/es5/processors/pipelines/index.d.ts new file mode 100644 index 0000000..7c722c6 --- /dev/null +++ b/es5/processors/pipelines/index.d.ts @@ -0,0 +1,2 @@ +export { Pipeline } from './Pipeline'; +export { WebGL2Pipeline } from './webgl2pipeline'; diff --git a/es5/processors/pipelines/index.js b/es5/processors/pipelines/index.js new file mode 100644 index 0000000..68e665c --- /dev/null +++ b/es5/processors/pipelines/index.js @@ -0,0 +1,8 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2Pipeline = exports.Pipeline = void 0; +var Pipeline_1 = require("./Pipeline"); +Object.defineProperty(exports, "Pipeline", { enumerable: true, get: function () { return Pipeline_1.Pipeline; } }); +var webgl2pipeline_1 = require("./webgl2pipeline"); +Object.defineProperty(exports, "WebGL2Pipeline", { enumerable: true, get: function () { return webgl2pipeline_1.WebGL2Pipeline; } }); +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/es5/processors/pipelines/index.js.map b/es5/processors/pipelines/index.js.map new file mode 100644 index 0000000..cc90a87 --- /dev/null +++ b/es5/processors/pipelines/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../lib/processors/pipelines/index.ts"],"names":[],"mappings":";;;AAAA,uCAAsC;AAA7B,oGAAA,QAAQ,OAAA;AACjB,mDAAkD;AAAzC,gHAAA,cAAc,OAAA","sourcesContent":["export { Pipeline } from './Pipeline';\nexport { WebGL2Pipeline } from './webgl2pipeline';\n"]} \ No newline at end of file diff --git a/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.d.ts b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.d.ts new file mode 100644 index 0000000..ec2e83a --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.d.ts @@ -0,0 +1,13 @@ +import { InputFrame } from '../../../types'; +import { Pipeline } from '../Pipeline'; +/** + * @private + */ +export declare class WebGL2PipelineInputStage implements Pipeline.Stage { + private readonly _glOut; + private readonly _inputFrameTexture; + private _inputTexture; + constructor(glOut: WebGL2RenderingContext); + cleanUp(): void; + render(inputFrame?: InputFrame, inputTextureData?: ImageData): void; +} diff --git a/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.js b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.js new file mode 100644 index 0000000..ec53dbf --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.js @@ -0,0 +1,46 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2PipelineInputStage = void 0; +var webgl2PipelineHelpers_1 = require("./webgl2PipelineHelpers"); +/** + * @private + */ +var WebGL2PipelineInputStage = /** @class */ (function () { + function WebGL2PipelineInputStage(glOut) { + this._inputTexture = null; + var _a = glOut.canvas, height = _a.height, width = _a.width; + this._glOut = glOut; + this._inputFrameTexture = (0, webgl2PipelineHelpers_1.createTexture)(glOut, glOut.RGBA8, width, height, glOut.NEAREST, glOut.NEAREST); + } + WebGL2PipelineInputStage.prototype.cleanUp = function () { + var _a = this, _glOut = _a._glOut, _inputFrameTexture = _a._inputFrameTexture, _inputTexture = _a._inputTexture; + _glOut.deleteTexture(_inputFrameTexture); + _glOut.deleteTexture(_inputTexture); + }; + WebGL2PipelineInputStage.prototype.render = function (inputFrame, inputTextureData) { + var _a = this, _glOut = _a._glOut, _inputFrameTexture = _a._inputFrameTexture; + var _b = _glOut.canvas, height = _b.height, width = _b.width; + _glOut.viewport(0, 0, width, height); + _glOut.clearColor(0, 0, 0, 0); + _glOut.clear(_glOut.COLOR_BUFFER_BIT); + if (inputFrame) { + _glOut.activeTexture(_glOut.TEXTURE0); + _glOut.bindTexture(_glOut.TEXTURE_2D, _inputFrameTexture); + _glOut.texSubImage2D(_glOut.TEXTURE_2D, 0, 0, 0, width, height, _glOut.RGBA, _glOut.UNSIGNED_BYTE, inputFrame); + } + if (!inputTextureData) { + return; + } + var data = inputTextureData.data, textureHeight = inputTextureData.height, textureWidth = inputTextureData.width; + if (!this._inputTexture) { + this._inputTexture = (0, webgl2PipelineHelpers_1.createTexture)(_glOut, _glOut.RGBA8, textureWidth, textureHeight, _glOut.NEAREST, _glOut.NEAREST); + } + _glOut.viewport(0, 0, textureWidth, textureHeight); + _glOut.activeTexture(_glOut.TEXTURE1); + _glOut.bindTexture(_glOut.TEXTURE_2D, this._inputTexture); + _glOut.texSubImage2D(_glOut.TEXTURE_2D, 0, 0, 0, textureWidth, textureHeight, _glOut.RGBA, _glOut.UNSIGNED_BYTE, data); + }; + return WebGL2PipelineInputStage; +}()); +exports.WebGL2PipelineInputStage = WebGL2PipelineInputStage; +//# sourceMappingURL=WebGL2PipelineInputStage.js.map \ No newline at end of file diff --git a/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.js.map b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.js.map new file mode 100644 index 0000000..4233e7c --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.js.map @@ -0,0 +1 @@ +{"version":3,"file":"WebGL2PipelineInputStage.js","sourceRoot":"","sources":["../../../../lib/processors/pipelines/webgl2pipeline/WebGL2PipelineInputStage.ts"],"names":[],"mappings":";;;AAEA,iEAAwD;AAExD;;GAEG;AACH;IAKE,kCAAY,KAA6B;QAFjC,kBAAa,GAAwB,IAAI,CAAC;QAG1C,IAAA,KAAoB,KAAK,CAAC,MAAM,EAA9B,MAAM,YAAA,EAAE,KAAK,WAAiB,CAAC;QACvC,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,kBAAkB,GAAG,IAAA,qCAAa,EACrC,KAAK,EACL,KAAK,CAAC,KAAK,EACX,KAAK,EACL,MAAM,EACN,KAAK,CAAC,OAAO,EACb,KAAK,CAAC,OAAO,CACb,CAAC;IACL,CAAC;IAED,0CAAO,GAAP;QACQ,IAAA,KAIF,IAAI,EAHN,MAAM,YAAA,EACN,kBAAkB,wBAAA,EAClB,aAAa,mBACP,CAAC;QACT,MAAM,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC;QACzC,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC;IACtC,CAAC;IAED,yCAAM,GAAN,UACE,UAAuB,EACvB,gBAA4B;QAEtB,IAAA,KAGF,IAAI,EAFN,MAAM,YAAA,EACN,kBAAkB,wBACZ,CAAC;QAEH,IAAA,KAAoB,MAAM,CAAC,MAAM,EAA/B,MAAM,YAAA,EAAE,KAAK,WAAkB,CAAC;QACxC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACrC,MAAM,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAEtC,IAAI,UAAU,EAAE;YACd,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACtC,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,UAAU,EACjB,kBAAkB,CACnB,CAAC;YACF,MAAM,CAAC,aAAa,CAClB,MAAM,CAAC,UAAU,EACjB,CAAC,EACD,CAAC,EACD,CAAC,EACD,KAAK,EACL,MAAM,EACN,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,aAAa,EACpB,UAAU,CACX,CAAC;SACH;QAED,IAAI,CAAC,gBAAgB,EAAE;YACrB,OAAO;SACR;QAEC,IAAA,IAAI,GAGF,gBAAgB,KAHd,EACI,aAAa,GAEnB,gBAAgB,OAFG,EACd,YAAY,GACjB,gBAAgB,MADC,CACA;QAErB,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE;YACvB,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAa,EAChC,MAAM,EACN,MAAM,CAAC,KAAK,EACZ,YAAY,EACZ,aAAa,EACb,MAAM,CAAC,OAAO,EACd,MAAM,CAAC,OAAO,CACf,CAAC;SACH;QAED,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,YAAY,EAAE,aAAa,CAAC,CAAC;QACnD,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAEtC,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,UAAU,EACjB,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,MAAM,CAAC,aAAa,CAClB,MAAM,CAAC,UAAU,EACjB,CAAC,EACD,CAAC,EACD,CAAC,EACD,YAAY,EACZ,aAAa,EACb,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,aAAa,EACpB,IAAI,CACL,CAAC;IACJ,CAAC;IACH,+BAAC;AAAD,CAAC,AApGD,IAoGC;AApGY,4DAAwB","sourcesContent":["import { InputFrame } from '../../../types';\nimport { Pipeline } from '../Pipeline';\nimport { createTexture } from './webgl2PipelineHelpers';\n\n/**\n * @private\n */\nexport class WebGL2PipelineInputStage implements Pipeline.Stage {\n private readonly _glOut: WebGL2RenderingContext;\n private readonly _inputFrameTexture: WebGLTexture;\n private _inputTexture: WebGLTexture | null = null;\n\n constructor(glOut: WebGL2RenderingContext) {\n const { height, width } = glOut.canvas;\n this._glOut = glOut;\n this._inputFrameTexture = createTexture(\n glOut,\n glOut.RGBA8,\n width,\n height,\n glOut.NEAREST,\n glOut.NEAREST\n )!;\n }\n\n cleanUp(): void {\n const {\n _glOut,\n _inputFrameTexture,\n _inputTexture\n } = this;\n _glOut.deleteTexture(_inputFrameTexture);\n _glOut.deleteTexture(_inputTexture);\n }\n\n render(\n inputFrame?: InputFrame,\n inputTextureData?: ImageData\n ): void {\n const {\n _glOut,\n _inputFrameTexture\n } = this;\n\n const { height, width } = _glOut.canvas;\n _glOut.viewport(0, 0, width, height);\n _glOut.clearColor(0, 0, 0, 0);\n _glOut.clear(_glOut.COLOR_BUFFER_BIT);\n\n if (inputFrame) {\n _glOut.activeTexture(_glOut.TEXTURE0);\n _glOut.bindTexture(\n _glOut.TEXTURE_2D,\n _inputFrameTexture\n );\n _glOut.texSubImage2D(\n _glOut.TEXTURE_2D,\n 0,\n 0,\n 0,\n width,\n height,\n _glOut.RGBA,\n _glOut.UNSIGNED_BYTE,\n inputFrame\n );\n }\n\n if (!inputTextureData) {\n return;\n }\n const {\n data,\n height: textureHeight,\n width: textureWidth\n } = inputTextureData;\n\n if (!this._inputTexture) {\n this._inputTexture = createTexture(\n _glOut,\n _glOut.RGBA8,\n textureWidth,\n textureHeight,\n _glOut.NEAREST,\n _glOut.NEAREST\n );\n }\n\n _glOut.viewport(0, 0, textureWidth, textureHeight);\n _glOut.activeTexture(_glOut.TEXTURE1);\n\n _glOut.bindTexture(\n _glOut.TEXTURE_2D,\n this._inputTexture\n );\n _glOut.texSubImage2D(\n _glOut.TEXTURE_2D,\n 0,\n 0,\n 0,\n textureWidth,\n textureHeight,\n _glOut.RGBA,\n _glOut.UNSIGNED_BYTE,\n data\n );\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.d.ts b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.d.ts new file mode 100644 index 0000000..2494008 --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.d.ts @@ -0,0 +1,41 @@ +import { Dimensions } from '../../../types'; +import { Pipeline } from '../Pipeline'; +interface InputConfig { + textureName: string; + textureUnit: number; +} +interface OutputConfig { + fragmentShaderSource: string; + glOut: WebGL2RenderingContext; + height?: number; + textureUnit?: number; + type: 'canvas' | 'texture'; + uniformVars?: UniformVarInfo[]; + vertexShaderSource?: string; + width?: number; +} +interface UniformVarInfo { + name: string; + type: 'float' | 'int' | 'uint' | 'float:v'; + values: number[]; +} +/**; + * @private + */ +export declare class WebGL2PipelineProcessingStage implements Pipeline.Stage { + protected readonly _outputDimensions: Dimensions; + private readonly _fragmentShader; + private readonly _glOut; + private readonly _outputFramebuffer; + private readonly _outputTexture; + private readonly _outputTextureUnit; + private readonly _positionBuffer; + private readonly _program; + private readonly _texCoordBuffer; + private readonly _vertexShader; + constructor(inputConfig: InputConfig, outputConfig: OutputConfig); + cleanUp(): void; + render(): void; + protected _setUniformVars(uniformVars: UniformVarInfo[]): void; +} +export {}; diff --git a/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.js b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.js new file mode 100644 index 0000000..88cc6ed --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.js @@ -0,0 +1,103 @@ +"use strict"; +var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) { + if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) { + if (ar || !(i in from)) { + if (!ar) ar = Array.prototype.slice.call(from, 0, i); + ar[i] = from[i]; + } + } + return to.concat(ar || Array.prototype.slice.call(from)); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2PipelineProcessingStage = void 0; +var webgl2PipelineHelpers_1 = require("./webgl2PipelineHelpers"); +/**; + * @private + */ +var WebGL2PipelineProcessingStage = /** @class */ (function () { + function WebGL2PipelineProcessingStage(inputConfig, outputConfig) { + this._outputFramebuffer = null; + this._outputTexture = null; + var textureName = inputConfig.textureName, textureUnit = inputConfig.textureUnit; + var glOut = outputConfig.glOut; + this._glOut = glOut; + var fragmentShaderSource = outputConfig.fragmentShaderSource, _a = outputConfig.height, height = _a === void 0 ? glOut.canvas.height : _a, _b = outputConfig.textureUnit, outputTextureUnit = _b === void 0 ? textureUnit + 1 : _b, outputType = outputConfig.type, _c = outputConfig.uniformVars, uniformVars = _c === void 0 ? [] : _c, _d = outputConfig.vertexShaderSource, vertexShaderSource = _d === void 0 ? "#version 300 es\n in vec2 a_position;\n in vec2 a_texCoord;\n\n out vec2 v_texCoord;\n\n void main() {\n gl_Position = vec4(a_position".concat(outputType === 'canvas' + ? ' * vec2(1.0, -1.0)' + : '', ", 0.0, 1.0);\n v_texCoord = a_texCoord;\n }\n ") : _d, _e = outputConfig.width, width = _e === void 0 ? glOut.canvas.width : _e; + this._outputDimensions = { + height: height, + width: width + }; + this._outputTextureUnit = outputTextureUnit; + this._fragmentShader = (0, webgl2PipelineHelpers_1.compileShader)(glOut, glOut.FRAGMENT_SHADER, fragmentShaderSource); + this._vertexShader = (0, webgl2PipelineHelpers_1.compileShader)(glOut, glOut.VERTEX_SHADER, vertexShaderSource); + this._positionBuffer = (0, webgl2PipelineHelpers_1.initBuffer)(glOut, [ + -1.0, -1.0, + 1.0, -1.0, + -1.0, 1.0, + 1.0, 1.0, + ]); + this._texCoordBuffer = (0, webgl2PipelineHelpers_1.initBuffer)(glOut, [ + 0.0, 0.0, + 1.0, 0.0, + 0.0, 1.0, + 1.0, 1.0, + ]); + if (outputType === 'texture') { + this._outputTexture = (0, webgl2PipelineHelpers_1.createTexture)(glOut, glOut.RGBA8, width, height); + this._outputFramebuffer = glOut.createFramebuffer(); + glOut.bindFramebuffer(glOut.FRAMEBUFFER, this._outputFramebuffer); + glOut.framebufferTexture2D(glOut.FRAMEBUFFER, glOut.COLOR_ATTACHMENT0, glOut.TEXTURE_2D, this._outputTexture, 0); + } + var program = (0, webgl2PipelineHelpers_1.createPipelineStageProgram)(glOut, this._vertexShader, this._fragmentShader, this._positionBuffer, this._texCoordBuffer); + this._program = program; + this._setUniformVars(__spreadArray([ + { + name: textureName, + type: 'int', + values: [textureUnit] + } + ], uniformVars, true)); + } + WebGL2PipelineProcessingStage.prototype.cleanUp = function () { + var _a = this, _fragmentShader = _a._fragmentShader, _glOut = _a._glOut, _positionBuffer = _a._positionBuffer, _program = _a._program, _texCoordBuffer = _a._texCoordBuffer, _vertexShader = _a._vertexShader; + _glOut.deleteProgram(_program); + _glOut.deleteBuffer(_texCoordBuffer); + _glOut.deleteBuffer(_positionBuffer); + _glOut.deleteShader(_vertexShader); + _glOut.deleteShader(_fragmentShader); + }; + WebGL2PipelineProcessingStage.prototype.render = function () { + var _a = this, _glOut = _a._glOut, _b = _a._outputDimensions, height = _b.height, width = _b.width, _outputFramebuffer = _a._outputFramebuffer, _outputTexture = _a._outputTexture, _outputTextureUnit = _a._outputTextureUnit, _program = _a._program; + _glOut.viewport(0, 0, width, height); + _glOut.useProgram(_program); + if (_outputTexture) { + _glOut.activeTexture(_glOut.TEXTURE0 + + _outputTextureUnit); + _glOut.bindTexture(_glOut.TEXTURE_2D, _outputTexture); + } + _glOut.bindFramebuffer(_glOut.FRAMEBUFFER, _outputFramebuffer); + _glOut.drawArrays(_glOut.TRIANGLE_STRIP, 0, 4); + }; + WebGL2PipelineProcessingStage.prototype._setUniformVars = function (uniformVars) { + var _a = this, _glOut = _a._glOut, _program = _a._program; + _glOut.useProgram(_program); + uniformVars.forEach(function (_a) { + var name = _a.name, type = _a.type, values = _a.values; + var uniformVarLocation = _glOut + .getUniformLocation(_program, name); + var isVector = type.split(':')[1] === 'v'; + if (isVector) { + // @ts-ignore + _glOut["uniform1".concat(type[0], "v")](uniformVarLocation, values); + } + else { + // @ts-ignore + _glOut["uniform".concat(values.length).concat(type[0])].apply(_glOut, __spreadArray([uniformVarLocation], values, false)); + } + }); + }; + return WebGL2PipelineProcessingStage; +}()); +exports.WebGL2PipelineProcessingStage = WebGL2PipelineProcessingStage; +//# sourceMappingURL=WebGL2PipelineProcessingStage.js.map \ No newline at end of file diff --git a/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.js.map b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.js.map new file mode 100644 index 0000000..96884bb --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.js.map @@ -0,0 +1 @@ +{"version":3,"file":"WebGL2PipelineProcessingStage.js","sourceRoot":"","sources":["../../../../lib/processors/pipelines/webgl2pipeline/WebGL2PipelineProcessingStage.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,iEAA+G;AAwB/G;;GAEG;AACH;IAYE,uCACE,WAAwB,EACxB,YAA0B;QAVX,uBAAkB,GAAuB,IAAI,CAAC;QAC9C,mBAAc,GAAwB,IAAI,CAAC;QAYxD,IAAA,WAAW,GAET,WAAW,YAFF,EACX,WAAW,GACT,WAAW,YADF,CACG;QAER,IAAA,KAAK,GAAK,YAAY,MAAjB,CAAkB;QAC/B,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QAGlB,IAAA,oBAAoB,GAqBlB,YAAY,qBArBM,EACpB,KAoBE,YAAY,OApBc,EAA5B,MAAM,mBAAG,KAAK,CAAC,MAAM,CAAC,MAAM,KAAA,EAC5B,KAmBE,YAAY,YAnBkC,EAAnC,iBAAiB,mBAAG,WAAW,GAAG,CAAC,KAAA,EAC1C,UAAU,GAkBd,YAAY,KAlBE,EAChB,KAiBE,YAAY,YAjBE,EAAhB,WAAW,mBAAG,EAAE,KAAA,EAChB,KAgBE,YAAY,mBAFb,EAdD,kBAAkB,mBAAG,qLAQnB,UAAU,KAAK,QAAQ;YACrB,CAAC,CAAC,oBAAoB;YACtB,CAAC,CAAC,EAAE,wEAIP,KAAA,EACD,KACE,YAAY,MADY,EAA1B,KAAK,mBAAG,KAAK,CAAC,MAAM,CAAC,KAAK,KAAA,CACX;QAEjB,IAAI,CAAC,iBAAiB,GAAG;YACvB,MAAM,QAAA;YACN,KAAK,OAAA;SACN,CAAC;QAEF,IAAI,CAAC,kBAAkB,GAAG,iBAAiB,CAAC;QAE5C,IAAI,CAAC,eAAe,GAAG,IAAA,qCAAa,EAClC,KAAK,EACL,KAAK,CAAC,eAAe,EACrB,oBAAoB,CACrB,CAAC;QAEF,IAAI,CAAC,aAAa,GAAG,IAAA,qCAAa,EAChC,KAAK,EACL,KAAK,CAAC,aAAa,EACnB,kBAAkB,CACnB,CAAC;QAEF,IAAI,CAAC,eAAe,GAAG,IAAA,kCAAU,EAC/B,KAAK,EACL;YACE,CAAC,GAAG,EAAE,CAAC,GAAG;YACV,GAAG,EAAE,CAAC,GAAG;YACT,CAAC,GAAG,EAAE,GAAG;YACT,GAAG,EAAE,GAAG;SACT,CACD,CAAC;QAEH,IAAI,CAAC,eAAe,GAAG,IAAA,kCAAU,EAC/B,KAAK,EACL;YACE,GAAG,EAAE,GAAG;YACR,GAAG,EAAE,GAAG;YACR,GAAG,EAAE,GAAG;YACR,GAAG,EAAE,GAAG;SACT,CACD,CAAC;QAEH,IAAI,UAAU,KAAK,SAAS,EAAE;YAC5B,IAAI,CAAC,cAAc,GAAG,IAAA,qCAAa,EACjC,KAAK,EACL,KAAK,CAAC,KAAK,EACX,KAAK,EACL,MAAM,CACP,CAAC;YACF,IAAI,CAAC,kBAAkB,GAAG,KAAK,CAAC,iBAAiB,EAAE,CAAC;YACpD,KAAK,CAAC,eAAe,CACnB,KAAK,CAAC,WAAW,EACjB,IAAI,CAAC,kBAAkB,CACxB,CAAC;YACF,KAAK,CAAC,oBAAoB,CACxB,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,iBAAiB,EACvB,KAAK,CAAC,UAAU,EAChB,IAAI,CAAC,cAAc,EACnB,CAAC,CACF,CAAC;SACH;QAED,IAAM,OAAO,GAAG,IAAA,kDAA0B,EACxC,KAAK,EACL,IAAI,CAAC,aAAa,EAClB,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,eAAe,CACrB,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QAExB,IAAI,CAAC,eAAe;YAClB;gBACE,IAAI,EAAE,WAAW;gBACjB,IAAI,EAAE,KAAK;gBACX,MAAM,EAAE,CAAC,WAAW,CAAC;aACtB;WACE,WAAW,QACd,CAAC;IACL,CAAC;IAED,+CAAO,GAAP;QACQ,IAAA,KAOF,IAAI,EANN,eAAe,qBAAA,EACf,MAAM,YAAA,EACN,eAAe,qBAAA,EACf,QAAQ,cAAA,EACR,eAAe,qBAAA,EACf,aAAa,mBACP,CAAC;QACT,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC/B,MAAM,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;QACrC,MAAM,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;QACrC,MAAM,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;QACnC,MAAM,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;IACvC,CAAC;IAED,8CAAM,GAAN;QACQ,IAAA,KAUF,IAAI,EATN,MAAM,YAAA,EACN,yBAGC,EAFC,MAAM,YAAA,EACN,KAAK,WAAA,EAEP,kBAAkB,wBAAA,EAClB,cAAc,oBAAA,EACd,kBAAkB,wBAAA,EAClB,QAAQ,cACF,CAAC;QAET,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACrC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAE5B,IAAI,cAAc,EAAE;YAClB,MAAM,CAAC,aAAa,CAClB,MAAM,CAAC,QAAQ;kBACb,kBAAkB,CACrB,CAAC;YACF,MAAM,CAAC,WAAW,CAChB,MAAM,CAAC,UAAU,EACjB,cAAc,CACf,CAAC;SACH;QACD,MAAM,CAAC,eAAe,CACpB,MAAM,CAAC,WAAW,EAClB,kBAAkB,CACnB,CAAC;QACF,MAAM,CAAC,UAAU,CACf,MAAM,CAAC,cAAc,EACrB,CAAC,EACD,CAAC,CACF,CAAC;IACJ,CAAC;IAES,uDAAe,GAAzB,UAA0B,WAA6B;QAC/C,IAAA,KAGF,IAAI,EAFN,MAAM,YAAA,EACN,QAAQ,cACF,CAAC;QAET,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAE5B,WAAW,CAAC,OAAO,CAAC,UAAC,EAAsB;gBAApB,IAAI,UAAA,EAAE,IAAI,UAAA,EAAE,MAAM,YAAA;YACvC,IAAM,kBAAkB,GAAG,MAAM;iBAC9B,kBAAkB,CACjB,QAAQ,EACR,IAAI,CACL,CAAC;YACJ,IAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC;YAC5C,IAAI,QAAQ,EAAE;gBACZ,aAAa;gBACb,MAAM,CAAC,kBAAW,IAAI,CAAC,CAAC,CAAC,MAAG,CAAC,CAC3B,kBAAkB,EAClB,MAAM,CACP,CAAC;aACH;iBAAM;gBACL,aAAa;gBACb,MAAM,CAAC,iBAAU,MAAM,CAAC,MAAM,SAAG,IAAI,CAAC,CAAC,CAAC,CAAE,CAAC,OAA3C,MAAM,iBACJ,kBAAkB,GACf,MAAM,UACT;aACH;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IACH,oCAAC;AAAD,CAAC,AAlND,IAkNC;AAlNY,sEAA6B","sourcesContent":["import { Dimensions } from '../../../types';\nimport { Pipeline } from '../Pipeline';\nimport { compileShader, createPipelineStageProgram, createTexture, initBuffer } from './webgl2PipelineHelpers';\n\ninterface InputConfig {\n textureName: string;\n textureUnit: number;\n}\n\ninterface OutputConfig {\n fragmentShaderSource: string;\n glOut: WebGL2RenderingContext;\n height?: number;\n textureUnit?: number;\n type: 'canvas' | 'texture';\n uniformVars?: UniformVarInfo[];\n vertexShaderSource?: string;\n width?: number;\n}\n\ninterface UniformVarInfo {\n name: string;\n type: 'float' | 'int' | 'uint' | 'float:v';\n values: number[];\n}\n\n/**;\n * @private\n */\nexport class WebGL2PipelineProcessingStage implements Pipeline.Stage {\n protected readonly _outputDimensions: Dimensions;\n private readonly _fragmentShader: WebGLSampler;\n private readonly _glOut: WebGL2RenderingContext;\n private readonly _outputFramebuffer: WebGLBuffer | null = null;\n private readonly _outputTexture: WebGLTexture | null = null;\n private readonly _outputTextureUnit: number;\n private readonly _positionBuffer: WebGLBuffer;\n private readonly _program: WebGLProgram;\n private readonly _texCoordBuffer: WebGLBuffer;\n private readonly _vertexShader: WebGLShader;\n\n constructor(\n inputConfig: InputConfig,\n outputConfig: OutputConfig\n ) {\n const {\n textureName,\n textureUnit,\n } = inputConfig;\n\n const { glOut } = outputConfig;\n this._glOut = glOut;\n\n const {\n fragmentShaderSource,\n height = glOut.canvas.height,\n textureUnit: outputTextureUnit = textureUnit + 1,\n type: outputType,\n uniformVars = [],\n vertexShaderSource = `#version 300 es\n in vec2 a_position;\n in vec2 a_texCoord;\n\n out vec2 v_texCoord;\n\n void main() {\n gl_Position = vec4(a_position${\n outputType === 'canvas'\n ? ' * vec2(1.0, -1.0)'\n : ''\n }, 0.0, 1.0);\n v_texCoord = a_texCoord;\n }\n `,\n width = glOut.canvas.width\n } = outputConfig;\n\n this._outputDimensions = {\n height,\n width\n };\n\n this._outputTextureUnit = outputTextureUnit;\n\n this._fragmentShader = compileShader(\n glOut,\n glOut.FRAGMENT_SHADER,\n fragmentShaderSource\n );\n\n this._vertexShader = compileShader(\n glOut,\n glOut.VERTEX_SHADER,\n vertexShaderSource\n );\n\n this._positionBuffer = initBuffer(\n glOut,\n [\n -1.0, -1.0,\n 1.0, -1.0,\n -1.0, 1.0,\n 1.0, 1.0,\n ]\n )!;\n\n this._texCoordBuffer = initBuffer(\n glOut,\n [\n 0.0, 0.0,\n 1.0, 0.0,\n 0.0, 1.0,\n 1.0, 1.0,\n ]\n )!;\n\n if (outputType === 'texture') {\n this._outputTexture = createTexture(\n glOut,\n glOut.RGBA8,\n width,\n height\n );\n this._outputFramebuffer = glOut.createFramebuffer();\n glOut.bindFramebuffer(\n glOut.FRAMEBUFFER,\n this._outputFramebuffer\n );\n glOut.framebufferTexture2D(\n glOut.FRAMEBUFFER,\n glOut.COLOR_ATTACHMENT0,\n glOut.TEXTURE_2D,\n this._outputTexture,\n 0\n );\n }\n\n const program = createPipelineStageProgram(\n glOut,\n this._vertexShader,\n this._fragmentShader,\n this._positionBuffer,\n this._texCoordBuffer\n );\n this._program = program;\n\n this._setUniformVars([\n {\n name: textureName,\n type: 'int',\n values: [textureUnit]\n },\n ...uniformVars\n ]);\n }\n\n cleanUp(): void {\n const {\n _fragmentShader,\n _glOut,\n _positionBuffer,\n _program,\n _texCoordBuffer,\n _vertexShader\n } = this;\n _glOut.deleteProgram(_program);\n _glOut.deleteBuffer(_texCoordBuffer);\n _glOut.deleteBuffer(_positionBuffer);\n _glOut.deleteShader(_vertexShader);\n _glOut.deleteShader(_fragmentShader);\n }\n\n render(): void {\n const {\n _glOut,\n _outputDimensions: {\n height,\n width\n },\n _outputFramebuffer,\n _outputTexture,\n _outputTextureUnit,\n _program\n } = this;\n\n _glOut.viewport(0, 0, width, height);\n _glOut.useProgram(_program);\n\n if (_outputTexture) {\n _glOut.activeTexture(\n _glOut.TEXTURE0\n + _outputTextureUnit\n );\n _glOut.bindTexture(\n _glOut.TEXTURE_2D,\n _outputTexture\n );\n }\n _glOut.bindFramebuffer(\n _glOut.FRAMEBUFFER,\n _outputFramebuffer\n );\n _glOut.drawArrays(\n _glOut.TRIANGLE_STRIP,\n 0,\n 4\n );\n }\n\n protected _setUniformVars(uniformVars: UniformVarInfo[]) {\n const {\n _glOut,\n _program\n } = this;\n\n _glOut.useProgram(_program);\n\n uniformVars.forEach(({ name, type, values }) => {\n const uniformVarLocation = _glOut\n .getUniformLocation(\n _program,\n name\n );\n const isVector = type.split(':')[1] === 'v';\n if (isVector) {\n // @ts-ignore\n _glOut[`uniform1${type[0]}v`](\n uniformVarLocation,\n values\n );\n } else {\n // @ts-ignore\n _glOut[`uniform${values.length}${type[0]}`](\n uniformVarLocation,\n ...values\n );\n }\n });\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/pipelines/webgl2pipeline/index.d.ts b/es5/processors/pipelines/webgl2pipeline/index.d.ts new file mode 100644 index 0000000..355fafe --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/index.d.ts @@ -0,0 +1,14 @@ +import { InputFrame } from '../../../types'; +import { Pipeline } from '../Pipeline'; +import { WebGL2PipelineInputStage } from './WebGL2PipelineInputStage'; +import { WebGL2PipelineProcessingStage } from './WebGL2PipelineProcessingStage'; +/** + * @private + */ +export declare class WebGL2Pipeline extends Pipeline { + static InputStage: typeof WebGL2PipelineInputStage; + static ProcessingStage: typeof WebGL2PipelineProcessingStage; + protected readonly _stages: (WebGL2PipelineInputStage | WebGL2PipelineProcessingStage)[]; + cleanUp(): void; + render(inputFrame?: InputFrame, inputTextureData?: ImageData): void; +} diff --git a/es5/processors/pipelines/webgl2pipeline/index.js b/es5/processors/pipelines/webgl2pipeline/index.js new file mode 100644 index 0000000..2544771 --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/index.js @@ -0,0 +1,46 @@ +"use strict"; +var __extends = (this && this.__extends) || (function () { + var extendStatics = function (d, b) { + extendStatics = Object.setPrototypeOf || + ({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) || + function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; }; + return extendStatics(d, b); + }; + return function (d, b) { + if (typeof b !== "function" && b !== null) + throw new TypeError("Class extends value " + String(b) + " is not a constructor or null"); + extendStatics(d, b); + function __() { this.constructor = d; } + d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __()); + }; +})(); +Object.defineProperty(exports, "__esModule", { value: true }); +exports.WebGL2Pipeline = void 0; +var Pipeline_1 = require("../Pipeline"); +var WebGL2PipelineInputStage_1 = require("./WebGL2PipelineInputStage"); +var WebGL2PipelineProcessingStage_1 = require("./WebGL2PipelineProcessingStage"); +/** + * @private + */ +var WebGL2Pipeline = /** @class */ (function (_super) { + __extends(WebGL2Pipeline, _super); + function WebGL2Pipeline() { + var _this = _super !== null && _super.apply(this, arguments) || this; + _this._stages = []; + return _this; + } + WebGL2Pipeline.prototype.cleanUp = function () { + this._stages.forEach(function (stage) { return stage.cleanUp(); }); + }; + WebGL2Pipeline.prototype.render = function (inputFrame, inputTextureData) { + var _a = this._stages, inputStage = _a[0], otherStages = _a.slice(1); + inputStage.render(inputFrame, inputTextureData); + otherStages.forEach(function (stage) { return stage + .render(); }); + }; + WebGL2Pipeline.InputStage = WebGL2PipelineInputStage_1.WebGL2PipelineInputStage; + WebGL2Pipeline.ProcessingStage = WebGL2PipelineProcessingStage_1.WebGL2PipelineProcessingStage; + return WebGL2Pipeline; +}(Pipeline_1.Pipeline)); +exports.WebGL2Pipeline = WebGL2Pipeline; +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/es5/processors/pipelines/webgl2pipeline/index.js.map b/es5/processors/pipelines/webgl2pipeline/index.js.map new file mode 100644 index 0000000..ed0beb2 --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../lib/processors/pipelines/webgl2pipeline/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;AACA,wCAAuC;AACvC,uEAAsE;AACtE,iFAAgF;AAEhF;;GAEG;AACH;IAAoC,kCAAQ;IAA5C;QAAA,qEAsBC;QAnBoB,aAAO,GAAiE,EAAE,CAAC;;IAmBhG,CAAC;IAjBC,gCAAO,GAAP;QACE,IAAI,CAAC,OAAO,CAAC,OAAO,CAClB,UAAC,KAAK,IAAK,OAAA,KAAK,CAAC,OAAO,EAAE,EAAf,CAAe,CAC3B,CAAC;IACJ,CAAC;IAED,+BAAM,GAAN,UACE,UAAuB,EACvB,gBAA4B;QAEtB,IAAA,KAA+B,IAAI,CAAC,OAAO,EAA1C,UAAU,QAAA,EAAK,WAAW,cAAgB,CAAC;QAClD,UAAU,CAAC,MAAM,CAAC,UAAU,EAAE,gBAAgB,CAAC,CAAC;QAChD,WAAW,CAAC,OAAO,CACjB,UAAC,KAAK,IAAK,OAAC,KAAuC;aAChD,MAAM,EAAE,EADA,CACA,CACZ,CAAC;IACJ,CAAC;IApBM,yBAAU,GAAG,mDAAwB,AAA3B,CAA4B;IACtC,8BAAe,GAAG,6DAA6B,AAAhC,CAAiC;IAoBzD,qBAAC;CAAA,AAtBD,CAAoC,mBAAQ,GAsB3C;AAtBY,wCAAc","sourcesContent":["import { InputFrame } from '../../../types';\nimport { Pipeline } from '../Pipeline';\nimport { WebGL2PipelineInputStage } from './WebGL2PipelineInputStage';\nimport { WebGL2PipelineProcessingStage } from './WebGL2PipelineProcessingStage';\n\n/**\n * @private\n */\nexport class WebGL2Pipeline extends Pipeline {\n static InputStage = WebGL2PipelineInputStage;\n static ProcessingStage = WebGL2PipelineProcessingStage;\n protected readonly _stages: (WebGL2PipelineInputStage | WebGL2PipelineProcessingStage)[] = [];\n\n cleanUp(): void {\n this._stages.forEach(\n (stage) => stage.cleanUp()\n );\n }\n\n render(\n inputFrame?: InputFrame,\n inputTextureData?: ImageData\n ): void {\n const [inputStage, ...otherStages] = this._stages;\n inputStage.render(inputFrame, inputTextureData);\n otherStages.forEach(\n (stage) => (stage as WebGL2PipelineProcessingStage)\n .render()\n );\n }\n}\n"]} \ No newline at end of file diff --git a/es5/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.d.ts b/es5/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.d.ts new file mode 100644 index 0000000..e53bb28 --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.d.ts @@ -0,0 +1,20 @@ +/** + * @private + */ +export declare function createPipelineStageProgram(gl: WebGL2RenderingContext, vertexShader: WebGLShader, fragmentShader: WebGLShader, positionBuffer: WebGLBuffer, texCoordBuffer: WebGLBuffer): WebGLProgram; +/** + * @private + */ +export declare function createProgram(gl: WebGL2RenderingContext, vertexShader: WebGLShader, fragmentShader: WebGLShader): WebGLProgram; +/** + * @private + */ +export declare function compileShader(gl: WebGL2RenderingContext, shaderType: number, shaderSource: string): WebGLShader; +/** + * @private + */ +export declare function createTexture(gl: WebGL2RenderingContext, internalformat: number, width: number, height: number, minFilter?: GLint, magFilter?: GLint): WebGLTexture | null; +/** + * @private + */ +export declare function initBuffer(gl: WebGL2RenderingContext, data: number[]): WebGLBuffer | null; diff --git a/es5/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.js b/es5/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.js new file mode 100644 index 0000000..5fc1d82 --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.js @@ -0,0 +1,73 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.initBuffer = exports.createTexture = exports.compileShader = exports.createProgram = exports.createPipelineStageProgram = void 0; +/** + * @private + */ +function createPipelineStageProgram(gl, vertexShader, fragmentShader, positionBuffer, texCoordBuffer) { + var program = createProgram(gl, vertexShader, fragmentShader); + var positionAttributeLocation = gl.getAttribLocation(program, 'a_position'); + gl.enableVertexAttribArray(positionAttributeLocation); + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + gl.vertexAttribPointer(positionAttributeLocation, 2, gl.FLOAT, false, 0, 0); + var texCoordAttributeLocation = gl.getAttribLocation(program, 'a_texCoord'); + gl.enableVertexAttribArray(texCoordAttributeLocation); + gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer); + gl.vertexAttribPointer(texCoordAttributeLocation, 2, gl.FLOAT, false, 0, 0); + return program; +} +exports.createPipelineStageProgram = createPipelineStageProgram; +/** + * @private + */ +function createProgram(gl, vertexShader, fragmentShader) { + var program = gl.createProgram(); + gl.attachShader(program, vertexShader); + gl.attachShader(program, fragmentShader); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + throw new Error("Could not link WebGL program: ".concat(gl.getProgramInfoLog(program))); + } + return program; +} +exports.createProgram = createProgram; +/** + * @private + */ +function compileShader(gl, shaderType, shaderSource) { + var shader = gl.createShader(shaderType); + gl.shaderSource(shader, shaderSource); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + throw new Error("Could not compile shader: ".concat(gl.getShaderInfoLog(shader))); + } + return shader; +} +exports.compileShader = compileShader; +/** + * @private + */ +function createTexture(gl, internalformat, width, height, minFilter, magFilter) { + if (minFilter === void 0) { minFilter = gl.NEAREST; } + if (magFilter === void 0) { magFilter = gl.NEAREST; } + var texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, minFilter); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, magFilter); + gl.texStorage2D(gl.TEXTURE_2D, 1, internalformat, width, height); + return texture; +} +exports.createTexture = createTexture; +/** + * @private + */ +function initBuffer(gl, data) { + var buffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(data), gl.STATIC_DRAW); + return buffer; +} +exports.initBuffer = initBuffer; +//# sourceMappingURL=webgl2PipelineHelpers.js.map \ No newline at end of file diff --git a/es5/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.js.map b/es5/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.js.map new file mode 100644 index 0000000..33a3b4b --- /dev/null +++ b/es5/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.js.map @@ -0,0 +1 @@ +{"version":3,"file":"webgl2PipelineHelpers.js","sourceRoot":"","sources":["../../../../lib/processors/pipelines/webgl2pipeline/webgl2PipelineHelpers.ts"],"names":[],"mappings":";;;AAAA;;GAEG;AACH,SAAgB,0BAA0B,CACxC,EAA0B,EAC1B,YAAyB,EACzB,cAA2B,EAC3B,cAA2B,EAC3B,cAA2B;IAE3B,IAAM,OAAO,GAAG,aAAa,CAAC,EAAE,EAAE,YAAY,EAAE,cAAc,CAAC,CAAC;IAEhE,IAAM,yBAAyB,GAAG,EAAE,CAAC,iBAAiB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAC9E,EAAE,CAAC,uBAAuB,CAAC,yBAAyB,CAAC,CAAC;IACtD,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;IAC/C,EAAE,CAAC,mBAAmB,CAAC,yBAAyB,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5E,IAAM,yBAAyB,GAAG,EAAE,CAAC,iBAAiB,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAC9E,EAAE,CAAC,uBAAuB,CAAC,yBAAyB,CAAC,CAAC;IACtD,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC;IAC/C,EAAE,CAAC,mBAAmB,CAAC,yBAAyB,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5E,OAAO,OAAO,CAAC;AACjB,CAAC;AApBD,gEAoBC;AAED;;GAEG;AACH,SAAgB,aAAa,CAC3B,EAA0B,EAC1B,YAAyB,EACzB,cAA2B;IAE3B,IAAM,OAAO,GAAG,EAAE,CAAC,aAAa,EAAG,CAAC;IACpC,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACvC,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IACzC,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACxB,IAAI,CAAC,EAAE,CAAC,mBAAmB,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,EAAE;QACpD,MAAM,IAAI,KAAK,CACb,wCAAiC,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAE,CACjE,CAAC;KACH;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAfD,sCAeC;AAED;;GAEG;AACH,SAAgB,aAAa,CAC3B,EAA0B,EAC1B,UAAkB,EAClB,YAAoB;IAEpB,IAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,CAAE,CAAC;IAC5C,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACtC,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACzB,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,EAAE;QACrD,MAAM,IAAI,KAAK,CAAC,oCAA6B,EAAE,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAE,CAAC,CAAC;KAC7E;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAZD,sCAYC;AAED;;GAEG;AACH,SAAgB,aAAa,CAC3B,EAA0B,EAC1B,cAAsB,EACtB,KAAa,EACb,MAAc,EACd,SAA6B,EAC7B,SAA6B;IAD7B,0BAAA,EAAA,YAAmB,EAAE,CAAC,OAAO;IAC7B,0BAAA,EAAA,YAAmB,EAAE,CAAC,OAAO;IAE7B,IAAM,OAAO,GAAG,EAAE,CAAC,aAAa,EAAE,CAAC;IACnC,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACvC,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC;IACrE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,cAAc,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC;IACrE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAC;IAClE,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAC;IAClE,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;IACjE,OAAO,OAAO,CAAC;AACjB,CAAC;AAhBD,sCAgBC;AAED;;GAEG;AACH,SAAgB,UAAU,CACxB,EAA0B,EAC1B,IAAc;IAEd,IAAM,MAAM,GAAG,EAAE,CAAC,YAAY,EAAE,CAAC;IACjC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACvC,EAAE,CAAC,UAAU,CACX,EAAE,CAAC,YAAY,EACf,IAAI,YAAY,CAAC,IAAI,CAAC,EACtB,EAAE,CAAC,WAAW,CACf,CAAC;IACF,OAAO,MAAM,CAAC;AAChB,CAAC;AAZD,gCAYC","sourcesContent":["/**\n * @private\n */\nexport function createPipelineStageProgram(\n gl: WebGL2RenderingContext,\n vertexShader: WebGLShader,\n fragmentShader: WebGLShader,\n positionBuffer: WebGLBuffer,\n texCoordBuffer: WebGLBuffer\n) {\n const program = createProgram(gl, vertexShader, fragmentShader);\n\n const positionAttributeLocation = gl.getAttribLocation(program, 'a_position');\n gl.enableVertexAttribArray(positionAttributeLocation);\n gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer);\n gl.vertexAttribPointer(positionAttributeLocation, 2, gl.FLOAT, false, 0, 0);\n\n const texCoordAttributeLocation = gl.getAttribLocation(program, 'a_texCoord');\n gl.enableVertexAttribArray(texCoordAttributeLocation);\n gl.bindBuffer(gl.ARRAY_BUFFER, texCoordBuffer);\n gl.vertexAttribPointer(texCoordAttributeLocation, 2, gl.FLOAT, false, 0, 0);\n\n return program;\n}\n\n/**\n * @private\n */\nexport function createProgram(\n gl: WebGL2RenderingContext,\n vertexShader: WebGLShader,\n fragmentShader: WebGLShader\n) {\n const program = gl.createProgram()!;\n gl.attachShader(program, vertexShader);\n gl.attachShader(program, fragmentShader);\n gl.linkProgram(program);\n if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {\n throw new Error(\n `Could not link WebGL program: ${gl.getProgramInfoLog(program)}`\n );\n }\n return program;\n}\n\n/**\n * @private\n */\nexport function compileShader(\n gl: WebGL2RenderingContext,\n shaderType: number,\n shaderSource: string\n) {\n const shader = gl.createShader(shaderType)!;\n gl.shaderSource(shader, shaderSource);\n gl.compileShader(shader);\n if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {\n throw new Error(`Could not compile shader: ${gl.getShaderInfoLog(shader)}`);\n }\n return shader;\n}\n\n/**\n * @private\n */\nexport function createTexture(\n gl: WebGL2RenderingContext,\n internalformat: number,\n width: number,\n height: number,\n minFilter: GLint = gl.NEAREST,\n magFilter: GLint = gl.NEAREST\n) {\n const texture = gl.createTexture();\n gl.bindTexture(gl.TEXTURE_2D, texture);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, minFilter);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, magFilter);\n gl.texStorage2D(gl.TEXTURE_2D, 1, internalformat, width, height);\n return texture;\n}\n\n/**\n * @private\n */\nexport function initBuffer(\n gl: WebGL2RenderingContext,\n data: number[]\n) {\n const buffer = gl.createBuffer();\n gl.bindBuffer(gl.ARRAY_BUFFER, buffer);\n gl.bufferData(\n gl.ARRAY_BUFFER,\n new Float32Array(data),\n gl.STATIC_DRAW\n );\n return buffer;\n}\n"]} \ No newline at end of file diff --git a/es5/types.d.ts b/es5/types.d.ts new file mode 100644 index 0000000..a8b4e0e --- /dev/null +++ b/es5/types.d.ts @@ -0,0 +1,63 @@ +/** + * @private + */ +declare global { + interface Window { + chrome: any; + createTwilioTFLiteModule: () => Promise; + createTwilioTFLiteSIMDModule: () => Promise; + OffscreenCanvas: typeof OffscreenCanvas; + Twilio: object & { + VideoProcessors?: any; + }; + } +} +/** + * @private + */ +export interface Timing { + delay?: number; + end?: number; + start?: number; +} +/** + * @private + */ +export interface Dimensions { + height: number; + width: number; +} +/** + * @private + */ +export type BilateralFilterConfig = { + sigmaSpace?: number; +}; +/** + * @private + */ +export type InputFrame = OffscreenCanvas | HTMLCanvasElement | VideoFrame; +/** + * ImageFit specifies the positioning of an image inside a viewport. + */ +export declare enum ImageFit { + /** + * Scale the image up or down to fill the viewport while preserving the aspect ratio. + * The image will be fully visible but will add empty space in the viewport if + * aspect ratios do not match. + */ + Contain = "Contain", + /** + * Scale the image to fill both height and width of the viewport while preserving + * the aspect ratio, but will crop the image if aspect ratios do not match. + */ + Cover = "Cover", + /** + * Stretches the image to fill the viewport regardless of aspect ratio. + */ + Fill = "Fill", + /** + * Ignore height and width and use the original size. + */ + None = "None" +} diff --git a/es5/types.js b/es5/types.js new file mode 100644 index 0000000..9c9d022 --- /dev/null +++ b/es5/types.js @@ -0,0 +1,29 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.ImageFit = void 0; +/** + * ImageFit specifies the positioning of an image inside a viewport. + */ +var ImageFit; +(function (ImageFit) { + /** + * Scale the image up or down to fill the viewport while preserving the aspect ratio. + * The image will be fully visible but will add empty space in the viewport if + * aspect ratios do not match. + */ + ImageFit["Contain"] = "Contain"; + /** + * Scale the image to fill both height and width of the viewport while preserving + * the aspect ratio, but will crop the image if aspect ratios do not match. + */ + ImageFit["Cover"] = "Cover"; + /** + * Stretches the image to fill the viewport regardless of aspect ratio. + */ + ImageFit["Fill"] = "Fill"; + /** + * Ignore height and width and use the original size. + */ + ImageFit["None"] = "None"; +})(ImageFit || (exports.ImageFit = ImageFit = {})); +//# sourceMappingURL=types.js.map \ No newline at end of file diff --git a/es5/types.js.map b/es5/types.js.map new file mode 100644 index 0000000..b045ab7 --- /dev/null +++ b/es5/types.js.map @@ -0,0 +1 @@ +{"version":3,"file":"types.js","sourceRoot":"","sources":["../lib/types.ts"],"names":[],"mappings":";;;AA0CA;;GAEG;AACH,IAAY,QAuBX;AAvBD,WAAY,QAAQ;IAClB;;;;OAIG;IACH,+BAAmB,CAAA;IAEnB;;;OAGG;IACH,2BAAe,CAAA;IAEf;;OAEG;IACH,yBAAa,CAAA;IAEb;;OAEG;IACH,yBAAa,CAAA;AACf,CAAC,EAvBW,QAAQ,wBAAR,QAAQ,QAuBnB","sourcesContent":["/**\n * @private\n */\n declare global {\n interface Window {\n chrome: any;\n createTwilioTFLiteModule: () => Promise;\n createTwilioTFLiteSIMDModule: () => Promise;\n OffscreenCanvas: typeof OffscreenCanvas;\n Twilio: object & { VideoProcessors?: any };\n }\n}\n\n/**\n * @private\n */\nexport interface Timing {\n delay?: number;\n end?: number;\n start?: number;\n}\n\n/**\n * @private\n */\nexport interface Dimensions {\n height: number;\n width: number;\n}\n\n/**\n * @private\n */\nexport type BilateralFilterConfig = {\n sigmaSpace?: number\n};\n\n/**\n * @private\n */\nexport type InputFrame = OffscreenCanvas | HTMLCanvasElement | VideoFrame;\n\n/**\n * ImageFit specifies the positioning of an image inside a viewport.\n */\nexport enum ImageFit {\n /**\n * Scale the image up or down to fill the viewport while preserving the aspect ratio.\n * The image will be fully visible but will add empty space in the viewport if\n * aspect ratios do not match.\n */\n Contain = 'Contain',\n\n /**\n * Scale the image to fill both height and width of the viewport while preserving\n * the aspect ratio, but will crop the image if aspect ratios do not match.\n */\n Cover = 'Cover',\n\n /**\n * Stretches the image to fill the viewport regardless of aspect ratio.\n */\n Fill = 'Fill',\n\n /**\n * Ignore height and width and use the original size.\n */\n None = 'None'\n}\n"]} \ No newline at end of file diff --git a/es5/utils/Benchmark.d.ts b/es5/utils/Benchmark.d.ts new file mode 100644 index 0000000..2b37fd4 --- /dev/null +++ b/es5/utils/Benchmark.d.ts @@ -0,0 +1,16 @@ +/** + * @private + */ +export declare class Benchmark { + static readonly cacheSize = 41; + private _timingCache; + private _timings; + constructor(); + end(name: string): void; + getAverageDelay(name: string): number | undefined; + getNames(): string[]; + getRate(name: string): number | undefined; + merge(benchmark: Benchmark): void; + start(name: string): void; + private _save; +} diff --git a/es5/utils/Benchmark.js b/es5/utils/Benchmark.js new file mode 100644 index 0000000..0ac91db --- /dev/null +++ b/es5/utils/Benchmark.js @@ -0,0 +1,85 @@ +"use strict"; +var __assign = (this && this.__assign) || function () { + __assign = Object.assign || function(t) { + for (var s, i = 1, n = arguments.length; i < n; i++) { + s = arguments[i]; + for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p)) + t[p] = s[p]; + } + return t; + }; + return __assign.apply(this, arguments); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Benchmark = void 0; +/** + * @private + */ +var Benchmark = /** @class */ (function () { + function Benchmark() { + this._timingCache = new Map(); + this._timings = new Map(); + } + Benchmark.prototype.end = function (name) { + var timing = this._timings.get(name); + if (!timing) { + return; + } + timing.end = Date.now(); + timing.delay = timing.end - timing.start; + this._save(name, __assign({}, timing)); + }; + Benchmark.prototype.getAverageDelay = function (name) { + var timingCache = this._timingCache.get(name); + if (!timingCache || !timingCache.length) { + return; + } + return timingCache.map(function (timing) { return timing.delay; }) + .reduce(function (total, value) { return total += value; }, 0) / timingCache.length; + }; + Benchmark.prototype.getNames = function () { + return Array.from(this._timingCache.keys()); + }; + Benchmark.prototype.getRate = function (name) { + var timingCache = this._timingCache.get(name); + if (!timingCache || timingCache.length < 2) { + return; + } + var totalDelay = timingCache[timingCache.length - 1].end - timingCache[0].start; + return (timingCache.length / totalDelay) * 1000; + }; + Benchmark.prototype.merge = function (benchmark) { + var _this = this; + var _timingCache = benchmark._timingCache, _timings = benchmark._timings; + _timingCache.forEach(function (cache, name) { return _this._timingCache.set(name, cache); }); + _timings.forEach(function (timing, name) { return _this._timings.set(name, timing); }); + }; + Benchmark.prototype.start = function (name) { + var timing = this._timings.get(name); + if (!timing) { + timing = {}; + this._timings.set(name, timing); + } + timing.start = Date.now(); + delete timing.end; + delete timing.delay; + }; + Benchmark.prototype._save = function (name, timing) { + var timingCache = this._timingCache.get(name); + if (!timingCache) { + timingCache = []; + this._timingCache.set(name, timingCache); + } + timingCache.push(timing); + if (timingCache.length > Benchmark.cacheSize) { + timingCache.splice(0, timingCache.length - Benchmark.cacheSize); + } + }; + // NOTE (csantos): How many timing information to save per benchmark. + // This is about the amount of timing info generated on a 24fps input. + // Enough samples to calculate fps + Benchmark.cacheSize = 41; + return Benchmark; +}()); +exports.Benchmark = Benchmark; +//# sourceMappingURL=Benchmark.js.map \ No newline at end of file diff --git a/es5/utils/Benchmark.js.map b/es5/utils/Benchmark.js.map new file mode 100644 index 0000000..0f7ca4d --- /dev/null +++ b/es5/utils/Benchmark.js.map @@ -0,0 +1 @@ +{"version":3,"file":"Benchmark.js","sourceRoot":"","sources":["../../lib/utils/Benchmark.ts"],"names":[],"mappings":";;;;;;;;;;;;;;AAEA;;GAEG;AACH;IAUE;QACE,IAAI,CAAC,YAAY,GAAG,IAAI,GAAG,EAAE,CAAC;QAC9B,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;IAC5B,CAAC;IAED,uBAAG,GAAH,UAAI,IAAY;QACd,IAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,EAAE;YACX,OAAO;SACR;QACD,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxB,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,GAAG,MAAM,CAAC,KAAM,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,IAAI,eAAM,MAAM,EAAE,CAAC;IAChC,CAAC;IAED,mCAAe,GAAf,UAAgB,IAAY;QAC1B,IAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,WAAW,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;YACvC,OAAO;SACR;QACD,OAAO,WAAW,CAAC,GAAG,CAAC,UAAA,MAAM,IAAI,OAAA,MAAM,CAAC,KAAM,EAAb,CAAa,CAAC;aAC5C,MAAM,CAAC,UAAC,KAAa,EAAE,KAAa,IAAK,OAAA,KAAK,IAAI,KAAK,EAAd,CAAc,EAAE,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC;IACtF,CAAC;IAED,4BAAQ,GAAR;QACE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,2BAAO,GAAP,UAAQ,IAAY;QAClB,IAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE;YAC1C,OAAO;SACR;QACD,IAAM,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAI,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,KAAM,CAAC;QACpF,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC;IAClD,CAAC;IAED,yBAAK,GAAL,UAAM,SAAoB;QAA1B,iBAQC;QAPS,IAAA,YAAY,GAAe,SAAS,aAAxB,EAAE,QAAQ,GAAK,SAAS,SAAd,CAAe;QAC7C,YAAY,CAAC,OAAO,CAClB,UAAC,KAAK,EAAE,IAAI,IAAK,OAAA,KAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,EAAlC,CAAkC,CACpD,CAAC;QACF,QAAQ,CAAC,OAAO,CACd,UAAC,MAAM,EAAE,IAAI,IAAK,OAAA,KAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,EAA/B,CAA+B,CAClD,CAAC;IACJ,CAAC;IAED,yBAAK,GAAL,UAAM,IAAY;QAChB,IAAI,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,EAAE;YACX,MAAM,GAAG,EAAE,CAAC;YACZ,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;SACjC;QACD,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC,GAAG,CAAC;QAClB,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAEO,yBAAK,GAAb,UAAc,IAAY,EAAE,MAAc;QACxC,IAAI,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,CAAC,WAAW,EAAE;YAChB,WAAW,GAAG,EAAE,CAAC;YACjB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;SAC1C;QAED,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEzB,IAAI,WAAW,CAAC,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE;YAC5C,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,WAAW,CAAC,MAAM,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;SACjE;IACH,CAAC;IA9ED,qEAAqE;IACrE,sEAAsE;IACtE,kCAAkC;IAClB,mBAAS,GAAG,EAAE,CAAC;IA4EjC,gBAAC;CAAA,AAjFD,IAiFC;AAjFY,8BAAS","sourcesContent":["import { Timing } from '../types';\n\n/**\n * @private\n */\nexport class Benchmark {\n\n // NOTE (csantos): How many timing information to save per benchmark.\n // This is about the amount of timing info generated on a 24fps input.\n // Enough samples to calculate fps\n static readonly cacheSize = 41;\n\n private _timingCache: Map;\n private _timings: Map;\n\n constructor() {\n this._timingCache = new Map();\n this._timings = new Map();\n }\n\n end(name: string) {\n const timing = this._timings.get(name);\n if (!timing) {\n return;\n }\n timing.end = Date.now();\n timing.delay = timing.end - timing.start!;\n this._save(name, {...timing});\n }\n\n getAverageDelay(name: string): number | undefined {\n const timingCache = this._timingCache.get(name);\n if (!timingCache || !timingCache.length) {\n return;\n }\n return timingCache.map(timing => timing.delay!)\n .reduce((total: number, value: number) => total += value, 0) / timingCache.length;\n }\n\n getNames(): string[] {\n return Array.from(this._timingCache.keys());\n }\n\n getRate(name: string): number | undefined {\n const timingCache = this._timingCache.get(name);\n if (!timingCache || timingCache.length < 2) {\n return;\n }\n const totalDelay = timingCache[timingCache.length - 1].end! - timingCache[0].start!;\n return (timingCache.length / totalDelay) * 1000;\n }\n\n merge(benchmark: Benchmark) {\n const { _timingCache, _timings } = benchmark;\n _timingCache.forEach(\n (cache, name) => this._timingCache.set(name, cache)\n );\n _timings.forEach(\n (timing, name) => this._timings.set(name, timing)\n );\n }\n\n start(name: string) {\n let timing = this._timings.get(name);\n if (!timing) {\n timing = {};\n this._timings.set(name, timing);\n }\n timing.start = Date.now();\n delete timing.end;\n delete timing.delay;\n }\n\n private _save(name: string, timing: Timing) {\n let timingCache = this._timingCache.get(name);\n if (!timingCache) {\n timingCache = [];\n this._timingCache.set(name, timingCache);\n }\n\n timingCache.push(timing);\n\n if (timingCache.length > Benchmark.cacheSize) {\n timingCache.splice(0, timingCache.length - Benchmark.cacheSize);\n }\n }\n}\n"]} \ No newline at end of file diff --git a/es5/utils/TwilioTFLite.d.ts b/es5/utils/TwilioTFLite.d.ts new file mode 100644 index 0000000..762d5b8 --- /dev/null +++ b/es5/utils/TwilioTFLite.d.ts @@ -0,0 +1,14 @@ +/** + * @private + */ +export declare class TwilioTFLite { + private _inputBuffer; + private _isSimdEnabled; + private _tflite; + get isSimdEnabled(): boolean | null; + initialize(assetsPath: string, modelName: string, moduleLoaderName: string, moduleSimdLoaderName: string): Promise; + loadInputBuffer(inputBuffer: Uint8ClampedArray): void; + runInference(): Uint8ClampedArray; + private _loadScript; + private _loadWasmModule; +} diff --git a/es5/utils/TwilioTFLite.js b/es5/utils/TwilioTFLite.js new file mode 100644 index 0000000..2b26fb1 --- /dev/null +++ b/es5/utils/TwilioTFLite.js @@ -0,0 +1,187 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.TwilioTFLite = void 0; +var isWebWorker = typeof WorkerGlobalScope !== 'undefined' + && self instanceof WorkerGlobalScope; +var loadedScripts = new Set(); +var model; +/** + * @private + */ +var TwilioTFLite = /** @class */ (function () { + function TwilioTFLite() { + this._inputBuffer = null; + this._isSimdEnabled = null; + this._tflite = null; + } + Object.defineProperty(TwilioTFLite.prototype, "isSimdEnabled", { + get: function () { + return this._isSimdEnabled; + }, + enumerable: false, + configurable: true + }); + TwilioTFLite.prototype.initialize = function (assetsPath, modelName, moduleLoaderName, moduleSimdLoaderName) { + return __awaiter(this, void 0, void 0, function () { + var _a, modelResponse, _b, tflite, modelBufferOffset; + return __generator(this, function (_c) { + switch (_c.label) { + case 0: + if (this._tflite) { + return [2 /*return*/]; + } + if (isWebWorker) { + // NOTE(mmalavalli): In a web worker, paths to other dependencies + // are determined relative to the assets path, so no need to append + // it to the file names of the dependencies. + assetsPath = ''; + } + return [4 /*yield*/, Promise.all([ + this._loadWasmModule(assetsPath, moduleLoaderName, moduleSimdLoaderName), + fetch("".concat(assetsPath).concat(modelName)), + ])]; + case 1: + _a = _c.sent(), modelResponse = _a[1]; + _b = model; + if (_b) return [3 /*break*/, 3]; + return [4 /*yield*/, modelResponse.arrayBuffer()]; + case 2: + _b = (_c.sent()); + _c.label = 3; + case 3: + model = _b; + tflite = this._tflite; + modelBufferOffset = tflite._getModelBufferMemoryOffset(); + tflite.HEAPU8.set(new Uint8Array(model), modelBufferOffset); + tflite._loadModel(model.byteLength); + return [2 /*return*/]; + } + }); + }); + }; + TwilioTFLite.prototype.loadInputBuffer = function (inputBuffer) { + var tflite = this._tflite; + var height = tflite._getInputHeight(); + var width = tflite._getInputWidth(); + var pixels = width * height; + var tfliteInputMemoryOffset = tflite._getInputMemoryOffset() / 4; + for (var i = 0; i < pixels; i++) { + var curTFLiteOffset = tfliteInputMemoryOffset + i * 3; + var curImageBufferOffset = i * 4; + tflite.HEAPF32[curTFLiteOffset] = inputBuffer[curImageBufferOffset] / 255; + tflite.HEAPF32[curTFLiteOffset + 1] = inputBuffer[curImageBufferOffset + 1] / 255; + tflite.HEAPF32[curTFLiteOffset + 2] = inputBuffer[curImageBufferOffset + 2] / 255; + } + this._inputBuffer = inputBuffer; + }; + TwilioTFLite.prototype.runInference = function () { + var tflite = this._tflite; + var height = tflite._getInputHeight(); + var width = tflite._getInputWidth(); + var pixels = width * height; + var tfliteOutputMemoryOffset = tflite._getOutputMemoryOffset() / 4; + tflite._runInference(); + var inputBuffer = this._inputBuffer || new Uint8ClampedArray(pixels * 4); + for (var i = 0; i < pixels; i++) { + inputBuffer[i * 4 + 3] = Math.round(tflite.HEAPF32[tfliteOutputMemoryOffset + i] * 255); + } + return inputBuffer; + }; + TwilioTFLite.prototype._loadScript = function (path) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + if (loadedScripts.has(path)) { + return [2 /*return*/]; + } + if (isWebWorker) { + importScripts(path); + loadedScripts.add(path); + return [2 /*return*/]; + } + return [2 /*return*/, new Promise(function (resolve, reject) { + var script = document.createElement('script'); + script.onload = function () { + loadedScripts.add(path); + resolve(); + }; + script.onerror = function () { + reject(); + }; + document.head.append(script); + script.src = path; + })]; + }); + }); + }; + TwilioTFLite.prototype._loadWasmModule = function (assetsPath, moduleLoaderName, moduleSimdLoaderName) { + return __awaiter(this, void 0, void 0, function () { + var _a, _b, _c; + return __generator(this, function (_d) { + switch (_d.label) { + case 0: + _d.trys.push([0, 3, , 6]); + return [4 /*yield*/, this._loadScript("".concat(assetsPath).concat(moduleSimdLoaderName))]; + case 1: + _d.sent(); + _a = this; + return [4 /*yield*/, createTwilioTFLiteSIMDModule()]; + case 2: + _a._tflite = _d.sent(); + this._isSimdEnabled = true; + return [3 /*break*/, 6]; + case 3: + _b = _d.sent(); + return [4 /*yield*/, this._loadScript("".concat(assetsPath).concat(moduleLoaderName))]; + case 4: + _d.sent(); + _c = this; + return [4 /*yield*/, createTwilioTFLiteModule()]; + case 5: + _c._tflite = _d.sent(); + this._isSimdEnabled = false; + return [3 /*break*/, 6]; + case 6: return [2 /*return*/]; + } + }); + }); + }; + return TwilioTFLite; +}()); +exports.TwilioTFLite = TwilioTFLite; +//# sourceMappingURL=TwilioTFLite.js.map \ No newline at end of file diff --git a/es5/utils/TwilioTFLite.js.map b/es5/utils/TwilioTFLite.js.map new file mode 100644 index 0000000..7d290c1 --- /dev/null +++ b/es5/utils/TwilioTFLite.js.map @@ -0,0 +1 @@ +{"version":3,"file":"TwilioTFLite.js","sourceRoot":"","sources":["../../lib/utils/TwilioTFLite.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKA,IAAM,WAAW,GAAG,OAAO,iBAAiB,KAAK,WAAW;OACvD,IAAI,YAAY,iBAAiB,CAAC;AAEvC,IAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;AACxC,IAAI,KAAkB,CAAC;AAEvB;;GAEG;AACH;IAAA;QACU,iBAAY,GAA6B,IAAI,CAAC;QAC9C,mBAAc,GAAmB,IAAI,CAAC;QACtC,YAAO,GAAQ,IAAI,CAAC;IA2G9B,CAAC;IAzGC,sBAAI,uCAAa;aAAjB;YACE,OAAO,IAAI,CAAC,cAAc,CAAC;QAC7B,CAAC;;;OAAA;IAEK,iCAAU,GAAhB,UACE,UAAkB,EAClB,SAAiB,EACjB,gBAAwB,EACxB,oBAA4B;;;;;;wBAE5B,IAAI,IAAI,CAAC,OAAO,EAAE;4BAChB,sBAAO;yBACR;wBACD,IAAI,WAAW,EAAE;4BACf,iEAAiE;4BACjE,mEAAmE;4BACnE,4CAA4C;4BAC5C,UAAU,GAAG,EAAE,CAAC;yBACjB;wBAC2C,qBAAM,OAAO,CAAC,GAAG,CAAC;gCAC5D,IAAI,CAAC,eAAe,CAClB,UAAU,EACV,gBAAgB,EAChB,oBAAoB,CACrB;gCACD,KAAK,CAAC,UAAG,UAAU,SAAG,SAAS,CAAE,CAAC;6BACnC,CAAC,EAAA;;wBAPI,KAAsC,SAO1C,EAPO,aAAa,QAAA;wBAQd,KAAA,KAAK,CAAA;gCAAL,wBAAK;wBAAI,qBAAM,aAAa,CAAC,WAAW,EAAE,EAAA;;8BAAjC,SAAiC;;;wBAAlD,KAAK,KAA6C,CAAC;wBAClC,MAAM,GAAK,IAAI,QAAT,CAAU;wBAC3B,iBAAiB,GAAG,MAAM,CAAC,2BAA2B,EAAE,CAAC;wBAC/D,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,EAAE,iBAAiB,CAAC,CAAC;wBAC5D,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;;;;;KACrC;IAED,sCAAe,GAAf,UAAgB,WAA8B;QACpC,IAAS,MAAM,GAAK,IAAI,QAAT,CAAU;QACjC,IAAM,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACxC,IAAM,KAAK,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;QACtC,IAAM,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;QAC9B,IAAM,uBAAuB,GAAG,MAAM,CAAC,qBAAqB,EAAE,GAAG,CAAC,CAAC;QAEnE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE;YAC/B,IAAM,eAAe,GAAG,uBAAuB,GAAG,CAAC,GAAG,CAAC,CAAC;YACxD,IAAM,oBAAoB,GAAG,CAAC,GAAG,CAAC,CAAC;YACnC,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,WAAW,CAAC,oBAAoB,CAAC,GAAG,GAAG,CAAC;YAC1E,MAAM,CAAC,OAAO,CAAC,eAAe,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,oBAAoB,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;YAClF,MAAM,CAAC,OAAO,CAAC,eAAe,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,oBAAoB,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;SACnF;QACD,IAAI,CAAC,YAAY,GAAG,WAAW,CAAC;IAClC,CAAC;IAED,mCAAY,GAAZ;QACU,IAAS,MAAM,GAAK,IAAI,QAAT,CAAU;QACjC,IAAM,MAAM,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;QACxC,IAAM,KAAK,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;QACtC,IAAM,MAAM,GAAG,KAAK,GAAG,MAAM,CAAC;QAC9B,IAAM,wBAAwB,GAAG,MAAM,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAC;QAErE,MAAM,CAAC,aAAa,EAAE,CAAC;QAEvB,IAAM,WAAW,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC3E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE;YAC/B,WAAY,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,wBAAwB,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;SAC1F;QACD,OAAO,WAAY,CAAC;IACtB,CAAC;IAEa,kCAAW,GAAzB,UAA0B,IAAY;;;gBACpC,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;oBAC3B,sBAAO;iBACR;gBACD,IAAI,WAAW,EAAE;oBACf,aAAa,CAAC,IAAI,CAAC,CAAC;oBACpB,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBACxB,sBAAO;iBACR;gBACD,sBAAO,IAAI,OAAO,CAAC,UAAC,OAAO,EAAE,MAAM;wBACjC,IAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;wBAChD,MAAM,CAAC,MAAM,GAAG;4BACd,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;4BACxB,OAAO,EAAE,CAAC;wBACZ,CAAC,CAAC;wBACF,MAAM,CAAC,OAAO,GAAG;4BACf,MAAM,EAAE,CAAC;wBACX,CAAC,CAAC;wBACF,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;wBAC7B,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC;oBACpB,CAAC,CAAC,EAAC;;;KACJ;IAEa,sCAAe,GAA7B,UACE,UAAkB,EAClB,gBAAwB,EACxB,oBAA4B;;;;;;;wBAG1B,qBAAM,IAAI,CAAC,WAAW,CAAC,UAAG,UAAU,SAAG,oBAAoB,CAAE,CAAC,EAAA;;wBAA9D,SAA8D,CAAC;wBAC/D,KAAA,IAAI,CAAA;wBAAW,qBAAM,4BAA4B,EAAE,EAAA;;wBAAnD,GAAK,OAAO,GAAG,SAAoC,CAAC;wBACpD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;;;;wBAE3B,qBAAM,IAAI,CAAC,WAAW,CAAC,UAAG,UAAU,SAAG,gBAAgB,CAAE,CAAC,EAAA;;wBAA1D,SAA0D,CAAC;wBAC3D,KAAA,IAAI,CAAA;wBAAW,qBAAM,wBAAwB,EAAE,EAAA;;wBAA/C,GAAK,OAAO,GAAG,SAAgC,CAAC;wBAChD,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;;;;;;KAE/B;IACH,mBAAC;AAAD,CAAC,AA9GD,IA8GC;AA9GY,oCAAY","sourcesContent":["declare const WorkerGlobalScope: any;\ndeclare function createTwilioTFLiteModule(): Promise;\ndeclare function createTwilioTFLiteSIMDModule(): Promise;\ndeclare function importScripts(path: string): any;\n\nconst isWebWorker = typeof WorkerGlobalScope !== 'undefined'\n && self instanceof WorkerGlobalScope;\n\nconst loadedScripts = new Set();\nlet model: ArrayBuffer;\n\n/**\n * @private\n */\nexport class TwilioTFLite {\n private _inputBuffer: Uint8ClampedArray | null = null;\n private _isSimdEnabled: boolean | null = null;\n private _tflite: any = null;\n\n get isSimdEnabled(): boolean | null {\n return this._isSimdEnabled;\n }\n\n async initialize(\n assetsPath: string,\n modelName: string,\n moduleLoaderName: string,\n moduleSimdLoaderName: string,\n ): Promise {\n if (this._tflite) {\n return;\n }\n if (isWebWorker) {\n // NOTE(mmalavalli): In a web worker, paths to other dependencies\n // are determined relative to the assets path, so no need to append\n // it to the file names of the dependencies.\n assetsPath = '';\n }\n const [, modelResponse]: [void, Response] = await Promise.all([\n this._loadWasmModule(\n assetsPath,\n moduleLoaderName,\n moduleSimdLoaderName,\n ),\n fetch(`${assetsPath}${modelName}`),\n ]);\n model = model || await modelResponse.arrayBuffer();\n const { _tflite: tflite } = this;\n const modelBufferOffset = tflite._getModelBufferMemoryOffset();\n tflite.HEAPU8.set(new Uint8Array(model), modelBufferOffset);\n tflite._loadModel(model.byteLength);\n }\n\n loadInputBuffer(inputBuffer: Uint8ClampedArray): void {\n const { _tflite: tflite } = this;\n const height = tflite._getInputHeight();\n const width = tflite._getInputWidth();\n const pixels = width * height;\n const tfliteInputMemoryOffset = tflite._getInputMemoryOffset() / 4;\n\n for (let i = 0; i < pixels; i++) {\n const curTFLiteOffset = tfliteInputMemoryOffset + i * 3;\n const curImageBufferOffset = i * 4;\n tflite.HEAPF32[curTFLiteOffset] = inputBuffer[curImageBufferOffset] / 255;\n tflite.HEAPF32[curTFLiteOffset + 1] = inputBuffer[curImageBufferOffset + 1] / 255;\n tflite.HEAPF32[curTFLiteOffset + 2] = inputBuffer[curImageBufferOffset + 2] / 255;\n }\n this._inputBuffer = inputBuffer;\n }\n\n runInference(): Uint8ClampedArray {\n const { _tflite: tflite } = this;\n const height = tflite._getInputHeight();\n const width = tflite._getInputWidth();\n const pixels = width * height;\n const tfliteOutputMemoryOffset = tflite._getOutputMemoryOffset() / 4;\n\n tflite._runInference();\n\n const inputBuffer = this._inputBuffer || new Uint8ClampedArray(pixels * 4);\n for (let i = 0; i < pixels; i++) {\n inputBuffer![i * 4 + 3] = Math.round(tflite.HEAPF32[tfliteOutputMemoryOffset + i] * 255);\n }\n return inputBuffer!;\n }\n\n private async _loadScript(path: string): Promise {\n if (loadedScripts.has(path)) {\n return;\n }\n if (isWebWorker) {\n importScripts(path);\n loadedScripts.add(path);\n return;\n }\n return new Promise((resolve, reject) => {\n const script = document.createElement('script');\n script.onload = () => {\n loadedScripts.add(path);\n resolve();\n };\n script.onerror = () => {\n reject();\n };\n document.head.append(script);\n script.src = path;\n });\n }\n\n private async _loadWasmModule(\n assetsPath: string,\n moduleLoaderName: string,\n moduleSimdLoaderName: string,\n ): Promise {\n try {\n await this._loadScript(`${assetsPath}${moduleSimdLoaderName}`);\n this._tflite = await createTwilioTFLiteSIMDModule();\n this._isSimdEnabled = true;\n } catch {\n await this._loadScript(`${assetsPath}${moduleLoaderName}`);\n this._tflite = await createTwilioTFLiteModule();\n this._isSimdEnabled = false;\n }\n }\n}\n"]} \ No newline at end of file diff --git a/es5/utils/support.d.ts b/es5/utils/support.d.ts new file mode 100644 index 0000000..329912d --- /dev/null +++ b/es5/utils/support.d.ts @@ -0,0 +1,28 @@ +/** + * @private + */ +export declare function isBrowserSupported(): boolean; +/** + * @private + */ +export declare function isChromiumImageBitmap(): boolean; +/** + * @private + */ +export declare const isCanvasBlurSupported: boolean; +/** + * Check if the current browser is officially supported by twilio-video-procesors.js. + * This is set to `true` for browsers that supports canvas + * [2D](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D) or + * [webgl2](https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext) + * rendering context. + * @example + * ```ts + * import { isSupported } from '@twilio/video-processors'; + * + * if (isSupported) { + * // Initialize the background processors + * } + * ``` + */ +export declare const isSupported: boolean; diff --git a/es5/utils/support.js b/es5/utils/support.js new file mode 100644 index 0000000..4c88a8f --- /dev/null +++ b/es5/utils/support.js @@ -0,0 +1,72 @@ +"use strict"; +var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) { + if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) { + if (ar || !(i in from)) { + if (!ar) ar = Array.prototype.slice.call(from, 0, i); + ar[i] = from[i]; + } + } + return to.concat(ar || Array.prototype.slice.call(from)); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.isSupported = exports.isCanvasBlurSupported = exports.isChromiumImageBitmap = exports.isBrowserSupported = void 0; +/** + * @private + */ +function getCanvas() { + return typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas'); +} +/** + * @private + */ +function isBrowserSupported() { + if (typeof window !== 'undefined' && typeof document !== 'undefined') { + return !!(getCanvas().getContext('2d') || getCanvas().getContext('webgl2')); + } + else { + return false; + } +} +exports.isBrowserSupported = isBrowserSupported; +/** + * @private + */ +function isChromiumImageBitmap() { + return /Chrome/.test(navigator.userAgent) + && typeof createImageBitmap === 'function'; +} +exports.isChromiumImageBitmap = isChromiumImageBitmap; +/** + * @private + */ +exports.isCanvasBlurSupported = (function () { + var blackPixel = [0, 0, 0, 255]; + var whitePixel = [255, 255, 255, 255]; + var inputImageData = new ImageData(new Uint8ClampedArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray(__spreadArray([], blackPixel, true), blackPixel, true), blackPixel, true), blackPixel, true), whitePixel, true), blackPixel, true), blackPixel, true), blackPixel, true), blackPixel, true)), 3, 3); + var canvas = getCanvas(); + var context = canvas.getContext('2d'); + canvas.width = 3; + canvas.height = 3; + context.putImageData(inputImageData, 0, 0); + context.filter = 'blur(1px)'; + context.drawImage(canvas, 0, 0); + var data = context.getImageData(0, 0, 3, 3).data; + return data[0] > 0; +})(); +/** + * Check if the current browser is officially supported by twilio-video-procesors.js. + * This is set to `true` for browsers that supports canvas + * [2D](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D) or + * [webgl2](https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext) + * rendering context. + * @example + * ```ts + * import { isSupported } from '@twilio/video-processors'; + * + * if (isSupported) { + * // Initialize the background processors + * } + * ``` + */ +exports.isSupported = isBrowserSupported(); +//# sourceMappingURL=support.js.map \ No newline at end of file diff --git a/es5/utils/support.js.map b/es5/utils/support.js.map new file mode 100644 index 0000000..88f3c07 --- /dev/null +++ b/es5/utils/support.js.map @@ -0,0 +1 @@ +{"version":3,"file":"support.js","sourceRoot":"","sources":["../../lib/utils/support.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA;;GAEG;AACH,SAAS,SAAS;IAChB,OAAO,OAAO,eAAe,KAAK,WAAW,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;AAC/G,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB;IAChC,IAAI,OAAO,MAAM,KAAK,WAAW,IAAI,OAAO,QAAQ,KAAK,WAAW,EAAE;QACpE,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,SAAS,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;KAC7E;SAAM;QACL,OAAO,KAAK,CAAC;KACd;AACH,CAAC;AAND,gDAMC;AAED;;GAEG;AACH,SAAgB,qBAAqB;IACnC,OAAO,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;WACpC,OAAO,iBAAiB,KAAK,UAAU,CAAC;AAC/C,CAAC;AAHD,sDAGC;AAED;;GAEG;AACU,QAAA,qBAAqB,GAAG,CAAC;IACpC,IAAM,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;IAClC,IAAM,UAAU,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IAExC,IAAM,cAAc,GAAI,IAAI,SAAS,CAAC,IAAI,iBAAiB,mIACtD,UAAU,SAAK,UAAU,SAAK,UAAU,SACxC,UAAU,SAAK,UAAU,SAAK,UAAU,SACxC,UAAU,SAAK,UAAU,SAAK,UAAU,QAC3C,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAEV,IAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,IAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,CAA6B,CAAC;IAEpE,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC;IACjB,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAClB,OAAO,CAAC,YAAY,CAAC,cAAc,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3C,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC;IAC7B,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAExB,IAAA,IAAI,GAAK,OAAO,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,KAArC,CAAsC;IAClD,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACrB,CAAC,CAAC,EAAE,CAAC;AAEL;;;;;;;;;;;;;;GAcG;AACU,QAAA,WAAW,GAAG,kBAAkB,EAAE,CAAC","sourcesContent":["/**\n * @private\n */\nfunction getCanvas() {\n return typeof OffscreenCanvas !== 'undefined' ? new OffscreenCanvas(1, 1) : document.createElement('canvas');\n}\n\n/**\n * @private\n */\nexport function isBrowserSupported() {\n if (typeof window !== 'undefined' && typeof document !== 'undefined') {\n return !!(getCanvas().getContext('2d') || getCanvas().getContext('webgl2'));\n } else {\n return false;\n }\n}\n\n/**\n * @private\n */\nexport function isChromiumImageBitmap() {\n return /Chrome/.test(navigator.userAgent)\n && typeof createImageBitmap === 'function';\n}\n\n/**\n * @private\n */\nexport const isCanvasBlurSupported = (() => {\n const blackPixel = [0, 0, 0, 255];\n const whitePixel = [255, 255, 255, 255];\n\n const inputImageData = new ImageData(new Uint8ClampedArray([\n ...blackPixel, ...blackPixel, ...blackPixel,\n ...blackPixel, ...whitePixel, ...blackPixel,\n ...blackPixel, ...blackPixel, ...blackPixel\n ]), 3, 3);\n\n const canvas = getCanvas();\n const context = canvas.getContext('2d') as CanvasRenderingContext2D;\n\n canvas.width = 3;\n canvas.height = 3;\n context.putImageData(inputImageData, 0, 0);\n context.filter = 'blur(1px)';\n context.drawImage(canvas, 0, 0);\n\n const { data } = context.getImageData(0, 0, 3, 3);\n return data[0] > 0;\n})();\n\n/**\n * Check if the current browser is officially supported by twilio-video-procesors.js.\n * This is set to `true` for browsers that supports canvas\n * [2D](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D) or\n * [webgl2](https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext)\n * rendering context.\n * @example\n * ```ts\n * import { isSupported } from '@twilio/video-processors';\n *\n * if (isSupported) {\n * // Initialize the background processors\n * }\n * ```\n */\nexport const isSupported = isBrowserSupported();\n"]} \ No newline at end of file diff --git a/es5/utils/version.d.ts b/es5/utils/version.d.ts new file mode 100644 index 0000000..f640437 --- /dev/null +++ b/es5/utils/version.d.ts @@ -0,0 +1,4 @@ +/** + * The current version of the library. + */ +export declare const version: string; diff --git a/es5/utils/version.js b/es5/utils/version.js new file mode 100644 index 0000000..8b5c036 --- /dev/null +++ b/es5/utils/version.js @@ -0,0 +1,9 @@ +"use strict"; +// This file is generated on build. To make changes, see scripts/version.js +Object.defineProperty(exports, "__esModule", { value: true }); +exports.version = void 0; +/** + * The current version of the library. + */ +exports.version = '3.0.0-preview.1'; +//# sourceMappingURL=version.js.map \ No newline at end of file diff --git a/es5/utils/version.js.map b/es5/utils/version.js.map new file mode 100644 index 0000000..b789383 --- /dev/null +++ b/es5/utils/version.js.map @@ -0,0 +1 @@ +{"version":3,"file":"version.js","sourceRoot":"","sources":["../../lib/utils/version.ts"],"names":[],"mappings":";AAAA,2EAA2E;;;AAE3E;;GAEG;AACU,QAAA,OAAO,GAAW,iBAAiB,CAAC","sourcesContent":["// This file is generated on build. To make changes, see scripts/version.js\n\n/**\n * The current version of the library.\n */\nexport const version: string = '3.0.0-preview.1';\n"]} \ No newline at end of file diff --git a/lib/utils/version.ts b/lib/utils/version.ts new file mode 100644 index 0000000..6795589 --- /dev/null +++ b/lib/utils/version.ts @@ -0,0 +1,6 @@ +// This file is generated on build. To make changes, see scripts/version.js + +/** + * The current version of the library. + */ +export const version: string = '3.0.0-preview.1'; diff --git a/package.json b/package.json index f2cc482..2824ba1 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@twilio/video-processors", "title": "Twilio Video Processors", "description": "Twilio Video Processors JavaScript Library", - "version": "2.2.1-dev", + "version": "3.0.0-preview.1", "homepage": "https://github.com/twilio/twilio-video-processors.js#readme", "author": "Charlie Santos ", "contributors": [