Prep for 2.2.0 + example app improvements. (#78)

twilio · Jul 16, 2024 · dc81239 · dc81239
1 parent f23709b
commit dc81239
Show file tree

Hide file tree

Showing 9 changed files with 287 additions and 166 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,11 +1,23 @@
-2.2.0 (in progress)
-===================
+2.2.0 (July 16, 2024)
+=====================
+
+Performance Improvements
+------------------------
+
+* The WebGL2 pipeline now has an overall higher output frame rate, even for 720p (HD) video.
+
+Changes
+-------
+
+* `BackgroundProcessorOptions.debounce` is now set to `false` by default.
+* `BackgroundProcessorOptions.maskBlurRadius` is now set to `8` as the default for the `WebGL2` pipeline, and `4` for the `Canvas2D` pipeline.
 
-#### Performance Improvements
+Bug Fixes
+---------
 
 * Fixed trailing effect of the person mask in both Canvas2D and WebGL2 pipelines.
+* Fixed a bug where changing the `maskBlurRadius` value on the `VideoProcessor` was not working.
 * TFLite module is loaded and initialized only once, no matter how many VideoProcessor instances are created.
-* Better performance with 720p (HD) LocalVideoTracks due to improvements in the WebGL2 pipeline.
 
 2.1.0 (December 12, 2023)
 =========================

diff --git a/examples/README.md b/examples/README.md
@@ -2,14 +2,15 @@
 
 ## Setup
 
-Checkout the latest release tag (see the latest releases [here](https://github.com/twilio/twilio-video-processors.js/releases)). For example, to checkout release 1.0.0:
+Go to `${PROJECT_ROOT}/examples` and run `npm install`. It will install the dependencies and run the application server.
 
-```
-git checkout 1.0.0
-```
+## Running the Demo
 
-Then, run `npm install` from the `examples` folder. It will install the dependencies and run the application server.
+Open `http://localhost:3000` in a Chrome tab. The app captures your camera upon loading and plays it in a `<video>` element. You can choose to enable, disable or update the background settings using the controls on the page. Additionally, you can specify the following url parameters:
 
-## App (Chrome Only)
-
-Open http://localhost:3000 in a Chrome tab. The app captures your camera upon loading and plays it in a `<video>` element. You can choose to enable, disable or update the background settings using the controls on the page.
+- `capFramerate` - Choose video capture frame rate (default: 30)
+- `capResolution=wxh` - Choose video capture resolution (default: 1280x720)
+- `debounce=true|false` - Whether to skip processing every other frame (default: false)
+- `maskBlurRadius` - Radius of the mask blur filter (default: 8 for WebGL2, 4 for Canvas2D)
+- `pipeline=Canvas2D|WebGL2` - Choose the canvas or webgl pipeline (default: WebGL2)
+- `stats=advanced|hide|show` - Show performance benchmarks (default: show)
diff --git a/examples/server.js b/examples/server.js
@@ -5,6 +5,10 @@ const app = express();
 app.use(express.static(resolve(__dirname, './virtualbackground')));
 app.use(express.static(resolve(__dirname, '../dist/build')));
 
+app.get('/README.md', (req, res) => {
+  res.sendFile(resolve(__dirname, './README.md'));
+});
+
 const port = parseInt(process.env.PORT || '3000');
 app.listen(port, () => {
   console.log(`App server started. Go to http://localhost:${port}`);

diff --git a/examples/virtualbackground/backgrounds/flag.jpg b/examples/virtualbackground/backgrounds/flag.jpg
diff --git a/examples/virtualbackground/backgrounds/mountain_lake.jpg b/examples/virtualbackground/backgrounds/mountain_lake.jpg
diff --git a/examples/virtualbackground/favicon.ico b/examples/virtualbackground/favicon.ico
diff --git a/examples/virtualbackground/index.css b/examples/virtualbackground/index.css
@@ -8,30 +8,36 @@ body {
   height: 100%;
 }
 
-#stats {
+pre#stats {
   display: none;
-  position: fixed;
-  background-color: white;
-  top: 42px;
-  left: 22px;
+  position: absolute;
+  border-radius: 5px;
+  background-color: rgba(255, 255, 255, 0.5);
+  top: 5px;
+  left: 5px;
   padding: 10px;
+  font-size: 0.7em;
+}
+
+div#backgrounds {
+  position: absolute;
+  top: 5px;
 }
 
 div.img-btn {
   cursor: pointer;
-  height: 65px;
-  width: 65px;
+  height: 50px;
+  width: 50px;
   background-size: cover;
   background-repeat: no-repeat;
   background-position: center;
   border-radius: 10px;
-  opacity: 0.6;
   display: inline-block;
   border: 1px solid gray;
 }
 
 div.img-btn:hover {
-  opacity: 1;
+  opacity: 0.6;
   transform: scale(1.1);
   border: 2px solid black;
 }
@@ -52,8 +58,8 @@ div.row.thin-gutters {
   margin: 0 2px 0 2px;
 }
 
-div.row.thin-gutters > .col,
-div.row.thin-gutters > [class*="col-"] {
+div.row.thin-gutters>.col,
+div.row.thin-gutters>[class*="col-"] {
   padding: 8px 8px;
 }
 
@@ -68,15 +74,16 @@ div.card-block {
   max-width: 100%;
 }
 
-div.left-form > div.card > div.card-block {
+div.left-form>div.card>div.card-block {
   margin: 10px;
 }
 
 .form-control {
   margin: 5px;
 }
 
-div.col-sm-6, div.col-sm-6 {
+div.col-sm-6,
+div.col-sm-6 {
   max-height: fit-content;
   overflow: auto;
 }
@@ -86,11 +93,15 @@ div.video-container {
 }
 
 video {
-  width: 100%    !important;
-  height: auto   !important;
+  border-radius: 5px;
+  max-width: 1280px !important;
+  min-width: 640px !important;
+  height: auto !important;
+  object-fit: cover;
 }
 
-div.col-sm-8, div.col-sm-4 {
+div.col-sm-8,
+div.col-sm-4 {
   max-width: 100%;
   flex: 100%;
 }
diff --git a/examples/virtualbackground/index.html b/examples/virtualbackground/index.html
@@ -1,50 +1,67 @@
 <!DOCTYPE html>
-<html>
-  <html lang="en">
-    <head>
-      <meta charset="UTF-8">
-      <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
-      <title>Virtual Background Example</title>
-      <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-wEmeIV1mKuiNpC+IOBjI7aAzPcEZeedi5yW5f2yOq55WWLwNGmvvx4Um1vskeMj0" crossorigin="anonymous">
-      <link rel="stylesheet" href="index.css">
-    </head>
-    <body>
-      <div class="container-fluid">
-        <div class="row thin-gutters align">
-          <div class="col-sm-8">
-            <div class="card video-container">
-              <div class="card-block">
-                <h4 class="card-title">Replace Your Background</h4>
-                <video id="video-input" autoplay></video>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <title>Virtual Background Demo</title>
+    <link rel="icon" type="image/png" href="favicon.ico">
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-wEmeIV1mKuiNpC+IOBjI7aAzPcEZeedi5yW5f2yOq55WWLwNGmvvx4Um1vskeMj0" crossorigin="anonymous">
+    <link rel="stylesheet" href="index.css">
+  </head>
+  <body>
+    <div class="container-fluid">
+      <div class="row thin-gutters align">
+        <div class="col-sm-4" id="instructions">
+        </div>
+        <div class="col-sm-4">
+          <div class="card video-container">
+            <div class="card-block">
+              <video id="video-input" autoplay></video>
+              <pre id="stats"></pre>
+              <div id="backgrounds">
+                <div id="none" class="img-btn" style="background-image: url('none.png');"></div>
+                <div id="blur" class="img-btn" style="background-image: url('blur.png');"></div>
+                <div id="living_room" class="img-btn" style="background-image: url('backgrounds/living_room.jpg');"></div>
+                <div id="office" class="img-btn" style="background-image: url('backgrounds/office.jpg');"></div>
+                <div id="vacation" class="img-btn" style="background-image: url('backgrounds/vacation.jpg');"></div>
+                <div id="flag" class="img-btn" style="background-image: url('backgrounds/flag.jpg');"></div>
+                <div id="mountain_lake" class="img-btn" style="background-image: url('backgrounds/mountain_lake.jpg');"></div>
               </div>
             </div>
           </div>
-          <div class="col-sm-4 left-form">
-            <div id="none" class="img-btn" style="background-image: url('none.png');"></div>
-            <div id="blur" class="img-btn" style="background-image: url('blur.png');"></div>
-            <div id="living_room" class="img-btn" style="background-image: url('backgrounds/living_room.jpg');"></div>
-            <div id="office" class="img-btn" style="background-image: url('backgrounds/office.jpg');"></div>
-            <div id="vacation" class="img-btn" style="background-image: url('backgrounds/vacation.jpg');"></div>
-          </div>
         </div>
       </div>
-      <div class="modal fade" id="errorModal" tabindex="-1" role="dialog" aria-labelledby="errorModalLabel" aria-hidden="true" data-bs-backdrop="static" data-bs-keyboard="false">
-        <div class="modal-dialog" role="document">
-          <div class="modal-content">
-            <div class="modal-header">
-              <h5 class="modal-title" id="errorModalLabel">Error</h5>
-            </div>
-            <div class="modal-body">
-              ...
-            </div>
+    </div>
+    <div class="modal fade" id="errorModal" tabindex="-1" role="dialog" aria-labelledby="errorModalLabel" aria-hidden="true" data-bs-backdrop="static" data-bs-keyboard="false">
+      <div class="modal-dialog" role="document">
+        <div class="modal-content">
+          <div class="modal-header">
+            <h5 class="modal-title" id="errorModalLabel">Error</h5>
+          </div>
+          <div class="modal-body">
+            ...
           </div>
         </div>
       </div>
-    <pre id="stats"></pre>
-  <script src="https://unpkg.com/wasm-feature-detect/dist/umd/index.js"></script>
-  <script src="https://sdk.twilio.com/js/video/releases/2.28.1/twilio-video.js"></script>
-  <script src="twilio-video-processors.js"></script>
-  <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0/dist/js/bootstrap.bundle.min.js" integrity="sha384-p34f1UUtsS3wqzfto5wAAmdvj+osOnFyQFpp4Ua3gs/ZVWx6oOypYoCJhGGScy+8" crossorigin="anonymous"></script>
-  <script src="index.js"></script>
-</body>
+    </div>
+    <script src="https://sdk.twilio.com/js/video/releases/2.28.1/twilio-video.js"></script>
+    <script src="twilio-video-processors.js"></script>
+    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.0/dist/js/bootstrap.bundle.min.js" integrity="sha384-p34f1UUtsS3wqzfto5wAAmdvj+osOnFyQFpp4Ua3gs/ZVWx6oOypYoCJhGGScy+8" crossorigin="anonymous"></script>
+    <script type="module" src="https://cdn.jsdelivr.net/gh/zerodevx/zero-md@2/dist/zero-md.min.js"></script>
+    <script>
+      (async () => {
+        const response = await fetch('/README.md');
+        const text = await response.text();
+        const $instructions = document.getElementById('instructions');
+        const $zeroMd = document.createElement('zero-md');
+        $zeroMd.setAttribute('no-shadow', '');
+        const $script = document.createElement('script');
+        $script.type = 'text/markdown';
+        $script.appendChild(document.createTextNode(text));
+        $zeroMd.appendChild($script);
+        $instructions.append($zeroMd);
+      })();
+    </script>
+    <script src="index.js"></script>
+  </body>
 </html>