Merge pull request #9 from CCInc/main

Add editor

app.js

@@ -7,17 +7,30 @@ var DEFAULT_SIZE = {
  width: 640,
  height: 360,
 }
+var stats = null;
 
-window.onload = (event) => {
+function loadapp() { 
  indicator = videoIndicator(getCanvas('outputVideo'))
  indicator.loading()
  start()
 }
 
+window.onload = (event) => {
+
+ if (typeof Stats !== 'undefined' && Stats) {
+ stats = new Stats();
+ stats.showPanel(0);
+ document.getElementById("stats").appendChild(stats.dom);
+ }
+ loadapp()
+}
+
 function saveVideoSize(mediaStream) {
  var s = mediaStream.getVideoTracks()[0].getSettings()
  VIDEO_SIZE.width = s.width;
  VIDEO_SIZE.height = s.height;
+ console.log("setting webcame size to: " + s.width + "x" + s.height);
+ camSizeChange(s.width, s.height);
 }
 
 function resizeVideo(size, targets) {
@@ -50,6 +63,46 @@ function getCanvas(id, getCtx) {
  return canvas;
 }
 
+function errorMessage(message) {
+ var error = document.getElementById('error');
+ error.textContent = message;
+}
+
+function camSizeChange(width, height) {
+ var elem = document.getElementById("cam_size");
+ if (elem) {
+ elem.textContent = "Webcam size: " + width + "x" + height;
+ }
+}
+
+var loadParams = getLoadParams();
+var modelParams = getModelParams();
+
+function getLoadParams() {
+ const urlParams = new URLSearchParams(window.location.search);
+ var loadParams = { 
+ architecture: urlParams.get('architecture') || 'MobileNetV1',
+ outputStride: parseInt(urlParams.get('outputStride')) || 16,
+ multiplier: parseFloat(urlParams.get('multiplier')) || 0.75,
+ quantBytes: parseInt(urlParams.get('quantBytes')) || 2,
+ };
+ return loadParams;
+}
+
+function getModelParams() {
+ const urlParams = new URLSearchParams(window.location.search);
+ var modelParams = {
+ flipHorizontal: urlParams.get('flipHorizontal') == null ? true : urlParams.get('flipHorizontal') == "true",
+ internalResolution: urlParams.get('internalResolution') || "medium",
+ segmentationThreshold: parseFloat(urlParams.get('segmentationThreshold')) || 0.7,
+ maxDetections: parseInt(urlParams.get('maxDetections')) || 10,
+ scoreThreshold: parseFloat(urlParams.get('scoreThreshold')) || 0.3,
+ nmsRadius: parseInt(urlParams.get('nmsRadius')) || 20,
+ maskBlurAmount : parseInt(urlParams.get('maskBlurAmount')) || 3,
+ };
+ return modelParams;
+}
+
 function start() {
  var video = getVideo();
  if (navigator.mediaDevices.getUserMedia) {
@@ -62,6 +115,7 @@ function start() {
  }
  navigator.mediaDevices.getUserMedia(videoConstraint)
  .then(function (stream) {
+ errorMessage('');
  saveVideoSize(stream);
  resizeVideo(VIDEO_SIZE, ['input']);
  resizeVideo(VIDEO_SIZE, ['output','buffer']);
@@ -72,45 +126,40 @@ function start() {
  }
  })
  .catch(function (err) {
+ errorMessage("Couldn't get the webcam object! " + err.message);
  console.log("Something went wrong!");
  console.error(err);
  });
  }
 }
 
-function initMLModel() {
+async function initMLModel() {
  var video = getVideo();
  var canvas = getCanvas("outputVideo");
- bodyPix.load({
- architechture: 'MobileNetV1',
- outputStride: 16,
- multiplier: 0.75,
- quantBytes: 2
- }).then(model => {
+ console.log('Loading model with params:');
+ console.log(loadParams);
+ await bodyPix.load(loadParams).then(model => {
  console.log('BodyPix model loaded.');
  indicator.stop();
+
+ console.log('Model parameters:');
+ console.log(modelParams);
  transformFrame(model, video, canvas);
  }).catch(err => {
+ indicator.stop();
+ errorMessage(err.message);
  console.error(err);
  })
 }
 
-function transformFrame(model, sourceVideo, targetCanvas) {
+async function transformFrame(model, sourceVideo, targetCanvas) {
  var w = VIDEO_SIZE.width;
  var h = VIDEO_SIZE.height;
  var tempCtx = getCanvas('bufferVideo', true);
  tempCtx.drawImage(sourceVideo, 0, 0, w, h);
  var frame = getCanvas('bufferVideo');
 
- model.segmentMultiPerson(frame, {
- flipHorizontal: true,
- internalResolution: 'medium',
- segmentationThreshold: 0.6,
- scoreThreshold: 0.3,
- maxDetections: 3,
- nmsRadius: 20
- }).then(segments => {
- if (typeof segments !== 'undefined' && segments.length > 0) {
+ await model.segmentPerson(frame, modelParams).then(segments => {
  const foregroundColor = {r: 0, g: 0, b: 0, a: 0};
  const backgroundColor = {r: 0, g: 255, b: 0, a: 255};
  var maskSegmentation = bodyPix.toMask( segments, foregroundColor, backgroundColor);
@@ -119,17 +168,13 @@ function transformFrame(model, sourceVideo, targetCanvas) {
  frame,
  maskSegmentation,
  1, // opacity
- 3, // mask blur amount,
- true // flip horizontal
+ modelParams.maskBlurAmount, // mask blur amount,
+ modelParams.flipHorizontal // flip horizontal
  );
- } else {
- // In case of nobody detected, make all green
- tempCtx.fillStyle = "#00FF00";
- tempCtx.fillRect(0,0, VIDEO_SIZE.width, VIDEO_SIZE.height);
- getCanvas('outputVideo', true).drawImage(frame, 0,0);
- }
  window.requestAnimationFrame(()=>{
+ if (stats) stats.begin();
  transformFrame(model, sourceVideo, targetCanvas)
+ if (stats) stats.end();
  });
  }).catch(err => {
  console.error(err);

index.html

@@ -6,12 +6,78 @@
  <script src="node_modules/@tensorflow/tfjs/dist/tf.min.js" charset="utf-8"></script>
  <script src="node_modules/@tensorflow-models/body-pix/dist/body-pix.min.js" charset="utf-8"></script>
  <script src="node_modules/bluebird/js/browser/bluebird.min.js" charset="utf-8"></script>
+ <script src="node_modules/stats.js/build/stats.min.js" charset="utf-8"></script>
+ <script>
+ const urlParams = new URLSearchParams(window.location.search);
+ var urlBase = location.href.substring(0, location.href.lastIndexOf("/")+1);
+ function updateUrl() {
+ var url = urlBase + 'player.html?' + new URLSearchParams(Object.assign({}, loadParams, modelParams)).toString();
+ document.getElementById('paramsUrl').href = url;
+ document.getElementById('paramsUrl').innerHTML = url; 
+ }
+ function updateLoadParam(key, val){
+ loadParams[key] = val;
+ updateUrl();
+ }
+ function updateModelParam(key, val){
+ modelParams[key] = val;
+ updateUrl();
+ }
+ </script>
  <script src="app.js" charset="utf-8"></script>
  <link rel="stylesheet" href="style.css">
  </head>
  <body>
+ <p id="error"></p>
+ <div id="stats"></div>
  <video id="inputVideo" width="640" height="480"></video>
  <canvas id="outputVideo" width="640" height="480"></canvas>
  <canvas id="bufferVideo" width="640" height="480"></canvas>
+ <div id="cam_size"></div>
+ <label for="net">Network:</label>
+ <select name="net" id="net" onchange="updateLoadParam('architecture', this.value);">
+ <option value="MobileNetV1" selected="selected">MobileNetV1</option>
+ <option value="ResNet50">ResNet50</option>
+ </select>
+ <label for="stride">Output Stride:</label>
+ <select name="stride" id="stride" onchange="updateLoadParam('outputStride', parseInt(this.value));">
+ <option value="8">8 (mobilenet only)</option>
+ <option value="16" selected="selected">16</option>
+ <option value="32">32 (resnet only)</option>
+ </select>
+ <label for="multiplier">Multiplier:</label>
+ <select name="multiplier" id="multiplier" onchange="updateLoadParam('multiplier', parseFloat(this.value));">
+ <option value="0.5">0.5 (mobilenet only)</option>
+ <option value="0.75" selected="selected">0.75 (mobilenet only)</option>
+ <option value="1.0">1.0</option>
+ </select>
+ <label for="quantBytes">Quantization Bytes:</label>
+ <select name="quantBytes" id="quantBytes" onchange="updateLoadParam('quantBytes', parseInt(this.value));">
+ <option value="1">1</option>
+ <option value="2" selected="selected">2</option>
+ <option value="4">4</option>
+ </select>
+ <button type="button" onclick="loadapp()">Reload Model</button>
+ </br>
+ <label for="internalResolution">internalResolution:</label>
+ <select name="internalResolution" id="internalResolution" onchange="updateModelParam('internalResolution', this.value);">
+ <option value="low">low</option>
+ <option value="medium" selected="selected">medium</option>
+ <option value="high">high</option>
+ <option value="full">full</option>
+ </select>
+ <label for="segmentationThreshold">segmentationThreshold:</label>
+ <input type="number" step=".1" min="0" max="1" name="segmentationThreshold" value="0.7" id="segmentationThreshold" onchange="updateModelParam('segmentationThreshold', parseFloat(this.value));"/>
+ <label for="maxDetections">maxDetections:</label>
+ <input type="number" step="1" name="maxDetections" value="10" id="maxDetections" onchange="updateModelParam('maxDetections', parseInt(this.value));"/>
+ <label for="scoreThreshold">scoreThreshold:</label>
+ <input type="number" min="0" max="1" step=".1" name="scoreThreshold" value="0.3" id="scoreThreshold" onchange="updateModelParam('scoreThreshold', parseFloat(this.value));"/>
+ <label for="nmsRadius">nmsRadius:</label>
+ <input type="number" step="1" name="nmsRadius" value="20" id="nmsRadius" onchange="updateModelParam('nmsRadius', parseInt(this.value));"/>
+ <label for="maskBlurAmount">maskBlurAmount:</label>
+ <input type="number" step="1" name="maskBlurAmount" value="3" id="maskBlurAmount" onchange="updateModelParam('maskBlurAmount', parseInt(this.value));"/>
+ </br>
+ <label for="paramsUrl">Url String:</label>
+ <a id="paramsUrl" name="paramsUrl" /><br><br>
  </body>
 </html>

package.json

@@ -4,15 +4,16 @@
  "description": "Virtual Green Background for Webcam",
  "main": "app.js",
  "scripts": {
- "build":"rm -rf dist && mkdir dist && rsync -rv * dist --exclude ./dist",
+ "build": "rm -rf dist && mkdir dist && rsync -rv * dist --exclude ./dist",
  "test": "echo \"No test!\""
  },
  "repository": "https://github.com/nhtua/greencam/",
  "author": "Liam Nguyen <liam102ca@outlook.com>",
  "license": "Apache-2.0",
  "dependencies": {
- "@tensorflow/tfjs": "^1.2.0",
  "@tensorflow-models/body-pix": "^2.0.0",
- "bluebird": "^3.7.2"
+ "@tensorflow/tfjs": "^1.2.0",
+ "bluebird": "^3.7.2",
+ "stats.js": "^0.17.0"
  }
 }

player.html

@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html lang="en" dir="ltr">
+ <head>
+ <meta charset="utf-8">
+ <title>Virtual Green Background</title>
+ <script src="node_modules/@tensorflow/tfjs/dist/tf.min.js" charset="utf-8"></script>
+ <script src="node_modules/@tensorflow-models/body-pix/dist/body-pix.min.js" charset="utf-8"></script>
+ <script src="node_modules/bluebird/js/browser/bluebird.min.js" charset="utf-8"></script>
+ <script src="app.js" charset="utf-8"></script>
+ <link rel="stylesheet" href="style.css">
+ </head>
+ <body>
+ <p id="error"></p>
+ <video id="inputVideo" width="640" height="480"></video>
+ <canvas id="outputVideo" width="640" height="480"></canvas>
+ <canvas id="bufferVideo" width="640" height="480"></canvas>
+ </body>
+</html>