Improve emoji hunt ui (#26)

* fixed centered bg video and text * move button to bottom * improve font choice & resize text
PAIR-code · Jul 26, 2018 · 1069a85 · 1069a85
1 parent e40746e
commit 1069a85
Show file tree

Hide file tree

Showing 3 changed files with 89 additions and 20 deletions.
diff --git a/demo/emoji_hunt/client/index.html b/demo/emoji_hunt/client/index.html
@@ -29,21 +29,63 @@
   #override {
     width: 100%;
     height: 100px;
+    font-family: 'San Francisco', Roboto, sans-serif;
+    font-size: 3em;
+    font-weight: bold;
   }
+
+  #findme {
+    font-size: 4em;
+  }
+
+  #status {
+    font-size: 3em;
+  }
+
+  #webcamvideo {
+    background-size:cover;
+    overflow: hidden;
+    position: absolute;
+    top: 0px;
+    left: 0px;
+    min-height: 100%;
+    width: 100%;
+    z-index: -1;
+    overflow: hidden;
+    object-fit: cover;
+  }
+
+  .transparent-bg {
+    background-color: rgba(255, 255, 255, 0.51);
+    color: black;
+    font-family: 'San Francisco', Roboto, sans-serif;
+  }
+
+  @media (orientation: portrait) {
+    #override {
+      position: fixed;
+      bottom: 0px;
+      left: 0px;
+      height: 10%;
+    }
+  }
+
   </style>
   <body>
-    <h1 class="centered">Demo client</h1>
-    <h1 class="centered" id='modelversion' class="centered"></h1>
-    <h1 class="centered" id='status' class="centered"></h1>
-    <h1 class="centered" id='findme' class="centered"></h1>
+    <div class="transparent-bg">
+        <h1 class="centered" id='modelversion' class="centered"></h1>
+        <h1 class="centered" id='findme' class="centered"></h1>
+        <h1 class="centered" id='status' class="centered"></h1>
+      </div>
     <button class="centered" id="override">I'm looking at it!</button>
-    <video playsinline muted autoplay id='webcamvideo' width="100%" height="224"></video>
     <div>
-      <div class="centered" >
+      <div id="inputcheckbox" class="centered" >
         <input type="checkbox" id="uploaddata" checked>
         <label for="uploaddata">Upload collected training data to server?</label>
       </div>
     </div>
+    <video class="centered" playsinline muted autoplay id="webcamvideo" width="100%" height="224"></video>
+    <!--canvas id="debugCanvas"></canvas-->
     <script src="index.js"></script>
 
   </body>

diff --git a/demo/emoji_hunt/client/index.js b/demo/emoji_hunt/client/index.js
@@ -31,7 +31,7 @@ const WEIGHT_MANIFEST =
 
 const SERVER_URL = `//${location.hostname}:3000`;
 const UPLOAD_URL = `//${location.hostname}:3000/data`;
-const USE_OAUTH = true;
+const USE_OAUTH = false;
 
 console.log('server url:', SERVER_URL)
 
@@ -86,11 +86,28 @@ async function getTopPred(preds) {
   return {index: top, label: SCAVENGER_HUNT_LABELS[top]};
 }
 
+// center-crop input Tensor3D into a square
+function squareCrop(frame) {
+  return tf.tidy(() => {
+    const [h, w] = frame.shape;
+    if(h > w) {
+      const halfW = Math.floor(w / 2);
+      const halfH = Math.floor(h / 2);
+      return frame.slice([halfH - halfW, 0], [halfW * 2, -1]);
+    } else {
+      const halfH = Math.floor(h / 2);
+      const halfW = Math.floor(w / 2);
+      return frame.slice([0, halfW - halfH], [-1, halfH * 2]);
+    }
+  })
+}
+
 function preprocess(webcam) {
   return tf.tidy(() => {
-    const frame = tf.fromPixels(webcam).toFloat();
+    const frame = tf.fromPixels(webcam);
+    const cropped = squareCrop(frame).toFloat();
     const scaled =
-        tf.image.resizeBilinear(frame, [MODEL_INPUT_WIDTH, MODEL_INPUT_WIDTH]);
+        tf.image.resizeBilinear(cropped, [MODEL_INPUT_WIDTH, MODEL_INPUT_WIDTH]);
     const prepped = scaled.sub(255 / 2).div(255 / 2).expandDims(0);
     return prepped;
   });
@@ -103,6 +120,10 @@ async function main() {
     return;
   }
 
+  ui.status('trying to get access to webcam...');
+
+  const webcam = await ui.webcam();
+
   ui.status('loading model...');
 
   const {model, varsAndLoss, optimizer} = await setupModel();
@@ -121,15 +142,6 @@ async function main() {
 
   optimizer.setLearningRate(hyperparams['learningRate']);
 
-  ui.status('trying to get access to webcam...');
-
-  const webcam = await ui.webcam();
-
-  while (webcam.videoHeight === 0) {
-    ui.status('waiting for video to initialise...');
-    await tf.nextFrame();
-  }
-
   let isTraining = false;
 
   ui.overrideButton(evt => {
@@ -196,7 +208,7 @@ async function main() {
 
     tf.dispose(preds);
 
-    ui.status(`its a ${label}`);
+    ui.status(`i see a ${label}...`);
     if (label === lookingFor.name) {
       ui.status(`congrats! u did it !`);
       for (let i = 0; i < 30; i++) {

diff --git a/demo/emoji_hunt/client/ui.js b/demo/emoji_hunt/client/ui.js
@@ -15,6 +15,8 @@
  * =============================================================================
  */
 
+import { nextFrame } from '@tensorflow/tfjs';
+
 function sel(str) {
   return document.querySelector(String.raw(...arguments));
 }
@@ -51,9 +53,22 @@ export async function webcam() {
   }
   try {
     const stream =
-      await navigator.mediaDevices.getUserMedia({audio: false, video: true});
+      await navigator.mediaDevices.getUserMedia({
+        audio: false,
+        video: {facingMode: 'environment'}
+      });
+
     video.srcObject = stream;
+
+    while (video.videoHeight === 0 || video.videoWidth === 0) {
+      status('waiting for video to initialise...');
+      await nextFrame();
+    }
+
+    video.width = video.videoWidth;
+    video.height = video.videoHeight;
     webcamSetup = true;
+
   } catch (exn) {
     status(`Error in accessing webcam: ${exn.toString()}`);
     throw exn;