demo_util.js

/**
 * @license
 * Copyright 2019 Google LLC. All Rights Reserved.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 * =============================================================================
 */
import * as posenet from '@tensorflow-models/posenet';
import * as tf from '@tensorflow/tfjs';

const color = 'aqua';
const boundingBoxColor = 'red';
const lineWidth = 2;

export const tryResNetButtonName = 'tryResNetButton';
export const tryResNetButtonText = '[New] Try ResNet50';
const tryResNetButtonTextCss = 'width:100%;text-decoration:underline;';
const tryResNetButtonBackgroundCss = 'background:#e61d5f;';

function isAndroid() {
  return /Android/i.test(navigator.userAgent);
}

function isiOS() {
  return /iPhone|iPad|iPod/i.test(navigator.userAgent);
}

export function isMobile() {
  return isAndroid() || isiOS();
}

function setDatGuiPropertyCss(propertyText, liCssString, spanCssString = '') {
  let spans = document.getElementsByClassName('property-name');
  for (let i = 0; i < spans.length; i++) {
    let text = spans[i].textContent || spans[i].innerText;
    if (text == propertyText) {
      spans[i].parentNode.parentNode.style = liCssString;
      if (spanCssString !== '') {
        spans[i].style = spanCssString;
      }
    }
  }
}

export function updateTryResNetButtonDatGuiCss() {
  setDatGuiPropertyCss(
    tryResNetButtonText, tryResNetButtonBackgroundCss,
    tryResNetButtonTextCss);
}

/**
 * Toggles between the loading UI and the main canvas UI.
 */
export function toggleLoadingUI(
  showLoadingUI, loadingDivId = 'loading', mainDivId = 'main') {
  if (showLoadingUI) {
    document.getElementById(loadingDivId).style.display = 'block';
    document.getElementById(mainDivId).style.display = 'none';
  } else {
    document.getElementById(loadingDivId).style.display = 'none';
    document.getElementById(mainDivId).style.display = 'block';
  }
}

function toTuple({ y, x }) {
  return [y, x];
}

export function drawPoint(ctx, y, x, r, color) {
  ctx.beginPath();
  ctx.arc(x, y, r, 0, 2 * Math.PI);
  ctx.fillStyle = color;
  ctx.fill();
}

/**
 * Draws a line on a canvas, i.e. a joint
 */
export function drawSegment([ay, ax], [by, bx], color, scale, ctx) {
  ctx.beginPath();
  ctx.moveTo(ax * scale, ay * scale);
  ctx.lineTo(bx * scale, by * scale);
  ctx.lineWidth = lineWidth;
  ctx.strokeStyle = color;
  ctx.stroke();
}

/**
 * Draws a pose skeleton by looking up all adjacent keypoints/joints
 */
export function drawSkeleton(keypoints, minConfidence, ctx, scale = 1) {
  const adjacentKeyPoints =
    posenet.getAdjacentKeyPoints(keypoints, minConfidence);

  adjacentKeyPoints.forEach((keypoints) => {
    drawSegment(
      toTuple(keypoints[0].position), toTuple(keypoints[1].position), color,
      scale, ctx);
  });
}

/**
 * Draw pose keypoints onto a canvas
 */
export function drawKeypoints(keypoints, minConfidence, ctx, scale = 1) {
  for (let i = 0; i < keypoints.length; i++) {
    const keypoint = keypoints[i];

    if (keypoint.score < minConfidence) {
      continue;
    }

    const { y, x } = keypoint.position;
    drawPoint(ctx, y * scale, x * scale, 3, color);
  }
}

/** */
export function keepYourDistance(poses, minPoseConfidence, minPartConfidence) {
  const avgShoulderWidth = 0.457; // meter - Source: Kleine Ergonomische Datensammlung, 16. Auflage, Bundesanstalt für Arbeitsschutz und Arbeitsmedizin
  const minDist = 1.5; // meter - minimum distance between two people according to corona regulations

  // For each pose (i.e. person) detected in an image, compute the center of their shoulder
  const shoulderCenters = poses.map(({ score, keypoints }) => {
    if (score >= minPoseConfidence) {
      const lShoulderVisible = keypoints[5].score > minPartConfidence;
      const rShoulderVisible = keypoints[6].score > minPartConfidence;
      if (!lShoulderVisible && !rShoulderVisible)
        return null;

      const lShoulder = keypoints[5].position;
      const rShoulder = keypoints[6].position;

      const shoulderCenter = lShoulderVisible
        ? (
          rShoulderVisible
            ? { x: (lShoulder.x + rShoulder.x) / 2, y: (lShoulder.y + rShoulder.y) / 2, d2: d2(lShoulder, rShoulder) }
            : lShoulder
        ) : rShoulder;

      return shoulderCenter;
    }
  }).filter((x) => !!x);

  // check pairwise if minimum distance is maintained
  shoulderCenters.forEach((c1, i1) => {
    const l = shoulderCenters.length;
    for (let i2 = i1 + 1; i2 < l; i2++) {
      const c2 = shoulderCenters[i2];
      if (!c1.d2 && !c2.d2) continue; // not possible to establish scale of image since neither shoulder is fully visible

      const shoulderWidth = !!c1.d2 // avg in pixel
        ? (
          !!c2.d2
            ? (Math.sqrt(c1.d2) + Math.sqrt(c2.d2)) / 2
            : Math.sqrt(c1.d2)
        ) : Math.sqrt(c2.d2);

      const pixelPerMeter = shoulderWidth / avgShoulderWidth; // avg number of pixels per Meter physical distance for those two entities (assuming shoulders are perpendicular to camera)

      const dPixel = Math.sqrt(d2(c1, c2)); // distance between shoulder centers in pixel
      const dMeter = dPixel/pixelPerMeter; // and approximately in meters

      if (dMeter < minDist) {
        console.log('Too close to each other!!!');
        beep();
      }
    }
  });
}

function d2(p1, p2) {
  return square(p1.x - p2.x) + square(p1.y - p2.y);
}

function square(x) {
  return x * x;
}

/**
 * Makes a sound if wrists are raised above shoulder line
 */
export function keepYourHandsOffYourFace(keypoints, minConfidence, showShoulderline, ctx) {
  const lShoulderVisible = keypoints[5].score > minConfidence;
  const lShoulderY = keypoints[5].position.y;
  const rShoulderVisible = keypoints[6].score > minConfidence;
  const rShoulderY = keypoints[6].position.y;

  // estimate shoulder hight
  if (!lShoulderVisible && !rShoulderVisible) return;
  const shoulderY = (lShoulderVisible
    ? (rShoulderVisible ? Math.max(lShoulderY, rShoulderY) : lShoulderY)
    : rShoulderY) + 40; // fixed offset to account for fingers

  if (showShoulderline) {
    drawSegment([shoulderY, 0], [shoulderY, (ctx.context || {}).width || 600], 'red', 1, ctx);
  }

  // left Hand in your face?
  const lWristVisible = keypoints[9].score > minConfidence;
  const lWristY = keypoints[9].position.y;
  if (lWristVisible) {
    if (lWristY <= shoulderY) beep();
  }

  // right Hand in your face?
  const rWristVisible = keypoints[10].score > minConfidence;
  const rWristY = keypoints[10].position.y;
  if (rWristVisible) {
    if (rWristY <= shoulderY) beep();
  }
  // console.log('ShoulderY:', shoulderY, 'lWristY:', lWristY, 'rWristY', rWristY);
}

const beep = (function () {
  // console.log('Keep your hands off your face!!!!');
  let ctxClass = window.audioContext || window.AudioContext || window.AudioContext || window.webkitAudioContext;
  let ctx = new ctxClass();
  return function () {
    let osc = ctx.createOscillator();
    osc.type = 'sawtooth';
    osc.frequency.setValueAtTime(800, ctx.currentTime); // value in hertz
    osc.detune.setValueAtTime(200, ctx.currentTime); // value in cents
    osc.connect(ctx.destination);
    if (osc.noteOn) osc.noteOn(0);
    if (osc.start) osc.start();

    setTimeout(function () {
      if (osc.noteOff) osc.noteOff(0);
      if (osc.stop) osc.stop();
    }, 1000);
  };
})();

/**
 * Draw the bounding box of a pose. For example, for a whole person standing
 * in an image, the bounding box will begin at the nose and extend to one of
 * ankles
 */
export function drawBoundingBox(keypoints, ctx) {
  const boundingBox = posenet.getBoundingBox(keypoints);

  ctx.rect(
    boundingBox.minX, boundingBox.minY, boundingBox.maxX - boundingBox.minX,
    boundingBox.maxY - boundingBox.minY);

  ctx.strokeStyle = boundingBoxColor;
  ctx.stroke();
}

/**
 * Converts an arary of pixel data into an ImageData object
 */
export async function renderToCanvas(a, ctx) {
  const [height, width] = a.shape;
  const imageData = new ImageData(width, height);

  const data = await a.data();

  for (let i = 0; i < height * width; ++i) {
    const j = i * 4;
    const k = i * 3;

    imageData.data[j + 0] = data[k + 0];
    imageData.data[j + 1] = data[k + 1];
    imageData.data[j + 2] = data[k + 2];
    imageData.data[j + 3] = 255;
  }

  ctx.putImageData(imageData, 0, 0);
}

/**
 * Draw an image on a canvas
 */
export function renderImageToCanvas(image, size, canvas) {
  canvas.width = size[0];
  canvas.height = size[1];
  const ctx = canvas.getContext('2d');

  ctx.drawImage(image, 0, 0);
}

/**
 * Draw heatmap values, one of the model outputs, on to the canvas
 * Read our blog post for a description of PoseNet's heatmap outputs
 * https://medium.com/tensorflow/real-time-human-pose-estimation-in-the-browser-with-tensorflow-js-7dd0bc881cd5
 */
export function drawHeatMapValues(heatMapValues, outputStride, canvas) {
  const ctx = canvas.getContext('2d');
  const radius = 5;
  const scaledValues = heatMapValues.mul(tf.scalar(outputStride, 'int32'));

  drawPoints(ctx, scaledValues, radius, color);
}

/**
 * Used by the drawHeatMapValues method to draw heatmap points on to
 * the canvas
 */
function drawPoints(ctx, points, radius, color) {
  const data = points.buffer().values;

  for (let i = 0; i < data.length; i += 2) {
    const pointY = data[i];
    const pointX = data[i + 1];

    if (pointX !== 0 && pointY !== 0) {
      ctx.beginPath();
      ctx.arc(pointX, pointY, radius, 0, 2 * Math.PI);
      ctx.fillStyle = color;
      ctx.fill();
    }
  }
}

/**
 * Draw offset vector values, one of the model outputs, on to the canvas
 * Read our blog post for a description of PoseNet's offset vector outputs
 * https://medium.com/tensorflow/real-time-human-pose-estimation-in-the-browser-with-tensorflow-js-7dd0bc881cd5
 */
export function drawOffsetVectors(
  heatMapValues, offsets, outputStride, scale = 1, ctx) {
  const offsetPoints =
    posenet.singlePose.getOffsetPoints(heatMapValues, outputStride, offsets);

  const heatmapData = heatMapValues.buffer().values;
  const offsetPointsData = offsetPoints.buffer().values;

  for (let i = 0; i < heatmapData.length; i += 2) {
    const heatmapY = heatmapData[i] * outputStride;
    const heatmapX = heatmapData[i + 1] * outputStride;
    const offsetPointY = offsetPointsData[i];
    const offsetPointX = offsetPointsData[i + 1];

    drawSegment(
      [heatmapY, heatmapX], [offsetPointY, offsetPointX], color, scale, ctx);
  }
}