diff --git a/.gitignore b/.gitignore index 0b877840..ff1ed836 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules/ dist/*.worker.js +typings/ diff --git a/examples/components/FaceMaskExample.js b/examples/components/FaceMaskExample.js index 00ebd97f..2b475920 100644 --- a/examples/components/FaceMaskExample.js +++ b/examples/components/FaceMaskExample.js @@ -1,5 +1,6 @@ import React from 'react'; import _ from 'lodash'; +import { disposableEvent } from 'jsio-event-kit'; import DropDownMenu from 'material-ui/DropDownMenu'; import MenuItem from 'material-ui/MenuItem'; @@ -72,15 +73,15 @@ export default class FaceMaskExample extends VideoExample { // draw face deformation model const tracker = this.state.tracker; const deformer = this.state.deformer; - deformer.load( - video, - tracker.getCurrentPosition(), - tracker, - video - ); - - // hide video and score - this.setState({ hideMedia: true, showScore: false }); + deformer.setTracker(tracker); + deformer.setMaskTexture(video); + deformer.setBackground(video); + + const disposable = disposableEvent(tracker, 'converged', () => { + // hide video and score + this.setState({ hideMedia: true, showScore: false }); + disposable.dispose(); + }); } _drawGridLoop () { diff --git a/examples/reducers/examples.js b/examples/reducers/examples.js index 6c643217..721da99c 100644 --- a/examples/reducers/examples.js +++ b/examples/reducers/examples.js @@ -95,7 +95,7 @@ export const setExample = (example) => { // The reducers const DEFAULT_STATE = { - activeExample: EXAMPLES[0].id + activeExample: EXAMPLES[7].id }; export default function (state = DEFAULT_STATE, action) { diff --git a/js/deformers/Deformer.ts b/js/deformers/Deformer.ts new file mode 100644 index 00000000..d989a663 --- /dev/null +++ b/js/deformers/Deformer.ts @@ -0,0 +1,143 @@ +import { EventEmitter } from 'events'; + +import { getWebGLContext } from 'clmtrackr/js/utils/webgl'; +import { + getBoundingBox, + generateTextureVertices +} from 'clmtrackr/js/utils/points'; +import { getImageData } from 'clmtrackr/js/utils/image'; + + +abstract class Deformer extends EventEmitter { + protected _gl: WebGLRenderingContext; + + protected _tracker; + protected _verticeMap; + + protected _dynamicMaskTexture: boolean; + protected _maskTextureSrcElement: HTMLElement; + protected _maskTextureCanvas: HTMLCanvasElement; + + protected _pointBB; + protected _maskTextureCoord; + + + abstract setBackground (element: HTMLElement): void; + abstract draw (points: number[][]): void; + abstract drawGrid (): void; + + + constructor () { + super(); + + this._dynamicMaskTexture = false; + this._maskTextureSrcElement = null; + this._maskTextureCanvas = document.createElement('canvas'); + this._pointBB = null; + + this._maskTextureCoord = null; + } + + + public init (canvas: HTMLCanvasElement): void { + if (!canvas) { + throw new Error('canvas parameter is falsey'); + } + this._gl = getWebGLContext(canvas); + if (!this._gl) { + throw new Error('Could not get a webgl context; have you already tried getting a 2d context on this canvas?'); + } + } + + public setTracker (tracker): void { + this._tracker = tracker; + // Set verts for this mask + this._verticeMap = tracker.model.path.vertices; + } + + public setMaskTexture (element: HTMLElement): void { + this._maskTextureSrcElement = element; + + const tagName = this._maskTextureSrcElement.tagName; + if (tagName === 'CANVAS') { + // Use the element as texture (its dynamic!) + this._dynamicMaskTexture = true; + } else { + // We need a still frame from it + this._dynamicMaskTexture = false; + this.updateMaskTexture(); + } + } + + public setPoints (points: number[][]): void { + if (!points) { + throw new Error('points is falsey'); + } + + // Find texture cropping from mask points + this._pointBB = getBoundingBox(points); + + // offset points by bounding box + const nupoints = points.map(p => [ + p[0] - this._pointBB.minX, + p[1] - this._pointBB.minY + ]); + + // create UVs based on map points + this._maskTextureCoord = generateTextureVertices( + nupoints, + this._verticeMap, + 1 / this._pointBB.width, + 1 / this._pointBB.height + ); + + this.updateMaskTexture(); + } + + protected updateMaskTexture (): HTMLElement { + if ( + !this._maskTextureSrcElement || + !this._pointBB + ) { + return null; + } + + this.emit('maskReady'); + + if (!this._dynamicMaskTexture) { + // Draw the srcElement to the mask texture canvas + const { + minX, minY, width, height + } = this._pointBB; + + const maskImage = getImageData( + this._maskTextureSrcElement, + minX, + minY, + width, + height + ); + + const canvas = this._maskTextureCanvas; + const ctx = canvas.getContext('2d'); + canvas.width = width; + canvas.height = height; + ctx.putImageData(maskImage, 0, 0); + + return canvas; + } else { + return this._maskTextureSrcElement; + } + } + + public getGLContext (): WebGLRenderingContext { + return this._gl; + } + + public clear (): void { + const gl = this.getGLContext(); + gl.clear(gl.COLOR_BUFFER_BIT); + } +} + +export default Deformer; diff --git a/js/deformers/three/index.ts b/js/deformers/three/index.ts new file mode 100644 index 00000000..83001cab --- /dev/null +++ b/js/deformers/three/index.ts @@ -0,0 +1,207 @@ +import { + WebGLRenderer, + Scene, + PerspectiveCamera, + Mesh, + MeshBasicMaterial, + PlaneGeometry, + Texture, + LinearFilter, + NearestFilter, + DoubleSide, + ShaderMaterial, + BufferGeometry, + BufferAttribute, + WebGLRenderTarget +} from 'three'; + +import { generateTextureVertices } from 'clmtrackr/js/utils/points'; +import Deformer from '../Deformer'; + +import createMaskVS from './shaders/mask.vert'; +import createMaskFS from './shaders/mask.frag'; + + +const RAD_TO_DEG = 180 / Math.PI; +const DEG_TO_RAD = Math.PI / 180; + + +export default class ThreeDeformer extends Deformer { + + private scene: Scene; + + private camera: PerspectiveCamera; + + private renderer: WebGLRenderer; + + private maskMesh: Mesh; + private bgMesh: Mesh; + + private bgScaleX: number; + private bgScaleY: number; + + + constructor () { + super(); + } + + public init (canvas: HTMLCanvasElement): void { + super.init(canvas); + + this.renderer = new WebGLRenderer({ + canvas, + preserveDrawingBuffer: true + }); + this.renderer.autoClear = false; + this.renderer.setSize(canvas.width, canvas.height); + + this.scene = new Scene(); + + this.camera = new PerspectiveCamera( + 75, + canvas.width / canvas.height, + 1, + 10000 + ); + this.camera.position.z = 1000; + + // Make the background + const tan = Math.tan(this.camera.fov / 2 * DEG_TO_RAD) * (2 * this.camera.position.z); + const bgGeom = new PlaneGeometry( + tan * this.camera.aspect, + tan + ); + const bgMat = new MeshBasicMaterial({ + color: 0x00ff00, + wireframe: true + }); + this.bgMesh = new Mesh(bgGeom, bgMat); + this.scene.add(this.bgMesh); + + this.bgScaleX = bgGeom.parameters.width / canvas.width; + this.bgScaleY = bgGeom.parameters.height / canvas.height; + + // Mask the mask geometry + const maskGeom = new BufferGeometry(); + const maskMat = new ShaderMaterial({ + uniforms: { + texture: { value: null }, + bgTexture: { value: null }, + bgWidth: { value: canvas.width }, + bgHeight: { value: canvas.height } + }, + vertexShader: createMaskVS(), + fragmentShader: createMaskFS() + }); + + this.maskMesh = new Mesh(maskGeom, maskMat); + + // Dont add mask to scene until it is ready + this.once('maskReady', () => { + this.scene.add(this.maskMesh); + }) + } + + public setBackground (element: HTMLElement): void { + const texture = new Texture(element); + texture.minFilter = LinearFilter; + const bgMaterial = this.bgMesh.material; + bgMaterial.map = texture; + + const maskBgTexture = this.maskMesh.material.uniforms.bgTexture; + maskBgTexture.value = texture; + maskBgTexture.needsUpdate = true; + + // Un-set the defaults + bgMaterial.wireframe = false; + bgMaterial.color.set(0xffffff); + } + + protected updateMaskTexture (): HTMLElement { + const srcElement = super.updateMaskTexture(); + if (!srcElement) { return; } + // Update mask texture + const texture = new Texture(srcElement); + texture.minFilter = LinearFilter; + texture.needsUpdate = true; + + const maskMaterial = this.maskMesh.material; + maskMaterial.map = texture; + maskMaterial.side = DoubleSide; + // Un-set the defaults + maskMaterial.wireframe = false; + + // Update the shader uniform + const uTexture = maskMaterial.uniforms.texture; + uTexture.value = texture; + uTexture.needsUpdate = true; + + return; + } + + public setPoints (points: number[][]): void { + super.setPoints(points); + + const geom = this.maskMesh.geometry; + + const faceCount = Math.floor(this._maskTextureCoord.length / 6 * 3) + // Initialize the verts + geom.addAttribute( + 'position', + new BufferAttribute(new Float32Array(faceCount * 3), 3) + ); + + // Initialize the UVs + const faceVertexUvs = new Float32Array(faceCount * 3); + for (let i = 0; i < this._maskTextureCoord.length; i += 2) { + faceVertexUvs[i] = this._maskTextureCoord[i]; + faceVertexUvs[i + 1] = 1 - this._maskTextureCoord[i + 1]; + } + + geom.addAttribute( + 'uv', + new BufferAttribute(faceVertexUvs, 2) + ); + geom.attributes.uv.needsUpdate = true; + } + + private updateMaskGeom (points: number[][]): void { + const maskVertices = generateTextureVertices(points, this._verticeMap); + + const geom = this.maskMesh.geometry; + const position = geom.attributes.position; + + const bgW = this.bgMesh.geometry.parameters.width; + const bgH = this.bgMesh.geometry.parameters.height; + const offsetX = bgW * -0.5; + const offsetY = bgH * -0.5; + + const verts = position.array; + let vertIndex = 0; + for (let i = 0; i < maskVertices.length; i += 2) { + verts[vertIndex++] = (maskVertices[i] * this.bgScaleX) + offsetX; + verts[vertIndex++] = (bgH - (maskVertices[i + 1] * this.bgScaleY)) + offsetY; + verts[vertIndex++] = 1; + } + + position.needsUpdate = true; + } + + public draw (points: number[][]): void { + // Update the scene + // TODO: this should move to a separate tick function to avoid rendering + // hiccups + this.updateMaskGeom(points); + + // Update bg texture + const bgTex = this.bgMesh.material.map; + if (bgTex) { + bgTex.needsUpdate = true; + this.maskMesh.material.uniforms.bgTexture.needsUpdate = true; + } + + this.renderer.render(this.scene, this.camera); + } + + public drawGrid (): void { } +} diff --git a/js/deformers/three/shaders/ColorSpaces.glsl b/js/deformers/three/shaders/ColorSpaces.glsl new file mode 100644 index 00000000..2a191c97 --- /dev/null +++ b/js/deformers/three/shaders/ColorSpaces.glsl @@ -0,0 +1,62 @@ +/* +GLSL Color Space Utility Functions +(c) 2015 tobspr +------------------------------------------------------------------------------- +The MIT License (MIT) +Copyright (c) 2015 +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------- +Most formulars / matrices are from: +https://en.wikipedia.org/wiki/SRGB +Some are from: +http://www.chilliant.com/rgb2hsv.html + +https://github.com/tobspr/GLSL-Color-Spaces/blob/master/ColorSpaces.inc.glsl +*/ + +const float HCV_EPSILON = 1e-10; + +vec3 rgb_to_hcv(vec3 rgb) +{ + // Based on work by Sam Hocevar and Emil Persson + vec4 P = (rgb.y < rgb.z) ? vec4(rgb.zy, -1.0, 2.0/3.0) : vec4(rgb.yz, 0.0, -1.0/3.0); + vec4 Q = (rgb.x < P.x) ? vec4(P.xyw, rgb.x) : vec4(rgb.x, P.yzx); + float C = Q.x - min(Q.w, Q.y); + float H = abs((Q.w - Q.y) / (6.0 * C + HCV_EPSILON) + Q.z); + return vec3(H, C, Q.x); +} + +vec3 rgb_to_hsv(vec3 rgb) +{ + vec3 HCV = rgb_to_hcv(rgb); + float S = HCV.y / (HCV.z + HCV_EPSILON); + return vec3(HCV.x, S, HCV.z); +} + +vec3 hue_to_rgb(float hue) +{ + float R = abs(hue * 6.0 - 3.0) - 1.0; + float G = 2.0 - abs(hue * 6.0 - 2.0); + float B = 2.0 - abs(hue * 6.0 - 4.0); + return saturate(vec3(R,G,B)); +} + +vec3 hsv_to_rgb(vec3 hsv) +{ + vec3 rgb = hue_to_rgb(hsv.x); + return ((rgb - 1.0) * hsv.y + 1.0) * hsv.z; +} \ No newline at end of file diff --git a/js/deformers/three/shaders/mask.frag b/js/deformers/three/shaders/mask.frag new file mode 100644 index 00000000..3f4d04f8 --- /dev/null +++ b/js/deformers/three/shaders/mask.frag @@ -0,0 +1,33 @@ +uniform sampler2D texture; +uniform sampler2D bgTexture; + +uniform float bgWidth; +uniform float bgHeight; + +varying vec2 vUv; + + +#include ./ColorSpaces; + + +void main() { + vec4 bgColor = texture2D( + bgTexture, + vec2( + gl_FragCoord.x / bgWidth, + gl_FragCoord.y / bgHeight + ) + ); + vec4 texColor = texture2D(texture, vUv); + + vec3 texHSV = rgb_to_hsv(vec3(texColor.x, texColor.y, texColor.z)); + vec3 bgHSV = rgb_to_hsv(vec3(bgColor.x, bgColor.y, bgColor.z)); + + float vDiff = bgHSV.z - texHSV.z; + + gl_FragColor = vec4(hsv_to_rgb(vec3( + texHSV.x, + texHSV.y, + texHSV.z + vDiff * 0.4 + )), 1.0); +} diff --git a/js/deformers/three/shaders/mask.vert b/js/deformers/three/shaders/mask.vert new file mode 100644 index 00000000..26c7af11 --- /dev/null +++ b/js/deformers/three/shaders/mask.vert @@ -0,0 +1,7 @@ +varying vec2 vUv; + +void main() { + vUv = uv; + + gl_Position = projectionMatrix * modelViewMatrix * vec4(position, 1.0); +} diff --git a/js/deformers/twgl/Background.js b/js/deformers/twgl/Background.ts similarity index 86% rename from js/deformers/twgl/Background.js rename to js/deformers/twgl/Background.ts index 055558fd..9cc2aa1a 100644 --- a/js/deformers/twgl/Background.js +++ b/js/deformers/twgl/Background.ts @@ -3,8 +3,19 @@ import twgl from 'twgl.js/dist/twgl'; import createBgVert from './shaders/background.vert'; import createBgFrag from './shaders/background.frag'; +import IDeformer from '../IDeformer'; + export default class Background { + private _deformer: IDeformer; + + private _element: HTMLElement; + + private _bgBufferInfo; + private _bgProgramInfo; + private _bgTextures; + private _bgUniforms; + constructor (deformer) { this._deformer = deformer; @@ -19,7 +30,7 @@ export default class Background { /** * @param {*} element - This will be the source for the background */ - setElement (element) { + public setElement (element): void { this._element = element; if (!this._element) { this._bgBufferInfo = null; @@ -56,7 +67,7 @@ export default class Background { }; } - draw () { + public draw (): void { if (!this._element) { return; } const gl = this._deformer.getGLContext(); diff --git a/js/deformers/twgl/index.js b/js/deformers/twgl/index.js deleted file mode 100644 index ed428681..00000000 --- a/js/deformers/twgl/index.js +++ /dev/null @@ -1,228 +0,0 @@ -import twgl from 'twgl.js/dist/twgl'; - -import { getImageData } from '../../utils/image'; -import { getBoundingBox } from '../../utils/points'; -import { getWebGLContext } from '../../utils/webgl'; -import Background from './Background'; - -import createDeformVert from './shaders/deform.vert'; -import createDeformFrag from './shaders/deform.frag'; - - -export default class Deformer { - constructor (params = {}) { - twgl.setDefaults({ attribPrefix: 'a_' }); - - this._isLoaded = false; - - this.background = new Background(this); - this.debug = new Background(this); - - this._tracker = null; - this._gl = null; - - this._verticeMap = null; - - this._dynamicMaskTexture = false; - this._maskTextureSrcElement = null; - this._maskTextureCanvas = document.createElement('canvas'); - this._pointBB = null; - - this._maskTextureCoord = null; - this._maskProgramInfo = null; - this._maskTextures = null; - this._maskUniforms = null; - } - - getGLContext () { - return this._gl; - } - - init (canvas) { - if (!canvas) { - throw new Error('canvas parameter is falsey'); - } - this._gl = getWebGLContext(canvas); - if (!this._gl) { - throw new Error('Could not get a webgl context; have you already tried getting a 2d context on this canvas?'); - } - } - - loadTexture (texPath, points, tracker, bgElement) { - const image = new Image(); - image.addEventListener('load', () => { - this.load(image, points, tracker, bgElement); - }); - image.src = texPath; - } - - _generateTextureVertices (points, vertMap, scaleX = 1, scaleY = 1) { - const tvs = []; - for (let i = 0; i < vertMap.length; i++) { - tvs.push(points[vertMap[i][0]][0] * scaleX); - tvs.push(points[vertMap[i][0]][1] * scaleY); - tvs.push(points[vertMap[i][1]][0] * scaleX); - tvs.push(points[vertMap[i][1]][1] * scaleY); - tvs.push(points[vertMap[i][2]][0] * scaleX); - tvs.push(points[vertMap[i][2]][1] * scaleY); - } - return tvs; - } - - setMaskTexture (element) { - this._maskTextureSrcElement = element; - - const tagName = this._maskTextureSrcElement.tagName; - if (tagName === 'CANVAS') { - // Use the element as texture (its dynamic!) - this._dynamicMaskTexture = true; - } else { - // We need a still frame from it - this._dynamicMaskTexture = false; - this.updateMaskTexture(); - } - } - - updateMaskTexture () { - if ( - !this._maskTextureSrcElement || - !this._pointBB - ) { return; } - - let srcElement; - if (!this._dynamicMaskTexture) { - // Draw the srcElement to the mask texture canvas - const { - minX, minY, width, height - } = this._pointBB; - - const maskImage = getImageData( - this._maskTextureSrcElement, - minX, - minY, - width, - height - ); - - const canvas = this._maskTextureCanvas; - const ctx = canvas.getContext('2d'); - canvas.width = width; - canvas.height = height; - ctx.putImageData(maskImage, 0, 0); - - srcElement = canvas; - } else { - srcElement = this._maskTextureSrcElement - } - - // Update the webgl stuff - const gl = this.getGLContext(); - this._maskTextures = twgl.createTextures(gl, { - mask: { - mag: gl.LINEAR, - min: gl.LINEAR, - src: srcElement - } - }); - - this._maskUniforms = { - u_resolution: [gl.drawingBufferWidth, gl.drawingBufferHeight], - u_sampler: this._maskTextures.mask - }; - } - - setPoints (points) { - if (!points) { - throw new Error('points is falsey'); - } - - // Find texture cropping from mask points - this._pointBB = getBoundingBox(points); - - // correct points - let nupoints = points.map(p => [ - p[0] - this._pointBB.minX, - p[1] - this._pointBB.minY - ]); - - // create vertices based on points - this._maskTextureCoord = this._generateTextureVertices( - nupoints, - this._verticeMap, - 1 / this._pointBB.width, - 1 / this._pointBB.height - ); - - this.updateMaskTexture(); - } - - setTracker (tracker) { - this._tracker = tracker; - // Set verts for this mask - this._verticeMap = tracker.model.path.vertices; - } - - load (element, points, tracker, bgElement) { - this.setTracker(tracker); - this.setMaskTexture(element); - points && this.setPoints(points); - - this.background.setElement(bgElement); - - const gl = this.getGLContext(); - this._maskProgramInfo = twgl.createProgramInfo(gl, [ - createDeformVert(), - createDeformFrag() - ]); - - this._isLoaded = true; - } - - draw (points) { - if (!this._isLoaded) { - console.warn('deformer not yet loaded, draw cannot run'); - return; - } - - const gl = this.getGLContext(); - twgl.resizeCanvasToDisplaySize(gl.canvas); - gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); - - this.background.draw(); - this.drawMask(points); - this.debug.draw(); - } - - drawMask (points) { - const gl = this.getGLContext(); - - // create drawvertices based on points - let vertices = this._generateTextureVertices(points, this._verticeMap); - - this._maskBufferInfo = twgl.createBufferInfoFromArrays(gl, { - position: { - numComponents: 2, - data: vertices - }, - texcoord: { - numComponents: 2, - data: this._maskTextureCoord - } - }); - - gl.useProgram(this._maskProgramInfo.program); - - twgl.setBuffersAndAttributes(gl, this._maskProgramInfo, this._maskBufferInfo); - twgl.setUniforms(this._maskProgramInfo, this._maskUniforms); - twgl.drawBufferInfo(gl, gl.TRIANGLES, this._maskBufferInfo); - } - - drawGrid () { - // TODO: implement (what is drawGrid?) - } - - clear () { - const gl = this.getGLContext(); - gl.clear(gl.COLOR_BUFFER_BIT); - } -} diff --git a/js/deformers/twgl/index.ts b/js/deformers/twgl/index.ts new file mode 100644 index 00000000..af7adadf --- /dev/null +++ b/js/deformers/twgl/index.ts @@ -0,0 +1,107 @@ +import twgl from 'twgl.js/dist/twgl'; + +import { generateTextureVertices } from 'clmtrackr/js/utils/points'; + +import Background from './Background'; + +import createDeformVert from './shaders/deform.vert'; +import createDeformFrag from './shaders/deform.frag'; + +import Deformer from '../Deformer'; + + +export default class TwglDeformer extends Deformer { + private background: Background; + private debug: Background; + + private _maskProgramInfo; + private _maskTextures; + private _maskUniforms; + private _maskBufferInfo; + + constructor (params = {}) { + super(); + + twgl.setDefaults({ attribPrefix: 'a_' }); + + this.background = new Background(this); + this.debug = new Background(this); + + this._maskProgramInfo = null; + this._maskTextures = null; + this._maskUniforms = null; + this._maskBufferInfo = null; + } + + public init (canvas: HTMLCanvasElement): void { + super.init(canvas); + + const gl = this.getGLContext(); + this._maskProgramInfo = twgl.createProgramInfo(gl, [ + createDeformVert(), + createDeformFrag() + ]); + } + + protected updateMaskTexture (): HTMLElement { + const srcElement = super.updateMaskTexture(); + if (!srcElement) { return; } + + // Update the webgl stuff + const gl = this.getGLContext(); + this._maskTextures = twgl.createTextures(gl, { + mask: { + mag: gl.LINEAR, + min: gl.LINEAR, + src: srcElement + } + }); + + this._maskUniforms = { + u_resolution: [gl.drawingBufferWidth, gl.drawingBufferHeight], + u_sampler: this._maskTextures.mask + }; + } + + public setBackground (bgElement: HTMLElement): void { + this.background.setElement(bgElement); + } + + public draw (points: number[][]): void { + const gl = this.getGLContext(); + twgl.resizeCanvasToDisplaySize(gl.canvas); + gl.viewport(0, 0, gl.canvas.width, gl.canvas.height); + + this.background.draw(); + this.drawMask(points); + this.debug.draw(); + } + + private drawMask (points: number[][]): void { + const gl = this.getGLContext(); + + // create drawvertices based on points + let vertices = generateTextureVertices(points, this._verticeMap); + + this._maskBufferInfo = twgl.createBufferInfoFromArrays(gl, { + position: { + numComponents: 2, + data: vertices + }, + texcoord: { + numComponents: 2, + data: this._maskTextureCoord + } + }); + + gl.useProgram(this._maskProgramInfo.program); + + twgl.setBuffersAndAttributes(gl, this._maskProgramInfo, this._maskBufferInfo); + twgl.setUniforms(this._maskProgramInfo, this._maskUniforms); + twgl.drawBufferInfo(gl, gl.TRIANGLES, this._maskBufferInfo); + } + + public drawGrid () { + // TODO: implement (what is drawGrid?) + } +} diff --git a/js/utils/points.js b/js/utils/points.js index 5df8bbb9..4bea6667 100644 --- a/js/utils/points.js +++ b/js/utils/points.js @@ -28,3 +28,18 @@ export const getBoundingBox = (points) => { height: maxY - minY }; }; + + +export const generateTextureVertices = (points, vertMap, scaleX = 1, scaleY = 1) => { + const tvs = new Float32Array(vertMap.length * 6); + for (let i = 0; i < vertMap.length; i++) { + const j = i * 6; + tvs[j] = points[vertMap[i][0]][0] * scaleX; + tvs[j + 1] = points[vertMap[i][0]][1] * scaleY; + tvs[j + 2] = points[vertMap[i][1]][0] * scaleX; + tvs[j + 3] = points[vertMap[i][1]][1] * scaleY; + tvs[j + 4] = points[vertMap[i][2]][0] * scaleX; + tvs[j + 5] = points[vertMap[i][2]][1] * scaleY; + } + return tvs; +}; diff --git a/package.json b/package.json index ab3573fb..13c790a0 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,10 @@ "description": "Javascript library for precise tracking of facial features via Constrained Local Models", "scripts": { "build": "NODE_ENV=production webpack --optimize-minimize --optimize-dedupe", - "watch": "webpack --debug --devtool inline-source-map --output-pathinfo --progress --colors --watch", - "serve": "webpack-dev-server --debug --devtool inline-source-map --output-pathinfo --inline --hot --host localhost", + "watch": "webpack --debug --devtool eval-source-map --output-pathinfo --progress --colors --watch", + "serve": "webpack-dev-server --debug --devtool eval-source-map --output-pathinfo --inline --hot --host localhost", "clean": "rm -rf dist", - "postinstall": "npm run build", + "postinstall": "typings install && npm run build", "publish": "./publish.sh", "generateModelJsons": "node ./generateModelJsons.js" }, @@ -26,13 +26,16 @@ "filesaver.js": "andyinabox/FileSaver.js", "getusermedia": "^1.3.5", "gh-pages": "^0.11.0", - "glsl-template-loader": "git+https://github.com/jsio-private/glsl-template-loader#ac20d0e2d88d2e231d2ea4c271e56205b8ea437f", + "glsl-template-loader": "git+https://github.com/jsio-private/glsl-template-loader#0c1b2cbaa6b1020de78527b37dcd2ce51a5e89bc", "html-webpack-plugin": "^2.22.0", "json-loader": "^0.5.4", "raf": "^3.1.0", "stylus": "^0.54.5", "stylus-loader": "^2.3.1", + "ts-loader": "^0.8.2", "twgl.js": "^1.9.0", + "typescript": "^1.8.10", + "typings": "^1.3.3", "uglify": "^0.1.5", "webpack": "^1.13.2", "webpack-dev-server": "^1.15.1", @@ -41,6 +44,7 @@ "dependencies": { "classnames": "^2.2.5", "jsfeat": "git+https://github.com/inspirit/jsfeat#f584f93c78b5085ef141eb1dc66d9d06a7123ec3", + "jsio-event-kit": "git+https://github.com/jsio-private/event-kit.git", "lodash": "^4.15.0", "material-design-color": "^2.3.1", "material-ui": "^0.15.4", @@ -52,6 +56,7 @@ "react-redux": "^4.4.5", "react-tap-event-plugin": "^1.0.0", "redux": "^3.6.0", - "redux-logger": "^2.6.1" + "redux-logger": "^2.6.1", + "three": "^0.81.0" } } diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000..bb3999f5 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "target": "es6", + "sourceMap": true, + "removeComments": false + }, + "exclude": [ + "node_modules" + ] +} diff --git a/typings.json b/typings.json new file mode 100644 index 00000000..ee49af2d --- /dev/null +++ b/typings.json @@ -0,0 +1,5 @@ +{ + "globalDependencies": { + "node": "registry:env/node#4.0.0+20160918225031" + } +} diff --git a/webpack.config.js b/webpack.config.js index 84b41093..867a70bf 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -22,7 +22,8 @@ const config = { alias: { 'clmtrackr': path.resolve(__dirname), 'stats.js': path.resolve(__dirname, 'lib', 'stats.js') - } + }, + extensions: ['', '.webpack.js', '.web.js', '.ts', '.tsx', '.js', '.vert', '.frag', '.glsl'] }, module: { loaders: [ @@ -34,6 +35,11 @@ const config = { test: /\.worker\.js$/, loader: 'worker-loader?inline=true' }, + { + test: /\.tsx?$/, + exclude: /node_modules/, + loader: 'babel-loader?presets[]=es2015&presets[]=react!ts-loader?ignoreDiagnostics[]=2307' + }, { test: /\.js$/, loader: 'babel-loader',