Video annotation: Extending the Point Tool

packages/lib-classifier/src/components/Classifier/components/SubjectViewer/components/InteractionLayer/InteractionLayer.js

@@ -6,15 +6,20 @@ import SVGContext from '@plugins/drawingTools/shared/SVGContext'
 import DrawingToolMarks from './components/DrawingToolMarks'
 import TranscribedLines from './components/TranscribedLines'
 import SubTaskPopup from './components/SubTaskPopup'
+import getFixedNumber from '../../helpers/getFixedNumber'
 
 const DrawingCanvas = styled('rect')`
-  ${props => props.disabled ?
-    css`cursor: not-allowed;` :
-    css`cursor: crosshair;`
-  }
+  ${(props) =>
+    props.disabled
+      ? css`
+          cursor: not-allowed;
+        `
+      : css`
+          cursor: crosshair;
+        `}
 `
 
-function InteractionLayer ({
+function InteractionLayer({
   activeMark,
   activeTool,
   activeToolIndex,
@@ -25,18 +30,22 @@ function InteractionLayer ({
   move,
   setActiveMark,
   scale,
-  width
+  width,
+  played
 }) {
-  const [ creating, setCreating ] = React.useState(false)
+  const [creating, setCreating] = React.useState(false)
   const { svg, getScreenCTM } = React.useContext(SVGContext)
 
-  useEffect(function onDeleteMark() {
-    if (creating && !activeMark) {
-      setCreating(false)
-    }
-  }, [activeMark])
+  useEffect(
+    function onDeleteMark() {
+      if (creating && !activeMark) {
+        setCreating(false)
+      }
+    },
+    [activeMark]
+  )
 
-  function convertEvent (event) {
+  function convertEvent(event) {
     const type = event.type
 
     const svgEventOffset = getEventOffset(event)
@@ -51,15 +60,17 @@ function InteractionLayer ({
     return svgCoordinateEvent
   }
 
-  function getEventOffset (event) {
+  function getEventOffset(event) {
     const svgPoint = svg.createSVGPoint()
     svgPoint.x = event.clientX
     svgPoint.y = event.clientY
     const svgEventOffset = svgPoint.matrixTransform(getScreenCTM().inverse())
     return svgEventOffset
   }
 
-  function createMark (event) {
+  function createMark(event) {
+    // TODO: add case for played = undefined
+    const timeStamp = getFixedNumber(played, 5)
     const mark = activeTool.createMark({
       id: cuid(),
       frame,
@@ -70,21 +81,24 @@ function InteractionLayer ({
     setActiveMark(mark)
     setCreating(true)
     mark.setSubTaskVisibility(false)
+    // Add a time value for tools that care about time. For most tools, this value is ignored.
+    mark.setVideoTime(timeStamp)
   }
 
-  function onPointerDown (event) {
+  function onPointerDown(event) {
     if (disabled || move) {
       return true
     }
     const { target, pointerId } = event
     target.setPointerCapture(pointerId)
 
     if (!activeTool.type) {
-      return false;
+      return false
     }
 
     if (creating) {
-      activeTool.handlePointerDown && activeTool.handlePointerDown(convertEvent(event), activeMark)
+      activeTool.handlePointerDown &&
+        activeTool.handlePointerDown(convertEvent(event), activeMark)
       if (activeMark.finished) onFinish(event)
       return true
     }
@@ -93,13 +107,14 @@ function InteractionLayer ({
     return false
   }
 
-  function onPointerMove (event) {
+  function onPointerMove(event) {
     if (creating) {
-      activeTool.handlePointerMove && activeTool.handlePointerMove(convertEvent(event), activeMark)
+      activeTool.handlePointerMove &&
+        activeTool.handlePointerMove(convertEvent(event), activeMark)
     }
   }
 
-  function onFinish (event) {
+  function onFinish(event) {
     if (event.preventDefault) event.preventDefault()
     setCreating(false)
     if (activeMark && !activeMark.isValid) {
@@ -110,12 +125,13 @@ function InteractionLayer ({
 
   function onPointerUp(event) {
     if (creating) {
-      activeTool.handlePointerUp && activeTool.handlePointerUp(convertEvent(event), activeMark)
+      activeTool.handlePointerUp &&
+        activeTool.handlePointerUp(convertEvent(event), activeMark)
       if (activeMark.finished) onFinish(event)
     }
   }
 
-  function inactivateMark () {
+  function inactivateMark() {
     setActiveMark(undefined)
   }
 
@@ -131,24 +147,21 @@ function InteractionLayer ({
         onPointerMove={onPointerMove}
         onPointerUp={onPointerUp}
       />
-      <TranscribedLines
-        scale={scale}
-      />
-      <SubTaskPopup
-        onDelete={inactivateMark}
-      />
-      {marks &&
+      <TranscribedLines scale={scale} />
+      <SubTaskPopup onDelete={inactivateMark} />
+      {marks && (
         <DrawingToolMarks
           activeMark={activeMark}
           marks={marks}
           onDelete={inactivateMark}
           onDeselectMark={inactivateMark}
           onFinish={onFinish}
-          onSelectMark={mark => setActiveMark(mark)}
+          onSelectMark={(mark) => setActiveMark(mark)}
           onMove={(mark, difference) => mark.move(difference)}
           scale={scale}
+          played={played}
         />
-      }
+      )}
     </g>
   )
 }

packages/lib-classifier/src/components/Classifier/components/SubjectViewer/components/InteractionLayer/InteractionLayer.spec.js

@@ -7,7 +7,11 @@ import InteractionLayer, { DrawingCanvas } from './InteractionLayer'
 import TranscribedLines from './components/TranscribedLines'
 import SubTaskPopup from './components/SubTaskPopup'
 import DrawingTask from '@plugins/tasks/DrawingTask'
-import { Line, Point, TranscriptionLine } from '@plugins/drawingTools/components'
+import {
+  Line,
+  Point,
+  TranscriptionLine
+} from '@plugins/drawingTools/components'
 
 describe('Component > InteractionLayer', function () {
   let wrapper
@@ -17,7 +21,8 @@ describe('Component > InteractionLayer', function () {
     initialDrag: sinon.stub(),
     initialPosition: sinon.stub(),
     setCoordinates: sinon.stub(),
-    setSubTaskVisibility: sinon.stub()
+    setSubTaskVisibility: sinon.stub(),
+    setVideoTime: sinon.stub()
   }
   const mockSVGPoint = {
     x: 100,
@@ -112,7 +117,9 @@ describe('Component > InteractionLayer', function () {
       describe('onPointerDown', function () {
         let mockedContext
         before(function () {
-          mockedContext = sinon.stub(React, 'useContext').callsFake(() => { return { svg, getScreenCTM } })
+          mockedContext = sinon.stub(React, 'useContext').callsFake(() => {
+            return { svg, getScreenCTM }
+          })
         })
 
         after(function () {
@@ -184,7 +191,9 @@ describe('Component > InteractionLayer', function () {
           }
           wrapper.find(DrawingCanvas).simulate('pointerdown', fakeEvent)
           wrapper.simulate('pointermove', fakeEvent)
-          expect(fakeEvent.target.setPointerCapture.withArgs('fakePointer')).to.have.been.calledOnce()
+          expect(
+            fakeEvent.target.setPointerCapture.withArgs('fakePointer')
+          ).to.have.been.calledOnce()
         })
 
         describe('with a TranscriptionLine mark already in progress', function () {
@@ -205,7 +214,9 @@ describe('Component > InteractionLayer', function () {
             })
             activeTool = mockDrawingTask.activeTool
             sinon.stub(activeTool, 'createMark').callsFake(() => mockMark)
-            sinon.stub(activeTool, 'handlePointerDown').callsFake(() => mockMark)
+            sinon
+              .stub(activeTool, 'handlePointerDown')
+              .callsFake(() => mockMark)
 
             wrapper = shallow(
               <InteractionLayer
@@ -236,7 +247,9 @@ describe('Component > InteractionLayer', function () {
     describe('onPointerUp', function () {
       let mockedContext
       before(function () {
-        mockedContext = sinon.stub(React, 'useContext').callsFake(() => { return { svg, getScreenCTM } })
+        mockedContext = sinon.stub(React, 'useContext').callsFake(() => {
+          return { svg, getScreenCTM }
+        })
       })
 
       after(function () {
@@ -284,12 +297,13 @@ describe('Component > InteractionLayer', function () {
       sinon.stub(activeTool, 'createMark').callsFake(() => mockMark)
       activeTool.createMark.resetHistory()
       wrapper = shallow(
-          <InteractionLayer
-            activeTool={activeTool}
-            disabled
-            height={400}
-            width={600}
-          />, {
+        <InteractionLayer
+          activeTool={activeTool}
+          disabled
+          height={400}
+          width={600}
+        />,
+        {
           wrappingComponent: SVGContext.Provider,
           wrappingComponentProps: { value: { svg, getScreenCTM } }
         }

packages/lib-classifier/src/components/Classifier/components/SubjectViewer/components/InteractionLayer/InteractionLayerContainer.js

@@ -1,25 +1,18 @@
 import { inject, observer } from 'mobx-react'
-import { getType } from 'mobx-state-tree'
 import PropTypes from 'prop-types'
 import React, { Component } from 'react'
 
 import InteractionLayer from './InteractionLayer'
 import PreviousMarks from './components/PreviousMarks'
 import SHOWN_MARKS from '@helpers/shownMarks'
 
-function storeMapper (stores) {
-  const {
-    activeStepTasks
-  } = stores.classifierStore.workflowSteps
-  const {
-    frame,
-    move
-  } = stores.classifierStore.subjectViewer
-  const {
-    active: classification
-  } = stores.classifierStore.classifications
+function storeMapper(stores) {
+  const { activeStepTasks } = stores.classifierStore.workflowSteps
+  const { frame, move } = stores.classifierStore.subjectViewer
 
-  const [activeInteractionTask] = activeStepTasks.filter(task => task.type === 'drawing' || task.type === 'transcription')
+  const [activeInteractionTask] = activeStepTasks.filter(
+    (task) => task.type === 'drawing' || task.type === 'transcription'
+  )
 
   return {
     activeInteractionTask,
@@ -31,14 +24,15 @@ function storeMapper (stores) {
 @inject(storeMapper)
 @observer
 class InteractionLayerContainer extends Component {
-  render () {
+  render() {
     const {
       activeInteractionTask,
       frame,
       height,
       move,
       scale,
-      width
+      width,
+      played
     } = this.props
 
     const {
@@ -52,12 +46,15 @@ class InteractionLayerContainer extends Component {
       taskKey
     } = activeInteractionTask
 
-    const newMarks = shownMarks === SHOWN_MARKS.NONE ? marks.slice(hidingIndex) : marks
-    const visibleMarksPerFrame = newMarks?.filter(mark => mark.frame === frame)
+    const newMarks =
+      shownMarks === SHOWN_MARKS.NONE ? marks.slice(hidingIndex) : marks
+    const visibleMarksPerFrame = newMarks?.filter(
+      (mark) => mark.frame === frame
+    )
 
     return (
       <>
-        {activeInteractionTask && activeTool &&
+        {activeInteractionTask && activeTool && (
           <InteractionLayer
             activeMark={activeMark}
             activeTool={activeTool}
@@ -69,10 +66,11 @@ class InteractionLayerContainer extends Component {
             marks={visibleMarksPerFrame}
             move={move}
             scale={scale}
+            played={played}
             setActiveMark={setActiveMark}
             width={width}
           />
-        }
+        )}
         <PreviousMarks scale={scale} />
       </>
     )
@@ -96,6 +94,7 @@ InteractionLayerContainer.wrappedComponent.propTypes = {
   interactionTaskAnnotations: PropTypes.array,
   move: PropTypes.bool,
   scale: PropTypes.number,
+  played: PropTypes.number,
   width: PropTypes.number.isRequired
 }
 
@@ -114,7 +113,8 @@ InteractionLayerContainer.wrappedComponent.defaultProps = {
   frame: 0,
   interactionTaskAnnotations: [],
   move: false,
-  scale: 1
+  scale: 1,
+  played: undefined // used for tracking video progress from 0 - 1. if undefined, subject is not a video
 }
 
 export default InteractionLayerContainer