Face detection stops when back/forwarding on video (React)

Hi! I’ve written this piece of code to have a video player with some basic functions: go 5 seconds backward-forward, play/pause buttons, and a checkbox to show/hide the face-api canvas with its detections.

import {React, useRef} from "react";
import './App.css';
import * as faceapi from "face-api.js/dist/face-api.min.js";

function App() {

    let container = useRef();
    let video = useRef();
    let modelsLoaded = false;
    let checkboxChecked = false;
    let canvas;
    let displaySize;

    function backward(){
        video.current.currentTime -= 5;
    }

    function forward(){
        video.current.currentTime += 5;
    }

    function play(){
        video.current.play();
    }

    function pause(){
        video.current.pause();
    }

    function checkboxChanged() {
        checkboxChecked = !checkboxChecked;
        if(checkboxChecked === true)
            loadModels();
    }


    function loadModels(){
        if(modelsLoaded === false){
            console.log("Loading models...")

            Promise.all([
                faceapi.nets.tinyFaceDetector.loadFromUri('./models'),
                faceapi.nets.faceLandmark68Net.loadFromUri('./models'),
                faceapi.nets.faceRecognitionNet.loadFromUri('./models'),
                faceapi.nets.faceExpressionNet.loadFromUri('./models')
            ])
                .then(() => {
                    console.log("Models loaded");
                    modelsLoaded = true;
                    createCanvas();
                    drawCanvas();
                })
                .catch((err) => console.log(err));
        }
        else{
            drawCanvas();
        }
    }

    function createCanvas(){
        canvas = faceapi.createCanvasFromMedia(video.current);
        canvas.style.position = "absolute";
        container.current.append(canvas);
        displaySize = { width: video.current.videoWidth, height: video.current.videoHeight };
        faceapi.matchDimensions(canvas, displaySize);
    }

    async function drawCanvas(){
        if(checkboxChecked){
            canvas.style.display = "block";

            const detections = await faceapi
                .detectAllFaces(video.current, new faceapi.TinyFaceDetectorOptions())
                .withFaceLandmarks()
                .withFaceExpressions();
            const resizedDetections = faceapi.resizeResults(detections, displaySize);
            canvas.getContext('2d').clearRect(0, 0, canvas.width, canvas.height);
            faceapi.draw.drawDetections(canvas, resizedDetections);
            faceapi.draw.drawFaceLandmarks(canvas, resizedDetections);
            faceapi.draw.drawFaceExpressions(canvas, resizedDetections);

            requestAnimationFrame(drawCanvas);
        }
        else{
            canvas.style.display = "none";
        }


    }

    return (
        <>
            <div ref={container} id={"face-api-container"} style={{display: "flex"}}>
                <video ref={video} id={"video"} controls autoPlay={true} muted={true} src="sintel.mp4"></video>
            </div>
            <button id={"backward_btn"} onClick={backward}>Bwd 5s</button>
            <button id={"forward_btn"} onClick={forward}>Fwd 5s</button>
            <button id={"play_btn"} onClick={play}>Play</button>
            <button id={"pause_btn"} onClick={pause}>Pause</button>
            <br/>
            <input type={"checkbox"} onChange={checkboxChanged}
            />face-api
        </>
    );
}

export default App;

This works fine, despite a slight framerate drop in the video while face-api is enabled. When I press the backward (or forward) button, though, the face detection stops: the canvas stops being updated. Curiously, this does not happen with the play and pause buttons.

How should I change my code in order to prevent this behaviour and keep the face-api canvas updated?

Thanks in advance.

[vladmandic]

re: video.paused - yes, it’s ok to display canvas while paused, but you don’t want to run processing again and again on the same still frame. and since if video.paused you’d be also skipping call to clearRect so last frame will stay displayed.

but anyhow, glad it helped 😃

off-topic: i’m maintaining newer fork of face-api.js, optimized for ES2018 and TensorFlow 3.x
https://github.com/vladmandic/face-api

[vladmandic]

there are better solutions, but this would be quick & easy - inside your drawCanvas function have that if statement just before calling faceapi.detectAllFaces so if it’s false, it basically skips entire faceapi processing block and immediately triggers requestAnimationFrame.

since requestAnimationFrame is frame-limited, its ok.

also extend the if with null check and paused check - something like

if (video && !video.paused && video.readyState > 2) {
  const detections = await faceapi
  ...
}
requestAnimationFrame(drawCanvas);

but on the other hand, you’re calling drawCanvas immeditely when models are loaded and it’s luck that loading of models takes longer than preparing of video - you should call that from video.onloadeddata event instead to make sure you don’t even trigger drawCanvas before video is ready.