I'm trying to lip sync a 3d model using BabylonJS but not all lip positions are happening at the time their supposed to - babylonjs

Question

I'm trying to lip sync a 3D model using babylonJS, morph targets, and audio obtained from Amazon Polly but the lip positions aren't happening at the exact time their supposed to (for example when the speech starts the lip positions start as well but not all the lip positions occur in sync with the sound or the speech ends before the lip positions end.) I tried creating a time counter variable called alpha and tried to increase the amount it's incremented by but there are times when it skips some of the lip position occurrences. The time (in milliseconds) that I want the lip positions to occur are shown in the variable jsonParsed. Does anyone know of a way to fix this. If you'd like to see my source code I've pasted it below.
var createScene = function () {
// This creates a basic Babylon Scene object (non-mesh)
var scene = new BABYLON.Scene(engine);
// This creates and positions a free camera (non-mesh)
var camera = new BABYLON.FreeCamera("camera1", new BABYLON.Vector3(0, 5, -10), scene);
// This creates a light, aiming 0,1,0 - to the sky (non-mesh)
var light = new BABYLON.HemisphericLight("light", new BABYLON.Vector3(0, 1, 0), scene);
// Default intensity is 1. Let's dim the light a small amount
light.intensity = 0.7;
url = 'https://raw.githubusercontent.com/chris45242/BabylonModel/main/';
fileName = "project.blend1.gltf";
const Casi = BABYLON.SceneLoader.Append(url, fileName, scene, function (s){
// Create a default arc rotate camera and light.
scene.createDefaultCameraOrLight(true, true, true);
scene.activeCamera.alpha += Math.PI;
scene.stopAllAnimations();
//scene.animationGroups[0].start(true);
scene.animationGroups[1].start(true);
var casiVisor = scene.getMeshByName("Casi's Visor");
casiVisor.setEnabled(true);
casiVisor.visibility = 0.7;
var casiBody = scene.getMeshByName("Casi's Body.001_primitive0");
var casiInnerMouth = scene.getMeshByName("Casi's Body.001_primitive2");
var casiTeeth = scene.getMeshByName("Casi's Teeth");
var primitive = scene.getMeshByName("Primitives.001");
var jsonParsed = //'[{"time":6,"type":"word","start":0,"end":4,"value":"it\'s"},'+
'[{"time":6,"type":"viseme","value":"i"},' +
'{"time":64,"type":"viseme","value":"t"},' +
'{"time":122,"type":"viseme","value":"s"},' +
//'{"time":190,"type":"word","start":5,"end":9,"value":"very"},' +
'{"time":190,"type":"viseme","value":"f"},' +
'{"time":241,"type":"viseme","value":"E"},' +
'{"time":282,"type":"viseme","value":"r"},' +
'{"time":376,"type":"viseme","value":"i"},' +
//'{"time":416,"type":"word","start":10,"end":14,"value":"good"},' +
'{"time":416,"type":"viseme","value":"k"},' +
'{"time":481,"type":"viseme","value":"u"},' +
'{"time":576,"type":"viseme","value":"t"},' +
//'{"time":627,"type":"word","start":15,"end":17,"value":"to"},' +
'{"time":627,"type":"viseme","value":"t"},' +
'{"time":674,"type":"viseme","value":"u"},' +
//'{"time":716,"type":"word","start":18,"end":22,"value":"meet"},' +
'{"time":716,"type":"viseme","value":"p"},' +
'{"time":801,"type":"viseme","value":"i"},' +
'{"time":923,"type":"viseme","value":"t"},' +
//'{"time":999,"type":"word","start":23,"end":26,"value":"you"},' +
'{"time":999,"type":"viseme","value":"i"},' +
'{"time":1135,"type":"viseme","value":"u"},' +
'{"time":1357,"type":"viseme","value":"sil"}]';
const obj = JSON.parse(jsonParsed);
let t = 0;
//const lipSync = 'a';
const lipSync = obj[t]['value'];
const vowelInput = {
value: lipSync,
}
// Create custom observable for the 'vowelInput'
vowelInput.onValueChange = new BABYLON.Observable();
// Create function to set current vowel
const setCurrentVowel = (value) => {
if (value === vowelInput.value) {
return value
} else {
vowelInput.value = value
console.log("test", value, vowelInput.value)
vowelInput.onValueChange.notifyObservers(vowelInput.value)
}
}
// Add callback function to the custom observable
vowelInput.onValueChange.add((value) => {
switch (value) {
case 'a': {
// Instead of changing position, you can all lip animations according to the vowel.
casiBody.morphTargetManager.getTarget(12).influence = 1;
casiInnerMouth.morphTargetManager.getTarget(12).influence = 1;
primitive.morphTargetManager.getTarget(3).influence = 1;
casiBody.morphTargetManager.getTarget(11).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(11).influence = 0;
casiBody.morphTargetManager.getTarget(13).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(13).influence = 0;
casiBody.morphTargetManager.getTarget(14).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(14).influence = 0;
casiTeeth.morphTargetManager.getTarget(0).influence = 0;
casiBody.morphTargetManager.getTarget(15).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(15).influence = 0;
casiBody.morphTargetManager.getTarget(16).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(16).influence = 0;
casiTeeth.morphTargetManager.getTarget(0).influence = 0;
casiTeeth.morphTargetManager.getTarget(1).influence = 0;
primitive.morphTargetManager.getTarget(2).influence = 0;
primitive.morphTargetManager.getTarget(4).influence = 0;
primitive.morphTargetManager.getTarget(5).influence = 0;
primitive.morphTargetManager.getTarget(6).influence = 0;
primitive.morphTargetManager.getTarget(7).influence = 0;
break
}
case 'e':
case 'E': {
casiBody.morphTargetManager.getTarget(14).influence = 1;
casiInnerMouth.morphTargetManager.getTarget(14).influence = 1;
casiTeeth.morphTargetManager.getTarget(0).influence = 1;
primitive.morphTargetManager.getTarget(5).influence = 1;
casiBody.morphTargetManager.getTarget(11).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(11).influence = 0;
casiBody.morphTargetManager.getTarget(12).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(12).influence = 0;
casiBody.morphTargetManager.getTarget(13).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(13).influence = 0;
casiBody.morphTargetManager.getTarget(15).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(15).influence = 0;
casiBody.morphTargetManager.getTarget(16).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(16).influence = 0;
casiTeeth.morphTargetManager.getTarget(1).influence = 0;
primitive.morphTargetManager.getTarget(2).influence = 0;
primitive.morphTargetManager.getTarget(3).influence = 0;
primitive.morphTargetManager.getTarget(4).influence = 0;
primitive.morphTargetManager.getTarget(6).influence = 0;
primitive.morphTargetManager.getTarget(7).influence = 0;
break
}
case 'o':
case 'r': {
casiBody.morphTargetManager.getTarget(11).influence = 1;
casiInnerMouth.morphTargetManager.getTarget(11).influence = 1;
primitive.morphTargetManager.getTarget(2).influence = 1;
casiBody.morphTargetManager.getTarget(12).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(12).influence = 0;
casiBody.morphTargetManager.getTarget(13).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(13).influence = 0;
casiBody.morphTargetManager.getTarget(14).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(14).influence = 0;
casiBody.morphTargetManager.getTarget(15).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(15).influence = 0;
casiBody.morphTargetManager.getTarget(16).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(16).influence = 0;
casiTeeth.morphTargetManager.getTarget(0).influence = 0;
casiTeeth.morphTargetManager.getTarget(1).influence = 0;
primitive.morphTargetManager.getTarget(3).influence = 0;
primitive.morphTargetManager.getTarget(4).influence = 0;
primitive.morphTargetManager.getTarget(5).influence = 0;
primitive.morphTargetManager.getTarget(6).influence = 0;
primitive.morphTargetManager.getTarget(7).influence = 0;
break
}
case 'u': {
casiBody.morphTargetManager.getTarget(13).influence = 1;
casiInnerMouth.morphTargetManager.getTarget(13).influence = 1;
primitive.morphTargetManager.getTarget(4).influence = 1;
casiBody.morphTargetManager.getTarget(11).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(11).influence = 0;
casiBody.morphTargetManager.getTarget(12).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(12).influence = 0;
casiBody.morphTargetManager.getTarget(14).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(14).influence = 0;
casiBody.morphTargetManager.getTarget(15).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(15).influence = 0;
casiBody.morphTargetManager.getTarget(16).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(16).influence = 0;
casiTeeth.morphTargetManager.getTarget(0).influence = 0;
casiTeeth.morphTargetManager.getTarget(1).influence = 0;
primitive.morphTargetManager.getTarget(2).influence = 0;
primitive.morphTargetManager.getTarget(3).influence = 0;
primitive.morphTargetManager.getTarget(5).influence = 0;
primitive.morphTargetManager.getTarget(6).influence = 0;
primitive.morphTargetManager.getTarget(7).influence = 0;
break
}
case 'i': {
casiBody.morphTargetManager.getTarget(15).influence = 1;
casiInnerMouth.morphTargetManager.getTarget(15).influence = 1;
primitive.morphTargetManager.getTarget(6).influence = 1;
casiBody.morphTargetManager.getTarget(11).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(11).influence = 0;
casiBody.morphTargetManager.getTarget(12).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(12).influence = 0;
casiBody.morphTargetManager.getTarget(13).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(13).influence = 0;
casiBody.morphTargetManager.getTarget(14).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(14).influence = 0;
casiBody.morphTargetManager.getTarget(16).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(16).influence = 0;
casiTeeth.morphTargetManager.getTarget(0).influence = 0;
casiTeeth.morphTargetManager.getTarget(1).influence = 0;
primitive.morphTargetManager.getTarget(2).influence = 0;
primitive.morphTargetManager.getTarget(3).influence = 0;
primitive.morphTargetManager.getTarget(4).influence = 0;
primitive.morphTargetManager.getTarget(5).influence = 0;
primitive.morphTargetManager.getTarget(7).influence = 0;
break
}
case 'f':
case 'v': {
casiBody.morphTargetManager.getTarget(16).influence = 1;
casiInnerMouth.morphTargetManager.getTarget(16).influence = 1;
casiTeeth.morphTargetManager.getTarget(1).influence = 1;
primitive.morphTargetManager.getTarget(7).influence = 1;
casiBody.morphTargetManager.getTarget(11).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(11).influence = 0;
casiBody.morphTargetManager.getTarget(12).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(12).influence = 0;
casiTeeth.morphTargetManager.getTarget(1).influence = 0;
casiBody.morphTargetManager.getTarget(13).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(13).influence = 0;
casiBody.morphTargetManager.getTarget(14).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(14).influence = 0;
casiBody.morphTargetManager.getTarget(15).influence = 0;
casiInnerMouth.morphTargetManager.getTarget(15).influence = 0;
casiTeeth.morphTargetManager.getTarget(0).influence = 0;
primitive.morphTargetManager.getTarget(2).influence = 0;
primitive.morphTargetManager.getTarget(3).influence = 0;
primitive.morphTargetManager.getTarget(4).influence = 0;
primitive.morphTargetManager.getTarget(5).influence = 0;
primitive.morphTargetManager.getTarget(6).influence = 0;
break
}
}
})
let idx = 0
const vowels = ['a', 'e', 'o', 'u', 'i', 'f']
var soundURL = "speech.mp3";
//alpha is the timer I use
let alpha = 0;
// Change current vowel repeatedly
var speech = new BABYLON.Sound("speech", url + soundURL, scene, function(){
setTimeout(speech.play(), 10000);
});
//const inter = setInterval(() => {
var intervalID = scene.onBeforeRenderObservable.add(function(){
alpha += 1;//continuously add 1 to the timer
if(alpha === obj[t]['time']){//if we've reached a time shown in jsonParsed array
console.log(alpha, t, obj[t]['time']);
setCurrentVowel(obj[t]['value']);
t = (t + 1) % obj.length;
console.log(alpha, t, obj[t]['time']);
}
/*setTimeout(() => {
setInterval(inter, obj[t]['time']);
});*/
//setCurrentVowel(obj[t]['value']);
//t = (t + 1) % obj.length;
//idx = (idx + 1) % 6
});
//}, 1000)
/*function inter(){
setCurrentVowel(obj[t]['value']);
}*/
scene.onDisposeObservable.add(() => {
clearInterval(inter)
})
});
// Our built-in 'sphere' shape.
//var sphere = BABYLON.MeshBuilder.CreateSphere("sphere", {diameter: 2, segments: 32}, scene);
// Move the sphere upward 1/2 its height
//sphere.position.y = 1;
// Our built-in 'ground' shape.
var ground = BABYLON.MeshBuilder.CreateGround("ground", {width: 6, height: 6}, scene);
return scene;
};