feat: made avatar mesh transitions smoother and added emotion animati…

…on based on chat tag

src/components/Avatar/Avatar.tsx

@@ -32,6 +32,7 @@ export interface Props {
   apiUrl?: string;
   animation?: string;
   isZoomed?: boolean;
+  chatProps?: any;
 }
 
 const Avatar: React.FC<Props> = ({
@@ -49,6 +50,7 @@ const Avatar: React.FC<Props> = ({
   apiUrl,
   animation,
   isZoomed = false,
+  chatProps,
 }) => {
   const { t } = useTranslation();
   const [isClient, setIsClient] = useState(false);
@@ -136,6 +138,7 @@ const Avatar: React.FC<Props> = ({
             loading={loading}
             style={getAvatarStyle()}
             isZoomed={isZoomed}
+            chatEmission={chatProps?.dialogState?.emission}
           />
         </ErrorBoundary>
       );

src/components/Avatar/AvatarView/AvatarComponent/avatarComponent.tsx

@@ -2,7 +2,7 @@ import React, { useState, useEffect, useCallback } from 'react';
 import AnimationControlPanel from './components/controls';
 import FullbodyAvatar from './components/fullbodyAvatar';
 import HalfBodyAvatar from './components/halfbodyAvatar';
-
+import { useViseme } from '../utils/useViseme';
 
 interface Props {
   showControls: boolean;
@@ -14,6 +14,7 @@ interface Props {
   headMovement: boolean;
   speaking: boolean;
   isZoomed: boolean;
+  chatEmission: any;
 }
 
 interface BaseAction {
@@ -30,7 +31,6 @@ const baseActions: Record<string, BaseAction> = {
   Idle3: { weight: 0 },
   Idle4: { weight: 0 },
   Idle5: { weight: 0 },
-  Loading: { weight: 0 },
   Rabbia1: { weight: 0 },
   Rabbia2: { weight: 0 },
   Rabbia3: { weight: 0 },
@@ -45,10 +45,8 @@ const baseActions: Record<string, BaseAction> = {
   Tristezza3: { weight: 0 },
 };
 
-
-
-
 export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
+  chatEmission,
   showControls,
   animation,
   loading,
@@ -74,42 +72,27 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
 
   const [timeScale, setTimeScale] = useState(0.8);
 
-
+  const { createVisemeSequence, currentVisemes, clearVisemes } = useViseme();
+
+  // Set the morph target influences for the given emotions
+  const setEmotion = useCallback(
+    (action: string) => {
+      const emotionMap = {
+        Gioia: { eyesClosed: 0.5, mouthSmile: 1 },
+        Rabbia: { eyesClosed: 1, mouthSmile: -0.5 },
+        Sorpresa: { mouthSmile: 0.5, eyesClosed: -0.2 },
+        Tristezza: { mouthSmile: -0.6, eyesClosed: 0.5 },
+        Timore: { mouthSmile: -0.5, eyesClosed: 1 },
+        default: { mouthSmile: 0, eyesClosed: 0 }
+      };
+      const emotion = Object.keys(emotionMap).find(key => action.startsWith(key)) || 'default';
+      setMorphTargetInfluences(emotionMap[emotion as keyof typeof emotionMap]);
+    },
+    []
+  );
 
   const onBaseActionChange = useCallback((action: string) => {
-
-    // set the morph target influences for the given emotions
-    if(action === 'Gioia1' || action === 'Gioia2' || action === 'Gioia3'){
-      setMorphTargetInfluences({
-        'eyesClosed': 0.5,
-        'mouthSmile': 1,
-      })
-    }else if(action === 'Rabbia1' || action === 'Rabbia2' || action === 'Rabbia3'){
-      setMorphTargetInfluences({
-        'eyesClosed': 1,
-        'mouthSmile': -0.5,
-      })
-    }else if(action === 'Sorpresa1' || action === 'Sorpresa2' || action === 'Sorpresa3'){
-      setMorphTargetInfluences({
-        'mouthSmile': 0.5,
-        'eyesClosed': -0.5,
-      })
-    } else if(action === 'Tristezza1' || action === 'Tristezza2' || action === 'Tristezza3'){
-      setMorphTargetInfluences({
-        'mouthSmile': -0.6,
-        'eyesClosed': 0.5,
-      })
-    } else if(action === 'Timore1' || action === 'Timore2' || action === 'Timore3'){
-      setMorphTargetInfluences({
-        'mouthSmile': -0.5,
-        'eyesClosed': 1,
-      })
-    } else {
-      setMorphTargetInfluences({
-        'mouthSmile': 0,
-        'eyesClosed': 0,
-      })
-    }
+    setEmotion(action);
     setCurrentBaseAction({
       action,
       weight: 1,
@@ -137,23 +120,44 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
     setTimeScale(value);
   }, []);
 
+  // Set the emotion based on the chatEmission
+  useEffect(() => {
+
+    if(chatEmission){
+      createVisemeSequence(chatEmission);
+    }
+
+    //Check if chatEmission has a tag
+    const hasOutputTag = chatEmission?.includes(
+      '<output class="memori-emotion">'
+    );
+    const outputContent = hasOutputTag
+      ? chatEmission
+          ?.split('<output class="memori-emotion">')[1]
+          ?.split('</output>')[0]
+          ?.trim()
+      : null;
+
+    if (outputContent) {
+      //Based on the outputContent, set the emotion
+      //The outputContent could be: "Gioia", "Sorpresa", "Tristezza", "Rabbia", "Timore"
+      //Choose a random number between 1 and 3
+      const randomNumber = Math.floor(Math.random() * 3) + 1;
+      const emotion = `${outputContent}${randomNumber}`;
+      setEmotion(emotion);
+    }
+  }, [chatEmission]);
+
+  //Set a loading state to true if the avatar is loading
   useEffect(() => {
-    if(loading){
-      setMorphTargetInfluences({
-        'mouthSmile': 0,
-        'eyesClosed': 0,
-      })
+    if (loading) {
       setCurrentBaseAction({
-        action: 'Loading',
+        action: 'Idle1',
         weight: 1,
-      })
+      });
     }
   }, [loading]);
 
-
-
-
-
   return (
     <>
       {showControls && (
@@ -168,30 +172,29 @@ export const AvatarView: React.FC<Props & { halfBody: boolean }> = ({
           modifyTimeScale={modifyTimeScale}
         />
       )}
-      {
-        halfBody ? (
-          <HalfBodyAvatar
-            url={url}
-            setMorphTargetInfluences={setMorphTargetInfluences}
-            headMovement={headMovement}
-            speaking={speaking}
-          />
-        ) : (
-          <FullbodyAvatar
-            url={url}
-            sex={sex}
-            eyeBlink={eyeBlink}
-            speaking={speaking}
-            currentBaseAction={currentBaseAction}
-            timeScale={timeScale}
-            setMorphTargetInfluences={setMorphTargetInfluences}
-            setMorphTargetDictionary={setMorphTargetDictionary}
-            morphTargetInfluences={morphTargetInfluences}
-            morphTargetDictionary={morphTargetDictionary}
-            isZoomed={isZoomed}
-          />
-        )
-      }
+      {halfBody ? (
+        <HalfBodyAvatar
+          url={url}
+          setMorphTargetInfluences={setMorphTargetInfluences}
+          headMovement={headMovement}
+          speaking={speaking}
+        />
+      ) : (
+        <FullbodyAvatar
+          url={url}
+          sex={sex}
+          eyeBlink={eyeBlink}
+          speaking={speaking}
+          currentBaseAction={currentBaseAction}
+          timeScale={timeScale}
+          setMorphTargetInfluences={setMorphTargetInfluences}
+          setMorphTargetDictionary={setMorphTargetDictionary}
+          morphTargetInfluences={morphTargetInfluences}
+          morphTargetDictionary={morphTargetDictionary}
+          isZoomed={isZoomed}
+          currentVisemes={currentVisemes}
+        />
+      )}
     </>
   );
 };

src/components/Avatar/AvatarView/AvatarComponent/components/fullbodyAvatar.tsx

@@ -1,17 +1,14 @@
 // Import necessary dependencies
 import React, { useEffect, useRef, useState } from 'react';
-import {
-  Vector3,
-  Euler,
-  AnimationMixer,
-  SkinnedMesh,
-  Object3D
-} from 'three';
+import { Vector3, Euler, AnimationMixer, SkinnedMesh, Object3D } from 'three';
 import { useAnimations, useGLTF } from '@react-three/drei';
 import { useGraph, dispose, useFrame } from '@react-three/fiber';
 import { correctMaterials, isSkinnedMesh } from '../../../../../helpers/utils';
 import { useAvatarBlink } from '../../utils/useEyeBlink';
-import { useViseme } from '../../utils/useViseme';
+
+const lerp = (start: number, end: number, alpha: number): number => {
+  return start * (1 - alpha) + end * alpha;
+};
 
 // Define the props interface for the FullbodyAvatar component
 interface FullbodyAvatarProps {
@@ -41,7 +38,8 @@ const AVATAR_POSITION_ZOOMED = new Vector3(0, -1.45, 0);
 // Define URLs for male and female animation assets
 const ANIMATION_URLS = {
   MALE: 'https://assets.memori.ai/api/v2/asset/1c350a21-97d8-4add-82cc-9dc10767a26b.glb',
-  FEMALE: 'https://assets.memori.ai/api/v2/asset/a1908dbf-8ce8-438d-90df-acf9dc2604ad.glb',
+  FEMALE:
+    'https://assets.memori.ai/api/v2/asset/a1908dbf-8ce8-438d-90df-acf9dc2604ad.glb',
 };
 
 // Define the FullbodyAvatar component
@@ -55,15 +53,15 @@ export default function FullbodyAvatar({
   setMorphTargetInfluences,
   setMorphTargetDictionary,
   morphTargetInfluences,
-  eyeBlink
+  eyeBlink,
 }: FullbodyAvatarProps) {
   // Load the 3D model and animations
   const { scene } = useGLTF(url);
   const { animations } = useGLTF(ANIMATION_URLS[sex]);
   const { nodes, materials } = useGraph(scene);
   const { actions } = useAnimations(animations, scene);
   const [mixer] = useState(() => new AnimationMixer(scene));
-  
+
   // Create a ref for the SkinnedMesh
   const avatarMeshRef = useRef<SkinnedMesh>();
 
@@ -73,31 +71,28 @@ export default function FullbodyAvatar({
     config: {
       minInterval: 1500,
       maxInterval: 4000,
-      blinkDuration: 120
-    }
+      blinkDuration: 120,
+    },
   });
 
-
-
   // Effect to setup morphTargets and cleanup
   useEffect(() => {
     // Correct materials for the avatar
     correctMaterials(materials);
-    
+
     // Find the avatar mesh
     scene.traverse((object: Object3D) => {
       if (object instanceof SkinnedMesh && object.name === 'Wolf3D_Avatar020') {
         avatarMeshRef.current = object;
-        
+
         // Set up morph target dictionary and influences
         if (object.morphTargetDictionary && object.morphTargetInfluences) {
           setMorphTargetDictionary(object.morphTargetDictionary);
-          
+
           // Create an object with all morph target influences set to 0
-          const initialInfluences = Object.keys(object.morphTargetDictionary).reduce(
-            (acc, key) => ({ ...acc, [key]: 0 }),
-            {}
-          );
+          const initialInfluences = Object.keys(
+            object.morphTargetDictionary
+          ).reduce((acc, key) => ({ ...acc, [key]: 0 }), {});
           setMorphTargetInfluences(initialInfluences);
         }
       }
@@ -111,15 +106,24 @@ export default function FullbodyAvatar({
       Object.values(materials).forEach(dispose);
       Object.values(nodes).filter(isSkinnedMesh).forEach(dispose);
     };
-  }, [materials, nodes, url, onLoaded, setMorphTargetDictionary, setMorphTargetInfluences]);
+  }, [
+    materials,
+    nodes,
+    url,
+    onLoaded,
+    setMorphTargetDictionary,
+    setMorphTargetInfluences,
+  ]);
 
   // Effect to handle animation changes
   useEffect(() => {
     if (!actions || !currentBaseAction.action) return;
-    
+
     const newAction = actions[currentBaseAction.action];
     if (!newAction) {
-      console.warn(`Animation "${currentBaseAction.action}" not found in actions.`);
+      console.warn(
+        `Animation "${currentBaseAction.action}" not found in actions.`
+      );
       return;
     }
 
@@ -142,12 +146,17 @@ export default function FullbodyAvatar({
     if (avatarMeshRef.current && avatarMeshRef.current.morphTargetDictionary) {
       Object.entries(morphTargetInfluences).forEach(([key, value]) => {
         const index = avatarMeshRef.current!.morphTargetDictionary![key];
-        if (typeof index === 'number' && avatarMeshRef.current!.morphTargetInfluences) {
-          avatarMeshRef.current!.morphTargetInfluences[index] = value;
+        if (
+          typeof index === 'number' &&
+          avatarMeshRef.current!.morphTargetInfluences
+        ) {
+          const currentValue =
+            avatarMeshRef.current!.morphTargetInfluences[index];
+          const smoothValue = lerp(currentValue, value, 0.1);
+          avatarMeshRef.current!.morphTargetInfluences[index] = smoothValue;
         }
       });
     }
-
     // Update the animation mixer
     mixer.update(delta * 0.001);
   });
@@ -161,4 +170,4 @@ export default function FullbodyAvatar({
       <primitive object={scene} />
     </group>
   );
-}
+}