Convai Integration

var ConvaiNpc = pc.createScript('convaiNpc');

/** Loading convai-sdk cdn */
var convaiClient = null;
var userTextStream = ""; // Stores the user's text input
var npcTextStream = ""; // Stores the NPC's text response
var keyPressed = false; // Keeps track of whether the "T" key is pressed
var isTalking = false; // Indicates if the NPC is currently talking
var conversationActive = false; // Indicates if a conversation is currently active
var visemeData = []; // Stores viseme data for lip sync animation
var visemeDataActive = false; // Indicates if viseme data is available
var formLoaded = false; // Indicates if the form has been loaded
let timeoutId;
/**
 * Initializes the ConvaiClient with an API key, a character ID, and flags for enabling the audio recorder and player. Call this first from Start()
 * Sets up several callbacks for handling different types of responses from the Convai Client.
 * @param {string} apiKey - The API key for the Convai service.
 * @param {string} characterId - The ID of the character to use in the Convai service.
 * @param {boolean} enableAudioRecorder - Flag to enable the audio recorder.
 * @param {boolean} enableAudioPlayer - Flag to enable the audio player.
 * @param {number} faceModal - The face modal to use for the character.
 * @param {boolean} enableFacialData - Flag to enable facial data.
*/
function initializeConvaiClient () {
    console.log("Convai client initiated.");
    convaiClient = new ConvaiClient({
        apiKey: "", // Replace with your API key
        characterId: "", // Replace with your character ID
        enableAudio: true,
        faceModal: 3,
        enableFacialData: true,
    });

    // Error callback
    convaiClient.setErrorCallback(function(type, statusMessage) {
        console.log("Error Callback", type, statusMessage);
    });

    // Response callback
    convaiClient.setResponseCallback(function(response) {
        // Handle user query
        if (response.hasUserQuery) {
            var transcript = response.getUserQuery();
            if (transcript) {
                var isFinal = transcript?.getIsFinal();
                var transcriptText = transcript.getTextData();
                if (isFinal) {
                    userTextStream += " " + transcriptText;
                }
                userTextStream = transcriptText;
            }
        }

        // Handle audio response
        if (response.hasAudioResponse()) {
            var audioResponse = response.getAudioResponse();
            npcTextStream += " " + audioResponse.getTextData();

            // Handle viseme data for lip sync animation
            if (audioResponse.hasVisemesData()) {
                let lipsyncData = audioResponse.getVisemesData().array[0];
                if (lipsyncData[0] !== -2) {
                    visemeData.push(lipsyncData);
                    visemeDataActive = true;
                }
            } else {
                visemeDataActive = false;
            }
        }
    });
}

// Handle key down event for starting audio input
function handleKeyDown(e) {
    const convaiFormEl = document.getElementById("convai-input");
    if (convaiClient && e.keyCode === 84 && !keyPressed && !(document.activeElement === convaiFormEl)) {
        e.stopPropagation();
        e.preventDefault();
        keyPressed = true;
        userTextStream = "";
        npcTextStream = "";
        convaiClient.startAudioChunk();
        conversationActive = true;
    }
}

// Handle key up event for stopping audio input
function handleKeyUp(e) {
    const convaiFormEl = document.getElementById("convai-input");
    if (convaiClient && e.keyCode === 84 && keyPressed && !(document.activeElement === convaiFormEl)) {
        e.preventDefault();
        keyPressed = false;
        clearTimeout(timeoutId);
        timeoutId = setTimeout(() => {
        convaiClient.endAudioChunk();
        }, 500);
    }
}

// Add event listeners for key down and key up
window.addEventListener("keydown", handleKeyDown);
window.addEventListener("keyup", handleKeyUp);

// Initialize code called once per entity
ConvaiNpc.prototype.initialize = function() {
    initializeConvaiClient();

    // Set callbacks for audio playback
    convaiClient.onAudioPlay(() => {
        isTalking = true;
    });
    convaiClient.onAudioStop(() => {
        isTalking = false;
        conversationActive = false;
    });
};

// Handle animation based on the talking state
ConvaiNpc.prototype.handleAnimation = function() {
    this.entity.anim.setBoolean("Talking", isTalking);
};

// Update code called every frame
ConvaiNpc.prototype.update = function(dt) {
    this.handleAnimation();

    // Check if the form is loaded and add event listener
    if (!formLoaded && document.getElementById("convai-form")) {
        this.formListener();
        formLoaded = true;
    }
};

// Add event listener for form submission
ConvaiNpc.prototype.formListener = function() {
    document.getElementById("convai-form").addEventListener("submit", (e) => {
        e.preventDefault();
        var inputValue = document.getElementById("convai-input").value;
        window.convaiClient.sendTextChunk(inputValue);
        userTextStream = inputValue;
        document.getElementById("convai-form").reset();
        conversationActive = true;
        npcTextStream = "";
    });
}

// Create a new script called 'playerAnimationHandler'
var PlayerAnimationHandler = pc.createScript('playerAnimationHandler');

// Add an attribute called 'blendTime' with a default value of 0.2
// This attribute controls the blend time between animations
PlayerAnimationHandler.attributes.add('blendTime', { type: 'number', default: 0.2 });

// Add an attribute called 'velMin' with a default value of 10
// This attribute represents the minimum velocity for a specific animation
PlayerAnimationHandler.attributes.add('velMin', { type: 'number', default: 10 });

// Add an attribute called 'velMax' with a default value of 50
// This attribute represents the maximum velocity for a specific animation
PlayerAnimationHandler.attributes.add('velMax', { type: 'number', default: 50 });

// This function is called when the script is initialized
PlayerAnimationHandler.prototype.initialize = function() {
    // Play the 'Idle' animation with the specified blend time
    this.entity.animation.play('Idle', this.blendTime);
};

// Create a new script called 'lipsync'
var Lipsync = pc.createScript('lipsync');

// Array to store the lipsync data
var lipsyncData = [];

// initialize code called once per entity
Lipsync.prototype.initialize = function() {
    // Find the head component of the entity
    this.headComponent = this.entity.findByName("Wolf3D_Head").render;

    // Find the teeth component of the entity
    this.teethComponent = this.entity.findByName("Wolf3D_Teeth").render;

    // Initialize the current viseme frame to 0
    this.currentVisemeFrame = 0;

    // Initialize the timer to 0
    this.timer = 0;
};

// Run morph targets on the viseme data received
Lipsync.prototype.runVisemeData = function(dt) {
    // Sync frames with time
    if (this.currentVisemeFrame <= lipsyncData.length && lipsyncData.length !== 0) {
        if (this.headComponent && this.teethComponent) {
            // Get the morph instance for the head and teeth components
            var headMorphInstance = this.headComponent.meshInstances[0].morphInstance;
            var teethMorphInstance = this.teethComponent.meshInstances[0].morphInstance;

            if (lipsyncData[this.currentVisemeFrame] !== undefined) {
                // Loop through the morph targets and set their weights
                for (let i = 0; i < 15; i++) {
                    if (lipsyncData[this.currentVisemeFrame][i] !== undefined) {
                        headMorphInstance.setWeight(i, lipsyncData[this.currentVisemeFrame][i]);
                        teethMorphInstance.setWeight(i, lipsyncData[this.currentVisemeFrame][i]);
                    }
                }
            }

            // Update the current viseme frame based on the timer
            this.currentVisemeFrame = Math.floor(this.timer * 100);
        }
    }
}

// Reset all morph targets to 0
Lipsync.prototype.zeroMorphs = function(dt) {
    if (this.headComponent && this.teethComponent) {
        // Get the morph instance for the head and teeth components
        var headMorphInstance = this.headComponent.meshInstances[0].morphInstance;
        var teethMorphInstance = this.teethComponent.meshInstances[0].morphInstance;

        // Loop through the morph targets and set their weights to 0
        for (let i = 0; i < 15; i++) {
            headMorphInstance.setWeight(i, 0);
            teethMorphInstance.setWeight(i, 0);
        }
    }
}

// update code called every frame
Lipsync.prototype.update = function(dt) {
    // Check if there is new viseme data and update the lipsyncData array
    if (window.visemeData && window.visemeDataActive) {
        lipsyncData = window.visemeData;
    }

    if (lipsyncData.length > 0) {
        // Update the timer
        this.timer += dt;

        // Run the viseme data
        this.runVisemeData();

        // Check if the viseme data has finished playing
        if (this.timer * 100 >= lipsyncData.length) {
            // Reset all morph targets to 0
            this.zeroMorphs();

            // Clear the lipsyncData array and reset the timer and currentVisemeFrame
            lipsyncData = [];
            this.timer = 0;
            this.currentVisemeFrame = 0;
            window.visemeData = [];
        }
    }
};

// Create a new script called 'headTracking'
var HeadTracking = pc.createScript('headTracking');

// Helper function to find a child entity by name recursively
HeadTracking.prototype.findChild = function(children, matchName) {
    for (let child of children) {
        if (child.name.includes(matchName)) {
            return child;
        }

        if (child.children) {
            const foundChild = this.findChild(child.children, matchName);
            if (foundChild) {
                return foundChild;
            }
        }
    }

    return null;
};

// Function to calculate the radian angle between two vectors
HeadTracking.prototype.getRadianAngle = function(vecA, vecB) {
    dot = vecA.normalize().dot(vecB.normalize());
    return Math.acos(dot);
};

// initialize code called once per entity
HeadTracking.prototype.initialize = function() {
    // Find the head, left eye, and right eye components
    this.headComponent = this.entity.findByName("Head");
    this.leftEye = this.entity.findByName("LeftEye");
    this.rightEye = this.entity.findByName("RightEye");
};

// Function called after the update loop
HeadTracking.prototype.postUpdate = function(dt) {
    // Check if the camera position is available
    if (window.fpsCamera?.position !== undefined) {
        const cameraPosition = window.fpsCamera.getPosition().clone();
        const angle = this.getRadianAngle(this.headComponent.forward, window.fpsCamera.forward);
        const degree = angle * pc.math.RAD_TO_DEG;

        // If the angle is within 45 degrees, look at the camera position
        if (degree <= 45) {
            this.headComponent.lookAt(-cameraPosition.x, cameraPosition.y, -cameraPosition.z);
            this.leftEye.lookAt(-cameraPosition.x, cameraPosition.y, -cameraPosition.z);
            this.rightEye.lookAt(-cameraPosition.x, cameraPosition.y, -cameraPosition.z);
        }
    }
};