Enable an avatar

import asyncio, json, uuid, websockets

TOKEN = "sgemb_…"                          # from step 2
BOT_SLUG = "poolnoodle"                    # bot's slug from the picker

async def main():
    url = f"wss://scaigrid.scailabs.ai/v1/public/scaibot/avatar/ws-interactive?token={TOKEN}"
    async with websockets.connect(url) as ws:
        # Open a session
        await ws.send(json.dumps({
            "type": "request",
            "request_id": str(uuid.uuid4()),
            "payload": {
                "action": "start_session",
                "params": {"avatar_id": BOT_SLUG},
            },
        }))
        ready = json.loads(await ws.recv())
        session_id = ready["payload"]["data"]["session_id"]

        # The server emits an `avatar_update` event right after
        # start_session succeeds.  It carries the 3D config
        # (model_url, blendshape_profile, idle_animation) so the
        # client can load the avatar without a separate HTTP call.
        config_frame = json.loads(await ws.recv())
        if (
            config_frame.get("type") == "event"
            and config_frame["payload"]["event_type"] == "avatar_update"
        ):
            avatar_config = config_frame["payload"]["data"]
            print(f"3D model: {avatar_config.get('model_url')}")
            print(f"idle anim: {avatar_config.get('idle_animation')}")

        # Ask the avatar to say something — text in, voice + blendshapes back
        await ws.send(json.dumps({
            "type": "request",
            "request_id": str(uuid.uuid4()),
            "payload": {
                "action": "text_to_speech",
                "params": {
                    "session_id": session_id,
                    "text": "Hello, world.",
                },
                "response_config": {
                    "include_text": True,
                    "include_voice": True,
                    "include_blendshapes": True,
                },
            },
        }))

        # Receive chunks
        while True:
            frame = json.loads(await ws.recv())
            if frame.get("type") == "response":
                data = frame["payload"]["data"]
                voice_b64 = data.get("voice", {}).get("data", "")
                n_frames = len(data.get("blendshapes", []) or [])
                print(f"got chunk: text={data.get('text')!r}, "
                      f"voice_bytes={len(voice_b64)}, blendshape_frames={n_frames}")
                if frame["payload"].get("chunk_info", {}).get("is_final"):
                    break

asyncio.run(main())

await ws.send(json.dumps({
    "type": "request",
    "request_id": str(uuid.uuid4()),
    "payload": {
        "action": "send_text",
        "params": {
            "session_id": session_id,
            "text": "What's the difference between a GLB and a GLTF?",
        },
        "response_config": {
            "include_text": True,
            "include_voice": True,
            "include_blendshapes": True,
        },
    },
}))

import * as THREE from 'three';
import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader.js';

// The avatar config arrives via the `avatar_update` event right
// after start_session succeeds.  Register the handler before you
// send start_session.
let avatarConfig = null;
ws.addEventListener('message', (e) => {
  const frame = JSON.parse(e.data);
  if (
    frame.type === 'event' &&
    frame.payload.event_type === 'avatar_update'
  ) {
    avatarConfig = frame.payload.data;
    // Load the model the server picked for this bot:
    loader.loadAsync(avatarConfig.model_url).then((gltf) => {
      scene.add(gltf.scene);
      // Play the bot's configured idle animation, if any:
      if (avatarConfig.idle_animation) {
        playAnimation(gltf, avatarConfig.idle_animation);
      }
    });
  }
});

const loader = new GLTFLoader();
const gltf = await loader.loadAsync(avatarConfig.model_url);
const head = gltf.scene.getObjectByName('Head');  // mesh with morph targets

// You received `blendshapes` and `voice.data` from the WebSocket.
const audio = new Audio(`data:audio/wav;base64,${voice_b64}`);
audio.play();

const FPS = 60;
const startMs = performance.now();
function tick() {
  const elapsedMs = (performance.now() - startMs);
  const frameIdx = Math.floor((elapsedMs / 1000) * FPS);
  const frame = blendshapes[frameIdx];
  if (frame) {
    // frame = ["00:00:00.000", c0, c1, ..., c67]
    for (let i = 0; i < head.morphTargetInfluences.length; i++) {
      head.morphTargetInfluences[i] = frame[i + 1] ?? 0;
    }
  }
  if (frameIdx < blendshapes.length) requestAnimationFrame(tick);
}
requestAnimationFrame(tick);