Chat 模式（Turn-based Chat）

{
  "role": "system | user | assistant",
  "content": "纯文本字符串"
}

{
  "role": "user",
  "content": [
    {"type": "text", "text": "描述这张图片"},
    {"type": "image", "data": "<base64>"},
    {"type": "audio", "data": "<base64 PCM float32>", "sample_rate": 16000},
    {"type": "video", "data": "<base64 video file>", "stack_frames": 1}
  ]
}

{
  "messages": [
    {"role": "user", "content": "你好！"}
  ],
  "streaming": true,
  "generation": {"max_new_tokens": 256, "length_penalty": 1.1},
  "tts": {"enabled": true, "ref_audio_data": "<base64>"},
  "image": {"max_slice_nums": null},
  "omni_mode": false,
  "enable_thinking": false
}

{
  "type": "chunk",
  "text_delta": "你好",
  "audio_data": "<base64, 24kHz>"
}

{
  "type": "done",
  "text": "你好！有什么可以帮你的？",
  "generated_tokens": 15,
  "input_tokens": 42,
  "audio_data": "<base64, 24kHz>",
  "recording_session_id": "chat_abc123"
}

// -- 声音克隆参考音频 (base64 PCM float32, 16kHz) --
// 从用户上传的文件或服务器默认音频 (/api/default_ref_audio) 加载
const refAudioBase64 = getRefAudioBase64();

const ws = new WebSocket(`wss://${location.host}/ws/chat`);
let fullText = '';

ws.onopen = () => {
  const payload = {
    messages: [
      // System message 使用 content-list 格式：[text, audio, text]。
      // 其中的 audio 项将参考音色嵌入 LLM 上下文。
      { role: 'system', content: [
        { type: 'text', text: '模仿音频样本的音色并生成新的内容。' },
        { type: 'audio', data: refAudioBase64 },          // 参考音色
        { type: 'text', text: '你是一个有帮助的助手，请自然地回复。' },
      ]},
      { role: 'user', content: '你好！' },
    ],
    streaming: true,
    generation: { max_new_tokens: 256, temperature: 0.7 },
    tts: {
      enabled: true,
      mode: 'audio_assistant',
      ref_audio_data: refAudioBase64,  // 同一音频用于初始化 TTS vocoder
    },
  };
  ws.send(JSON.stringify(payload));
};

ws.onmessage = (event) => {
  const msg = JSON.parse(event.data);
  switch (msg.type) {
    case 'prefill_done':
      // 服务端已将 prompt 填入 KV Cache
      console.log(`Prefill 完成，${msg.input_tokens} 个输入 token`);
      break;
    case 'chunk':
      // 流式 token：增量文本和/或音频片段
      if (msg.text_delta) {
        fullText += msg.text_delta;
        console.log('流式输出:', fullText);
      }
      if (msg.audio_data) playAudio(msg.audio_data);  // PCM float32, 24kHz
      break;
    case 'done':
      // 生成完毕 — 完整文本和 token 统计
      console.log('最终文本:', msg.text);
      console.log(`Token 统计: ${msg.input_tokens} 输入, ${msg.generated_tokens} 输出`);
      ws.close();
      break;
    case 'error':
      console.error('错误:', msg.error);
      ws.close();
      break;
  }
};

ws.onerror = () => console.error('WebSocket 错误');
ws.onclose = () => console.log('连接已关闭');

import asyncio, json, base64
import numpy as np
import websockets

def load_ref_audio(path: str) -> str:
    """加载 WAV 文件，返回 base64 编码的 PCM float32 (16kHz)。"""
    import soundfile as sf
    audio, _ = sf.read(path, dtype="float32", samplerate=16000)
    return base64.b64encode(audio.tobytes()).decode()

async def chat_streaming(
    server_url="wss://localhost:8006/ws/chat",
    ref_audio_path: str | None = "ref.wav",
):
    async with websockets.connect(server_url) as ws:
        # 加载参考音频用于声音克隆
        ref_b64 = load_ref_audio(ref_audio_path) if ref_audio_path else None

        # System message 使用 content-list 格式：[text, audio, text]。
        # 其中的 audio 项将参考音色嵌入 LLM 上下文。
        system_content = "You are a helpful assistant."
        if ref_b64:
            system_content = [
                {"type": "text", "text": "模仿音频样本的音色并生成新的内容。"},
                {"type": "audio", "data": ref_b64},          # 参考音色
                {"type": "text", "text": "你是一个有帮助的助手，请自然地回复。"},
            ]

        # TTS 配置 — ref_audio_data 单独初始化 TTS vocoder
        tts_config = {"enabled": True, "mode": "audio_assistant"}
        if ref_b64:
            tts_config["ref_audio_data"] = ref_b64

        await ws.send(json.dumps({
            "messages": [
                {"role": "system", "content": system_content},
                {"role": "user", "content": "你好！"},
            ],
            "streaming": True,
            "generation": {"max_new_tokens": 256, "temperature": 0.7},
            "tts": tts_config,
        }))

        full_text = ""
        audio_chunks = []

        async for raw in ws:
            msg = json.loads(raw)
            if msg["type"] == "prefill_done":
                print(f"Prefill 完成，{msg['input_tokens']} 个输入 token")
            elif msg["type"] == "chunk":
                # 增量文本 token
                if msg.get("text_delta"):
                    full_text += msg["text_delta"]
                    print(f"流式输出: {full_text}")
                # 增量音频片段 (PCM float32, 24kHz)
                if msg.get("audio_data"):
                    audio_chunks.append(base64.b64decode(msg["audio_data"]))
            elif msg["type"] == "done":
                print(f"最终文本: {msg['text']}")
                print(f"Token 统计: {msg['input_tokens']} 输入, {msg['generated_tokens']} 输出")
                break
            elif msg["type"] == "error":
                print(f"错误: {msg['error']}")
                break

        if audio_chunks:
            pcm = np.frombuffer(b"".join(audio_chunks), dtype=np.float32)
            print(f"收到 {len(pcm)/24000:.1f} 秒音频")

asyncio.run(chat_streaming())