WebSocket API Reference

const ws = new WebSocket('wss://zeebee.ai/ws');

// Send initialization message after connection
ws.onopen = () => {
  ws.send(JSON.stringify({
    type: 'init',
    user_id: 'YOUR_USER_ID',    // Option 1: Using user_id
    api_key: 'YOUR_API_KEY',    // Option 2: Using API key (preferred for integrations)
    conversation_id: 'OPTIONAL_CONVERSATION_ID',
    model: 'gpt-4o',            // Optional: Specify model (defaults to gpt-4o)
    client_info: {}             // Optional: Additional client information
  }));
};

{
  "type": "init_ack",
  "status": "connected",
  "user_id": "user_12345"
}

{
  "type": "error",
  "message": "Invalid API key. Please check your credentials."
}

{
  "type": "error",
  "message": "Authentication required. Provide a valid user ID or API key."
}

const ws = new WebSocket('wss://zeebee.ai/ws');

ws.onopen = () => {
  console.log('Connected to ZeebeeAI WebSocket API');
  
  // Send initialization message
  ws.send(JSON.stringify({
    type: 'init',
    user_id: 'YOUR_USER_ID'  // or use api_key
  }));
};

ws.onclose = (event) => {
  console.log(`Connection closed: ${event.code} ${event.reason}`);
};

ws.onerror = (error) => {
  console.error('WebSocket error:', error);
};

ws.onmessage = (event) => {
  const data = JSON.parse(event.data);
  
  if (data.type === 'ping') {
    // Respond with a pong message
    ws.send(JSON.stringify({
      type: 'pong',
      ping_id: data.ping_id,
      timestamp: Date.now()
    }));
  }
  
  // Process other message types
  // ...
};

{
  "type": "connection_warning",
  "level": "warning",
  "message": "Your connection appears to be inactive. Please respond or send any message to keep the connection alive.",
  "time_remaining": 30,
  "reconnect_required": false,
  "ping_requested": true
}

// Send ping to server
ws.send(JSON.stringify({
  type: 'ping',
  timestamp: Date.now()
}));

{
  "type": "message_type",
  // Message-specific fields
}

// Start a new conversation
ws.send(JSON.stringify({
  type: 'audio',
  audio_data: 'BASE64_ENCODED_AUDIO',
  model: 'gpt-4o',
  stt_provider: 'openai',
  stt_options: {
    model: 'whisper-1'
  }
}));

// Transcription response
{
  "type": "transcript",
  "text": "What's the weather like today?"
}

// AI text response
{
  "type": "response",
  "text": "I don't have access to real-time weather data. To get the current weather, you can check a weather website or app, or look outside your window.",
  "conversation_id": "conv_12345"
}

// Initialize with a specific conversation
ws.send(JSON.stringify({
  type: 'init',
  user_id: 'YOUR_USER_ID',
  conversation_id: 'conv_12345'
}));

// Send a message to the existing conversation
ws.send(JSON.stringify({
  type: 'audio',
  audio_data: 'BASE64_ENCODED_AUDIO',
  conversation_id: 'conv_12345',
  model: 'gpt-4o'
}));

// Send audio data for transcription and response
ws.send(JSON.stringify({
  type: 'audio',
  audio_data: 'BASE64_ENCODED_AUDIO', // Base64-encoded audio data
  model: 'gpt-4o',                     // LLM model to use
  stt_provider: 'openai',              // Speech-to-Text provider (default: openai)
  stt_options: {                       // Speech-to-Text options
    model: 'whisper-1'                 // STT model to use
  },
  tts_provider: 'openai',              // Text-to-Speech provider (default: openai)
  tts_voice: 'alloy',                  // Voice for Text-to-Speech (default: alloy)
  tts_options: {},                     // Additional TTS options
  language_code: 'en-US'               // Language code for STT/TTS (default: en-US)
}));

ws.onmessage = (event) => {
  const data = JSON.parse(event.data);
  
  switch (data.type) {
    case 'transcript':
      console.log('Transcription:', data.text);
      // Update UI with transcribed text
      break;
      
    case 'response':
      console.log('AI response:', data.text);
      // Update UI with AI text response
      break;
      
    case 'audio_stream_start':
      console.log('Audio stream starting, chunks expected:', data.total_chunks);
      // Prepare audio player for streaming
      break;
      
    case 'audio_chunk':
      console.log(`Received audio chunk ${data.chunk_index}/${data.total_chunks}`);
      // Process audio chunk (e.g., add to buffer)
      const audioChunk = base64ToArrayBuffer(data.data);
      // Add chunk to audio buffer
      break;
      
    case 'audio_stream_end':
      console.log('Audio stream complete');
      // Play the complete audio
      break;
      
    case 'tts_error':
      console.error('TTS error:', data.message);
      // Handle TTS error
      break;
  }
};

ws.send(JSON.stringify({
  type: 'audio',
  audio_data: 'BASE64_ENCODED_AUDIO',
  language_code: 'fr-FR', // Use French
  model: 'gpt-4o'
}));

{
  "type": "error",
  "message": "Error message describing what went wrong"
}

ws.onmessage = (event) => {
  const data = JSON.parse(event.data);
  
  if (data.type === 'error') {
    console.error('Server error:', data.message);
    
    // Handle different error scenarios
    if (data.message.includes('Invalid API key')) {
      // Handle authentication error
      promptForAuthentication();
    } else if (data.message.includes('transcribe audio')) {
      // Handle transcription failure
      retryTranscription();
    } else {
      // Handle generic errors
      showErrorToUser(data.message);
    }
  }
  
  // Process other message types...
};

// Initialize WebSocket connection
const ws = new WebSocket('wss://zeebee.ai/ws');
let audioContext;
let mediaRecorder;
let recordedChunks = [];
let isRecording = false;
let conversationId = null;

// Handle connection events
ws.onopen = () => {
  console.log('Connected to ZeebeeAI WebSocket server');
  
  // Authenticate
  ws.send(JSON.stringify({
    type: 'init',
    api_key: 'YOUR_API_KEY'
  }));
  
  // Enable recording button after connection
  document.getElementById('recordButton').disabled = false;
};

ws.onclose = (event) => {
  console.log(`WebSocket connection closed: ${event.code} ${event.reason}`);
  document.getElementById('recordButton').disabled = true;
};

ws.onerror = (error) => {
  console.error('WebSocket error:', error);
};

// Handle incoming messages
ws.onmessage = (event) => {
  const data = JSON.parse(event.data);
  console.log('Received message:', data.type);
  
  switch (data.type) {
    case 'init_ack':
      console.log('Authentication successful');
      document.getElementById('status').textContent = 'Connected';
      break;
      
    case 'ping':
      // Respond to server pings
      ws.send(JSON.stringify({
        type: 'pong',
        ping_id: data.ping_id,
        timestamp: Date.now()
      }));
      break;
      
    case 'transcript':
      // Show transcribed text
      document.getElementById('transcript').textContent = data.text;
      break;
      
    case 'response':
      // Save conversation ID for later use
      if (data.conversation_id) {
        conversationId = data.conversation_id;
      }
      
      // Show AI response text
      document.getElementById('response').textContent = data.text;
      break;
      
    case 'audio_stream_start':
      console.log('Starting to receive audio stream');
      // Initialize audio buffer
      initializeAudioPlayback(data.total_chunks);
      break;
      
    case 'audio_chunk':
      // Process audio chunk
      processAudioChunk(data.data, data.chunk_index);
      break;
      
    case 'audio_stream_end':
      // Finalize and play the audio
      playAudioResponse();
      break;
      
    case 'error':
      console.error('Server error:', data.message);
      document.getElementById('error').textContent = data.message;
      break;
  }
};

// Record audio and send to server
async function startRecording() {
  try {
    // Get microphone access
    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
    
    // Initialize audio context and recorder
    audioContext = new AudioContext();
    mediaRecorder = new MediaRecorder(stream);
    recordedChunks = [];
    isRecording = true;
    
    // Update UI
    document.getElementById('status').textContent = 'Recording...';
    document.getElementById('recordButton').textContent = 'Stop';
    
    // Handle recorded data
    mediaRecorder.ondataavailable = (event) => {
      if (event.data.size > 0) {
        recordedChunks.push(event.data);
      }
    };
    
    // Handle recording stop
    mediaRecorder.onstop = async () => {
      // Create audio blob
      const audioBlob = new Blob(recordedChunks, { type: 'audio/webm' });
      
      // Convert to base64
      const base64Audio = await blobToBase64(audioBlob);
      
      // Send to server
      ws.send(JSON.stringify({
        type: 'audio',
        audio_data: base64Audio,
        model: 'gpt-4o',
        conversation_id: conversationId, // Include conversation ID if continuing a conversation
        language_code: 'en-US',
        tts_voice: 'alloy'
      }));
      
      // Update UI
      document.getElementById('status').textContent = 'Processing...';
    };
    
    // Start recording
    mediaRecorder.start();
    
  } catch (error) {
    console.error('Error starting recording:', error);
    document.getElementById('error').textContent = 'Could not access microphone.';
  }
}

// Stop recording
function stopRecording() {
  if (mediaRecorder && isRecording) {
    mediaRecorder.stop();
    isRecording = false;
    document.getElementById('recordButton').textContent = 'Record';
  }
}

// Toggle recording
function toggleRecording() {
  if (isRecording) {
    stopRecording();
  } else {
    startRecording();
  }
}

// Helper function to convert blob to base64
function blobToBase64(blob) {
  return new Promise((resolve, reject) => {
    const reader = new FileReader();
    reader.onloadend = () => resolve(reader.result.split(',')[1]);
    reader.onerror = reject;
    reader.readAsDataURL(blob);
  });
}

// Audio playback handling
let audioBuffer = [];
let totalChunksExpected = 0;

function initializeAudioPlayback(totalChunks) {
  audioBuffer = new Array(totalChunks);
  totalChunksExpected = totalChunks;
}

function processAudioChunk(base64Data, chunkIndex) {
  audioBuffer[chunkIndex] = base64Data;
}

function playAudioResponse() {
  // Check if all chunks received
  const allChunksReceived = !audioBuffer.includes(undefined);
  
  if (allChunksReceived) {
    // Combine all chunks
    const base64Audio = audioBuffer.join('');
    
    // Convert to blob
    const byteCharacters = atob(base64Audio);
    const byteNumbers = new Array(byteCharacters.length);
    
    for (let i = 0; i < byteCharacters.length; i++) {
      byteNumbers[i] = byteCharacters.charCodeAt(i);
    }
    
    const byteArray = new Uint8Array(byteNumbers);
    const audioBlob = new Blob([byteArray], { type: 'audio/mp3' });
    
    // Create audio element and play
    const audioUrl = URL.createObjectURL(audioBlob);
    const audio = new Audio(audioUrl);
    audio.play();
    
    document.getElementById('status').textContent = 'Playing response...';
  }
}

<div class="voice-chat-container">
  <h2>ZeebeeAI Voice Chat</h2>
  
  <div class="status-container">
    <p>Status: <span id="status">Connecting...</span></p>
    <p class="error" id="error"></p>
  </div>
  
  <div class="controls">
    <button id="recordButton" onclick="toggleRecording()" disabled>Record</button>
  </div>
  
  <div class="conversation">
    <div class="message-container">
      <h4>You said:</h4>
      <p id="transcript"></p>
    </div>
    
    <div class="message-container">
      <h4>Zeebee says:</h4>
      <p id="response"></p>
    </div>
  </div>
</div>