Interface AudioProcessor

You can add a local AudioProcessor to the SDK to access and process the audio input stream before sending it to Twilio. Similarly, you can add a remote AudioProcessor to access and process the audio output stream before it is rendered on the speaker.

  • To add a processor, implement the AudioProcessor interface and use AudioHelper.addProcessor.
  • To remove a processor, use AudioHelper.removeProcessor.
  • To specify whether the processor is local or remote, use the optional isRemote parameter.

Use cases include:

  • Background noise removal using a noise cancellation library of your choice
  • Music playback when putting the call on hold
  • Audio filters
  • AI audio classification
  • ... and more!

The following example demonstrates how to utilize AudioProcessor APIs to use background music for local audio instead of using a microphone.

import { AudioProcessor, Device } from '@twilio/voice-sdk';

let audioContext;

class BackgroundAudioProcessor implements AudioProcessor {

 private audioContext: AudioContext;
 private background: MediaElementAudioSourceNode;
 private destination: MediaStreamAudioDestinationNode;

 constructor() {
   if (!audioContext) {
     audioContext = new AudioContext();
   }
   this.audioContext = audioContext;
 }

 async createProcessedStream(stream: MediaStream): Promise<MediaStream> {
   // Create the source node
   const audioEl = new Audio('/background.mp3');
   audioEl.addEventListener('canplaythrough', () => audioEl.play());
   this.background = this.audioContext.createMediaElementSource(audioEl);

   // Create the destination node and connect the source node
   this.destination = this.audioContext.createMediaStreamDestination();
   this.background.connect(this.destination);

   // Return the resulting MediaStream
   return this.destination.stream;
 }

 async destroyProcessedStream(stream: MediaStream): Promise<void> {
   // Cleanup
   this.background.disconnect();
   this.destination.disconnect();
 }
}
// Construct a device object, passing your own token and desired options
const device = new Device(token, options);

// Construct the AudioProcessor
const processor = new BackgroundAudioProcessor();

// Add the local processor
await device.audio.addProcessor(processor, false);
// Remove the local processor later
// await device.audio.removeProcessor(processor, false);

// Or add remote processor
// await device.audio.addProcessor(processor, true);
// Remove remote processor
// await device.audio.removeProcessor(processor, true);
Copy