-
Notifications
You must be signed in to change notification settings - Fork 58
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
63 changed files
with
19,315 additions
and
83 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| 0.8.9 | ||
| 0.8.10 |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| 0.8.9 | ||
| 0.8.10 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| # Web UI Streaming Client | ||
|
|
||
| ## Introduction | ||
| LCA UI offers an option to stream audio from a browser tab+Microphone to LCA/Agent Assist. | ||
| Enable the feature `WebSocketAudioInput` when deploying/updating the LCA stack. | ||
|
|
||
| To use this feature: | ||
| 1. From the LCA UI, click on `Stream Audio` link from the left navigation menu as shown below.  | ||
| 2. Change default values for Call ID, Agent ID, Customer Phone, and System Phone as needed. Assign a role to the mic input - Agent vs. Caller. | ||
| 3. Click on `Start Streaming` as shown below. | ||
| 4. Share the browser tab that is playing the media (video/audio files, meeting, etc.) | ||
| 5. Speak into the microphone. | ||
| 6. The web streaming client combines the audio output from browser tab and the microphone input into a streo (two channel) audio stream. The client sends the stream to the websocket server for downstream processing (enrichment, agent assist, etc.). | ||
| 7. Click on `Stop Streaming` to end the streaming session.  | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
113 changes: 113 additions & 0 deletions
113
lca-ai-stack/source/ui/public/worklets/recording-processor.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| // Based on sample from | ||
| // https://github.com/GoogleChromeLabs/web-audio-samples/blob/main/src/audio-worklet/migration/worklet-recorder/recording-processor.js | ||
|
|
||
| class RecordingProcessor extends AudioWorkletProcessor { | ||
| constructor(options) { | ||
| super(); | ||
| this.sampleRate = 0; | ||
| this.maxRecordingFrames = 0; | ||
| this.numberOfChannels = 0; | ||
| this._frameSize = 128; | ||
|
|
||
| if (options && options.processorOptions) { | ||
| const { | ||
| numberOfChannels, | ||
| sampleRate, | ||
| maxFrameCount, | ||
| } = options.processorOptions; | ||
|
|
||
| this.sampleRate = sampleRate; | ||
| this.maxRecordingFrames = maxFrameCount; | ||
| this.numberOfChannels = numberOfChannels; | ||
| } | ||
|
|
||
| this._leftRecordingBuffer = new Float32Array(this.maxRecordingFrames); | ||
| this._rightRecordingBuffer = new Float32Array(this.maxRecordingFrames); | ||
|
|
||
| this.recordedFrames = 0; | ||
| this.isRecording = false; | ||
|
|
||
| this.framesSinceLastPublish = 0; | ||
| this.publishInterval = this.sampleRate * 5; | ||
|
|
||
| this.port.onmessage = (event) => { | ||
| if (event.data.message === 'UPDATE_RECORDING_STATE') { | ||
| this.isRecording = event.data.setRecording; | ||
| } | ||
| }; | ||
| } | ||
|
|
||
| process(inputs, outputs) { | ||
| let currentSample = 0.0; | ||
| for (let input = 0; input < 1; input++) { | ||
| for (let channel = 0; channel < this.numberOfChannels; channel++) { | ||
| for (let sample = 0; sample < inputs[input][channel].length; sample++) { | ||
|
|
||
| currentSample = inputs[input][channel][sample]; | ||
|
|
||
| if (this.isRecording) { | ||
| if (channel == 0) { | ||
| this._leftRecordingBuffer[sample+this.recordedFrames] = currentSample; | ||
| } else if (channel == 1) { | ||
| this._rightRecordingBuffer[sample+this.recordedFrames] = currentSample; | ||
| } | ||
| } | ||
| // Pass data directly to output, unchanged. | ||
| outputs[input][channel][sample] = currentSample; | ||
| } | ||
|
|
||
| } | ||
| } | ||
|
|
||
| const shouldPublish = this.framesSinceLastPublish >= this.publishInterval; | ||
|
|
||
| // Validate that recording hasn't reached its limit. | ||
| if (this.isRecording) { | ||
| if (this.recordedFrames + this._frameSize < this.maxRecordingFrames) { | ||
| this.recordedFrames += this._frameSize; | ||
|
|
||
| // Post a recording recording length update on the clock's schedule | ||
| if (shouldPublish) { | ||
| const recordingBuffer = new Array(this.numberOfChannels) | ||
| .fill(new Float32Array(this.maxRecordingFrames)); | ||
| recordingBuffer[0] = this._leftRecordingBuffer; | ||
| recordingBuffer[1] = this._rightRecordingBuffer; | ||
| this.port.postMessage({ | ||
| message: 'SHARE_RECORDING_BUFFER', | ||
| buffer: recordingBuffer, | ||
| recordingLength: this.recordedFrames | ||
| }); | ||
| this.framesSinceLastPublish = 0; | ||
| this.recordedFrames = 0 | ||
| } else { | ||
| this.framesSinceLastPublish += this._frameSize; | ||
| } | ||
| } else { | ||
| this.recordedFrames += this._frameSize; | ||
|
|
||
| const recordingBuffer = new Array(this.numberOfChannels) | ||
| .fill(new Float32Array(this.maxRecordingFrames)); | ||
| recordingBuffer[0] = this._leftRecordingBuffer; | ||
| recordingBuffer[1] = this._rightRecordingBuffer; | ||
|
|
||
| this.port.postMessage({ | ||
| message: 'SHARE_RECORDING_BUFFER', | ||
| buffer: recordingBuffer, | ||
| recordingLength: this.recordedFrames | ||
| }); | ||
|
|
||
| this.recordedFrames = 0; | ||
| this.framesSinceLastPublish = 0; | ||
| } | ||
| } else { | ||
| console.log('stopping worklet processor node') | ||
| this.recordedFrames = 0; | ||
| this.framesSinceLastPublish = 0; | ||
| return false; | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
| } | ||
|
|
||
| registerProcessor('recording-processor', RecordingProcessor); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.