Skip to content

Commit

Permalink
feat(AI-call-summary): Automatically summarize call transcript
Browse files Browse the repository at this point in the history 
Signed-off-by: Joas Schilling <coding@schilljs.com>
  • Loading branch information
@nickvergessen
nickvergessen committed Nov 18, 2024
1 parent 552f53d commit f39b736
Show file tree
Hide file tree
Showing 8 changed files with 286 additions and 64 deletions.
3 changes: 2 additions & 1 deletion docs/settings.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@ Legend:
| `calls_start_without_media` | string<br>`yes` or `no` | `no` | Yes | | Whether participants start with enabled or disabled audio and video by default |
| `breakout_rooms` | string<br>`yes` or `no` | `yes` | Yes | | Whether or not breakout rooms are allowed (Will only prevent creating new breakout rooms. Existing conversations are not modified.) |
| `call_recording` | string<br>`yes` or `no` | `yes` | Yes | | Enable call recording |
| `call_recording_transcription` | string<br>`yes` or `no` | `no` | No | | Whether call recordings should automatically be transcripted when a transcription provider is enabled. |
| `call_recording_summary` | string<br>`yes` or `no` | `no` | No | | Whether call recordings should automatically be summarized when a transcription and summary provider is enabled. |
| `call_recording_transcription` | string<br>`yes` or `no` | `no` | No | | Whether call recordings should automatically be transcribed when a transcription provider is enabled. |
| `sip_dialout` | string<br>`yes` or `no` | `no` | Yes | | SIP dial-out is allowed when a SIP bridge is configured |
| `federation_enabled` | string<br>`yes` or `no` | `no` | Yes | | 🏗️ *Work in progress:* Whether or not federation with this instance is allowed |
| `federation_incoming_enabled` | string<br>`1` or `0` | `1` | Yes | | 🏗️ *Work in progress:* Whether users of this instance can be invited to federated conversations |
Expand Down
8 changes: 4 additions & 4 deletions lib/AppInfo/Application.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,8 @@
use OCP\Share\Events\BeforeShareCreatedEvent;
use OCP\Share\Events\ShareCreatedEvent;
use OCP\Share\Events\VerifyMountPointEvent;
use OCP\SpeechToText\Events\TranscriptionFailedEvent;
use OCP\SpeechToText\Events\TranscriptionSuccessfulEvent;
use OCP\TaskProcessing\Events\TaskFailedEvent;
use OCP\TaskProcessing\Events\TaskSuccessfulEvent;
use OCP\User\Events\BeforeUserLoggedOutEvent;
use OCP\User\Events\UserChangedEvent;
use OCP\User\Events\UserDeletedEvent;
Expand Down Expand Up @@ -273,8 +273,8 @@ public function register(IRegistrationContext $context): void {
$context->registerEventListener(RoomDeletedEvent::class, RecordingListener::class);
$context->registerEventListener(CallEndedEvent::class, RecordingListener::class);
$context->registerEventListener(CallEndedForEveryoneEvent::class, RecordingListener::class);
$context->registerEventListener(TranscriptionSuccessfulEvent::class, RecordingListener::class);
$context->registerEventListener(TranscriptionFailedEvent::class, RecordingListener::class);
$context->registerEventListener(TaskSuccessfulEvent::class, RecordingListener::class);
$context->registerEventListener(TaskFailedEvent::class, RecordingListener::class);

// Federation listeners
$context->registerEventListener(BeforeRoomDeletedEvent::class, TalkV1BeforeRoomDeletedListener::class);
Expand Down
2 changes: 1 addition & 1 deletion lib/Controller/RecordingController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -446,7 +446,7 @@ public function shareToChat(int $fileId, int $timestamp): DataResponse {
$this->getRoom(),
$this->participant,
$fileId,
$timestamp
$timestamp,
);
} catch (InvalidArgumentException $e) {
return new DataResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST);
Expand Down
10 changes: 8 additions & 2 deletions lib/Notification/Notifier.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ public function prepare(INotification $notification, string $languageCode): INot
->setLink($this->url->linkToRouteAbsolute('spreed.Page.showCall', ['token' => $room->getToken()]));

$subject = $notification->getSubject();
if ($subject === 'record_file_stored' || $subject === 'transcript_file_stored' || $subject === 'transcript_failed') {
if ($subject === 'record_file_stored' || $subject === 'transcript_file_stored' || $subject === 'transcript_failed' || $subject === 'summary_file_stored' || $subject === 'summary_failed') {
return $this->parseStoredRecording($notification, $room, $participant, $l);
}
if ($subject === 'record_file_store_fail') {
Expand Down Expand Up @@ -363,9 +363,15 @@ protected function parseStoredRecording(
} elseif ($notification->getSubject() === 'transcript_file_stored') {
$subject = $l->t('Transcript now available');
$message = $l->t('The transcript for the call in {call} was uploaded to {file}.');
} else {
} elseif ($notification->getSubject() === 'transcript_failed') {
$subject = $l->t('Failed to transcript call recording');
$message = $l->t('The server failed to transcript the recording at {file} for the call in {call}. Please reach out to the administration.');
} elseif ($notification->getSubject() === 'summary_file_stored') {
$subject = $l->t('Call summary now available');
$message = $l->t('The summary for the call in {call} was uploaded to {file}.');
} else {
$subject = $l->t('Failed to summarize call recording');
$message = $l->t('The server failed to summarize the recording at {file} for the call in {call}. Please reach out to the administration.');
}

$notification
Expand Down
66 changes: 41 additions & 25 deletions lib/Recording/Listener.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@
use OCP\EventDispatcher\IEventListener;
use OCP\Files\File;
use OCP\Files\IRootFolder;
use OCP\SpeechToText\Events\AbstractTranscriptionEvent;
use OCP\SpeechToText\Events\TranscriptionFailedEvent;
use OCP\SpeechToText\Events\TranscriptionSuccessfulEvent;
use OCP\TaskProcessing\Events\AbstractTaskProcessingEvent;
use OCP\TaskProcessing\Events\TaskFailedEvent;
use OCP\TaskProcessing\Events\TaskSuccessfulEvent;
use Psr\Log\LoggerInterface;

/**
* @template-implements IEventListener<Event>
Expand All @@ -34,12 +35,17 @@ public function __construct(
protected RecordingService $recordingService,
protected ConsentService $consentService,
protected IRootFolder $rootFolder,
protected LoggerInterface $logger,
) {
}

public function handle(Event $event): void {
if ($event instanceof AbstractTranscriptionEvent) {
$this->handleTranscriptionEvents($event);
if ($event instanceof AbstractTaskProcessingEvent) {
try {
$this->handleTranscriptionEvents($event);
} catch (\Throwable $e) {
$this->logger->error('An error occurred while processing recording AI follow-up task', ['exception' => $e]);
}
return;
}

Expand All @@ -54,40 +60,50 @@ public function handle(Event $event): void {
};
}

public function handleTranscriptionEvents(AbstractTranscriptionEvent $event): void {
if ($event->getAppId() !== Application::APP_ID) {
public function handleTranscriptionEvents(AbstractTaskProcessingEvent $event): void {
$task = $event->getTask();
if ($task->getAppId() !== Application::APP_ID) {
return;
}

if ($event instanceof TranscriptionSuccessfulEvent) {
$this->successfulTranscript($event->getUserId(), $event->getFile(), $event->getTranscript());
} elseif ($event instanceof TranscriptionFailedEvent) {
$this->failedTranscript($event->getUserId(), $event->getFile());
}
}

protected function successfulTranscript(?string $owner, ?File $fileNode, string $transcript): void {
if (!$fileNode instanceof File) {
return;
}
// 'call/transcription/' . $room->getToken()
$customId = $task->getCustomId();
$isTranscription = $isSummary = false;
if (str_starts_with($customId, 'call/transcription/')) {
$isTranscription = true;
$roomToken = substr($customId, strlen('call/transcription/'));

if ($owner === null) {
$fileId = (int)($task->getInput()['input'] ?? null);
} elseif (str_starts_with($customId, 'call/summary/')) {
[$roomToken, $fileId] = explode('/', substr($customId, strlen('call/summary/')));
$fileId = (int)$fileId;
$isSummary = true;
} else {
return;
}

$this->recordingService->storeTranscript($owner, $fileNode, $transcript);
}

protected function failedTranscript(?string $owner, ?File $fileNode): void {
if (!$fileNode instanceof File) {
if ($fileId === 0) {
return;
}

if ($owner === null) {
if ($task->getUserId() === null) {
return;
}

$this->recordingService->notifyAboutFailedTranscript($owner, $fileNode);
if ($isTranscription) {
if ($event instanceof TaskSuccessfulEvent) {
$this->recordingService->storeTranscript($task->getUserId(), $roomToken, $fileId, $task->getOutput()['output'] ?? '', 'transcript');
} elseif ($event instanceof TaskFailedEvent) {
$this->recordingService->notifyAboutFailedTranscript($task->getUserId(), $roomToken, $fileId, 'transcript');
}
} elseif ($isSummary) {
if ($event instanceof TaskSuccessfulEvent) {
$this->recordingService->storeTranscript($task->getUserId(), $roomToken, $fileId, $task->getOutput()['output'] ?? '', 'summary');
} elseif ($event instanceof TaskFailedEvent) {
$this->recordingService->notifyAboutFailedTranscript($task->getUserId(), $roomToken, $fileId, 'summary');
}
}
}

protected function roomDeleted(RoomDeletedEvent $event): void {
Expand Down
Loading

0 comments on commit f39b736

Please sign in to comment.