feat(chat): Add API to summarize chat messages

appinfo/routes/routesChatController.php

@@ -21,6 +21,8 @@
 	'ocs' => [
 		/** @see \OCA\Talk\Controller\ChatController::receiveMessages() */
 		['name' => 'Chat#receiveMessages', 'url' => '/api/{apiVersion}/chat/{token}', 'verb' => 'GET', 'requirements' => $requirements],
+		/** @see \OCA\Talk\Controller\ChatController::summarizeChat() */
+		['name' => 'Chat#summarizeChat', 'url' => '/api/{apiVersion}/chat/{token}/summarize', 'verb' => 'POST', 'requirements' => $requirements],
 		/** @see \OCA\Talk\Controller\ChatController::sendMessage() */
 		['name' => 'Chat#sendMessage', 'url' => '/api/{apiVersion}/chat/{token}', 'verb' => 'POST', 'requirements' => $requirements],
 		/** @see \OCA\Talk\Controller\ChatController::clearHistory() */

docs/capabilities.md

@@ -160,5 +160,7 @@
 * `archived-conversations` (local) - Conversations can be marked as archived which will hide them from the conversation list by default
 * `talk-polls-drafts` - Whether moderators can store and retrieve poll drafts
 * `download-call-participants` - Whether the endpoints for moderators to download the call participants is available
+* `chat-summary-api` (local) - Whether the endpoint to get summarized chat messages in a conversation is available
+* `config => chat => summary-threshold` (local) - Number of unread messages that should exist to show a "Generate summary" option
 * `config => call => start-without-media` (local) - Boolean, whether media should be disabled when starting or joining a conversation
 * `config => call => max-duration` - Integer, maximum call duration in seconds. Please note that this should only be used with system cron and with a reasonable high value, due to the expended duration until the background job ran.

lib/Capabilities.php

@@ -18,6 +18,8 @@
 use OCP\IConfig;
 use OCP\IUser;
 use OCP\IUserSession;
+use OCP\TaskProcessing\IManager as ITaskProcessingManager;
+use OCP\TaskProcessing\TaskTypes\TextToTextSummary;
 use OCP\Translation\ITranslationManager;
 use OCP\Util;
 
@@ -108,6 +110,12 @@ class Capabilities implements IPublicCapability {
 		'download-call-participants',
 	];
 
+	public const CONDITIONAL_FEATURES = [
+		'message-expiration',
+		'reactions',
+		'chat-summary-api',
+	];
+
 	public const LOCAL_FEATURES = [
 		'favorites',
 		'chat-read-status',
@@ -119,6 +127,7 @@ class Capabilities implements IPublicCapability {
 		'remind-me-later',
 		'note-to-self',
 		'archived-conversations',
+		'chat-summary-api',
 	];
 
 	public const LOCAL_CONFIGS = [
@@ -135,6 +144,7 @@ class Capabilities implements IPublicCapability {
 			'read-privacy',
 			'has-translation-providers',
 			'typing-privacy',
+			'summary-threshold',
 		],
 		'conversations' => [
 			'can-create',
@@ -164,6 +174,7 @@ public function __construct(
 		protected IUserSession $userSession,
 		protected IAppManager $appManager,
 		protected ITranslationManager $translationManager,
+		protected ITaskProcessingManager $taskProcessingManager,
 		ICacheFactory $cacheFactory,
 	) {
 		$this->talkCache = $cacheFactory->createLocal('talk::');
@@ -207,6 +218,7 @@ public function getCapabilities(): array {
 					'read-privacy' => Participant::PRIVACY_PUBLIC,
 					'has-translation-providers' => $this->translationManager->hasProviders(),
 					'typing-privacy' => Participant::PRIVACY_PUBLIC,
+					'summary-threshold' => 100,
 				],
 				'conversations' => [
 					'can-create' => $user instanceof IUser && !$this->talkConfig->isNotAllowedToCreateConversations($user)
@@ -300,6 +312,11 @@ public function getCapabilities(): array {
 			$capabilities['config']['call']['can-enable-sip'] = $this->talkConfig->canUserEnableSIP($user);
 		}
 
+		$supportedTaskTypes = $this->taskProcessingManager->getAvailableTaskTypes();
+		if (isset($supportedTaskTypes[TextToTextSummary::ID])) {
+			$capabilities['features'][] = 'chat-summary-api';
+		}
+
 		return [
 			'spreed' => $capabilities,
 		];

lib/Controller/ChatController.php

@@ -8,13 +8,15 @@
 
 namespace OCA\Talk\Controller;
 
+use OCA\Talk\AppInfo\Application;
 use OCA\Talk\Chat\AutoComplete\SearchPlugin;
 use OCA\Talk\Chat\AutoComplete\Sorter;
 use OCA\Talk\Chat\ChatManager;
 use OCA\Talk\Chat\MessageParser;
 use OCA\Talk\Chat\Notifier;
 use OCA\Talk\Chat\ReactionManager;
 use OCA\Talk\Exceptions\CannotReachRemoteException;
+use OCA\Talk\Exceptions\ChatSummaryException;
 use OCA\Talk\Federation\Authenticator;
 use OCA\Talk\GuestManager;
 use OCA\Talk\MatterbridgeManager;
@@ -51,6 +53,7 @@
 use OCP\AppFramework\Http\Attribute\PublicPage;
 use OCP\AppFramework\Http\Attribute\UserRateLimit;
 use OCP\AppFramework\Http\DataResponse;
+use OCP\AppFramework\Services\IAppConfig;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\Collaboration\AutoComplete\IManager;
 use OCP\Collaboration\Collaborators\ISearchResult;
@@ -62,14 +65,20 @@
 use OCP\IRequest;
 use OCP\IUserManager;
 use OCP\RichObjectStrings\InvalidObjectExeption;
+use OCP\RichObjectStrings\IRichTextFormatter;
 use OCP\RichObjectStrings\IValidator;
 use OCP\Security\ITrustedDomainHelper;
 use OCP\Security\RateLimiting\IRateLimitExceededException;
 use OCP\Share\Exceptions\ShareNotFound;
 use OCP\Share\IShare;
+use OCP\TaskProcessing\Exception\Exception;
+use OCP\TaskProcessing\IManager as ITaskProcessingManager;
+use OCP\TaskProcessing\Task;
+use OCP\TaskProcessing\TaskTypes\TextToTextSummary;
 use OCP\User\Events\UserLiveStatusEvent;
 use OCP\UserStatus\IManager as IUserStatusManager;
 use OCP\UserStatus\IUserStatus;
+use Psr\Log\LoggerInterface;
 
 /**
  * @psalm-import-type TalkChatMentionSuggestion from ResponseDefinitions
@@ -114,6 +123,10 @@ public function __construct(
 		protected Authenticator $federationAuthenticator,
 		protected ProxyCacheMessageService $pcmService,
 		protected Notifier $notifier,
+		protected IRichTextFormatter $richTextFormatter,
+		protected ITaskProcessingManager $taskProcessingManager,
+		protected IAppConfig $appConfig,
+		protected LoggerInterface $logger,
 	) {
 		parent::__construct($appName, $request);
 	}
@@ -489,6 +502,138 @@ public function receiveMessages(int $lookIntoFuture,
 		return $this->prepareCommentsAsDataResponse($comments, $lastCommonReadId);
 	}
 
+	/**
+	 * Summarize the next bunch of chat messages from a given offset
+	 *
+	 * Required capability: `chat-summary-api`
+	 *
+	 * @param positive-int $fromMessageId Offset from where on the summary should be generated
+	 * @return DataResponse<Http::STATUS_CREATED, array{taskId: int, nextOffset?: int}, array{}>|DataResponse<Http::STATUS_BAD_REQUEST, array{error: 'ai-no-provider'|'ai-error'}, array{}>|DataResponse<Http::STATUS_NO_CONTENT, array<empty>, array{}>
+	 * @throws \InvalidArgumentException
+	 *
+	 * 201: Summary was scheduled, use the returned taskId to get the status
+	 *   information and output from the TaskProcessing API:
+	 *   https://docs.nextcloud.com/server/latest/developer_manual/client_apis/OCS/ocs-taskprocessing-api.html#fetch-a-task-by-id
+	 *   If the response data contains nextOffset, not all messages could be handled in a single request.
+	 *   After receiving the response a second summary should be requested with the provided nextOffset.
+	 * 204: No messages found to summarize
+	 * 400: No AI provider available or summarizing failed
+	 */
+	#[PublicPage]
+	#[RequireModeratorOrNoLobby]
+	#[RequireParticipant]
+	public function summarizeChat(
+		int $fromMessageId,
+	): DataResponse {
+		$fromMessageId = max(0, $fromMessageId);
+
+		$supportedTaskTypes = $this->taskProcessingManager->getAvailableTaskTypes();
+		if (!isset($supportedTaskTypes[TextToTextSummary::ID])) {
+			return new DataResponse([
+				'error' => ChatSummaryException::REASON_AI_ERROR,
+			], Http::STATUS_BAD_REQUEST);
+		}
+
+		// if ($this->room->isFederatedConversation()) {
+		// /** @var \OCA\Talk\Federation\Proxy\TalkV1\Controller\ChatController $proxy */
+		// $proxy = \OCP\Server::get(\OCA\Talk\Federation\Proxy\TalkV1\Controller\ChatController::class);
+		// return $proxy->summarizeChat(
+		// $this->room,
+		// $this->participant,
+		// $fromMessageId,
+		// );
+		// }
+
+		$currentUser = $this->userManager->get($this->userId);
+		$batchSize = $this->appConfig->getAppValueInt('ai_unread_summary_batch_size', 500);
+		$comments = $this->chatManager->waitForNewMessages($this->room, $fromMessageId, $batchSize, 0, $currentUser, true, false);
+		$this->preloadShares($comments);
+
+		$messages = [];
+		$nextOffset = 0;
+		foreach ($comments as $comment) {
+			$message = $this->messageParser->createMessage($this->room, $this->participant, $comment, $this->l);
+			$this->messageParser->parseMessage($message);
+
+			if (!$message->getVisibility()) {
+				continue;
+			}
+
+			if ($message->getMessageType() === ChatManager::VERB_SYSTEM
+				&& !in_array($message->getMessageRaw(), [
+					'call_ended',
+					'call_ended_everyone',
+					'file_shared',
+					'object_shared',
+				], true)) {
+				// Ignore system messages apart from calls, shared objects and files
+				continue;
+			}
+
+			$parsedMessage = $this->richTextFormatter->richToParsed(
+				$message->getMessage(),
+				$message->getMessageParameters(),
+			);
+
+			$displayName = $message->getActorDisplayName();
+			if (in_array($message->getActorType(), [
+				Attendee::ACTOR_GUESTS,
+				Attendee::ACTOR_EMAILS,
+			], true)) {
+				if ($displayName === '') {
+					$displayName = $this->l->t('Guest');
+				} else {
+					$displayName = $this->l->t('%s (guest)', $displayName);
+				}
+			}
+
+			if ($comment->getParentId() !== '0') {
+				// FIXME should add something?
+			}
+
+			$messages[] = $displayName . ': ' . $parsedMessage;
+			$nextOffset = (int)$comment->getId();
+		}
+
+		if (empty($messages)) {
+			return new DataResponse([], Http::STATUS_NO_CONTENT);
+		}
+
+		$task = new Task(
+			TextToTextSummary::ID,
+			['input' => implode("\n\n", $messages)],
+			Application::APP_ID,
+			$this->userId,
+			'summary/' . $this->room->getToken(),
+		);
+
+		try {
+			$this->taskProcessingManager->scheduleTask($task);
+		} catch (Exception $e) {
+			$this->logger->error('An error occurred while trying to summarize unread messages', ['exception' => $e]);
+			return new DataResponse([
+				'error' => ChatSummaryException::REASON_AI_ERROR,
+			], Http::STATUS_BAD_REQUEST);
+		}
+
+		$taskId = $task->getId();
+		if ($taskId === null) {
+			return new DataResponse([
+				'error' => ChatSummaryException::REASON_AI_ERROR,
+			], Http::STATUS_BAD_REQUEST);
+		}
+
+		$data = [
+			'taskId' => $taskId,
+		];
+
+		if ($nextOffset !== $this->room->getLastMessageId()) {
+			$data['nextOffset'] = $nextOffset;
+		}
+
+		return new DataResponse($data, Http::STATUS_CREATED);
+	}
+
 	/**
 	 * @return DataResponse<Http::STATUS_OK|Http::STATUS_NOT_MODIFIED, TalkChatMessageWithParent[], array{X-Chat-Last-Common-Read?: numeric-string, X-Chat-Last-Given?: numeric-string}>
 	 */

lib/Exceptions/ChatSummaryException.php

@@ -0,0 +1,30 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Talk\Exceptions;
+
+class ChatSummaryException extends \InvalidArgumentException {
+	public const REASON_NO_PROVIDER = 'ai-no-provider';
+	public const REASON_AI_ERROR = 'ai-error';
+
+	/**
+	 * @param self::REASON_* $reason
+	 */
+	public function __construct(
+		protected string $reason,
+	) {
+		parent::__construct($reason);
+	}
+
+	/**
+	 * @return self::REASON_*
+	 */
+	public function getReason(): string {
+		return $this->reason;
+	}
+}

lib/ResponseDefinitions.php

@@ -354,6 +354,7 @@
  *             read-privacy: int,
  *             has-translation-providers: bool,
  *             typing-privacy: int,
+ *             summary-threshold: positive-int,
  *         },
  *         conversations: array{
  *             can-create: bool,

openapi-administration.json

@@ -204,7 +204,8 @@
                                     "max-length",
                                     "read-privacy",
                                     "has-translation-providers",
-                                    "typing-privacy"
+                                    "typing-privacy",
+                                    "summary-threshold"
                                 ],
                                 "properties": {
                                     "max-length": {
@@ -221,6 +222,11 @@
                                     "typing-privacy": {
                                         "type": "integer",
                                         "format": "int64"
+                                    },
+                                    "summary-threshold": {
+                                        "type": "integer",
+                                        "format": "int64",
+                                        "minimum": 1
                                     }
                                 }
                             },

openapi-backend-recording.json

@@ -137,7 +137,8 @@
                                     "max-length",
                                     "read-privacy",
                                     "has-translation-providers",
-                                    "typing-privacy"
+                                    "typing-privacy",
+                                    "summary-threshold"
                                 ],
                                 "properties": {
                                     "max-length": {
@@ -154,6 +155,11 @@
                                     "typing-privacy": {
                                         "type": "integer",
                                         "format": "int64"
+                                    },
+                                    "summary-threshold": {
+                                        "type": "integer",
+                                        "format": "int64",
+                                        "minimum": 1
                                     }
                                 }
                             },

openapi-backend-signaling.json

@@ -137,7 +137,8 @@
                                     "max-length",
                                     "read-privacy",
                                     "has-translation-providers",
-                                    "typing-privacy"
+                                    "typing-privacy",
+                                    "summary-threshold"
                                 ],
                                 "properties": {
                                     "max-length": {
@@ -154,6 +155,11 @@
                                     "typing-privacy": {
                                         "type": "integer",
                                         "format": "int64"
+                                    },
+                                    "summary-threshold": {
+                                        "type": "integer",
+                                        "format": "int64",
+                                        "minimum": 1
                                     }
                                 }
                             },