feat(chat): Add API to summarize chat messages

Signed-off-by: Joas Schilling <coding@schilljs.com>

appinfo/routes/routesChatController.php

@@ -21,6 +21,8 @@ return [
 	'ocs' => [
 		/** @see \OCA\Talk\Controller\ChatController::receiveMessages() */
 		['name' => 'Chat#receiveMessages', 'url' => '/api/{apiVersion}/chat/{token}', 'verb' => 'GET', 'requirements' => $requirements],
+		/** @see \OCA\Talk\Controller\ChatController::summarizeChat() */
+		['name' => 'Chat#summarizeChat', 'url' => '/api/{apiVersion}/chat/{token}/summarize', 'verb' => 'POST', 'requirements' => $requirements],
 		/** @see \OCA\Talk\Controller\ChatController::sendMessage() */
 		['name' => 'Chat#sendMessage', 'url' => '/api/{apiVersion}/chat/{token}', 'verb' => 'POST', 'requirements' => $requirements],
 		/** @see \OCA\Talk\Controller\ChatController::clearHistory() */

docs/capabilities.md

@@ -160,5 +160,6 @@
 * `archived-conversations` (local) - Conversations can be marked as archived which will hide them from the conversation list by default
 * `talk-polls-drafts` - Whether moderators can store and retrieve poll drafts
 * `download-call-participants` - Whether the endpoints for moderators to download the call participants is available
+* `chat-summary-api` (local) - Whether the endpoint to get summarized chat messages in a conversation is available
 * `config => call => start-without-media` (local) - Boolean, whether media should be disabled when starting or joining a conversation
 * `config => call => max-duration` - Integer, maximum call duration in seconds. Please note that this should only be used with system cron and with a reasonable high value, due to the expended duration until the background job ran.

lib/Capabilities.php

@@ -18,6 +18,8 @@ use OCP\ICacheFactory;
 use OCP\IConfig;
 use OCP\IUser;
 use OCP\IUserSession;
+use OCP\TaskProcessing\IManager as ITaskProcessingManager;
+use OCP\TaskProcessing\TaskTypes\TextToTextSummary;
 use OCP\Translation\ITranslationManager;
 use OCP\Util;
 
@@ -108,6 +110,12 @@ class Capabilities implements IPublicCapability {
 		'download-call-participants',
 	];
 
+	public const CONDITIONAL_FEATURES = [
+		'message-expiration',
+		'reactions',
+		'chat-summary-api',
+	];
+
 	public const LOCAL_FEATURES = [
 		'favorites',
 		'chat-read-status',
@@ -119,6 +127,7 @@ class Capabilities implements IPublicCapability {
 		'remind-me-later',
 		'note-to-self',
 		'archived-conversations',
+		'chat-summary-api',
 	];
 
 	public const LOCAL_CONFIGS = [
@@ -164,6 +173,7 @@ class Capabilities implements IPublicCapability {
 		protected IUserSession $userSession,
 		protected IAppManager $appManager,
 		protected ITranslationManager $translationManager,
+		protected ITaskProcessingManager $taskProcessingManager,
 		ICacheFactory $cacheFactory,
 	) {
 		$this->talkCache = $cacheFactory->createLocal('talk::');
@@ -300,6 +310,11 @@ class Capabilities implements IPublicCapability {
 			$capabilities['config']['call']['can-enable-sip'] = $this->talkConfig->canUserEnableSIP($user);
 		}
 
+		$supportedTaskTypes = $this->taskProcessingManager->getAvailableTaskTypes();
+		if (isset($supportedTaskTypes[TextToTextSummary::ID])) {
+			$capabilities['features'][] = 'chat-summary-api';
+		}
+
 		return [
 			'spreed' => $capabilities,
 		];

lib/Controller/ChatController.php

@@ -8,6 +8,7 @@ declare(strict_types=1);
 
 namespace OCA\Talk\Controller;
 
+use OCA\Talk\AppInfo\Application;
 use OCA\Talk\Chat\AutoComplete\SearchPlugin;
 use OCA\Talk\Chat\AutoComplete\Sorter;
 use OCA\Talk\Chat\ChatManager;
@@ -15,6 +16,7 @@ use OCA\Talk\Chat\MessageParser;
 use OCA\Talk\Chat\Notifier;
 use OCA\Talk\Chat\ReactionManager;
 use OCA\Talk\Exceptions\CannotReachRemoteException;
+use OCA\Talk\Exceptions\ChatSummaryException;
 use OCA\Talk\Federation\Authenticator;
 use OCA\Talk\GuestManager;
 use OCA\Talk\MatterbridgeManager;
@@ -51,6 +53,7 @@ use OCP\AppFramework\Http\Attribute\NoAdminRequired;
 use OCP\AppFramework\Http\Attribute\PublicPage;
 use OCP\AppFramework\Http\Attribute\UserRateLimit;
 use OCP\AppFramework\Http\DataResponse;
+use OCP\AppFramework\Services\IAppConfig;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\Collaboration\AutoComplete\IManager;
 use OCP\Collaboration\Collaborators\ISearchResult;
@@ -62,14 +65,20 @@ use OCP\IL10N;
 use OCP\IRequest;
 use OCP\IUserManager;
 use OCP\RichObjectStrings\InvalidObjectExeption;
+use OCP\RichObjectStrings\IRichTextFormatter;
 use OCP\RichObjectStrings\IValidator;
 use OCP\Security\ITrustedDomainHelper;
 use OCP\Security\RateLimiting\IRateLimitExceededException;
 use OCP\Share\Exceptions\ShareNotFound;
 use OCP\Share\IShare;
+use OCP\TaskProcessing\Exception\Exception;
+use OCP\TaskProcessing\IManager as ITaskProcessingManager;
+use OCP\TaskProcessing\Task;
+use OCP\TaskProcessing\TaskTypes\TextToTextSummary;
 use OCP\User\Events\UserLiveStatusEvent;
 use OCP\UserStatus\IManager as IUserStatusManager;
 use OCP\UserStatus\IUserStatus;
+use Psr\Log\LoggerInterface;
 
 /**
  * @psalm-import-type TalkChatMentionSuggestion from ResponseDefinitions
@@ -114,6 +123,10 @@ class ChatController extends AEnvironmentAwareController {
 		protected Authenticator $federationAuthenticator,
 		protected ProxyCacheMessageService $pcmService,
 		protected Notifier $notifier,
+		protected IRichTextFormatter $richTextFormatter,
+		protected ITaskProcessingManager $taskProcessingManager,
+		protected IAppConfig $appConfig,
+		protected LoggerInterface $logger,
 	) {
 		parent::__construct($appName, $request);
 	}
@@ -489,6 +502,138 @@ class ChatController extends AEnvironmentAwareController {
 		return $this->prepareCommentsAsDataResponse($comments, $lastCommonReadId);
 	}
 
+	/**
+	 * Summarize the next bunch of chat messages from a given offset
+	 *
+	 * Required capability: `chat-summary-api`
+	 *
+	 * @param positive-int $fromMessageId Offset from where on the summary should be generated
+	 * @return DataResponse<Http::STATUS_CREATED, array{taskId: int, nextOffset?: int}, array{}>|DataResponse<Http::STATUS_BAD_REQUEST, array{error: 'ai-no-provider'|'ai-error'}, array{}>|DataResponse<Http::STATUS_NO_CONTENT, array<empty>, array{}>
+	 * @throws \InvalidArgumentException
+	 *
+	 * 201: Summary was scheduled, use the returned taskId to get the status
+	 *   information and output from the TaskProcessing API:
+	 *   https://docs.nextcloud.com/server/latest/developer_manual/client_apis/OCS/ocs-taskprocessing-api.html#fetch-a-task-by-id
+	 *   If the response data contains nextOffset, not all messages could be handled in a single request.
+	 *   After receiving the response a second summary should be requested with the provided nextOffset.
+	 * 204: No messages found to summarize
+	 * 400: No AI provider available or summarizing failed
+	 */
+	#[PublicPage]
+	#[RequireModeratorOrNoLobby]
+	#[RequireParticipant]
+	public function summarizeChat(
+		int $fromMessageId,
+	): DataResponse {
+		$fromMessageId = max(0, $fromMessageId);
+
+		$supportedTaskTypes = $this->taskProcessingManager->getAvailableTaskTypes();
+		if (!isset($supportedTaskTypes[TextToTextSummary::ID])) {
+			return new DataResponse([
+				'error' => ChatSummaryException::REASON_AI_ERROR,
+			], Http::STATUS_BAD_REQUEST);
+		}
+
+		// if ($this->room->isFederatedConversation()) {
+		// /** @var \OCA\Talk\Federation\Proxy\TalkV1\Controller\ChatController $proxy */
+		// $proxy = \OCP\Server::get(\OCA\Talk\Federation\Proxy\TalkV1\Controller\ChatController::class);
+		// return $proxy->summarizeChat(
+		// $this->room,
+		// $this->participant,
+		// $fromMessageId,
+		// );
+		// }
+
+		$currentUser = $this->userManager->get($this->userId);
+		$batchSize = $this->appConfig->getAppValueInt('ai_unread_summary_batch_size', 500);
+		$comments = $this->chatManager->waitForNewMessages($this->room, $fromMessageId, $batchSize, 0, $currentUser, true, false);
+		$this->preloadShares($comments);
+
+		$messages = [];
+		$nextOffset = 0;
+		foreach ($comments as $comment) {
+			$message = $this->messageParser->createMessage($this->room, $this->participant, $comment, $this->l);
+			$this->messageParser->parseMessage($message);
+
+			if (!$message->getVisibility()) {
+				continue;
+			}
+
+			if ($message->getMessageType() === ChatManager::VERB_SYSTEM
+				&& !in_array($message->getMessageRaw(), [
+					'call_ended',
+					'call_ended_everyone',
+					'file_shared',
+					'object_shared',
+				], true)) {
+				// Ignore system messages apart from calls, shared objects and files
+				continue;
+			}
+
+			$parsedMessage = $this->richTextFormatter->richToParsed(
+				$message->getMessage(),
+				$message->getMessageParameters(),
+			);
+
+			$displayName = $message->getActorDisplayName();
+			if (in_array($message->getActorType(), [
+				Attendee::ACTOR_GUESTS,
+				Attendee::ACTOR_EMAILS,
+			], true)) {
+				if ($displayName === '') {
+					$displayName = $this->l->t('Guest');
+				} else {
+					$displayName = $this->l->t('%s (guest)', $displayName);
+				}
+			}
+
+			if ($comment->getParentId() !== '0') {
+				// FIXME should add something?
+			}
+
+			$messages[] = $displayName . ': ' . $parsedMessage;
+			$nextOffset = (int)$comment->getId();
+		}
+
+		if (empty($messages)) {
+			return new DataResponse([], Http::STATUS_NO_CONTENT);
+		}
+
+		$task = new Task(
+			TextToTextSummary::ID,
+			['input' => implode("\n\n", $messages)],
+			Application::APP_ID,
+			$this->userId,
+			'summary/' . $this->room->getToken(),
+		);
+
+		try {
+			$this->taskProcessingManager->scheduleTask($task);
+		} catch (Exception $e) {
+			$this->logger->error('An error occurred while trying to summarize unread messages', ['exception' => $e]);
+			return new DataResponse([
+				'error' => ChatSummaryException::REASON_AI_ERROR,
+			], Http::STATUS_BAD_REQUEST);
+		}
+
+		$taskId = $task->getId();
+		if ($taskId === null) {
+			return new DataResponse([
+				'error' => ChatSummaryException::REASON_AI_ERROR,
+			], Http::STATUS_BAD_REQUEST);
+		}
+
+		$data = [
+			'taskId' => $taskId,
+		];
+
+		if ($nextOffset !== $this->room->getLastMessageId()) {
+			$data['nextOffset'] = $nextOffset;
+		}
+
+		return new DataResponse($data, Http::STATUS_CREATED);
+	}
+
 	/**
 	 * @return DataResponse<Http::STATUS_OK|Http::STATUS_NOT_MODIFIED, TalkChatMessageWithParent[], array{X-Chat-Last-Common-Read?: numeric-string, X-Chat-Last-Given?: numeric-string}>
 	 */

lib/Exceptions/ChatSummaryException.php

@@ -0,0 +1,30 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Talk\Exceptions;
+
+class ChatSummaryException extends \InvalidArgumentException {
+	public const REASON_NO_PROVIDER = 'ai-no-provider';
+	public const REASON_AI_ERROR = 'ai-error';
+
+	/**
+	 * @param self::REASON_* $reason
+	 */
+	public function __construct(
+		protected string $reason,
+	) {
+		parent::__construct($reason);
+	}
+
+	/**
+	 * @return self::REASON_*
+	 */
+	public function getReason(): string {
+		return $this->reason;
+	}
+}

tests/php/CapabilitiesTest.php

@@ -21,6 +21,9 @@ use OCP\ICacheFactory;
 use OCP\IConfig;
 use OCP\IUser;
 use OCP\IUserSession;
+use OCP\TaskProcessing\IManager as ITaskProcessingManager;
+use OCP\TaskProcessing\TaskTypes\TextToTextFormalization;
+use OCP\TaskProcessing\TaskTypes\TextToTextSummary;
 use OCP\Translation\ITranslationManager;
 use PHPUnit\Framework\MockObject\MockObject;
 use Test\TestCase;
@@ -33,6 +36,7 @@ class CapabilitiesTest extends TestCase {
 	protected IUserSession&MockObject $userSession;
 	protected IAppManager&MockObject $appManager;
 	protected ITranslationManager&MockObject $translationManager;
+	protected ITaskProcessingManager&MockObject $taskProcessingManager;
 	protected ICacheFactory&MockObject $cacheFactory;
 	protected ICache&MockObject $talkCache;
 
@@ -45,6 +49,7 @@ class CapabilitiesTest extends TestCase {
 		$this->userSession = $this->createMock(IUserSession::class);
 		$this->appManager = $this->createMock(IAppManager::class);
 		$this->translationManager = $this->createMock(ITranslationManager::class);
+		$this->taskProcessingManager = $this->createMock(ITaskProcessingManager::class);
 		$this->cacheFactory = $this->createMock(ICacheFactory::class);
 		$this->talkCache = $this->createMock(ICache::class);
 
@@ -62,8 +67,8 @@ class CapabilitiesTest extends TestCase {
 			->willReturn('1.2.3');
 	}
 
-	public function testGetCapabilitiesGuest(): void {
-		$capabilities = new Capabilities(
+	protected function getCapabilities(): Capabilities {
+		return new Capabilities(
 			$this->serverConfig,
 			$this->talkConfig,
 			$this->appConfig,
@@ -71,8 +76,13 @@ class CapabilitiesTest extends TestCase {
 			$this->userSession,
 			$this->appManager,
 			$this->translationManager,
+			$this->taskProcessingManager,
 			$this->cacheFactory,
 		);
+	}
+
+	public function testGetCapabilitiesGuest(): void {
+		$capabilities = $this->getCapabilities();
 
 		$this->userSession->expects($this->once())
 			->method('getUser')
@@ -172,16 +182,7 @@ class CapabilitiesTest extends TestCase {
 	 * @dataProvider dataGetCapabilitiesUserAllowed
 	 */
 	public function testGetCapabilitiesUserAllowed(bool $isNotAllowed, bool $canCreate, string $quota, bool $canUpload, int $readPrivacy): void {
-		$capabilities = new Capabilities(
-			$this->serverConfig,
-			$this->talkConfig,
-			$this->appConfig,
-			$this->commentsManager,
-			$this->userSession,
-			$this->appManager,
-			$this->translationManager,
-			$this->cacheFactory,
-		);
+		$capabilities = $this->getCapabilities();
 
 		$user = $this->createMock(IUser::class);
 		$user->expects($this->atLeastOnce())
@@ -218,6 +219,9 @@ class CapabilitiesTest extends TestCase {
 		$user->method('getQuota')
 			->willReturn($quota);
 
+		$this->taskProcessingManager->method('getAvailableTaskTypes')
+			->willReturn([TextToTextSummary::ID => true]);
+
 		$this->serverConfig->expects($this->any())
 			->method('getAppValue')
 			->willReturnMap([
@@ -236,6 +240,7 @@ class CapabilitiesTest extends TestCase {
 					Capabilities::FEATURES, [
 						'message-expiration',
 						'reactions',
+						'chat-summary-api',
 					]
 				),
 				'features-local' => Capabilities::LOCAL_FEATURES,
@@ -293,19 +298,35 @@ class CapabilitiesTest extends TestCase {
 				'version' => '1.2.3',
 			],
 		], $data);
+	}
 
-		foreach ($data['spreed']['features'] as $feature) {
+	public function testCapabilitiesDocumentation(): void {
+		foreach (Capabilities::FEATURES as $feature) {
 			$suffix = '';
-			if (in_array($feature, $data['spreed']['features-local'])) {
+			if (in_array($feature, Capabilities::LOCAL_FEATURES)) {
 				$suffix = ' (local)';
 			}
 			$this->assertCapabilityIsDocumented("`$feature`" . $suffix);
 		}
 
-		foreach ($data['spreed']['config'] as $feature => $configs) {
-			foreach ($configs as $config => $configData) {
+		foreach (Capabilities::CONDITIONAL_FEATURES as $feature) {
+			$suffix = '';
+			if (in_array($feature, Capabilities::LOCAL_FEATURES)) {
+				$suffix = ' (local)';
+			}
+			$this->assertCapabilityIsDocumented("`$feature`" . $suffix);
+		}
+
+		$openapi = json_decode(file_get_contents(__DIR__ . '/../../openapi.json'), true, flags: JSON_THROW_ON_ERROR);
+		$configDefinition = $openapi['components']['schemas']['Capabilities']['properties']['config']['properties'] ?? null;
+		$this->assertIsArray($configDefinition, 'Failed to read Capabilities config from openapi.json');
+
+		$configFeatures = array_keys($configDefinition);
+
+		foreach ($configFeatures as $feature) {
+			foreach (array_keys($configDefinition[$feature]['properties']) as $config) {
 				$suffix = '';
-				if (isset($data['spreed']['config-local'][$feature]) && in_array($config, $data['spreed']['config-local'][$feature])) {
+				if (isset($data['spreed']['config-local'][$feature]) && in_array($config, Capabilities::LOCAL_CONFIGS[$feature])) {
 					$suffix = ' (local)';
 				}
 				$this->assertCapabilityIsDocumented("`config => $feature => $config`" . $suffix);
@@ -319,16 +340,7 @@ class CapabilitiesTest extends TestCase {
 	}
 
 	public function testGetCapabilitiesUserDisallowed(): void {
-		$capabilities = new Capabilities(
-			$this->serverConfig,
-			$this->talkConfig,
-			$this->appConfig,
-			$this->commentsManager,
-			$this->userSession,
-			$this->appManager,
-			$this->translationManager,
-			$this->cacheFactory,
-		);
+		$capabilities = $this->getCapabilities();
 
 		$user = $this->createMock(IUser::class);
 		$this->userSession->expects($this->once())
@@ -345,16 +357,7 @@ class CapabilitiesTest extends TestCase {
 	}
 
 	public function testCapabilitiesHelloV2Key(): void {
-		$capabilities = new Capabilities(
-			$this->serverConfig,
-			$this->talkConfig,
-			$this->appConfig,
-			$this->commentsManager,
-			$this->userSession,
-			$this->appManager,
-			$this->translationManager,
-			$this->cacheFactory,
-		);
+		$capabilities = $this->getCapabilities();
 
 		$this->talkConfig->expects($this->once())
 			->method('getSignalingTokenPublicKey')
@@ -375,16 +378,7 @@ class CapabilitiesTest extends TestCase {
 	 * @dataProvider dataTestConfigRecording
 	 */
 	public function testConfigRecording(bool $enabled): void {
-		$capabilities = new Capabilities(
-			$this->serverConfig,
-			$this->talkConfig,
-			$this->appConfig,
-			$this->commentsManager,
-			$this->userSession,
-			$this->appManager,
-			$this->translationManager,
-			$this->cacheFactory,
-		);
+		$capabilities = $this->getCapabilities();
 
 		$this->talkConfig->expects($this->once())
 			->method('isRecordingEnabled')
@@ -395,16 +389,7 @@ class CapabilitiesTest extends TestCase {
 	}
 
 	public function testCapabilitiesTranslations(): void {
-		$capabilities = new Capabilities(
-			$this->serverConfig,
-			$this->talkConfig,
-			$this->appConfig,
-			$this->commentsManager,
-			$this->userSession,
-			$this->appManager,
-			$this->translationManager,
-			$this->cacheFactory,
-		);
+		$capabilities = $this->getCapabilities();
 
 		$this->translationManager->method('hasProviders')
 			->willReturn(true);
@@ -412,4 +397,14 @@ class CapabilitiesTest extends TestCase {
 		$data = json_decode(json_encode($capabilities->getCapabilities(), JSON_THROW_ON_ERROR), true);
 		$this->assertEquals(true, $data['spreed']['config']['chat']['has-translation-providers']);
 	}
+
+	public function testSummaryTaskProviders(): void {
+		$capabilities = $this->getCapabilities();
+
+		$this->taskProcessingManager->method('getAvailableTaskTypes')
+			->willReturn([TextToTextFormalization::ID => true]);
+
+		$data = json_decode(json_encode($capabilities->getCapabilities(), JSON_THROW_ON_ERROR), true);
+		$this->assertNotContains('chat-summary-api', $data['spreed']['features']);
+	}
 }

tests/php/Controller/ChatControllerTest.php

@@ -34,6 +34,7 @@ use OCA\Talk\Share\RoomShareProvider;
 use OCP\App\IAppManager;
 use OCP\AppFramework\Http;
 use OCP\AppFramework\Http\DataResponse;
+use OCP\AppFramework\Services\IAppConfig;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\Collaboration\AutoComplete\IManager;
 use OCP\Collaboration\Collaborators\ISearchResult;
@@ -43,12 +44,15 @@ use OCP\IL10N;
 use OCP\IRequest;
 use OCP\IUser;
 use OCP\IUserManager;
+use OCP\RichObjectStrings\IRichTextFormatter;
 use OCP\RichObjectStrings\IValidator;
 use OCP\Security\ITrustedDomainHelper;
+use OCP\TaskProcessing\IManager as ITaskProcessingManager;
 use OCP\UserStatus\IManager as IUserStatusManager;
 use PHPUnit\Framework\Assert;
 use PHPUnit\Framework\Constraint\Callback;
 use PHPUnit\Framework\MockObject\MockObject;
+use Psr\Log\LoggerInterface;
 use Test\TestCase;
 
 class ChatControllerTest extends TestCase {
@@ -81,6 +85,10 @@ class ChatControllerTest extends TestCase {
 	private Authenticator&MockObject $federationAuthenticator;
 	private ProxyCacheMessageService&MockObject $pcmService;
 	private Notifier&MockObject $notifier;
+	private IRichTextFormatter&MockObject $richTextFormatter;
+	private ITaskProcessingManager&MockObject $taskProcessingManager;
+	private IAppConfig&MockObject $appConfig;
+	private LoggerInterface&MockObject $logger;
 
 	protected Room&MockObject $room;
 
@@ -121,6 +129,10 @@ class ChatControllerTest extends TestCase {
 		$this->federationAuthenticator = $this->createMock(Authenticator::class);
 		$this->pcmService = $this->createMock(ProxyCacheMessageService::class);
 		$this->notifier = $this->createMock(Notifier::class);
+		$this->richTextFormatter = $this->createMock(IRichTextFormatter::class);
+		$this->taskProcessingManager = $this->createMock(ITaskProcessingManager::class);
+		$this->appConfig = $this->createMock(IAppConfig::class);
+		$this->logger = $this->createMock(LoggerInterface::class);
 
 		$this->room = $this->createMock(Room::class);
 
@@ -167,6 +179,10 @@ class ChatControllerTest extends TestCase {
 			$this->federationAuthenticator,
 			$this->pcmService,
 			$this->notifier,
+			$this->richTextFormatter,
+			$this->taskProcessingManager,
+			$this->appConfig,
+			$this->logger,
 		);
 	}