Merge pull request #10465 from nextcloud/feat/ai-summery

feat: ai message summary

lib/AppInfo/Application.php

@@ -46,10 +46,12 @@
 use OCA\Mail\Listener\MessageKnownSinceListener;
 use OCA\Mail\Listener\MoveJunkListener;
 use OCA\Mail\Listener\NewMessagesNotifier;
+use OCA\Mail\Listener\NewMessagesSummarizeListener;
 use OCA\Mail\Listener\OauthTokenRefreshListener;
 use OCA\Mail\Listener\OptionalIndicesListener;
 use OCA\Mail\Listener\OutOfOfficeListener;
 use OCA\Mail\Listener\SpamReportListener;
+use OCA\Mail\Listener\TaskProcessingListener;
 use OCA\Mail\Listener\UserDeletedListener;
 use OCA\Mail\Notification\Notifier;
 use OCA\Mail\Provider\MailProvider;
@@ -71,6 +73,7 @@
 use OCP\AppFramework\Bootstrap\IRegistrationContext;
 use OCP\DB\Events\AddMissingIndicesEvent;
 use OCP\IServerContainer;
+use OCP\TaskProcessing\Events\TaskSuccessfulEvent;
 use OCP\User\Events\OutOfOfficeChangedEvent;
 use OCP\User\Events\OutOfOfficeClearedEvent;
 use OCP\User\Events\OutOfOfficeEndedEvent;
@@ -131,6 +134,7 @@ public function register(IRegistrationContext $context): void {
 		$context->registerEventListener(MessageSentEvent::class, InteractionListener::class);
 		$context->registerEventListener(NewMessagesSynchronized::class, MessageKnownSinceListener::class);
 		$context->registerEventListener(NewMessagesSynchronized::class, NewMessagesNotifier::class);
+		$context->registerEventListener(NewMessagesSynchronized::class, NewMessagesSummarizeListener::class);
 		$context->registerEventListener(SynchronizationEvent::class, AccountSynchronizedThreadUpdaterListener::class);
 		$context->registerEventListener(UserDeletedEvent::class, UserDeletedListener::class);
 		$context->registerEventListener(NewMessagesSynchronized::class, FollowUpClassifierListener::class);
@@ -139,6 +143,7 @@ public function register(IRegistrationContext $context): void {
 		$context->registerEventListener(OutOfOfficeChangedEvent::class, OutOfOfficeListener::class);
 		$context->registerEventListener(OutOfOfficeClearedEvent::class, OutOfOfficeListener::class);
 		$context->registerEventListener(OutOfOfficeScheduledEvent::class, OutOfOfficeListener::class);
+		$context->registerEventListener(TaskSuccessfulEvent::class, TaskProcessingListener::class);
 
 		$context->registerMiddleWare(ErrorMiddleware::class);
 		$context->registerMiddleWare(ProvisioningMiddleware::class);

lib/Db/Message.php

@@ -58,6 +58,8 @@
  * @method bool|null getFlagMdnsent()
  * @method void setPreviewText(?string $subject)
  * @method null|string getPreviewText()
+ * @method void setSummary(?string $summary)
+ * @method null|string getSummary()
  * @method void setUpdatedAt(int $time)
  * @method int getUpdatedAt()
  * @method bool isImipMessage()
@@ -108,6 +110,7 @@ class Message extends Entity implements JsonSerializable {
 	protected $flagImportant = false;
 	protected $flagMdnsent;
 	protected $previewText;
+	protected $summary;
 	protected $imipMessage = false;
 	protected $imipProcessed = false;
 	protected $imipError = false;
@@ -325,6 +328,7 @@ static function (Tag $tag) {
 			'threadRootId' => $this->getThreadRootId(),
 			'imipMessage' => $this->isImipMessage(),
 			'previewText' => $this->getPreviewText(),
+			'summary' => $this->getSummary(),
 			'encrypted' => ($this->isEncrypted() === true),
 			'mentionsMe' => $this->getMentionsMe(),
 		];

lib/Listener/NewMessagesSummarizeListener.php

@@ -0,0 +1,51 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Mail\Listener;
+
+use OCA\Mail\Contracts\IMailManager;
+use OCA\Mail\Events\NewMessagesSynchronized;
+use OCA\Mail\Exception\ServiceException;
+use OCA\Mail\IMAP\IMAPClientFactory;
+use OCA\Mail\Service\AiIntegrations\AiIntegrationsService;
+use OCP\EventDispatcher\Event;
+use OCP\EventDispatcher\IEventListener;
+use Psr\Log\LoggerInterface;
+
+/**
+ * @template-implements IEventListener<Event>
+ */
+class NewMessagesSummarizeListener implements IEventListener {
+
+	public function __construct(
+		private LoggerInterface $logger,
+		private IMAPClientFactory $imapFactory,
+		private AiIntegrationsService $aiService,
+		private IMailManager $mailManager,
+	) {
+	}
+
+	public function handle(Event $event): void {
+
+		if (!($event instanceof NewMessagesSynchronized)) {
+			return;
+		}
+
+		try {
+			$this->aiService->summarizeMessages(
+				$event->getAccount(),
+				$event->getMessages(),
+			);
+		} catch (ServiceException $e) {
+			$this->logger->error('Could not initiate a message summarize task(s): ' . $e->getMessage(), [
+				'exception' => $e,
+			]);
+		}
+	}
+}

lib/Listener/TaskProcessingListener.php

@@ -0,0 +1,93 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Mail\Listener;
+
+use OCA\Mail\AppInfo\Application;
+use OCA\Mail\Db\MessageMapper;
+use OCP\EventDispatcher\Event;
+use OCP\EventDispatcher\IEventListener;
+use OCP\TaskProcessing\Events\TaskSuccessfulEvent;
+use OCP\TaskProcessing\TaskTypes\TextToTextSummary;
+use Psr\Log\LoggerInterface;
+
+/**
+ * @template-implements IEventListener<Event>
+ */
+class TaskProcessingListener implements IEventListener {
+
+	public function __construct(
+		private LoggerInterface $logger,
+		private MessageMapper $messageMapper,
+	) {
+	}
+
+	public function handle(Event $event): void {
+
+		if (!($event instanceof TaskSuccessfulEvent)) {
+			return;
+		}
+
+		$task = $event->getTask();
+
+		if ($task->getAppId() !== Application::APP_ID) {
+			return;
+		}
+
+		if ($task->getTaskTypeId() !== TextToTextSummary::ID) {
+			return;
+		}
+
+		if ($task->getCustomId() !== null) {
+			[$type, $id] = explode(':', $task->getCustomId());
+		} else {
+			$this->logger->info('Error handling task processing event custom id missing', ['taskCustomId' => $task->getCustomId()]);
+			return;
+		}
+		if ($type === null || $id === null) {
+			$this->logger->info('Error handling task processing event custom id is invalid', ['taskCustomId' => $task->getCustomId()]);
+			return;
+		}
+		if ($task->getUserId() !== null) {
+			$userId = $task->getUserId();
+		} else {
+			$this->logger->info('Error handling task processing event user id missing');
+			return;
+		}
+		if ($task->getOutput() !== null) {
+			$output = $task->getOutput();
+			if (isset($output['output']) && is_string($output['output'])) {
+				$summary = $output['output'];
+			} else {
+				$this->logger->info('Error handling task processing event output is invalid', ['taskOutput' => $output]);
+				return;
+			}
+		} else {
+			$this->logger->info('Error handling task processing event output missing');
+			return;
+		}
+
+		if ($type === 'message') {
+			$this->handleMessageSummary($userId, (int)$id, $summary);
+		}
+
+	}
+
+	private function handleMessageSummary(string $userId, int $id, string $summary): void {
+		$messages = $this->messageMapper->findByIds($userId, [$id], '');
+
+		if (count($messages) !== 1) {
+			return;
+		}
+
+		$message = $messages[0];
+		$message->setSummary(substr($summary, 0, 1024));
+		$this->messageMapper->update($message);
+	}
+}

lib/Migration/Version4100Date20241209000000.php

@@ -0,0 +1,38 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\Mail\Migration;
+
+use Closure;
+use OCP\DB\ISchemaWrapper;
+use OCP\DB\Types;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+class Version4100Date20241209000000 extends SimpleMigrationStep {
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure(): ISchemaWrapper $schemaClosure
+	 * @param array $options
+	 * @return null|ISchemaWrapper
+	 */
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
+		$schema = $schemaClosure();
+
+		$outboxTable = $schema->getTable('mail_messages');
+		if (!$outboxTable->hasColumn('summary')) {
+			$outboxTable->addColumn('summary', Types::STRING, [
+				'length' => 1024,
+				'notnull' => false,
+			]);
+		}
+		return $schema;
+	}
+}

lib/Service/AiIntegrations/AiIntegrationsService.php

@@ -20,47 +20,106 @@
 use OCA\Mail\Model\EventData;
 use OCA\Mail\Model\IMAPMessage;
 use OCP\IConfig;
+use OCP\TaskProcessing\Exception\Exception as TaskProcessingException;
+use OCP\TaskProcessing\IManager as TaskProcessingManager;
+use OCP\TaskProcessing\Task as TaskProcessingTask;
+use OCP\TaskProcessing\TaskTypes\TextToTextSummary;
 use OCP\TextProcessing\FreePromptTaskType;
 use OCP\TextProcessing\IManager;
 use OCP\TextProcessing\SummaryTaskType;
 use OCP\TextProcessing\Task;
 use Psr\Container\ContainerExceptionInterface;
 use Psr\Container\ContainerInterface;
+use Psr\Log\LoggerInterface;
+
 use function array_map;
 use function implode;
 use function in_array;
 use function json_decode;
 
 class AiIntegrationsService {
 
-	/** @var ContainerInterface */
-	private ContainerInterface $container;
-
-	/** @var Cache */
-	private Cache $cache;
-
-	/** @var IMAPClientFactory */
-	private IMAPClientFactory $clientFactory;
-
-	/** @var IMailManager */
-	private IMailManager $mailManager;
-
-	private IConfig $config;
-
 	private const EVENT_DATA_PROMPT_PREAMBLE = <<<PROMPT
 I am scheduling an event based on an email thread and need an event title and agenda. Provide the result as JSON with keys for "title" and "agenda". For example ```{ "title": "Project kick-off meeting", "agenda": "* Introduction\\n* Project goals\\n* Next steps" }```.
 
 The email contents are:
 
 PROMPT;
 
-	public function __construct(ContainerInterface $container, Cache $cache, IMAPClientFactory $clientFactory, IMailManager $mailManager, IConfig $config) {
-		$this->container = $container;
-		$this->cache = $cache;
-		$this->clientFactory = $clientFactory;
-		$this->mailManager = $mailManager;
-		$this->config = $config;
+	public function __construct(
+		private ContainerInterface $container,
+		private LoggerInterface $logger,
+		private IConfig $config,
+		private Cache $cache,
+		private IMAPClientFactory $clientFactory,
+		private IMailManager $mailManager,
+		private TaskProcessingManager $taskProcessingManager,
+	) {
+	}
+
+	/**
+	 * generates summary for each message
+	 *
+	 * @param Account $account
+	 * @param array<Message> $messages
+	 *
+	 * @return void
+	 */
+	public function summarizeMessages(Account $account, array $messages): void {
+		try {
+			$this->taskProcessingManager->getPreferredProvider(TextToTextSummary::ID);
+		} catch (TaskProcessingException $e) {
+			$this->logger->info('No text summary provider available');
+			return;
+		}
+
+		$client = $this->clientFactory->getClient($account);
+		try {
+			foreach ($messages as $entry) {
+
+				if (mb_strlen((string)$entry->getSummary()) !== 0) {
+					continue;
+				}
+				// retrieve full message from server
+				$userId = $account->getUserId();
+				$mailboxId = $entry->getMailboxId();
+				$messageLocalId = $entry->getId();
+				$messageRemoteId = $entry->getUid();
+				$mailbox = $this->mailManager->getMailbox($userId, $mailboxId);
+				$message = $this->mailManager->getImapMessage(
+					$client,
+					$account,
+					$mailbox,
+					$messageRemoteId,
+					true
+				);
+				// skip message if it is encrypted
+				if ($message->isEncrypted()) {
+					continue;
+				}
+				// construct prompt and task
+				$messageBody = $message->getPlainBody();
+				$prompt = "You are tasked with formulating a helpful summary of a email message. \r\n" .
+						  "The summary should be less than 1024 characters. \r\n" .
+						  "Here is the ***E-MAIL*** for which you must generate a helpful summary: \r\n" .
+						  "***START_OF_E-MAIL***\r\n$messageBody\r\n***END_OF_E-MAIL***\r\n";
+				$task = new TaskProcessingTask(
+					TextToTextSummary::ID,
+					[
+						'max_tokens' => 1024,
+						'input' => $prompt,
+					],
+					Application::APP_ID,
+					$userId,
+					'message:' . (string)$messageLocalId
+				);
+				$this->taskProcessingManager->scheduleTask($task);
+			}
+		} finally {
+			$client->logout();
+		}
 	}
+
 	/**
 	 * @param Account $account
 	 * @param string $threadId

package.json

@@ -1,7 +1,7 @@
 {
   "name": "nextcloud-mail",
   "description": "Nextcloud Mail",
-  "version": "4.2.0-alpha.0",
+  "version": "4.2.0-alpha1",
   "author": "Christoph Wurst <[email protected]>",
   "license": "agpl",
   "private": true,