diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 216d489af4..741c093b16 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -46,10 +46,12 @@ use OCA\Mail\Listener\MessageKnownSinceListener; use OCA\Mail\Listener\MoveJunkListener; use OCA\Mail\Listener\NewMessagesNotifier; +use OCA\Mail\Listener\NewMessagesSummarizeListener; use OCA\Mail\Listener\OauthTokenRefreshListener; use OCA\Mail\Listener\OptionalIndicesListener; use OCA\Mail\Listener\OutOfOfficeListener; use OCA\Mail\Listener\SpamReportListener; +use OCA\Mail\Listener\TaskProcessingListener; use OCA\Mail\Listener\UserDeletedListener; use OCA\Mail\Notification\Notifier; use OCA\Mail\Provider\MailProvider; @@ -71,6 +73,7 @@ use OCP\AppFramework\Bootstrap\IRegistrationContext; use OCP\DB\Events\AddMissingIndicesEvent; use OCP\IServerContainer; +use OCP\TaskProcessing\Events\TaskSuccessfulEvent; use OCP\User\Events\OutOfOfficeChangedEvent; use OCP\User\Events\OutOfOfficeClearedEvent; use OCP\User\Events\OutOfOfficeEndedEvent; @@ -131,6 +134,7 @@ public function register(IRegistrationContext $context): void { $context->registerEventListener(MessageSentEvent::class, InteractionListener::class); $context->registerEventListener(NewMessagesSynchronized::class, MessageKnownSinceListener::class); $context->registerEventListener(NewMessagesSynchronized::class, NewMessagesNotifier::class); + $context->registerEventListener(NewMessagesSynchronized::class, NewMessagesSummarizeListener::class); $context->registerEventListener(SynchronizationEvent::class, AccountSynchronizedThreadUpdaterListener::class); $context->registerEventListener(UserDeletedEvent::class, UserDeletedListener::class); $context->registerEventListener(NewMessagesSynchronized::class, FollowUpClassifierListener::class); @@ -139,6 +143,7 @@ public function register(IRegistrationContext $context): void { $context->registerEventListener(OutOfOfficeChangedEvent::class, OutOfOfficeListener::class); $context->registerEventListener(OutOfOfficeClearedEvent::class, OutOfOfficeListener::class); $context->registerEventListener(OutOfOfficeScheduledEvent::class, OutOfOfficeListener::class); + $context->registerEventListener(TaskSuccessfulEvent::class, TaskProcessingListener::class); $context->registerMiddleWare(ErrorMiddleware::class); $context->registerMiddleWare(ProvisioningMiddleware::class); diff --git a/lib/Db/Message.php b/lib/Db/Message.php index 229c26f395..3376f3742f 100644 --- a/lib/Db/Message.php +++ b/lib/Db/Message.php @@ -58,6 +58,8 @@ * @method bool|null getFlagMdnsent() * @method void setPreviewText(?string $subject) * @method null|string getPreviewText() + * @method void setSummary(?string $summary) + * @method null|string getSummary() * @method void setUpdatedAt(int $time) * @method int getUpdatedAt() * @method bool isImipMessage() @@ -108,6 +110,7 @@ class Message extends Entity implements JsonSerializable { protected $flagImportant = false; protected $flagMdnsent; protected $previewText; + protected $summary; protected $imipMessage = false; protected $imipProcessed = false; protected $imipError = false; @@ -325,6 +328,7 @@ static function (Tag $tag) { 'threadRootId' => $this->getThreadRootId(), 'imipMessage' => $this->isImipMessage(), 'previewText' => $this->getPreviewText(), + 'summary' => $this->getSummary(), 'encrypted' => ($this->isEncrypted() === true), 'mentionsMe' => $this->getMentionsMe(), ]; diff --git a/lib/Listener/NewMessagesSummarizeListener.php b/lib/Listener/NewMessagesSummarizeListener.php new file mode 100644 index 0000000000..6728c47767 --- /dev/null +++ b/lib/Listener/NewMessagesSummarizeListener.php @@ -0,0 +1,51 @@ + + */ +class NewMessagesSummarizeListener implements IEventListener { + + public function __construct( + private LoggerInterface $logger, + private IMAPClientFactory $imapFactory, + private AiIntegrationsService $aiService, + private IMailManager $mailManager, + ) { + } + + public function handle(Event $event): void { + + if (!($event instanceof NewMessagesSynchronized)) { + return; + } + + try { + $this->aiService->summarizeMessages( + $event->getAccount(), + $event->getMessages(), + ); + } catch (ServiceException $e) { + $this->logger->error('Could not initiate a message summarize task(s): ' . $e->getMessage(), [ + 'exception' => $e, + ]); + } + } +} diff --git a/lib/Listener/TaskProcessingListener.php b/lib/Listener/TaskProcessingListener.php new file mode 100644 index 0000000000..5a08f78e2c --- /dev/null +++ b/lib/Listener/TaskProcessingListener.php @@ -0,0 +1,93 @@ + + */ +class TaskProcessingListener implements IEventListener { + + public function __construct( + private LoggerInterface $logger, + private MessageMapper $messageMapper, + ) { + } + + public function handle(Event $event): void { + + if (!($event instanceof TaskSuccessfulEvent)) { + return; + } + + $task = $event->getTask(); + + if ($task->getAppId() !== Application::APP_ID) { + return; + } + + if ($task->getTaskTypeId() !== TextToTextSummary::ID) { + return; + } + + if ($task->getCustomId() !== null) { + [$type, $id] = explode(':', $task->getCustomId()); + } else { + $this->logger->info('Error handling task processing event custom id missing', ['taskCustomId' => $task->getCustomId()]); + return; + } + if ($type === null || $id === null) { + $this->logger->info('Error handling task processing event custom id is invalid', ['taskCustomId' => $task->getCustomId()]); + return; + } + if ($task->getUserId() !== null) { + $userId = $task->getUserId(); + } else { + $this->logger->info('Error handling task processing event user id missing'); + return; + } + if ($task->getOutput() !== null) { + $output = $task->getOutput(); + if (isset($output['output']) && is_string($output['output'])) { + $summary = $output['output']; + } else { + $this->logger->info('Error handling task processing event output is invalid', ['taskOutput' => $output]); + return; + } + } else { + $this->logger->info('Error handling task processing event output missing'); + return; + } + + if ($type === 'message') { + $this->handleMessageSummary($userId, (int)$id, $summary); + } + + } + + private function handleMessageSummary(string $userId, int $id, string $summary): void { + $messages = $this->messageMapper->findByIds($userId, [$id], ''); + + if (count($messages) !== 1) { + return; + } + + $message = $messages[0]; + $message->setSummary(substr($summary, 0, 1024)); + $this->messageMapper->update($message); + } +} diff --git a/lib/Migration/Version4100Date20241209000000.php b/lib/Migration/Version4100Date20241209000000.php new file mode 100644 index 0000000000..6161edaafc --- /dev/null +++ b/lib/Migration/Version4100Date20241209000000.php @@ -0,0 +1,38 @@ +getTable('mail_messages'); + if (!$outboxTable->hasColumn('summary')) { + $outboxTable->addColumn('summary', Types::STRING, [ + 'length' => 1024, + 'notnull' => false, + ]); + } + return $schema; + } +} diff --git a/lib/Service/AiIntegrations/AiIntegrationsService.php b/lib/Service/AiIntegrations/AiIntegrationsService.php index afd41fa19d..6c72adaaf5 100644 --- a/lib/Service/AiIntegrations/AiIntegrationsService.php +++ b/lib/Service/AiIntegrations/AiIntegrationsService.php @@ -20,12 +20,18 @@ use OCA\Mail\Model\EventData; use OCA\Mail\Model\IMAPMessage; use OCP\IConfig; +use OCP\TaskProcessing\Exception\Exception as TaskProcessingException; +use OCP\TaskProcessing\IManager as TaskProcessingManager; +use OCP\TaskProcessing\Task as TaskProcessingTask; +use OCP\TaskProcessing\TaskTypes\TextToTextSummary; use OCP\TextProcessing\FreePromptTaskType; use OCP\TextProcessing\IManager; use OCP\TextProcessing\SummaryTaskType; use OCP\TextProcessing\Task; use Psr\Container\ContainerExceptionInterface; use Psr\Container\ContainerInterface; +use Psr\Log\LoggerInterface; + use function array_map; use function implode; use function in_array; @@ -33,20 +39,6 @@ class AiIntegrationsService { - /** @var ContainerInterface */ - private ContainerInterface $container; - - /** @var Cache */ - private Cache $cache; - - /** @var IMAPClientFactory */ - private IMAPClientFactory $clientFactory; - - /** @var IMailManager */ - private IMailManager $mailManager; - - private IConfig $config; - private const EVENT_DATA_PROMPT_PREAMBLE = <<container = $container; - $this->cache = $cache; - $this->clientFactory = $clientFactory; - $this->mailManager = $mailManager; - $this->config = $config; + public function __construct( + private ContainerInterface $container, + private LoggerInterface $logger, + private IConfig $config, + private Cache $cache, + private IMAPClientFactory $clientFactory, + private IMailManager $mailManager, + private TaskProcessingManager $taskProcessingManager, + ) { + } + + /** + * generates summary for each message + * + * @param Account $account + * @param array $messages + * + * @return void + */ + public function summarizeMessages(Account $account, array $messages): void { + try { + $this->taskProcessingManager->getPreferredProvider(TextToTextSummary::ID); + } catch (TaskProcessingException $e) { + $this->logger->info('No text summary provider available'); + return; + } + + $client = $this->clientFactory->getClient($account); + try { + foreach ($messages as $entry) { + + if (mb_strlen((string)$entry->getSummary()) !== 0) { + continue; + } + // retrieve full message from server + $userId = $account->getUserId(); + $mailboxId = $entry->getMailboxId(); + $messageLocalId = $entry->getId(); + $messageRemoteId = $entry->getUid(); + $mailbox = $this->mailManager->getMailbox($userId, $mailboxId); + $message = $this->mailManager->getImapMessage( + $client, + $account, + $mailbox, + $messageRemoteId, + true + ); + // skip message if it is encrypted + if ($message->isEncrypted()) { + continue; + } + // construct prompt and task + $messageBody = $message->getPlainBody(); + $prompt = "You are tasked with formulating a helpful summary of a email message. \r\n" . + "The summary should be less than 1024 characters. \r\n" . + "Here is the ***E-MAIL*** for which you must generate a helpful summary: \r\n" . + "***START_OF_E-MAIL***\r\n$messageBody\r\n***END_OF_E-MAIL***\r\n"; + $task = new TaskProcessingTask( + TextToTextSummary::ID, + [ + 'max_tokens' => 1024, + 'input' => $prompt, + ], + Application::APP_ID, + $userId, + 'message:' . (string)$messageLocalId + ); + $this->taskProcessingManager->scheduleTask($task); + } + } finally { + $client->logout(); + } } + /** * @param Account $account * @param string $threadId diff --git a/package-lock.json b/package-lock.json index 317703db1a..f785352ea5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "nextcloud-mail", - "version": "4.2.0-alpha.0", + "version": "4.2.0-alpha1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "nextcloud-mail", - "version": "4.2.0-alpha.0", + "version": "4.2.0-alpha1", "license": "agpl", "dependencies": { "@ckeditor/ckeditor5-alignment": "37.1.0", diff --git a/package.json b/package.json index 93b5e0db63..c080a8cee4 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "nextcloud-mail", "description": "Nextcloud Mail", - "version": "4.2.0-alpha.0", + "version": "4.2.0-alpha1", "author": "Christoph Wurst ", "license": "agpl", "private": true, diff --git a/src/components/Envelope.vue b/src/components/Envelope.vue index 6e5ce77cdc..b4666ec3a9 100644 --- a/src/components/Envelope.vue +++ b/src/components/Envelope.vue @@ -79,8 +79,9 @@
- {{ isEncrypted ? t('mail', 'Encrypted message') : data.previewText.trim() }} + :title="data.summary ? t('mail', 'This summary was AI generated') : null"> + + {{ isEncrypted ? t('mail', 'Encrypted message') : data.summary ? data.summary.trim() : data.previewText.trim() }}
@@ -337,6 +338,7 @@ import EnvelopeSkeleton from './EnvelopeSkeleton.vue' import AlertOctagonIcon from 'vue-material-design-icons/AlertOctagon.vue' import Avatar from './Avatar.vue' import IconCreateEvent from 'vue-material-design-icons/Calendar.vue' +import SparkleIcon from 'vue-material-design-icons/Creation.vue' import ClockOutlineIcon from 'vue-material-design-icons/ClockOutline.vue' import CheckIcon from 'vue-material-design-icons/Check.vue' import ChevronLeft from 'vue-material-design-icons/ChevronLeft.vue' @@ -406,6 +408,7 @@ export default { PlusIcon, TagIcon, TagModal, + SparkleIcon, Star, StarOutline, EmailRead, @@ -919,11 +922,21 @@ export default { } &__preview-text { color: var(--color-text-maxcontrast); - white-space: nowrap; overflow: hidden; - text-overflow: ellipsis; font-weight: initial; - flex: 1 1; + max-height: calc(var(--default-font-size) * var(--default-line-height) * 2); + + /* Weird CSS hacks to make text ellipsize without white-space: nowrap */ + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + + .material-design-icon { + display: inline; + + position: relative; + top: 2px; + } } } diff --git a/tests/Unit/Service/AiIntegrationsServiceTest.php b/tests/Unit/Service/AiIntegrationsServiceTest.php index ffa8ba4b1a..e7a4271294 100644 --- a/tests/Unit/Service/AiIntegrationsServiceTest.php +++ b/tests/Unit/Service/AiIntegrationsServiceTest.php @@ -22,6 +22,9 @@ use OCA\Mail\Service\AiIntegrations\Cache; use OCP\AppFramework\QueryException; use OCP\IConfig; +use OCP\TaskProcessing\Exception\Exception as TaskProcessingException; +use OCP\TaskProcessing\IManager as TaskProcessingManager; +use OCP\TaskProcessing\IProvider as TaskProcessingProvider; use OCP\TextProcessing\FreePromptTaskType; use OCP\TextProcessing\IManager; use OCP\TextProcessing\SummaryTaskType; @@ -30,30 +33,22 @@ use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\MockObject\UnknownTypeException; use Psr\Container\ContainerInterface; +use Psr\Log\NullLogger; + use function interface_exists; class AiIntegrationsServiceTest extends TestCase { - /** @var ContainerInterface|MockObject */ - private $container; - - /** @var IManager|MockObject */ - private $manager; - - /** @var AiIntegrationsService */ - private $aiIntegrationsService; - - /** @var Cache */ - private $cache; - - /** @var IMAPClientFactory|MockObject */ - private $clientFactory; - - /** @var IMailManager|MockObject */ - private $mailManager; - - /** @var IConfig|MockObject */ - private $config; + private ContainerInterface|MockObject $container; + private IManager|MockObject $manager; + private IConfig|MockObject $config; + private NullLogger|MockObject $logger; + private AiIntegrationsService $aiIntegrationsService; + private Cache|MockObject $cache; + private IMAPClientFactory|MockObject $clientFactory; + private IMailManager|MockObject $mailManager; + private TaskProcessingManager|MockObject $taskProcessingManager; + private TaskProcessingProvider|MockObject $taskProcessingProvider; protected function setUp(): void { parent::setUp(); @@ -64,17 +59,23 @@ protected function setUp(): void { $this->manager = null; } + $this->logger = $this->createMock(NullLogger::class); + $this->config = $this->createMock(IConfig::class); $this->cache = $this->createMock(Cache::class); $this->clientFactory = $this->createMock(IMAPClientFactory::class); $this->mailManager = $this->createMock(IMailManager::class); - $this->config = $this->createMock(IConfig::class); + $this->taskProcessingManager = $this->createMock(TaskProcessingManager::class); $this->aiIntegrationsService = new AiIntegrationsService( $this->container, + $this->logger, + $this->config, $this->cache, $this->clientFactory, $this->mailManager, - $this->config, + $this->taskProcessingManager, ); + + $this->taskProcessingProvider = $this->createMock(TaskProcessingProvider::class); } public function testSummarizeThreadNoBackend() { @@ -374,4 +375,157 @@ public function testGenerateEventData(): void { self::assertSame('* Q&A', $result->getDescription()); } + public function testSummarizeMessagesNoProvider() { + $account = new Account(new MailAccount()); + $message = new Message(); + $this->taskProcessingManager->expects(self::once()) + ->method('getPreferredProvider') + ->willThrowException(new TaskProcessingException()); + $this->logger->expects(self::once()) + ->method('info') + ->with('No text summary provider available'); + + $this->aiIntegrationsService->summarizeMessages($account, [$message]); + } + + public function testSummarizeMessagesContainsSummary() { + $mailAccount = new MailAccount(); + $mailAccount->setId(123); + $mailAccount->setEmail('user@domain.tld'); + $mailAccount->setInboundHost('127.0.0.1'); + $mailAccount->setInboundPort(993); + $mailAccount->setInboundSslMode('ssl'); + $mailAccount->setInboundUser('user@domain.tld'); + $mailAccount->setInboundPassword('encrypted'); + $account = new Account($mailAccount); + $message = new Message(); + $message->setSummary('Test Summary'); + + $this->taskProcessingManager->expects(self::once()) + ->method('getPreferredProvider') + ->willReturn($this->taskProcessingProvider); + $this->clientFactory->expects(self::once()) + ->method('getClient'); + + $this->aiIntegrationsService->summarizeMessages($account, [$message]); + } + + public function testSummarizeMessagesIsEncrypted() { + $mailAccount = new MailAccount(); + $mailAccount->setId(123); + $mailAccount->setUserId('user1'); + $mailAccount->setEmail('user1@domain.tld'); + $mailAccount->setInboundHost('127.0.0.1'); + $mailAccount->setInboundPort(993); + $mailAccount->setInboundSslMode('ssl'); + $mailAccount->setInboundUser('user1@domain.tld'); + $mailAccount->setInboundPassword('encrypted'); + $account = new Account($mailAccount); + + $mailBox = new Mailbox(); + $mailBox->setId(1); + + $message = new Message(); + $message->setId(1); + $message->setUid(100); + $message->setMailboxId(1); + + $imapClient = $this->clientFactory->getClient($account); + + $imapMessage = $this->createMock(IMAPMessage::class); + $imapMessage->expects(self::never()) + ->method('getPlainBody') + ->willReturn('This is a test message'); + $imapMessage->expects(self::once()) + ->method('isEncrypted')->willReturn(true); + + $this->taskProcessingManager->expects(self::once()) + ->method('getPreferredProvider') + ->willReturn($this->taskProcessingProvider); + $this->taskProcessingManager->expects(self::never()) + ->method('scheduleTask'); + + $this->clientFactory->expects(self::once()) + ->method('getClient'); + + $this->mailManager->expects(self::once()) + ->method('getMailbox') + ->with( + $account->getUserId(), + $message->getMailboxId() + ) + ->willReturn($mailBox); + $this->mailManager->expects(self::once()) + ->method('getImapMessage') + ->with( + $imapClient, + $account, + $mailBox, + $message->getUid(), + true + ) + ->willReturn($imapMessage); + + $this->aiIntegrationsService->summarizeMessages($account, [$message]); + } + + public function testSummarizeMessages() { + $mailAccount = new MailAccount(); + $mailAccount->setId(123); + $mailAccount->setUserId('user1'); + $mailAccount->setEmail('user1@domain.tld'); + $mailAccount->setInboundHost('127.0.0.1'); + $mailAccount->setInboundPort(993); + $mailAccount->setInboundSslMode('ssl'); + $mailAccount->setInboundUser('user1@domain.tld'); + $mailAccount->setInboundPassword('encrypted'); + $account = new Account($mailAccount); + + $mailBox = new Mailbox(); + $mailBox->setId(1); + + $message = new Message(); + $message->setId(1); + $message->setUid(100); + $message->setMailboxId(1); + + $imapClient = $this->clientFactory->getClient($account); + + $imapMessage = $this->createMock(IMAPMessage::class); + $imapMessage->expects(self::once()) + ->method('getPlainBody') + ->willReturn('This is a test message'); + $imapMessage->expects(self::once()) + ->method('isEncrypted')->willReturn(false); + + $this->taskProcessingManager->expects(self::once()) + ->method('getPreferredProvider') + ->willReturn($this->taskProcessingProvider); + $this->taskProcessingManager->expects(self::once()) + ->method('scheduleTask'); + + $this->clientFactory->expects(self::once()) + ->method('getClient'); + + $this->mailManager->expects(self::once()) + ->method('getMailbox') + ->with( + $account->getUserId(), + $message->getMailboxId() + ) + ->willReturn($mailBox); + $this->mailManager->expects(self::once()) + ->method('getImapMessage') + ->with( + $imapClient, + $account, + $mailBox, + $message->getUid(), + true + ) + ->willReturn($imapMessage); + + $this->aiIntegrationsService->summarizeMessages($account, [$message]); + } + }