From b121d46dcc53c968ac0a72ad1de341af07cec47c Mon Sep 17 00:00:00 2001 From: Jasper Spaans Date: Tue, 9 Jul 2019 09:57:40 +0200 Subject: [PATCH 1/4] Add ESORT/ESEARCH support. ESORT and ESEARCH are extensions to SORT and SEARCH allowing users to also get the number of messages (COUNT) matching their criteria, the MIN and MAX ids of those messages, and to also only return part of the message ids for more efficient pagination. --- imapclient/imapclient.py | 66 ++++++++++++++++++++++++++++++++++- imapclient/response_parser.py | 50 ++++++++++++++++++++++++++ tests/test_response_parser.py | 14 ++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/imapclient/imapclient.py b/imapclient/imapclient.py index f3be35d4..a0e760eb 100644 --- a/imapclient/imapclient.py +++ b/imapclient/imapclient.py @@ -24,7 +24,12 @@ from . import tls from .datetime_util import datetime_to_INTERNALDATE, format_criteria_date from .imap_utf7 import encode as encode_utf7, decode as decode_utf7 -from .response_parser import parse_response, parse_message_list, parse_fetch_response +from .response_parser import ( + parse_esearch_response, + parse_fetch_response, + parse_message_list, + parse_response, +) from .util import to_bytes, to_unicode, assert_imap_protocol, chunk xrange = moves.xrange @@ -886,6 +891,48 @@ def unsubscribe_folder(self, folder): """ return self._command_and_check('unsubscribe', self._normalise_folder(folder)) + @require_capability('ESEARCH') + def esearch(self, criteria='ALL', returns=None, charset=None): + """Performs a search using the ESEARCH syntax as defined in :rfc:`4731`. + + See the :py:meth:`.search` method below for what the *criteria* + argument should contain; *returns* should be a string that + contains items to be returned. + + Currently supported are: + ALL + PARTIAL first:last + MIN + MAX + COUNT + + Combinations are possible, except you cannot have both ALL and PARTIAL. + + An example value could be: 'PARTIAL 1:50 COUNT' + + This will return a dictionary with keys matching the item names. The values will + be parsed as you would expect them, meaning PARTIAL and ALL will be a list of ints, + and the remaining three will be ints. + + For PARTIAL and ALL, there will also be matching PARTIAL_RAW and ALL_RAW values, that + contain the list of messages as returned by the server. This might be a more compact + representation and can be fed easily to :py:meth:`.fetch` without having to + (re)serialize the ids. + + Note that ESEARCH is an extension to the IMAP4 standard so it + may not be supported by all IMAP servers. + """ + args = [] + if returns: + args.extend([b'RETURN', to_bytes('('+ returns+')')]) + + if charset: + args.extend([b'CHARSET', to_bytes(charset)]) + args.extend(_normalise_search_criteria(criteria, charset)) + + data = self._raw_command_untagged(b'SEARCH', args, response_name='ESEARCH') + return parse_esearch_response(data) + def search(self, criteria='ALL', charset=None): """Return a list of messages ids from the currently selected folder matching *criteria*. @@ -994,6 +1041,23 @@ def _search(self, criteria, charset): return parse_message_list(data) + @require_capability('ESORT') + def esort(self, sort_criteria, criteria='ALL', returns=None, charset='UTF-8'): + """Performs a search and sorts the result using ESORT as defined in :rfc:`5267`. + + See the :py:meth:`.sort` method below for what the *criteria* and + *sort_criteria* arguments should contain; for the *returns* argument see + :py:meth:`.esearch*`. + """ + args = [] + if returns: + args.extend([b'RETURN', to_bytes('(' + returns + ')')]) + args.append(_normalise_sort_criteria(sort_criteria)) + args.append(to_bytes(charset)) + args.extend(_normalise_search_criteria(criteria, charset)) + data = self._raw_command_untagged(b'SORT', args, response_name='ESEARCH') + return parse_esearch_response(data) + @require_capability('SORT') def sort(self, sort_criteria, criteria='ALL', charset='UTF-8'): """Return a list of message ids from the currently selected diff --git a/imapclient/response_parser.py b/imapclient/response_parser.py index 6939fbbd..404980d4 100644 --- a/imapclient/response_parser.py +++ b/imapclient/response_parser.py @@ -152,6 +152,56 @@ def parse_fetch_response(text, normalise_times=True, uid_is_key=True): return parsed_response +def parse_esearch_response(data): + """Parses the IMAP ESEARCH responses as returned by imaplib. + + These are generated by ESORT and ESEARCH queries. This function will return + a dictionary, with the keys matching the *returns* value. + + See :py:meth:`ImapClient.esearch` for more info. + """ + print(repr(data)) + retval = {} + it = iter(parse_response(data)) + try: + while 1: + bite = six.next(it) + if isinstance(bite, tuple) and len(bite) == 2 and bite[0] == b'TAG': + # FIXME: should verify we only consume messages matching our tag + continue + elif bite == b'UID': # this is just a marker that we are using UIDs, ignore it. + continue + elif bite == b'ALL': + message_bite = six.next(it) + retval[bite + b'_RAW'] = message_bite + retval[bite] = _parse_compact_message_list(message_bite) + elif bite == b'PARTIAL': + message_bite = six.next(it)[1] + retval[bite + b'_RAW'] = message_bite + retval[bite] = _parse_compact_message_list(message_bite) + else: + retval[bite] = six.next(it) + except StopIteration: + pass + + return retval + +def _parse_compact_message_list(message_bite): + messages = [] + for message_atom in message_bite.split(b','): + first_b, sep, last_b = message_atom.partition(b':') + first = _int_or_error(first_b, 'invalid ID') + if sep: + last = _int_or_error(last_b, 'invalid ID') + if last < first: # 10:12 is equivalent to 12:10 (!) + first, last = last, first + messages.extend(range(first, last+1)) + else: + messages.append(first) + return messages + + + def _int_or_error(value, error_text): try: return int(value) diff --git a/tests/test_response_parser.py b/tests/test_response_parser.py index 0e87381d..5b546d84 100644 --- a/tests/test_response_parser.py +++ b/tests/test_response_parser.py @@ -13,6 +13,7 @@ from imapclient.datetime_util import datetime_to_native from imapclient.fixed_offset import FixedOffset from imapclient.response_parser import ( + parse_esearch_response, parse_response, parse_message_list, parse_fetch_response, @@ -191,6 +192,19 @@ def test_modseq_interleaved(self): self.assertEqual(out.modseq, 9) +class TestParseEsearchRespons(unittest.TestCase): + def test_esort(self): + self.assertEqual(parse_esearch_response([b'(TAG "KFOO6") UID PARTIAL (1:5 68669,69520,68831,68835,66540) COUNT 2216']), + {b'COUNT': 2216, + b'PARTIAL': [68669, 69520, 68831, 68835, 66540], + b'PARTIAL_RAW': b'68669,69520,68831,68835,66540'}) + + def test_esearch(self): + self.assertEqual(parse_esearch_response([b'(TAG "GJHF5") UID PARTIAL (1:5 69574,69590,69605,69607:69608) COUNT 2216']), + {b'COUNT': 2216, + b'PARTIAL': [69574, 69590, 69605, 69607, 69608], + b'PARTIAL_RAW': b'69574,69590,69605,69607:69608'}) + class TestParseFetchResponse(unittest.TestCase): def test_basic(self): From 58963f539485691f62536ffec60dc6f7dbafeddb Mon Sep 17 00:00:00 2001 From: Jasper Spaans Date: Wed, 10 Jul 2019 16:59:47 +0200 Subject: [PATCH 2/4] Support ESEARCH/ESORT returning an empty set of messages. --- imapclient/imapclient.py | 1 + imapclient/response_parser.py | 3 +++ tests/test_response_parser.py | 7 +++++++ 3 files changed, 11 insertions(+) diff --git a/imapclient/imapclient.py b/imapclient/imapclient.py index a0e760eb..2d027f89 100644 --- a/imapclient/imapclient.py +++ b/imapclient/imapclient.py @@ -918,6 +918,7 @@ def esearch(self, criteria='ALL', returns=None, charset=None): contain the list of messages as returned by the server. This might be a more compact representation and can be fed easily to :py:meth:`.fetch` without having to (re)serialize the ids. + Note that if no messages match, the _RAW versions will be set to None. Note that ESEARCH is an extension to the IMAP4 standard so it may not be supported by all IMAP servers. diff --git a/imapclient/response_parser.py b/imapclient/response_parser.py index 404980d4..0bfb1953 100644 --- a/imapclient/response_parser.py +++ b/imapclient/response_parser.py @@ -187,6 +187,9 @@ def parse_esearch_response(data): return retval def _parse_compact_message_list(message_bite): + if message_bite is None: + return [] + messages = [] for message_atom in message_bite.split(b','): first_b, sep, last_b = message_atom.partition(b':') diff --git a/tests/test_response_parser.py b/tests/test_response_parser.py index 5b546d84..82b98bda 100644 --- a/tests/test_response_parser.py +++ b/tests/test_response_parser.py @@ -205,6 +205,13 @@ def test_esearch(self): b'PARTIAL': [69574, 69590, 69605, 69607, 69608], b'PARTIAL_RAW': b'69574,69590,69605,69607:69608'}) + def test_partial_no_result(self): + self.assertEqual(parse_esearch_response([b'(TAG "no-exist") UID PARTIAL (1:5 NIL) COUNT 0']), + {b'COUNT': 0, + b'PARTIAL': [], + b'PARTIAL_RAW': None}) + + class TestParseFetchResponse(unittest.TestCase): def test_basic(self): From 53ee89757ae53f4bb3ea16bc0f39fc04bb7d800d Mon Sep 17 00:00:00 2001 From: Jasper Spaans Date: Wed, 10 Jul 2019 17:41:05 +0200 Subject: [PATCH 3/4] Support ESEARCH/ESORT returning a single messages as well. --- imapclient/response_parser.py | 17 ++++++++++++++--- tests/test_response_parser.py | 5 +++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/imapclient/response_parser.py b/imapclient/response_parser.py index 0bfb1953..a4584bed 100644 --- a/imapclient/response_parser.py +++ b/imapclient/response_parser.py @@ -173,11 +173,11 @@ def parse_esearch_response(data): continue elif bite == b'ALL': message_bite = six.next(it) - retval[bite + b'_RAW'] = message_bite + retval[bite + b'_RAW'] = _raw_as_bytes(message_bite) retval[bite] = _parse_compact_message_list(message_bite) elif bite == b'PARTIAL': message_bite = six.next(it)[1] - retval[bite + b'_RAW'] = message_bite + retval[bite + b'_RAW'] = _raw_as_bytes(message_bite) retval[bite] = _parse_compact_message_list(message_bite) else: retval[bite] = six.next(it) @@ -186,10 +186,21 @@ def parse_esearch_response(data): return retval + +def _raw_as_bytes(raw): + if raw is None: + return None + elif isinstance(raw, int): + return str(raw).encode('ascii') + else: + return raw + + def _parse_compact_message_list(message_bite): if message_bite is None: return [] - + if isinstance(message_bite, int): + return [message_bite] messages = [] for message_atom in message_bite.split(b','): first_b, sep, last_b = message_atom.partition(b':') diff --git a/tests/test_response_parser.py b/tests/test_response_parser.py index 82b98bda..d6488e78 100644 --- a/tests/test_response_parser.py +++ b/tests/test_response_parser.py @@ -211,6 +211,11 @@ def test_partial_no_result(self): b'PARTIAL': [], b'PARTIAL_RAW': None}) + def test_partial_single_result(self): + self.assertEqual(parse_esearch_response([b'(TAG "one-result") UID PARTIAL (1:5 69573) COUNT 1']), + {b'COUNT': 1, + b'PARTIAL': [69573], + b'PARTIAL_RAW': b'69573'}) class TestParseFetchResponse(unittest.TestCase): From 9ab229a5f2f818e783d95f87d8ab5929d8bed92d Mon Sep 17 00:00:00 2001 From: Jasper Spaans Date: Wed, 10 Jul 2019 18:15:10 +0200 Subject: [PATCH 4/4] Remove debug print. --- imapclient/response_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/imapclient/response_parser.py b/imapclient/response_parser.py index a4584bed..1dc49f4f 100644 --- a/imapclient/response_parser.py +++ b/imapclient/response_parser.py @@ -160,7 +160,6 @@ def parse_esearch_response(data): See :py:meth:`ImapClient.esearch` for more info. """ - print(repr(data)) retval = {} it = iter(parse_response(data)) try: