add iter_keys_upto_delimiter fastpath for FS store

simplekv/fs.py

@@ -2,6 +2,7 @@
 # coding=utf8
 
 import os
+import os.path
 import shutil
 
 from . import KeyValueStore, UrlMixin, CopyMixin
@@ -153,6 +154,39 @@ def keys(self, prefix=u""):
     def iter_keys(self, prefix=u""):
         return iter(self.keys(prefix))
 
+    def iter_keys_upto_delimiter(self, delimiter, prefix=u""):
+        if delimiter != os.sep:
+            return super(FilesystemStore, self).iter_keys_upto_delimiter(
+                delimiter,
+                prefix,
+            )
+        return self._iter_keys_upto_delimiter_efficient(delimiter, prefix)
+
+    def _iter_keys_upto_delimiter_efficient(self, delimiter, prefix=u""):
+        if delimiter in prefix:
+            pos = prefix.rfind(delimiter)
+            search_prefix = prefix[:pos]
+            path = os.path.join(self.root, search_prefix)
+        else:
+            search_prefix = None
+            path = self.root
+
+        try:
+            for k in os.listdir(path):
+                subpath = os.path.join(path, k)
+
+                if search_prefix is not None:
+                    k = os.path.join(search_prefix, k)
+
+                if os.path.isdir(subpath):
+                    k += delimiter
+
+                if k.startswith(prefix):
+                    yield k
+        except OSError:
+            # path does not exists
+            pass
+
 
 class WebFilesystemStore(FilesystemStore):
     """FilesystemStore that supports generating URLs suitable for web

tests/test_filesystem_store.py

@@ -173,3 +173,41 @@ def store(self, tmpdir):
         class ExtendedKeyspaceStore(ExtendedKeyspaceMixin, FilesystemStore):
             pass
         return ExtendedKeyspaceStore(tmpdir)
+
+    def test_key_iterator_upto_delimiter_ossep(self, store, value):
+        delimiter = u"X"
+        for k in [
+            u"a1" + os.sep + u"b1",
+            u"a1" + os.sep + u"b1",
+            u"a2" + os.sep + u"b1",
+            u"a3",
+            u"a4" + os.sep + u"b1" + os.sep + u"c1",
+            u"a4" + os.sep + u"b1" + os.sep + u"c2",
+            u"a4" + os.sep + u"b2" + os.sep + u"c1",
+            u"a4" + os.sep + u"b3",
+        ]:
+            store.put(k, value)
+
+        l = sorted(store.iter_keys_upto_delimiter(os.sep))
+        assert l == [
+            u"a1" + os.sep,
+            u"a2" + os.sep,
+            u"a3",
+            u"a4" + os.sep,
+        ]
+
+        l = sorted(store.iter_keys_upto_delimiter(
+            os.sep,
+            prefix=u"a4" + os.sep,
+        ))
+        assert l == [
+            u"a4" + os.sep + "b1" + os.sep,
+            u"a4" + os.sep + "b2" + os.sep,
+            u"a4" + os.sep + "b3",
+        ]
+
+        l = sorted(store.iter_keys_upto_delimiter(
+            os.sep,
+            prefix=u"foo" + os.sep,
+        ))
+        assert l == []