Merge pull request #7 from jotelha/2024-01-31-sorting

2024 01 31 sorting

.github/workflows/test.yml

@@ -24,7 +24,7 @@ jobs:
       uses: actions/checkout@v3
 
     - name: Set up python3 ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

dtool_lookup_server_search_plugin_mongo/__init__.py

@@ -1,3 +1,3 @@
 """dtool-lookup-server-search-plugin-mongo package."""
 
-__version__ = "0.1.0"
+__version__ = "0.2.0"

dtool_lookup_server_search_plugin_mongo/utils_search.py

@@ -7,13 +7,17 @@
 
 from pymongo import MongoClient
 
-from dtool_lookup_server import SearchABC, ValidationError
+from dtool_lookup_server import (
+    SearchABC, ValidationError, PaginationParameters, SortParameters)
 
 from dtool_lookup_server.date_utils import (
     extract_created_at_as_datetime,
     extract_frozen_at_as_datetime,
 )
 
+from dtool_lookup_server.sql_models import DatasetSchema
+from dtool_lookup_server.schemas import SearchDatasetSchema
+
 from dtool_lookup_server_search_plugin_mongo.config import (
     Config, CONFIG_SECRETS_TO_OBFUSCATE)
 
@@ -161,60 +165,44 @@ def register_dataset(self, dataset_info):
         except pymongo.errors.DocumentTooLarge as e:
             raise (ValidationError("Dataset has too much metadata: {}".format(e)))
 
-    def search(self, query):
+    def search(self, query: SearchDatasetSchema,
+               pagination_parameters: PaginationParameters = None,
+               sort_parameters: SortParameters = None) -> DatasetSchema(many=True):
 
         # Deal with edge case where a user has no access to any base URIs.
         if len(query["base_uris"]) == 0:
             return []
 
-        mongo_query = _dict_to_mongo_query(query)
-
-        cx = self.collection.find(
-            mongo_query,
-            {
-                "_id": False,
-                "readme": False,
-                "manifest": False,
-                "annotations": False,
-            },
-        )
-
-        datasets = []
-        for ds in cx:
-            # Convert datetime object to float timestamp.
-            for key in ("created_at", "frozen_at"):
-                datetime_obj = ds[key]
-                ds[key] = dtoolcore.utils.timestamp(datetime_obj)
-
-            datasets.append(ds)
-
-        return datasets
-
-    def lookup_uris(self, uuid, base_uris):
-
-        # Deal with edge case where a user has no access to any base URIs.
-        if len(base_uris) == 0:
-            return []
-
-        mongo_query = {
-            "uuid": uuid,
-            "$or": [{"base_uri": v} for v in base_uris]
-        }
-        cx = self.collection.find(
-            mongo_query,
-            {
-                "_id": False,
-                "uri": True,
-                "base_uri": True,
-                "uuid": True,
-                "name": True
-            },
-        )
+        mongo_sort = None
+        # assumes that order parameters are compatible with pymongo.ASCENDING and DESCENDING
+        if sort_parameters is not None:
+            mongo_sort = [(field, order) for field, order in sort_parameters.order().items()]
 
-        datasets = []
-        for ds in cx:
-            datasets.append(ds)
+        mongo_query = _dict_to_mongo_query(query)
+        mongo_projection = {
+                    "_id": False,
+                    "readme": False,
+                    "manifest": False,
+                    "annotations": False,
+                }
+
+        if pagination_parameters is None:
+            cx = self.collection.find(
+                filter=mongo_query,
+                projection=mongo_projection,
+                sort=mongo_sort
+            )
+        else:
+            pagination_parameters.item_count = self.collection.count_documents(filter=mongo_query)
+            cx = self.collection.find(
+                filter=mongo_query,
+                projection=mongo_projection,
+                sort=mongo_sort,
+                skip=(pagination_parameters.page-1)*pagination_parameters.page_size,
+                limit=pagination_parameters.page_size
+            )
 
+        datasets = [ds for ds in cx]
         return datasets
 
     def get_config(self):

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 
 url = "https://github.com/jic-dtool/dtool-lookup-server-search-plugin-mongo"
-version = "0.1.0"
+version = "0.2.0"
 readme = open('README.rst').read()
 
 setup(
@@ -10,6 +10,7 @@
     version=version,
     description="Search plugin for dtool-lookup-server using mongodb",
     long_description=readme,
+    long_description_content_type="text/x-rst",
     include_package_data=True,
     author="Tjelvar Olsson",
     author_email="[email protected]",

tests/conftest.py

@@ -30,7 +30,7 @@ def tmp_app_with_users(request):
     from dtool_lookup_server.utils import (
         register_users,
         register_base_uri,
-        update_permissions,
+        put_permissions,
     )
 
     tmp_mongo_db_name = random_string()
@@ -76,11 +76,10 @@ def tmp_app_with_users(request):
     register_base_uri(base_uri)
 
     permissions = {
-        "base_uri": base_uri,
         "users_with_search_permissions": ["grumpy", "sleepy"],
         "users_with_register_permissions": ["grumpy"]
     }
-    update_permissions(permissions)
+    put_permissions(base_uri, permissions)
 
     @request.addfinalizer
     def teardown():

tests/test_utils_search_standalone.py

@@ -90,64 +90,6 @@ class _MockApp(object):
 # Here are the tests
 ##############################################################################
 
-def test_lookup_uris(tmp_mongo_db):  # NOQA
-
-    ds_info_1 = create_dataset_info(
-        "s3://store1",
-        "apple-gala",
-        "---\ndescription: gala apples",
-        ["barrel1", "barrel2"],
-        ["red", "yellow"],
-        {"type": "fruit"},
-        "farmer"
-    )
-    ds_info_2 = update_base_uri(ds_info_1.copy(), "s3://store2")
-    uuid = ds_info_1["uuid"]
-
-    both_base_uris = ["s3://store1", "s3://store2"]
-    only_store2 = ["s3://store2"]
-
-    mongo_search = MongoSearch()
-    app = _MockApp()
-    app.config = {
-        "SEARCH_MONGO_URI": MONGO_URI,
-        "SEARCH_MONGO_DB": tmp_mongo_db,
-        "SEARCH_MONGO_COLLECTION": "datasets"
-    }
-    mongo_search.init_app(app)
-
-    # Should be empty. Nothing registered yet.
-    assert mongo_search.lookup_uris(uuid, both_base_uris) == []
-
-    # Register a dataset.
-    mongo_search.register_dataset(ds_info_1)
-    assert mongo_search.lookup_uris(uuid, both_base_uris) == [
-        {
-            "base_uri": ds_info_1["base_uri"],
-            "name": ds_info_1["name"],
-            "uri": ds_info_1["uri"],
-            "uuid": ds_info_1["uuid"]
-            }
-    ]
-
-    # Register the same dataset in different base URI
-    mongo_search.register_dataset(ds_info_2)
-    assert len(mongo_search.lookup_uris(uuid, both_base_uris)) == 2  # NOQA
-
-    # Make sure only the dataset from store2 is retrievd if limited
-    # that base URI.
-    assert mongo_search.lookup_uris(uuid, only_store2) == [
-        {
-            "base_uri": ds_info_2["base_uri"],
-            "name": ds_info_2["name"],
-            "uri": ds_info_2["uri"],
-            "uuid": ds_info_2["uuid"]
-            }
-    ]
-
-    # Make sure nothing is returned if there are no base URIs.
-    assert len(mongo_search.lookup_uris(uuid, [])) == 0  # NOQA
-
 
 def test_register_basic(tmp_mongo_db):  # NOQA