Add filters in DID search

.github/actions/test/action.yml

@@ -6,7 +6,7 @@ runs:
       run: jlpm run eslint:check
       shell: bash
     - name: Run Jest
-      run: jlpm jest
+      run: jlpm jest -u
       shell: bash
     - name: Run Pytest
       run: pytest rucio_jupyterlab/tests/

.gitignore

@@ -123,3 +123,7 @@ dmypy.json
 
 # Yarn cache
 .yarn/
+
+# jest files
+src/__tests__/__snapshots__/
+junit.xml

jest.config.js

@@ -23,6 +23,6 @@ module.exports = {
     '!src/**/.ipynb_checkpoints/*'
   ],
   coverageReporters: ['lcov', 'text'],
-  testRegex: 'src/.*/.*.spec.ts[x]?$',
+  testRegex: 'src/\_\_tests\_\_/.*\.tsx?$',
   transformIgnorePatterns: [`/node_modules/(?!${esModules}).+`]
-};
+};

package.json

@@ -102,6 +102,7 @@
     "@types/mocha": "^10.0.6",
     "@types/react": "^18.0.26",
     "@types/react-addons-linked-state-mixin": "^0.14.22",
+    "@types/react-test-renderer": "^18.3.0",
     "@typescript-eslint/eslint-plugin": "^7.2.0",
     "@typescript-eslint/parser": "^7.2.0",
     "css-loader": "^6.7.1",
@@ -113,6 +114,7 @@
     "mkdirp": "^3.0.1",
     "npm-run-all": "^4.1.5",
     "prettier": "^3.0.0",
+    "react-test-renderer": "^18.3.1",
     "rimraf": "^5.0.1",
     "source-map-loader": "^1.0.2",
     "style-loader": "^3.3.1",

rucio_jupyterlab/_version.py

@@ -1,4 +1,4 @@
 # This file is auto-generated by Hatchling. As such, do not:
 #   - modify
 #   - track in version control e.g. be sure to add to .gitignore
-__version__ = VERSION = '0.10.0'
+__version__ = VERSION = '1.0.0'

rucio_jupyterlab/handlers/did_search.py

@@ -28,14 +28,20 @@ def __init__(self, namespace, rucio):
         self.rucio = rucio
         self.db = get_db()  # pylint: disable=invalid-name
 
-    def search_did(self, scope, name, search_type, limit):
+    def search_did(self, scope, name, search_type, filters, limit):
         wildcard_enabled = self.rucio.instance_config.get('wildcard_enabled', False)
 
         if ('*' in name or '%' in name) and not wildcard_enabled:
             raise WildcardDisallowedException()
 
-        dids = self.rucio.search_did(scope, name, search_type, limit)
+        dids = self.rucio.search_did(scope, name, search_type, filters, limit)
 
+        for did in dids:
+            if did['did_type'] is None:  # JSON plugin was used lacking data
+                metadata = self.rucio.get_metadata(scope, did['name'])[0]
+                did['did_type'] = f"DIDType.{metadata['did_type']}"
+                did['bytes'] = metadata['bytes']
+                did['length'] = metadata['length']
 
         def mapper(entry, _):
             return {
@@ -54,13 +60,14 @@ def get(self):
         namespace = self.get_query_argument('namespace')
         search_type = self.get_query_argument('type', 'collection')
         did = self.get_query_argument('did')
+        filters = self.get_query_argument('filters', default=None)
         rucio = self.rucio.for_instance(namespace)
 
         (scope, name) = did.split(':')
         handler = DIDSearchHandlerImpl(namespace, rucio)
 
         try:
-            dids = handler.search_did(scope, name, search_type, ROW_LIMIT)
+            dids = handler.search_did(scope, name, search_type, filters, ROW_LIMIT)
             self.finish(json.dumps(dids))
         except RucioAuthenticationException:
             self.set_status(401)

rucio_jupyterlab/rucio/rucio.py

@@ -8,6 +8,7 @@
 # - Muhammad Aditya Hilmy, <[email protected]>, 2020
 
 import logging
+import re
 import time
 import json
 from urllib.parse import urlencode, quote
@@ -16,6 +17,129 @@
 from .authenticators import RucioAuthenticationException, authenticate_userpass, authenticate_x509, authenticate_oidc
 
 
+def parse_did_filter_from_string_fe(input_string, name='*', type='collection', omit_name=False):
+    """
+    Parse DID filter string for the filter engine (fe).
+
+    Should adhere to the following conventions:
+    - ';' represents the logical OR operator
+    - ',' represents the logical AND operator
+    - all operators belong to set of (<=, >=, ==, !=, >, <, =)
+    - there should be no duplicate key+operator criteria.
+
+    One sided and compound inequalities are supported.
+
+    Sanity checking of input is left to the filter engine.
+
+    :param input_string: String containing the filter options.
+    :param name: DID name.
+    :param type: The type of the did: all(container, dataset, file), collection(dataset or container), dataset, container.
+    :param omit_name: omit addition of name to filters.
+    :return: list of dictionaries with each dictionary as a separate OR expression.
+    """
+    # lookup table unifying all comprehended operators to a nominal suffix.
+    # note that the order matters as the regex engine is eager, e.g. don't want to evaluate '<=' as '<' and '='.
+    operators_suffix_LUT = dict({
+        '≤': 'lte',
+        '≥': 'gte',
+        '==': '',
+        '≠': 'ne',
+        '>': 'gt',
+        '<': 'lt',
+        '=': ''
+    })
+
+    # lookup table mapping operator opposites, used to reverse compound inequalities.
+    operator_opposites_LUT = {
+        'lt': 'gt',
+        'lte': 'gte'
+    }
+    operator_opposites_LUT.update({op2: op1 for op1, op2 in operator_opposites_LUT.items()})
+
+    filters = []
+    if input_string:
+        or_groups = list(filter(None, input_string.split(';')))     # split <input_string> into OR clauses
+        for or_group in or_groups:
+            or_group = or_group.strip()
+            and_groups = list(filter(None, or_group.split(',')))    # split <or_group> into AND clauses
+            and_group_filters = {}
+            for and_group in and_groups:
+                and_group = and_group.strip()
+                # tokenise this AND clause using operators as delimiters.
+                tokenisation_regex = "({})".format('|'.join(operators_suffix_LUT.keys()))
+                and_group_split_by_operator = list(filter(None, re.split(tokenisation_regex, and_group)))
+                if len(and_group_split_by_operator) == 3:       # this is a one-sided inequality or expression
+                    key, operator, value = [token.strip() for token in and_group_split_by_operator]
+
+                    # substitute input operator with the nominal operator defined by the LUT, <operators_suffix_LUT>.
+                    operator_mapped = operators_suffix_LUT.get(operator)
+
+                    filter_key_full = key = "'{}'".format(key)
+                    if operator_mapped is not None:
+                        if operator_mapped:
+                            filter_key_full = "{}.{}".format(key, operator_mapped)
+                    else:
+                        raise ValueError("{} operator not understood.".format(operator_mapped))
+
+                    if filter_key_full in and_group_filters:
+                        raise ValueError(filter_key_full)
+                    else:
+                        if not is_numeric(value):
+                            value = "'{}'".format(value)
+                        and_group_filters[filter_key_full] = value
+                elif len(and_group_split_by_operator) == 5:     # this is a compound inequality
+                    value1, operator1, key, operator2, value2 = [token.strip() for token in and_group_split_by_operator]
+
+                    # substitute input operator with the nominal operator defined by the LUT, <operators_suffix_LUT>.
+                    operator1_mapped = operator_opposites_LUT.get(operators_suffix_LUT.get(operator1))
+                    operator2_mapped = operators_suffix_LUT.get(operator2)
+
+                    key = "'{}'".format(key)
+                    filter_key1_full = filter_key2_full = key
+                    if operator1_mapped is not None and operator2_mapped is not None:
+                        if operator1_mapped:    # ignore '' operator (maps from equals)
+                            filter_key1_full = "{}.{}".format(key, operator1_mapped)
+                        if operator2_mapped:    # ignore '' operator (maps from equals)
+                            filter_key2_full = "{}.{}".format(key, operator2_mapped)
+                    else:
+                        raise ValueError("{} operator not understood.".format(operator_mapped))
+
+                    if filter_key1_full in and_group_filters:
+                        raise ValueError(filter_key1_full)
+                    else:
+                        if not is_numeric(value1):
+                            value1 = "'{}'".format(value1)
+                        and_group_filters[filter_key1_full] = value1
+                    if filter_key2_full in and_group_filters:
+                        raise ValueError(filter_key2_full)
+                    else:
+                        if not is_numeric(value2):
+                            value2 = "'{}'".format(value2)
+                        and_group_filters[filter_key2_full] = value2
+                else:
+                    raise ValueError(and_group)
+
+            # add name key to each AND clause if it hasn't already been populated from the filter and <omit_name> not set.
+            if not omit_name and 'name' not in and_group_filters:
+                and_group_filters['name'] = name
+
+            filters.append(and_group_filters)
+    else:
+        if not omit_name:
+            filters.append({
+                'name': name
+            })
+    return filters, type
+
+
+def is_numeric(value):
+    try:
+        float(value)
+        return True
+    except ValueError:
+        return False
+
+
 class RucioAPI:
     rucio_auth_token_cache = dict()
 
@@ -55,16 +179,20 @@ def get_rses(self, rse_expression=None):
 
         return results
 
-    def search_did(self, scope, name, search_type='collection', limit=None):
+    def search_did(self, scope, name, search_type='collection', filters=None, limit=None):
         token = self._get_auth_token()
         headers = {'X-Rucio-Auth-Token': token}
 
         scope = quote(scope)
-        urlencoded_params = urlencode({
+        params = {
             'type': search_type,
             'long': '1',
             'name': name
-        })
+        }
+        if filters:
+            filters, _ = parse_did_filter_from_string_fe(filters, name=name)
+            params['filters'] = filters
+        urlencoded_params = urlencode(params)
 
         response = requests.get(url=f'{self.base_url}/dids/{scope}/dids/search?{urlencoded_params}', headers=headers, verify=self.rucio_ca_cert)
 
@@ -80,6 +208,23 @@ def search_did(self, scope, name, search_type='collection', limit=None):
 
         return results
 
+    def get_metadata(self, scope, name):
+        token = self._get_auth_token()
+        headers = {'X-Rucio-Auth-Token': token}
+
+        scope = quote(scope)
+        name = quote(name)
+
+        response = requests.get(url=f'{self.base_url}/dids/{scope}/{name}/meta', headers=headers, verify=self.rucio_ca_cert)
+
+        if response.text == '':
+            return []
+
+        lines = response.text.rstrip('\n').splitlines()
+        results = [json.loads(l) for l in lines]
+
+        return results
+
     def get_files(self, scope, name):
         token = self._get_auth_token()
         headers = {'X-Rucio-Auth-Token': token}

rucio_jupyterlab/tests/test_handler_did_search.py

@@ -28,9 +28,9 @@ def test_search_did__with_wildcard__wildcard_enabled__should_return_correct_resp
     ])
 
     handler = DIDSearchHandlerImpl(MOCK_ACTIVE_INSTANCE, rucio)
-    result = handler.search_did('scope', 'name*', 'all', 100)
+    result = handler.search_did('scope', 'name*', 'all', None, 100)
 
-    rucio.search_did.assert_called_once_with('scope', 'name*', 'all', 100)
+    rucio.search_did.assert_called_once_with('scope', 'name*', 'all', None, 100)
 
     expected = [
         {'did': 'scope:name1', 'size': None, 'type': 'container'},
@@ -51,9 +51,9 @@ def test_search_did__without_wildcard__wildcard_disabled__should_return_correct_
     ])
 
     handler = DIDSearchHandlerImpl(MOCK_ACTIVE_INSTANCE, rucio)
-    result = handler.search_did('scope', 'name', 'all', 100)
+    result = handler.search_did('scope', 'name', 'all', None, 100)
 
-    rucio.search_did.assert_called_once_with('scope', 'name', 'all', 100)
+    rucio.search_did.assert_called_once_with('scope', 'name', 'all', None, 100)
 
     expected = [
         {'did': 'scope:name1', 'size': None, 'type': 'container'},
@@ -76,8 +76,8 @@ def test_search_did__with_wildcard__wildcard_disabled__should_raise_exception(mo
     handler = DIDSearchHandlerImpl(MOCK_ACTIVE_INSTANCE, rucio)
 
     with pytest.raises(WildcardDisallowedException):
-        handler.search_did('scope', 'name*', 'all', 100)
-        rucio.search_did.assert_called_once_with('scope', 'name', 'all', 100)
+        handler.search_did('scope', 'name*', 'all', None, 100)
+        rucio.search_did.assert_called_once_with('scope', 'name', 'all', None, 100)
 
 
 def test_search_did__with_percent_wildcard__wildcard_disabled__should_raise_exception(mocker, rucio):
@@ -92,8 +92,8 @@ def test_search_did__with_percent_wildcard__wildcard_disabled__should_raise_exce
     handler = DIDSearchHandlerImpl(MOCK_ACTIVE_INSTANCE, rucio)
 
     with pytest.raises(WildcardDisallowedException):
-        handler.search_did('scope', 'name%', 'all', 100)
-        rucio.search_did.assert_called_once_with('scope', 'name', 'all', 100)
+        handler.search_did('scope', 'name%', 'all', None, 100)
+        rucio.search_did.assert_called_once_with('scope', 'name', 'all', None, 100)
 
 
 def test_get_handler__inputs_correct__should_not_error(mocker, rucio):
@@ -111,7 +111,7 @@ def mock_get_query_argument(key, default=None):
 
     class MockDIDSearchHandler(DIDSearchHandlerImpl):
         @staticmethod
-        def search_did(scope, name, search_type='all', limit=100):
+        def search_did(scope, name, search_type='all', filter=None, limit=100):
             return [
                 {'did': 'scope:name1', 'size': None, 'type': 'container'},
                 {'did': 'scope:name2', 'size': None, 'type': 'dataset'},
@@ -156,7 +156,7 @@ def mock_get_query_argument(key, default=None):
 
     class MockDIDSearchHandler(DIDSearchHandlerImpl):
         @staticmethod
-        def search_did(scope, name, search_type='all', limit=100):
+        def search_did(scope, name, search_type='all', filter=None, limit=100):
             raise WildcardDisallowedException()
 
     mocker.patch('rucio_jupyterlab.handlers.did_search.DIDSearchHandlerImpl', MockDIDSearchHandler)

tests/Actions.test.ts → src/__tests__/Actions.test.ts

@@ -9,13 +9,13 @@
  * - Muhammad Aditya Hilmy, <[email protected]>, 2020
  */
 
-jest.mock('../src/utils/ApiRequest');
-jest.mock('../src/stores/UIStore');
+jest.mock('../utils/ApiRequest');
+jest.mock('../stores/UIStore');
 
-import { requestAPI } from '../src/utils/ApiRequest';
-import { Actions } from '../src/utils/Actions';
-import { IInstance, IAttachedFile, IFileDIDDetails, RucioAuthCredentials } from '../src/types';
-import { UIStore } from '../src/stores/UIStore';
+import { requestAPI } from '../utils/ApiRequest';
+import { Actions } from '../utils/Actions';
+import { IInstance, IAttachedFile, IFileDIDDetails, RucioAuthCredentials } from '../types';
+import { UIStore } from '../stores/UIStore';
 
 describe('fetchInstancesConfig', () => {
     test('should call /instances endpoint', async () => {

tests/ApiRequest.test.ts → src/__tests__/ApiRequest.test.ts

@@ -9,7 +9,7 @@
  * - Muhammad Aditya Hilmy, <[email protected]>, 2020
  */
 
-import { requestAPI } from '../src/utils/ApiRequest';
+import { requestAPI } from '../utils/ApiRequest';
 
 jest.mock('@jupyterlab/services', () => ({
     ServerConnection: {

src/__tests__/ExploreTab.test.tsx

@@ -0,0 +1,14 @@
+import React from 'react';
+import { useState } from 'react';
+import renderer from 'react-test-renderer';
+import { ExploreTab } from '../components/@Explore/ExploreTab';
+
+it('empty Tab created', () => {
+
+  const component = renderer.create(
+    <ExploreTab />
+  );
+
+  let tree = component.toJSON();
+  expect(tree).toMatchSnapshot()
+});

tests/Helpers.test.ts → src/__tests__/Helpers.test.ts

@@ -9,8 +9,8 @@
  * - Muhammad Aditya Hilmy, <[email protected]>, 2020-2021
  */
 
-import { computeCollectionState, toHumanReadableSize, checkVariableNameValid } from '../src/utils/Helpers';
-import { IFileDIDDetails } from '../src/types';
+import { computeCollectionState, toHumanReadableSize, checkVariableNameValid } from '../utils/Helpers';
+import { IFileDIDDetails } from '../types';
 
 describe('toHumanReadableSize', () => {
     test('return bytes value', () => {