Fixes #325

Implemented first version of FullSearchQueryParser to tokemize user input
OpenConext · Nov 18, 2024 · ea40586 · ea40586
1 parent 0b2de85
commit ea40586
Show file tree

Hide file tree

Showing 8 changed files with 92 additions and 12 deletions.
diff --git a/server/src/main/java/access/api/FullSearchQueryParser.java b/server/src/main/java/access/api/FullSearchQueryParser.java
@@ -0,0 +1,38 @@
+package access.api;
+
+import access.exception.InvalidInputException;
+import org.springframework.util.StringUtils;
+
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class FullSearchQueryParser {
+
+    //SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_DEFAULT_STOPWORD;
+    private static final List<String> stopWords = List.of(
+            "a", "about", "an", "are",
+            "as", "at", "be", "by",
+            "com", "de", "en", "for",
+            "from", "how", "i", "in",
+            "is", "it", "la", "of",
+            "on", "or", "that", "the",
+            "this", "to", "was", "what",
+            "when", "where", "who", "will",
+            "with", "und", "the", "www"
+    );
+
+    private FullSearchQueryParser() {
+    }
+
+    public static String parse(String query) {
+        if (!StringUtils.hasText(query)) {
+            throw new InvalidInputException("Full text query parameter has @NotNull @NotBlank requirement");
+        }
+        String parsedQuery = Stream.of(query.split("[ @.,+*]"))
+                .filter(part -> !(part.isEmpty() || stopWords.contains(part.toLowerCase())))
+                .map(part -> "+" + part)
+                .collect(Collectors.joining(" "));
+        return parsedQuery + "*";
+    }
+}
diff --git a/server/src/main/java/access/api/RoleController.java b/server/src/main/java/access/api/RoleController.java
@@ -146,7 +146,7 @@ public ResponseEntity<List<Role>> search(@RequestParam(value = "query") String q
                                              @Parameter(hidden = true) User user) {
         LOG.debug("/search");
         UserPermissions.assertSuperUser(user);
-        List<Role> roles = roleRepository.search(query + "*", 15);
+        List<Role> roles = roleRepository.search(FullSearchQueryParser.parse(query), 15);
         return ResponseEntity.ok(manage.addManageMetaData(roles));
     }
 
@@ -202,7 +202,7 @@ private ResponseEntity<Role> saveOrUpdate(Role role, User user) {
         boolean isNew = role.getId() == null;
         List<String> previousApplicationIdentifiers = new ArrayList<>();
         Optional<UserRole> optionalUserRole = user.userRoleForRole(role);
-        boolean immutableApplicationUsages = !user.isSuperUser() && 
+        boolean immutableApplicationUsages = !user.isSuperUser() &&
                 optionalUserRole.isPresent() && optionalUserRole.get().getAuthority().equals(Authority.MANAGER);
         boolean nameChanged = false;
         if (!isNew) {

diff --git a/server/src/main/java/access/api/UserController.java b/server/src/main/java/access/api/UserController.java
@@ -138,7 +138,7 @@ public ResponseEntity<List<User>> search(@RequestParam(value = "query") String q
         LOG.debug(String.format("/search for user %s", user.getEduPersonPrincipalName()));
         UserPermissions.assertSuperUser(user);
         List<User> users = query.equals("owl") ? userRepository.findAll() :
-                userRepository.search(query.replaceAll("@", " ") + "*", 15);
+                userRepository.search(FullSearchQueryParser.parse(query), 15);
         return ResponseEntity.ok(users);
     }
 
@@ -159,7 +159,7 @@ public ResponseEntity<Page<?>> searchPaginated(@RequestParam(value = "query", re
         Pageable pageable = PageRequest.of(pageNumber, pageSize, Sort.by(Sort.Direction.fromString(sortDirection), sort));
         Page<Map<String, Object>> page = StringUtils.hasText(query) ?
                 userRepository.searchByPage(pageable) :
-                userRepository.searchByPageWithKeyword(query.replaceAll("@", " ") + "*", pageable) ;
+                userRepository.searchByPageWithKeyword(FullSearchQueryParser.parse(query), pageable) ;
         return ResponseEntity.ok(page);
     }
 
@@ -182,7 +182,7 @@ public ResponseEntity<List<UserRoles>> searchByApplication(@RequestParam(value =
         }
         List<Map<String, Object>> results = query.equals("owl") ?
                 userRepository.searchByApplicationAllUsers(manageIdentifiers) :
-                userRepository.searchByApplication(manageIdentifiers, query.replaceAll("@", " ") + "*", 15);
+                userRepository.searchByApplication(manageIdentifiers, FullSearchQueryParser.parse(query), 15);
         //There are duplicate users in the results, need to group by ID
         Map<Long, List<Map<String, Object>>> groupedBy = results.stream().collect(Collectors.groupingBy(map -> (Long) map.get("id")));
         List<UserRoles> userRoles = groupedBy.values().stream()

diff --git a/server/src/main/java/access/api/UserRoleController.java b/server/src/main/java/access/api/UserRoleController.java
@@ -100,7 +100,7 @@ public ResponseEntity<List<Map<String, Object>>> consequencesDeleteRole(@PathVar
     @GetMapping("/search/{roleId}/{guests}")
     public ResponseEntity<Page<?>> searchPaginated(@PathVariable("roleId") Long roleId,
                                                    @PathVariable("guests") boolean guests,
-                                                   @RequestParam(value = "query", required = false, defaultValue = "") String query,
+                                                   @RequestParam(value = "query") String query,
                                                    @RequestParam(value = "pageNumber", required = false, defaultValue = "0") int pageNumber,
                                                    @RequestParam(value = "pageSize", required = false, defaultValue = "10") int pageSize,
                                                    @RequestParam(value = "sort", required = false, defaultValue = "id") String sort,
@@ -114,6 +114,7 @@ public ResponseEntity<Page<?>> searchPaginated(@PathVariable("roleId") Long role
 
         Pageable pageable = PageRequest.of(pageNumber, pageSize, Sort.by(Sort.Direction.fromString(sortDirection), sort));
         Page<Map<String, Object>> page;
+        query = FullSearchQueryParser.parse(query);
         if (StringUtils.hasText(query)) {
             page = guests ?
                     userRoleRepository.searchGuestsByPageWithKeyword(roleId, query, pageable) :

diff --git a/server/src/test/java/access/AbstractTest.java b/server/src/test/java/access/AbstractTest.java
@@ -557,21 +557,21 @@ public void doSeed() {
         this.apiTokenRepository.deleteAllInBatch();
 
         User superUser =
-                new User(true, SUPER_SUB, SUPER_SUB, "example.com", "David", "Doe", "david.doe@examole.com");
+                new User(true, SUPER_SUB, SUPER_SUB, "example.com", "David", "Doe", "david.doe@example.com");
         User institutionAdmin =
-                new User(false, INSTITUTION_ADMIN_SUB, INSTITUTION_ADMIN_SUB, "example.com", "Carl", "Doe", "carl.doe@examole.com");
+                new User(false, INSTITUTION_ADMIN_SUB, INSTITUTION_ADMIN_SUB, "example.com", "Carl", "Doe", "carl.doe@example.com");
         institutionAdmin.setInstitutionAdmin(true);
         institutionAdmin.setInstitutionAdminByInvite(true);
         institutionAdmin.setOrganizationGUID(ORGANISATION_GUID);
 
         User manager =
-                new User(false, MANAGE_SUB, MANAGE_SUB, "example.com", "Mary", "Doe", "mary.doe@examole.com");
+                new User(false, MANAGE_SUB, MANAGE_SUB, "example.com", "Mary", "Doe", "mary.doe@example.com");
         User inviter =
-                new User(false, INVITER_SUB, INVITER_SUB, "example.com", "Paul", "Doe", "paul.doe@examole.com");
+                new User(false, INVITER_SUB, INVITER_SUB, "example.com", "Paul", "Doe", "paul.doe@example.com");
         User wikiInviter =
-                new User(false, INVITER_WIKI_SUB, INVITER_WIKI_SUB, "example.com", "James", "Doe", "james.doe@examole.com");
+                new User(false, INVITER_WIKI_SUB, INVITER_WIKI_SUB, "example.com", "James", "Doe", "james.doe@example.com");
         User guest =
-                new User(false, GUEST_SUB, GUEST_SUB, "example.com", "Ann", "Doe", "ann.doe@examole.com");
+                new User(false, GUEST_SUB, GUEST_SUB, "example.com", "Ann", "Doe", "ann.doe@example.com");
         guest.setEduId(UUID.randomUUID().toString());
         doSave(this.userRepository, superUser, institutionAdmin, manager, inviter, wikiInviter, guest);
 

diff --git a/server/src/test/java/access/api/FullSearchQueryParserTest.java b/server/src/test/java/access/api/FullSearchQueryParserTest.java
@@ -0,0 +1,23 @@
+package access.api;
+
+import access.exception.InvalidInputException;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class FullSearchQueryParserTest {
+
+    @Test
+    void parse() {
+        String parsed = FullSearchQueryParser.parse("This *is+  +a ** test for + the [email protected] *query*");
+        assertEquals("+test +john +doe +example +query*", parsed);
+    }
+
+    @Test
+    void parseEmpty() {
+        assertThrows(InvalidInputException.class, () -> FullSearchQueryParser.parse(null));
+        assertThrows(InvalidInputException.class, () -> FullSearchQueryParser.parse(""));
+        assertThrows(InvalidInputException.class, () -> FullSearchQueryParser.parse("  "));
+    }
+
+}
diff --git a/server/src/test/java/access/api/UserControllerTest.java b/server/src/test/java/access/api/UserControllerTest.java
@@ -295,6 +295,22 @@ void search() throws Exception {
         assertEquals(1, users.size());
     }
 
+    @Test
+    void searchWithAtSign() throws Exception {
+        AccessCookieFilter accessCookieFilter = openIDConnectFlow("/api/v1/users/login", SUPER_SUB);
+
+        List<User> users = given()
+                .when()
+                .filter(accessCookieFilter.cookieFilter())
+                .accept(ContentType.JSON)
+                .contentType(ContentType.JSON)
+                .queryParam("query", "[email protected]")
+                .get("/api/v1/users/search")
+                .as(new TypeRef<>() {
+                });
+        assertEquals(1, users.size());
+    }
+
     @Test
     void searchPaginated() throws Exception {
         AccessCookieFilter accessCookieFilter = openIDConnectFlow("/api/v1/users/login", SUPER_SUB);

diff --git a/server/src/test/java/access/api/UserRoleControllerTest.java b/server/src/test/java/access/api/UserRoleControllerTest.java
@@ -59,6 +59,7 @@ void searchGuestsByPage() throws Exception {
                         "pageNumber", 0,
                         "pageSize", 1,
                         "sort", "end_date",
+                        "query","example.com",
                         "sortDirection", Sort.Direction.DESC
                 ))
                 .pathParams("roleId", role.getId())
@@ -83,6 +84,7 @@ void searchNonGuestsByPage() throws Exception {
                 .queryParams(Map.of(
                         "pageNumber", 0,
                         "pageSize", 1,
+                        "query","example",
                         "sort", "end_date",
                         "sortDirection", Sort.Direction.DESC
                 ))