From 2d424e33f5a05e1f00c69ebe6b3ad12f0270bde8 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Mon, 31 Jan 2022 17:52:58 +0000 Subject: [PATCH 1/9] Replace .travis.yml with ci-build.yml #23 --- .github/workflows/ci-build.yml | 64 +++++++++++++++++++ .travis.yml | 30 --------- README.md | 2 +- .../org/icatproject/icat/TestIcatClient.java | 2 +- 4 files changed, 66 insertions(+), 32 deletions(-) create mode 100755 .github/workflows/ci-build.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml new file mode 100755 index 0000000..d8c4542 --- /dev/null +++ b/.github/workflows/ci-build.yml @@ -0,0 +1,64 @@ +name: CI Build +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + continue-on-error: ${{ matrix.experimental }} + strategy: + fail-fast: false + matrix: + include: + - version: 8 + experimental: false + - version: 11 + experimental: true + + steps: + - name: Checkout ICAT Client + uses: actions/checkout@v2 + + - name: Set up JDK + uses: actions/setup-java@v2 + with: + java-version: ${{ matrix.version }} + distribution: 'temurin' + + - name: Installation + run: | + sudo pip install wheel ansible pycrypto + sudo add-apt-repository universe + sudo apt-get update + + - name: Setup python2 + run: | + sudo apt install python2 + curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py + sudo python2 get-pip.py + sudo python2 -m pip install requests requests_toolbelt + + - name: Checkout ICAT Ansible + uses: actions/checkout@v2 + with: + repository: icatproject-contrib/icat-ansible + path: icat-ansible + + - name: Setup Ansible + run: | + cd icat-ansible + echo -e "[icat-client-dev-hosts]\nlocalhost ansible_connection=local" > hosts + echo -e "icattravispw" > vault_pass.txt + mv ../vault.yml ./group_vars/all + mv ../icat-client-dev-hosts.yml . + sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"runner\"/" ./group_vars/all/vars.yml + sed -i -e "s/python-pip/python3-pip/" ./roles/dev-common/tasks/main.yml + ansible-playbook --vault-password-file ./vault_pass.txt --inventory ./hosts ./icat-client-dev-hosts.yml + cd .. + + - name: Test with Maven + run: mvn -DserverUrl=https://$(hostname -f):8181 test -B diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index ec439d4..0000000 --- a/.travis.yml +++ /dev/null @@ -1,30 +0,0 @@ -language: java -jdk: - - openjdk8 - - openjdk11 - -jobs: - allow_failures: - - jdk: openjdk11 - -dist: xenial - -cache: - directories: - - $HOME/.m2 - -install: - - sudo pip install wheel ansible pycrypto - - sudo add-apt-repository universe - - sudo apt-get update - - git clone https://github.com/icatproject-contrib/icat-ansible.git - - cd icat-ansible - - echo -e "[icat-client-dev-hosts]\nlocalhost ansible_connection=local" > hosts - - echo -e "icattravispw" > vault_pass.txt - - mv ../vault.yml ./group_vars/all - - mv ../icat-client-dev-hosts.yml . - - 'sed -i -e "s/^payara_user: \"glassfish\"/payara_user: \"travis\"/" ./group_vars/all/vars.yml' - - ansible-playbook --vault-password-file ./vault_pass.txt --inventory ./hosts ./icat-client-dev-hosts.yml - - cd .. - -script: 'mvn -DserverUrl=https://localhost:8181 test -B' diff --git a/README.md b/README.md index f5117b2..f3c93dc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # icat.client: Clients for ICAT -[![Build Status](https://travis-ci.org/icatproject/icat.client.svg?branch=master)](https://travis-ci.org/icatproject/icat.client) +[![Build Status](https://github.com/icatproject/icat.client/workflows/CI%20Build/badge.svg?branch=master)](https://github.com/icatproject/icat.client/actions?query=workflow%3A%22CI+Build%22) General installation instructions are at https://icatproject.org/installation/component diff --git a/src/test/java/org/icatproject/icat/TestIcatClient.java b/src/test/java/org/icatproject/icat/TestIcatClient.java index 7f43391..23782b3 100644 --- a/src/test/java/org/icatproject/icat/TestIcatClient.java +++ b/src/test/java/org/icatproject/icat/TestIcatClient.java @@ -113,7 +113,7 @@ public void testInfo() throws Exception { @Test public void python() { ShellCommand sc = new ShellCommand("bash", "-c", "serverUrl=" + System.getProperty("serverUrl") - + " PYTHONPATH=src/main/python/ python src/test/python/test.py"); + + " PYTHONPATH=src/main/python/ python2 src/test/python/test.py"); if (sc.getExitValue() != 0) { System.out.println(sc.getStdout()); System.out.println(sc.getStderr()); From f16bddbcb9415432ae88adb3e1ad5f5fb5ec0aed Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Sat, 30 Apr 2022 02:10:20 +0100 Subject: [PATCH 2/9] Support for new searc/documents #27 --- .../org/icatproject/icat/client/ICAT.java | 142 +++++++ .../org/icatproject/icat/client/Session.java | 376 +++++++++++++----- 2 files changed, 416 insertions(+), 102 deletions(-) diff --git a/src/main/java/org/icatproject/icat/client/ICAT.java b/src/main/java/org/icatproject/icat/client/ICAT.java index b78bfcc..3aff4ab 100644 --- a/src/main/java/org/icatproject/icat/client/ICAT.java +++ b/src/main/java/org/icatproject/icat/client/ICAT.java @@ -592,6 +592,61 @@ String searchInvestigations(String sessionId, String user, String text, Date low } } + String searchInvestigations(String sessionId, String user, String text, Date lower, Date upper, + List parameters, List samples, String userFullName, String searchAfter, int limit, String sort, String facets) + throws IcatException { + URIBuilder uriBuilder = getUriBuilder("search/documents"); + uriBuilder.setParameter("sessionId", sessionId); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("target", "Investigation"); + if (user != null) { + gen.write("user", user); + } + if (text != null) { + gen.write("text", text); + } + if (lower != null) { + gen.write("lower", DateTools.dateToString(lower, Resolution.MINUTE)); + } + if (upper != null) { + gen.write("upper", DateTools.dateToString(upper, Resolution.MINUTE)); + } + if (parameters != null && !parameters.isEmpty()) { + writeParameters(gen, parameters); + } + if (samples != null && !samples.isEmpty()) { + gen.writeStartArray("samples"); + for (String sample : samples) { + gen.write(sample); + } + gen.writeEnd(); + } + if (userFullName != null) { + gen.write("userFullName", userFullName); + } + gen.writeEnd(); + } + + uriBuilder.setParameter("query", baos.toString()); + uriBuilder.setParameter("search_after", searchAfter); + uriBuilder.setParameter("limit", Integer.toString(limit)); + uriBuilder.setParameter("sort", sort); + uriBuilder.setParameter("facets", facets); + URI uri = getUri(uriBuilder); + + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + HttpGet httpGet = new HttpGet(uri); + try (CloseableHttpResponse response = httpclient.execute(httpGet)) { + return getString(response); + } + } catch (IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + + } + } + String searchDatasets(String sessionId, String user, String text, Date lower, Date upper, List parameters, int maxResults) throws IcatException { URIBuilder uriBuilder = getUriBuilder("lucene/data"); @@ -633,6 +688,50 @@ String searchDatasets(String sessionId, String user, String text, Date lower, Da } } + String searchDatasets(String sessionId, String user, String text, Date lower, Date upper, + List parameters, String searchAfter, int limit, String sort, String facets) throws IcatException { + URIBuilder uriBuilder = getUriBuilder("search/documents"); + uriBuilder.setParameter("sessionId", sessionId); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("target", "Dataset"); + if (user != null) { + gen.write("user", user); + } + if (text != null) { + gen.write("text", text); + } + if (lower != null) { + gen.write("lower", DateTools.dateToString(lower, Resolution.MINUTE)); + } + if (upper != null) { + gen.write("upper", DateTools.dateToString(upper, Resolution.MINUTE)); + } + if (parameters != null && !parameters.isEmpty()) { + writeParameters(gen, parameters); + } + gen.writeEnd(); + } + + uriBuilder.setParameter("query", baos.toString()); + uriBuilder.setParameter("search_after", searchAfter); + uriBuilder.setParameter("limit", Integer.toString(limit)); + uriBuilder.setParameter("sort", sort); + uriBuilder.setParameter("facets", facets); + URI uri = getUri(uriBuilder); + + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + HttpGet httpGet = new HttpGet(uri); + try (CloseableHttpResponse response = httpclient.execute(httpGet)) { + return getString(response); + } + } catch (IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + + } + } + private void writeParameters(JsonGenerator gen, List parameters) { gen.writeStartArray("parameters"); for (ParameterForLucene parameter : parameters) { @@ -805,6 +904,49 @@ String searchDatafiles(String sessionId, String user, String text, Date lower, D } } + String searchDatafiles(String sessionId, String user, String text, Date lower, Date upper, + List parameters, String searchAfter, int limit, String sort, String facets) throws IcatException { + URIBuilder uriBuilder = getUriBuilder("search/documents"); + uriBuilder.setParameter("sessionId", sessionId); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (JsonGenerator gen = Json.createGenerator(baos)) { + gen.writeStartObject(); + gen.write("target", "Datafile"); + if (user != null) { + gen.write("user", user); + } + if (text != null) { + gen.write("text", text); + } + if (lower != null) { + gen.write("lower", DateTools.dateToString(lower, Resolution.MINUTE)); + } + if (upper != null) { + gen.write("upper", DateTools.dateToString(upper, Resolution.MINUTE)); + } + if (parameters != null && !parameters.isEmpty()) { + writeParameters(gen, parameters); + } + gen.writeEnd(); + } + + uriBuilder.setParameter("query", baos.toString()); + uriBuilder.setParameter("search_after", searchAfter); + uriBuilder.setParameter("limit", Integer.toString(limit)); + uriBuilder.setParameter("sort", sort); + uriBuilder.setParameter("facets", facets); + URI uri = getUri(uriBuilder); + + try (CloseableHttpClient httpclient = HttpClients.createDefault()) { + HttpGet httpGet = new HttpGet(uri); + try (CloseableHttpResponse response = httpclient.execute(httpGet)) { + return getString(response); + } + } catch (IOException e) { + throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); + } + } + long cloneEntity(String sessionId, String name, long id, Map keys) throws IcatException { URI uri = getUri(getUriBuilder("cloner")); List formparams = new ArrayList<>(); diff --git a/src/main/java/org/icatproject/icat/client/Session.java b/src/main/java/org/icatproject/icat/client/Session.java index 52a84de..1f704e7 100644 --- a/src/main/java/org/icatproject/icat/client/Session.java +++ b/src/main/java/org/icatproject/icat/client/Session.java @@ -145,14 +145,14 @@ public enum DuplicateAction { * Write (create or update) ICAT entities from a Json String. * * @param entities - * Json representation of ICAT entities and their related - * entities. If there is only one, the outer "[" "]" may be - * omitted. + * Json representation of ICAT entities and their related + * entities. If there is only one, the outer "[" "]" may be + * omitted. * * @return the ids of the top level entities created * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public List write(String entities) throws IcatException { return icat.write(sessionId, entities); @@ -163,14 +163,14 @@ public List write(String entities) throws IcatException { * * @deprecated Replace by {@link #write(String)} * @param entities - * Json representation of ICAT entities and their related - * entities. If there is only one, the outer "[" "]" may be - * omitted. + * Json representation of ICAT entities and their related + * entities. If there is only one, the outer "[" "]" may be + * omitted. * * @return the ids of the top level entities created * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ @Deprecated public List create(String entities) throws IcatException { @@ -181,11 +181,11 @@ public List create(String entities) throws IcatException { * Delete ICAT entities specified by a Json String. * * @param entities - * Json representation of ICAT entities. If there is only one the - * outer "[" "]" may be omitted. + * Json representation of ICAT entities. If there is only one + * the outer "[" "]" may be omitted. * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public void delete(String entities) throws IcatException { icat.delete(sessionId, entities); @@ -195,16 +195,16 @@ public void delete(String entities) throws IcatException { * Export all metadata from ICAT. * * @param attributes - * which attributes to export. If you don't plan to - * importMetaData as a "root user" there is no point in using - * {@link Attributes#ALL} and {@link Attributes#USER} is to be - * preferred. + * which attributes to export. If you don't plan to + * importMetaData as a "root user" there is no point in using + * {@link Attributes#ALL} and {@link Attributes#USER} is to be + * preferred. * * @return an OutputStream. The structure of the OutputStream is described * at {@link Session} * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public InputStream exportMetaData(Attributes attributes) throws IcatException { return icat.exportMetaData(sessionId, null, attributes); @@ -214,19 +214,19 @@ public InputStream exportMetaData(Attributes attributes) throws IcatException { * Export metadata from ICAT as specified in the query * * @param query - * a normal ICAT query which may have an INCLUDE clause. This is - * used to define the metadata to export. + * a normal ICAT query which may have an INCLUDE clause. This + * is used to define the metadata to export. * @param attributes - * which attributes to export. If you don't plan to - * importMetaData as a "root user" there is no point in using - * {@link Attributes#ALL} and {@link Attributes#USER} is to be - * preferred. + * which attributes to export. If you don't plan to + * importMetaData as a "root user" there is no point in using + * {@link Attributes#ALL} and {@link Attributes#USER} is to be + * preferred. * * @return an OutputStream. The structure of the OutputStream is described * at {@link Session} * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public InputStream exportMetaData(String query, Attributes attributes) throws IcatException { return icat.exportMetaData(sessionId, query, attributes); @@ -238,7 +238,7 @@ public InputStream exportMetaData(String query, Attributes attributes) throws Ic * @return the time remaining * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public double getRemainingMinutes() throws IcatException { return icat.getRemainingMinutes(sessionId); @@ -259,7 +259,7 @@ public String getId() { * @return the user name * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public String getUserName() throws IcatException { return icat.getUserName(sessionId); @@ -269,19 +269,21 @@ public String getUserName() throws IcatException { * Import metadata into ICAT for a file specified by a Path * * @param path - * the path of the import file. The structure of the import file - * is described at {@link Session} + * the path of the import file. The structure of the + * import file is described at {@link Session} * @param duplicateAction - * what to do when a duplicate is encountered + * what to do when a duplicate is encountered * @param attributes - * which attributes to import. Only a "root user" can specify - * {@link Attributes#ALL} to respect those fields specified in - * the import file which are not settable by normal users: - * createId, createTime, modId and modTime. This is to allow an - * ICAT to be accurately exported and imported. + * which attributes to import. Only a "root user" can + * specify {@link Attributes#ALL} to respect those fields + * specified in the import file which are not settable by + * normal + * users: createId, createTime, modId and modTime. This + * is to + * allow an ICAT to be accurately exported and imported. * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public void importMetaData(Path path, DuplicateAction duplicateAction, Attributes attributes) throws IcatException { icat.importMetaData(sessionId, path, duplicateAction, attributes); @@ -291,7 +293,7 @@ public void importMetaData(Path path, DuplicateAction duplicateAction, Attribute * Logout of the session after which the session cannot be re-used * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public void logout() throws IcatException { icat.logout(sessionId); @@ -301,7 +303,7 @@ public void logout() throws IcatException { * Refresh the session by resetting the time remaining * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public void refresh() throws IcatException { icat.refresh(sessionId); @@ -314,12 +316,12 @@ public void refresh() throws IcatException { * continue in its present form. * * @param query - * a normal ICAT query with optional INCLUDE and LIMIT clauses. + * a normal ICAT query with optional INCLUDE and LIMIT clauses. * * @return the Json holding the results * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public String search(String query) throws IcatException { return icat.search(sessionId, query); @@ -332,14 +334,14 @@ public String search(String query) throws IcatException { * continue in its present form. * * @param query - * a normal ICAT get query with an optional INCLUDE clause. + * a normal ICAT get query with an optional INCLUDE clause. * @param id - * the id of the entity to be returned + * the id of the entity to be returned * * @return the Json holding the result * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public String get(String query, long id) throws IcatException { return icat.get(sessionId, query, id); @@ -349,40 +351,48 @@ public String get(String query, long id) throws IcatException { * Return a set of investigations satisfying the constraints * * @param user - * If not null must exactly match the name of a user related via - * the investigation user to the investigation. + * If not null must exactly match the name of a user related + * via + * the investigation user to the investigation. * @param text - * If not null a text search (with ANDs ORs etc) for any text in - * the investigation fields. This is understood by the lucene parser but avoid trying to use fields. + * If not null a text search (with ANDs ORs etc) for any + * text in + * the investigation fields. This is understood by the + * lucene parser but avoid trying to use fields. * @param lower - * If not null investigation end date must be greater than or - * equal to this. + * If not null investigation end date must be greater than + * or + * equal to this. * @param upper - * If not null investigation start date must be less than or - * equal to this. + * If not null investigation start date must be less than or + * equal to this. * @param parameters - * If not null all the parameters must match. + * If not null all the parameters must match. * @param samples - * If not null all the specified samples, using a text search - * (with ANDs ORs etc) must be related to the investigation. This - * is understood by the lucene parser but avoid trying to use fields. + * If not null all the specified samples, using a text + * search + * (with ANDs ORs etc) must be related to the investigation. + * This + * is understood by the lucene parser but avoid trying to use fields. * @param userFullName - * If not null a text search is made against the full name of a - * user related via the investigation user to the investigation. - * This is understood by the lucene parser but avoid trying to use fields. + * If not null a text search is made against the full name + * of a + * user related via the investigation user to the + * investigation. + * This is understood by the lucene parser but avoid trying to use fields. * @param maxResults - * The maximum number of results to return. + * The maximum number of results to return. * * @return the Json holding the result. * * @throws IcatException - * For various ICAT errors. + * For various ICAT errors. */ public String searchInvestigations(String user, String text, Date lower, Date upper, List parameters, List samples, String userFullName, int maxResults) @@ -391,43 +401,155 @@ public String searchInvestigations(String user, String text, Date lower, Date up maxResults); } + /** + * Return a set of indexed documents representing Investigations that satisfy + * the search constraints + * + * @param user + * If not null must exactly match the name of a user related + * via the investigation user to the investigation. + * @param text + * If not null a text search (with ANDs ORs etc) for any + * text in the investigation fields. This is understood by + * the lucene parser but avoid trying to use fields. + * @param lower + * If not null investigation end date must be greater than + * or equal to this. + * @param upper + * If not null investigation start date must be less than or + * equal to this. + * @param parameters + * If not null all the parameters must match. + * @param samples + * If not null all the specified samples, using a text + * search (with ANDs ORs etc) must be related to the + * investigation. + * This is understood by the lucene parser but avoid trying to use fields. + * @param userFullName + * If not null a text search is made against the full name + * of a user related via the investigation user to the + * investigation. This is understood by the lucene parser but avoid trying to use fields. + * @param searchAfter + * String representing the last document of a previous + * search, so that results from this search will only + * include results from after the document + * @param limit + * The maximum number of results to return. + * @param sort + * String representing a JSON object which contains key + * value pairs of the field(s) to sort on and their + * direction + * @param facets + * String representing a JsonArray of JsonObjects. Each + * should define the "target" entity name, and optionally + * another JsonArray of "dimensions", which are specific + * fields to facet. If absent, then all applicable fields + * will be faceted. + * + * @return the Json holding the result. + * + * @throws IcatException + * For various ICAT errors. + */ + public String searchInvestigations(String user, String text, Date lower, Date upper, + List parameters, List samples, String userFullName, String searchAfter, + int limit, String sort, String facets) + throws IcatException { + return icat.searchInvestigations(sessionId, user, text, lower, upper, parameters, samples, userFullName, + searchAfter, limit, sort, facets); + } + /** * Return a set of datasets satisfying the constraints * * @param user - * If not null must exactly match the name of a user related via - * the investigation user and the investigation to the data set. + * If not null must exactly match the name of a user related + * via the investigation user and the investigation to the + * data set. * @param text - * If not null a text search (with ANDs ORs etc) for any text in - * the data set fields. This is understood by the lucene parser but avoid trying to use fields. + * If not null a text search (with ANDs ORs etc) for any text + * in the data set fields. This is understood by the lucene parser but avoid trying to use fields. * @param lower - * If not null data set end date must be greater than or equal to - * this. + * If not null data set end date must be greater than or equal + * to this. * @param upper - * If not null data set start date must be less than or equal to - * this. + * If not null data set start date must be less than or equal + * to this. * @param parameters - * If not null all the parameters must match. + * If not null all the parameters must match. * @param maxResults - * The maximum number of results to return. + * The maximum number of results to return. * * @return the Json holding the result. * * @throws IcatException - * For various ICAT errors. + * For various ICAT errors. */ public String searchDatasets(String user, String text, Date lower, Date upper, List parameters, int maxResults) throws IcatException { return icat.searchDatasets(sessionId, user, text, lower, upper, parameters, maxResults); } + /** + * Return a set of indexed documents representing Datasets that satisfy the + * search constraints + * + * @param user + * If not null must exactly match the name of a user related + * via the investigation user and the investigation to the + * data set. + * @param text + * If not null a text search (with ANDs ORs etc) for any text + * in the data set fields. This is understood by the lucene parser but avoid trying to use fields. + * @param lower + * If not null data set end date must be greater than or + * equal to this. + * @param upper + * If not null data set start date must be less than or equal + * to this. + * @param parameters + * If not null all the parameters must match. + * @param searchAfter + * String representing the last document of a previous + * search, so that results from this search will only include + * results from after the document + * @param limit + * The maximum number of results to return. + * @param sort + * String representing a JSON object which contains key value + * pairs of the field(s) to sort on and their direction + * @param facets + * String representing a JsonArray of JsonObjects. Each + * should define the "target" entity name, and optionally + * another JsonArray of "dimensions", which are specific + * fields to facet. If absent, then all applicable fields + * will be faceted. + * + * @return the Json holding the result. + * + * @throws IcatException + * For various ICAT errors. + */ + public String searchDatasets(String user, String text, Date lower, Date upper, List parameters, + String searchAfter, int limit, String sort, String facets) throws IcatException { + return icat.searchDatasets(sessionId, user, text, lower, upper, parameters, searchAfter, limit, sort, facets); + } + /** * Clear the lucene populating list * * @throws IcatException - * For various ICAT errors. + * For various ICAT errors. */ public void luceneClear() throws IcatException { icat.luceneClear(sessionId); @@ -437,7 +559,7 @@ public void luceneClear() throws IcatException { * Force a commit of the lucene database * * @throws IcatException - * For various ICAT errors. + * For various ICAT errors. */ public void luceneCommit() throws IcatException { icat.luceneCommit(sessionId); @@ -447,10 +569,10 @@ public void luceneCommit() throws IcatException { * Clear and repopulate lucene documents for the specified entityName. * * @param entityName - * the name of the entity + * the name of the entity * * @throws IcatException - * For various ICAT errors. + * For various ICAT errors. */ public void lucenePopulate(String entityName, long minid) throws IcatException { icat.lucenePopulate(sessionId, entityName, minid); @@ -462,7 +584,7 @@ public void lucenePopulate(String entityName, long minid) throws IcatException { * @return list of class names * * @throws IcatException - * For various ICAT errors. + * For various ICAT errors. */ public List luceneGetPopulating() throws IcatException { return icat.luceneGetPopulating(sessionId); @@ -472,48 +594,98 @@ public List luceneGetPopulating() throws IcatException { * Return a set of data files satisfying the constraints * * @param user - * If not null must exactly match the name of a user related via - * the investigation user and the investigation to the data set. + * If not null must exactly match the name of a user related + * via + * the investigation user and the investigation to the data + * set. * @param text - * If not null a text search (with ANDs ORs etc) for any text in - * the data file fields. This is understood by the lucene parser but avoid trying to use fields. + * If not null a text search (with ANDs ORs etc) for any text + * in the data file fields. This is understood by the lucene parser but avoid trying to use fields. * @param lower - * If not null data file date must be greater than or equal to - * this. + * If not null data file date must be greater than or equal to + * this. * @param upper - * If not null data file date must be less than or equal to - * thilist of class namess. + * If not null data file date must be less than or equal to + * this. * @param parameters - * If not null all the parameters must match. + * If not null all the parameters must match. * @param maxResults - * The maximum number of results to return. + * The maximum number of results to return. * * @return the Json holding the result. * * @throws IcatException - * For various ICAT errors. + * For various ICAT errors. */ public String searchDatafiles(String user, String text, Date lower, Date upper, List parameters, int maxResults) throws IcatException { return icat.searchDatafiles(sessionId, user, text, lower, upper, parameters, maxResults); } + /** + * Return a set of indexed documents representing Datafiles that satisfy the + * search constraints + * + * @param user + * If not null must exactly match the name of a user related + * via the investigation user and the investigation to the + * data set. + * @param text + * If not null a text search (with ANDs ORs etc) for any text + * in the data file fields. This is understood by the + * lucene parser but avoid trying to use fields. + * @param lower + * If not null data file date must be greater than or equal + * to this. + * @param upper + * If not null data file date must be less than or equal to + * this. + * @param parameters + * If not null all the parameters must match. + * @param searchAfter + * String representing the last document of a previous + * search, so that results from this search will only include + * results from after the document + * @param limit + * The maximum number of results to return. + * @param sort + * String representing a JSON object which contains key value + * pairs of the field(s) to sort on and their direction + * @param facets + * String representing a JsonArray of JsonObjects. Each + * should define the "target" entity name, and optionally + * another JsonArray of "dimensions", which are specific + * fields to facet. If absent, then all applicable fields + * will be faceted. + * + * @return the Json holding the result. + * + * @throws IcatException + * For various ICAT errors. + */ + public String searchDatafiles(String user, String text, Date lower, Date upper, List parameters, + String searchAfter, int limit, String sort, String facets) throws IcatException { + return icat.searchDatafiles(sessionId, user, text, lower, upper, parameters, searchAfter, limit, sort, facets); + } + /** * Clone an entity and return the id of the clone * * @param name - * the name of the type of entity + * the name of the type of entity * @param id - * the id value of the entity to be cloned + * the id value of the entity to be cloned * @param keys - * a map of field names and values to be different in the clone + * a map of field names and values to be different in the clone * * @return the id of the clone * * @throws IcatException - * For various ICAT errors. + * For various ICAT errors. */ public long cloneEntity(String name, long id, Map keys) throws IcatException { return icat.cloneEntity(sessionId, name, id, keys); @@ -525,10 +697,10 @@ public long cloneEntity(String name, long id, Map keys) throws I * This is only used in testing * * @param ms - * number of milliseconds to wait + * number of milliseconds to wait * * @throws IcatException - * For various ICAT errors. + * For various ICAT errors. */ public void waitMillis(long ms) throws IcatException { icat.waitMillis(sessionId, ms); @@ -537,7 +709,7 @@ public void waitMillis(long ms) throws IcatException { /** * @param path input path to find contents of * @return json describing the contents - * @throws IcatException + * @throws IcatException */ public String list(String path) throws IcatException { return icat.list(sessionId, path); From 50dbf3269cbf234733e86e8a252131347073ade1 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Wed, 13 Jul 2022 15:46:23 +0100 Subject: [PATCH 3/9] Update parameters for new free text search #27 --- .../org/icatproject/icat/client/ICAT.java | 34 +++++++++---------- .../org/icatproject/icat/client/Session.java | 25 +++++++------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/icatproject/icat/client/ICAT.java b/src/main/java/org/icatproject/icat/client/ICAT.java index 3aff4ab..7a9667c 100644 --- a/src/main/java/org/icatproject/icat/client/ICAT.java +++ b/src/main/java/org/icatproject/icat/client/ICAT.java @@ -18,6 +18,7 @@ import java.util.Map.Entry; import javax.json.Json; +import javax.json.JsonArray; import javax.json.JsonException; import javax.json.JsonReader; import javax.json.JsonString; @@ -593,8 +594,8 @@ String searchInvestigations(String sessionId, String user, String text, Date low } String searchInvestigations(String sessionId, String user, String text, Date lower, Date upper, - List parameters, List samples, String userFullName, String searchAfter, int limit, String sort, String facets) - throws IcatException { + List parameters, String userFullName, String searchAfter, int maxCount, String sort, + JsonArray facets) throws IcatException { URIBuilder uriBuilder = getUriBuilder("search/documents"); uriBuilder.setParameter("sessionId", sessionId); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -616,24 +617,19 @@ String searchInvestigations(String sessionId, String user, String text, Date low if (parameters != null && !parameters.isEmpty()) { writeParameters(gen, parameters); } - if (samples != null && !samples.isEmpty()) { - gen.writeStartArray("samples"); - for (String sample : samples) { - gen.write(sample); - } - gen.writeEnd(); - } if (userFullName != null) { gen.write("userFullName", userFullName); } + if (facets != null) { + gen.write("facets", facets); + } gen.writeEnd(); } uriBuilder.setParameter("query", baos.toString()); uriBuilder.setParameter("search_after", searchAfter); - uriBuilder.setParameter("limit", Integer.toString(limit)); + uriBuilder.setParameter("maxCount", Integer.toString(maxCount)); uriBuilder.setParameter("sort", sort); - uriBuilder.setParameter("facets", facets); URI uri = getUri(uriBuilder); try (CloseableHttpClient httpclient = HttpClients.createDefault()) { @@ -689,7 +685,7 @@ String searchDatasets(String sessionId, String user, String text, Date lower, Da } String searchDatasets(String sessionId, String user, String text, Date lower, Date upper, - List parameters, String searchAfter, int limit, String sort, String facets) throws IcatException { + List parameters, String searchAfter, int maxCount, String sort, JsonArray facets) throws IcatException { URIBuilder uriBuilder = getUriBuilder("search/documents"); uriBuilder.setParameter("sessionId", sessionId); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -711,14 +707,16 @@ String searchDatasets(String sessionId, String user, String text, Date lower, Da if (parameters != null && !parameters.isEmpty()) { writeParameters(gen, parameters); } + if (facets != null) { + gen.write("facets", facets); + } gen.writeEnd(); } uriBuilder.setParameter("query", baos.toString()); uriBuilder.setParameter("search_after", searchAfter); - uriBuilder.setParameter("limit", Integer.toString(limit)); + uriBuilder.setParameter("maxCount", Integer.toString(maxCount)); uriBuilder.setParameter("sort", sort); - uriBuilder.setParameter("facets", facets); URI uri = getUri(uriBuilder); try (CloseableHttpClient httpclient = HttpClients.createDefault()) { @@ -905,7 +903,7 @@ String searchDatafiles(String sessionId, String user, String text, Date lower, D } String searchDatafiles(String sessionId, String user, String text, Date lower, Date upper, - List parameters, String searchAfter, int limit, String sort, String facets) throws IcatException { + List parameters, String searchAfter, int maxCount, String sort, JsonArray facets) throws IcatException { URIBuilder uriBuilder = getUriBuilder("search/documents"); uriBuilder.setParameter("sessionId", sessionId); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -927,14 +925,16 @@ String searchDatafiles(String sessionId, String user, String text, Date lower, D if (parameters != null && !parameters.isEmpty()) { writeParameters(gen, parameters); } + if (facets != null) { + gen.write("facets", facets); + } gen.writeEnd(); } uriBuilder.setParameter("query", baos.toString()); uriBuilder.setParameter("search_after", searchAfter); - uriBuilder.setParameter("limit", Integer.toString(limit)); + uriBuilder.setParameter("maxCount", Integer.toString(maxCount)); uriBuilder.setParameter("sort", sort); - uriBuilder.setParameter("facets", facets); URI uri = getUri(uriBuilder); try (CloseableHttpClient httpclient = HttpClients.createDefault()) { diff --git a/src/main/java/org/icatproject/icat/client/Session.java b/src/main/java/org/icatproject/icat/client/Session.java index 1f704e7..4abc364 100644 --- a/src/main/java/org/icatproject/icat/client/Session.java +++ b/src/main/java/org/icatproject/icat/client/Session.java @@ -6,6 +6,8 @@ import java.util.List; import java.util.Map; +import javax.json.JsonArray; + /** * A RESTful ICAT session. *

@@ -439,7 +441,7 @@ public String searchInvestigations(String user, String text, Date lower, Date up * String representing the last document of a previous * search, so that results from this search will only * include results from after the document - * @param limit + * @param maxCount * The maximum number of results to return. * @param sort * String representing a JSON object which contains key @@ -458,11 +460,10 @@ public String searchInvestigations(String user, String text, Date lower, Date up * For various ICAT errors. */ public String searchInvestigations(String user, String text, Date lower, Date upper, - List parameters, List samples, String userFullName, String searchAfter, - int limit, String sort, String facets) - throws IcatException { - return icat.searchInvestigations(sessionId, user, text, lower, upper, parameters, samples, userFullName, - searchAfter, limit, sort, facets); + List parameters, String userFullName, String searchAfter, int maxCount, String sort, + JsonArray facets) throws IcatException { + return icat.searchInvestigations(sessionId, user, text, lower, upper, parameters, userFullName, + searchAfter, maxCount, sort, facets); } /** @@ -523,7 +524,7 @@ public String searchDatasets(String user, String text, Date lower, Date upper, L * String representing the last document of a previous * search, so that results from this search will only include * results from after the document - * @param limit + * @param maxCount * The maximum number of results to return. * @param sort * String representing a JSON object which contains key value @@ -541,8 +542,8 @@ public String searchDatasets(String user, String text, Date lower, Date upper, L * For various ICAT errors. */ public String searchDatasets(String user, String text, Date lower, Date upper, List parameters, - String searchAfter, int limit, String sort, String facets) throws IcatException { - return icat.searchDatasets(sessionId, user, text, lower, upper, parameters, searchAfter, limit, sort, facets); + String searchAfter, int maxCount, String sort, JsonArray facets) throws IcatException { + return icat.searchDatasets(sessionId, user, text, lower, upper, parameters, searchAfter, maxCount, sort, facets); } /** @@ -650,7 +651,7 @@ public String searchDatafiles(String user, String text, Date lower, Date upper, * String representing the last document of a previous * search, so that results from this search will only include * results from after the document - * @param limit + * @param maxCount * The maximum number of results to return. * @param sort * String representing a JSON object which contains key value @@ -668,8 +669,8 @@ public String searchDatafiles(String user, String text, Date lower, Date upper, * For various ICAT errors. */ public String searchDatafiles(String user, String text, Date lower, Date upper, List parameters, - String searchAfter, int limit, String sort, String facets) throws IcatException { - return icat.searchDatafiles(sessionId, user, text, lower, upper, parameters, searchAfter, limit, sort, facets); + String searchAfter, int maxCount, String sort, JsonArray facets) throws IcatException { + return icat.searchDatafiles(sessionId, user, text, lower, upper, parameters, searchAfter, maxCount, sort, facets); } /** From 87f699fcefa93a6382531f7d081d3f2c51414c27 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Sun, 24 Jul 2022 08:02:18 +0100 Subject: [PATCH 4/9] Update lucenePopulate with new format #27 --- src/main/java/org/icatproject/icat/client/ICAT.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/icatproject/icat/client/ICAT.java b/src/main/java/org/icatproject/icat/client/ICAT.java index 7a9667c..0bd497d 100644 --- a/src/main/java/org/icatproject/icat/client/ICAT.java +++ b/src/main/java/org/icatproject/icat/client/ICAT.java @@ -847,9 +847,11 @@ List luceneGetPopulating(String sessionId) throws IcatException { } void lucenePopulate(String sessionId, String entityName, long minid) throws IcatException { - URI uri = getUri(getUriBuilder("lucene/db/" + entityName + "/" + minid)); + URI uri = getUri(getUriBuilder("lucene/db/" + entityName)); List formparams = new ArrayList<>(); formparams.add(new BasicNameValuePair("sessionId", sessionId)); + formparams.add(new BasicNameValuePair("minId", Long.toString(minid))); + formparams.add(new BasicNameValuePair("delete", "true")); try (CloseableHttpClient httpclient = HttpClients.createDefault()) { HttpPost httpPost = new HttpPost(uri); From 07cdc5b2762b4d5f89d82cd89da929a3e616c77d Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 9 Sep 2022 05:04:14 +0100 Subject: [PATCH 5/9] Add deprecation warnings #27 --- pom.xml | 2 +- .../org/icatproject/icat/client/ICAT.java | 9 ++- .../org/icatproject/icat/client/Session.java | 67 +++++++++++++++---- 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/pom.xml b/pom.xml index 4c2da8b..6aba862 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ org.icatproject icat.client - 5.0.0-SNAPSHOT + 5.1.0-SNAPSHOT jar ICAT Clients Provides facilities to login to an ICAT instance after which one can create, read, update or delete entities according to your permissions. This includes a Java client using SOAP access and a Java client using the REST interface of the icat.server. diff --git a/src/main/java/org/icatproject/icat/client/ICAT.java b/src/main/java/org/icatproject/icat/client/ICAT.java index 0bd497d..0b8802e 100644 --- a/src/main/java/org/icatproject/icat/client/ICAT.java +++ b/src/main/java/org/icatproject/icat/client/ICAT.java @@ -846,12 +846,15 @@ List luceneGetPopulating(String sessionId) throws IcatException { } } - void lucenePopulate(String sessionId, String entityName, long minid) throws IcatException { + void lucenePopulate(String sessionId, String entityName, Long minId, Long maxId, Boolean delete) throws IcatException { URI uri = getUri(getUriBuilder("lucene/db/" + entityName)); List formparams = new ArrayList<>(); formparams.add(new BasicNameValuePair("sessionId", sessionId)); - formparams.add(new BasicNameValuePair("minId", Long.toString(minid))); - formparams.add(new BasicNameValuePair("delete", "true")); + formparams.add(new BasicNameValuePair("minId", minId.toString())); + if (maxId != null) { + formparams.add(new BasicNameValuePair("maxId", maxId.toString())); + } + formparams.add(new BasicNameValuePair("delete", delete.toString())); try (CloseableHttpClient httpclient = HttpClients.createDefault()) { HttpPost httpPost = new HttpPost(uri); diff --git a/src/main/java/org/icatproject/icat/client/Session.java b/src/main/java/org/icatproject/icat/client/Session.java index 4abc364..ce97ed6 100644 --- a/src/main/java/org/icatproject/icat/client/Session.java +++ b/src/main/java/org/icatproject/icat/client/Session.java @@ -352,6 +352,10 @@ public String get(String query, long id) throws IcatException { /** * Return a set of investigations satisfying the constraints * + * @deprecated in favour of {@link #searchInvestigations(String, String, Date, Date, List, String, int, String, JsonArray)}, which allows an upper limit + * on population to be set and makes deletion of existing documents + * optional. + * * @param user * If not null must exactly match the name of a user related * via @@ -396,6 +400,7 @@ public String get(String query, long id) throws IcatException { * @throws IcatException * For various ICAT errors. */ + @Deprecated public String searchInvestigations(String user, String text, Date lower, Date upper, List parameters, List samples, String userFullName, int maxResults) throws IcatException { @@ -448,11 +453,11 @@ public String searchInvestigations(String user, String text, Date lower, Date up * value pairs of the field(s) to sort on and their * direction * @param facets - * String representing a JsonArray of JsonObjects. Each - * should define the "target" entity name, and optionally - * another JsonArray of "dimensions", which are specific - * fields to facet. If absent, then all applicable fields - * will be faceted. + * String representing a JsonArray of JsonObjects. Each + * should define the "target" entity name, and optionally + * another JsonArray of "dimensions", which are specific + * fields to facet. If absent, then all applicable fields + * will be faceted. * * @return the Json holding the result. * @@ -469,6 +474,10 @@ public String searchInvestigations(String user, String text, Date lower, Date up /** * Return a set of datasets satisfying the constraints * + * @deprecated in favour of {@link #searchDatasets(String, String, Date, Date, List, String, int, String, JsonArray)}, which allows an upper limit + * on population to be set and makes deletion of existing documents + * optional. + * * @param user * If not null must exactly match the name of a user related * via the investigation user and the investigation to the @@ -494,6 +503,7 @@ public String searchInvestigations(String user, String text, Date lower, Date up * @throws IcatException * For various ICAT errors. */ + @Deprecated public String searchDatasets(String user, String text, Date lower, Date upper, List parameters, int maxResults) throws IcatException { return icat.searchDatasets(sessionId, user, text, lower, upper, parameters, maxResults); @@ -543,7 +553,8 @@ public String searchDatasets(String user, String text, Date lower, Date upper, L */ public String searchDatasets(String user, String text, Date lower, Date upper, List parameters, String searchAfter, int maxCount, String sort, JsonArray facets) throws IcatException { - return icat.searchDatasets(sessionId, user, text, lower, upper, parameters, searchAfter, maxCount, sort, facets); + return icat.searchDatasets(sessionId, user, text, lower, upper, parameters, searchAfter, maxCount, sort, + facets); } /** @@ -569,14 +580,38 @@ public void luceneCommit() throws IcatException { /** * Clear and repopulate lucene documents for the specified entityName. * - * @param entityName - * the name of the entity + * @deprecated in favour of {@link #searchPopulate}, which allows an upper limit + * on population to be set and makes deletion of existing documents + * optional. * - * @throws IcatException - * For various ICAT errors. + * @param entityName the name of the entity + * @param minId Process entities with id values greater than (NOT equal to) + * this value + * + * @throws IcatException For various ICAT errors. */ - public void lucenePopulate(String entityName, long minid) throws IcatException { - icat.lucenePopulate(sessionId, entityName, minid); + @Deprecated + public void lucenePopulate(String entityName, long minId) throws IcatException { + icat.lucenePopulate(sessionId, entityName, minId, null, true); + } + + /** + * Populates search engine documents for the specified entityName. + * + * Optionally, this will also delete all existing documents of entityName. This + * should only be used when repopulating from scratch is needed. + * + * @param entityName the name of the entity + * @param minId Process entities with id values greater than (NOT equal to) + * this value + * @param maxId Process entities up to and including with id up to and + * including this value + * @param delete If true, then all existing documents of this type will be + * deleted before adding new ones. + * @throws IcatException For various ICAT errors. + */ + public void searchPopulate(String entityName, long minId, long maxId, boolean delete) throws IcatException { + icat.lucenePopulate(sessionId, entityName, minId, maxId, delete); } /** @@ -594,6 +629,10 @@ public List luceneGetPopulating() throws IcatException { /** * Return a set of data files satisfying the constraints * + * @deprecated in favour of {@link #searchDatafiles(String, String, Date, Date, List, String, int, String, JsonArray)}, which allows an upper limit + * on population to be set and makes deletion of existing documents + * optional. + * * @param user * If not null must exactly match the name of a user related * via @@ -620,6 +659,7 @@ public List luceneGetPopulating() throws IcatException { * @throws IcatException * For various ICAT errors. */ + @Deprecated public String searchDatafiles(String user, String text, Date lower, Date upper, List parameters, int maxResults) throws IcatException { return icat.searchDatafiles(sessionId, user, text, lower, upper, parameters, maxResults); @@ -670,7 +710,8 @@ public String searchDatafiles(String user, String text, Date lower, Date upper, */ public String searchDatafiles(String user, String text, Date lower, Date upper, List parameters, String searchAfter, int maxCount, String sort, JsonArray facets) throws IcatException { - return icat.searchDatafiles(sessionId, user, text, lower, upper, parameters, searchAfter, maxCount, sort, facets); + return icat.searchDatafiles(sessionId, user, text, lower, upper, parameters, searchAfter, maxCount, sort, + facets); } /** From 9a34808dd1cf7114c74c80068f63f4039e6a94c3 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Mon, 26 Sep 2022 11:43:19 +0100 Subject: [PATCH 6/9] Refactor into searchDocuments #27 --- .../org/icatproject/icat/client/ICAT.java | 131 +++++------------- 1 file changed, 31 insertions(+), 100 deletions(-) diff --git a/src/main/java/org/icatproject/icat/client/ICAT.java b/src/main/java/org/icatproject/icat/client/ICAT.java index 0b8802e..5bcf592 100644 --- a/src/main/java/org/icatproject/icat/client/ICAT.java +++ b/src/main/java/org/icatproject/icat/client/ICAT.java @@ -65,10 +65,10 @@ public class ICAT { * URI * * @param urlString - * The URI of a server in the form https://example.com:443. + * The URI of a server in the form https://example.com:443. * * @throws URISyntaxException - * If the urlString is not a valid URI + * If the urlString is not a valid URI */ public ICAT(String urlString) throws URISyntaxException { this.uri = new URI(urlString); @@ -382,13 +382,13 @@ void importMetaData(String sessionId, Path path, DuplicateAction duplicate, Attr * See whether or not someone is logged in. * * @param userName - * which must include mnemonic if the authenticator plugin is - * configured to return them. + * which must include mnemonic if the authenticator plugin is + * configured to return them. * * @return true if at least one session exists else false. * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public boolean isLoggedIn(String userName) throws IcatException { URI uri = getUri(getUriBuilder("user/" + userName)); @@ -407,13 +407,13 @@ public boolean isLoggedIn(String userName) throws IcatException { * Login to a RESTful ICAT instance and return a Session * * @param plugin - * The mnemonic of the authentication plugin + * The mnemonic of the authentication plugin * @param credentials - * A map holding credential key/value pairs + * A map holding credential key/value pairs * @return A RESTful ICAT Session * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public Session login(String plugin, Map credentials) throws IcatException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -447,7 +447,7 @@ public Session login(String plugin, Map credentials) throws Icat * No check is made on the validity of the sessionId. * * @param sessionId - * the sessionId to hold in the session. + * the sessionId to hold in the session. * * @return the new session */ @@ -503,7 +503,7 @@ String search(String sessionId, String query) throws IcatException { * @return the version of the ICAT server * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ @Deprecated public String getApiVersion() throws IcatException { @@ -525,7 +525,7 @@ public String getApiVersion() throws IcatException { * @return the version of the ICAT server * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public String getVersion() throws IcatException { URI uri = getUri(getUriBuilder("version")); @@ -593,7 +593,7 @@ String searchInvestigations(String sessionId, String user, String text, Date low } } - String searchInvestigations(String sessionId, String user, String text, Date lower, Date upper, + private String searchDocuments(String target, String sessionId, String user, String text, Date lower, Date upper, List parameters, String userFullName, String searchAfter, int maxCount, String sort, JsonArray facets) throws IcatException { URIBuilder uriBuilder = getUriBuilder("search/documents"); @@ -601,7 +601,7 @@ String searchInvestigations(String sessionId, String user, String text, Date low ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (JsonGenerator gen = Json.createGenerator(baos)) { gen.writeStartObject(); - gen.write("target", "Investigation"); + gen.write("target", target); if (user != null) { gen.write("user", user); } @@ -643,6 +643,13 @@ String searchInvestigations(String sessionId, String user, String text, Date low } } + String searchInvestigations(String sessionId, String user, String text, Date lower, Date upper, + List parameters, String userFullName, String searchAfter, int maxCount, String sort, + JsonArray facets) throws IcatException { + return searchDocuments("Investigation", sessionId, user, text, lower, upper, parameters, userFullName, + searchAfter, maxCount, sort, facets); + } + String searchDatasets(String sessionId, String user, String text, Date lower, Date upper, List parameters, int maxResults) throws IcatException { URIBuilder uriBuilder = getUriBuilder("lucene/data"); @@ -685,49 +692,10 @@ String searchDatasets(String sessionId, String user, String text, Date lower, Da } String searchDatasets(String sessionId, String user, String text, Date lower, Date upper, - List parameters, String searchAfter, int maxCount, String sort, JsonArray facets) throws IcatException { - URIBuilder uriBuilder = getUriBuilder("search/documents"); - uriBuilder.setParameter("sessionId", sessionId); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("target", "Dataset"); - if (user != null) { - gen.write("user", user); - } - if (text != null) { - gen.write("text", text); - } - if (lower != null) { - gen.write("lower", DateTools.dateToString(lower, Resolution.MINUTE)); - } - if (upper != null) { - gen.write("upper", DateTools.dateToString(upper, Resolution.MINUTE)); - } - if (parameters != null && !parameters.isEmpty()) { - writeParameters(gen, parameters); - } - if (facets != null) { - gen.write("facets", facets); - } - gen.writeEnd(); - } - - uriBuilder.setParameter("query", baos.toString()); - uriBuilder.setParameter("search_after", searchAfter); - uriBuilder.setParameter("maxCount", Integer.toString(maxCount)); - uriBuilder.setParameter("sort", sort); - URI uri = getUri(uriBuilder); - - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - HttpGet httpGet = new HttpGet(uri); - try (CloseableHttpResponse response = httpclient.execute(httpGet)) { - return getString(response); - } - } catch (IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - - } + List parameters, String searchAfter, int maxCount, String sort, JsonArray facets) + throws IcatException { + return searchDocuments("Dataset", sessionId, user, text, lower, upper, parameters, null, searchAfter, maxCount, + sort, facets); } private void writeParameters(JsonGenerator gen, List parameters) { @@ -779,7 +747,7 @@ private void writeParameters(JsonGenerator gen, List paramet * @return the json string * * @throws IcatException - * For various ICAT errors + * For various ICAT errors */ public String getProperties() throws IcatException { URI uri = getUri(getUriBuilder("properties")); @@ -846,7 +814,8 @@ List luceneGetPopulating(String sessionId) throws IcatException { } } - void lucenePopulate(String sessionId, String entityName, Long minId, Long maxId, Boolean delete) throws IcatException { + void lucenePopulate(String sessionId, String entityName, Long minId, Long maxId, Boolean delete) + throws IcatException { URI uri = getUri(getUriBuilder("lucene/db/" + entityName)); List formparams = new ArrayList<>(); formparams.add(new BasicNameValuePair("sessionId", sessionId)); @@ -908,48 +877,10 @@ String searchDatafiles(String sessionId, String user, String text, Date lower, D } String searchDatafiles(String sessionId, String user, String text, Date lower, Date upper, - List parameters, String searchAfter, int maxCount, String sort, JsonArray facets) throws IcatException { - URIBuilder uriBuilder = getUriBuilder("search/documents"); - uriBuilder.setParameter("sessionId", sessionId); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (JsonGenerator gen = Json.createGenerator(baos)) { - gen.writeStartObject(); - gen.write("target", "Datafile"); - if (user != null) { - gen.write("user", user); - } - if (text != null) { - gen.write("text", text); - } - if (lower != null) { - gen.write("lower", DateTools.dateToString(lower, Resolution.MINUTE)); - } - if (upper != null) { - gen.write("upper", DateTools.dateToString(upper, Resolution.MINUTE)); - } - if (parameters != null && !parameters.isEmpty()) { - writeParameters(gen, parameters); - } - if (facets != null) { - gen.write("facets", facets); - } - gen.writeEnd(); - } - - uriBuilder.setParameter("query", baos.toString()); - uriBuilder.setParameter("search_after", searchAfter); - uriBuilder.setParameter("maxCount", Integer.toString(maxCount)); - uriBuilder.setParameter("sort", sort); - URI uri = getUri(uriBuilder); - - try (CloseableHttpClient httpclient = HttpClients.createDefault()) { - HttpGet httpGet = new HttpGet(uri); - try (CloseableHttpResponse response = httpclient.execute(httpGet)) { - return getString(response); - } - } catch (IOException e) { - throw new IcatException(IcatExceptionType.INTERNAL, e.getClass() + " " + e.getMessage()); - } + List parameters, String searchAfter, int maxCount, String sort, JsonArray facets) + throws IcatException { + return searchDocuments("Datafile", sessionId, user, text, lower, upper, parameters, null, searchAfter, maxCount, + sort, facets); } long cloneEntity(String sessionId, String name, long id, Map keys) throws IcatException { From b87802fa1d4d50be7012d18ddfbde60e9c6ab7c9 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Wed, 12 Oct 2022 19:19:58 +0100 Subject: [PATCH 7/9] Remove dependency on Lucene date tools #27 --- pom.xml | 7 ---- .../org/icatproject/icat/client/ICAT.java | 32 +++++++++++-------- .../org/icatproject/icat/TestIcatClient.java | 2 +- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/pom.xml b/pom.xml index f0608e1..0a0a07f 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,6 @@ UTF-8 github https://repo.icatproject.org/repo/ - 5.3.0 @@ -62,12 +61,6 @@ - - org.apache.lucene - lucene-core - ${luceneVersion} - - javax javaee-api diff --git a/src/main/java/org/icatproject/icat/client/ICAT.java b/src/main/java/org/icatproject/icat/client/ICAT.java index 5bcf592..2986acd 100644 --- a/src/main/java/org/icatproject/icat/client/ICAT.java +++ b/src/main/java/org/icatproject/icat/client/ICAT.java @@ -47,8 +47,6 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; -import org.apache.lucene.document.DateTools; -import org.apache.lucene.document.DateTools.Resolution; import org.icatproject.icat.client.IcatException.IcatExceptionType; import org.icatproject.icat.client.Session.Attributes; import org.icatproject.icat.client.Session.DuplicateAction; @@ -556,11 +554,10 @@ String searchInvestigations(String sessionId, String user, String text, Date low gen.write("text", text); } if (lower != null) { - // TODO Remove DateTools as it is from a Lucene library! - gen.write("lower", DateTools.dateToString(lower, Resolution.MINUTE)); + gen.write("lower", roundMinute(lower, 0)); } if (upper != null) { - gen.write("upper", DateTools.dateToString(upper, Resolution.MINUTE)); + gen.write("upper", roundMinute(upper, 59999)); } if (parameters != null && !parameters.isEmpty()) { writeParameters(gen, parameters); @@ -609,10 +606,10 @@ private String searchDocuments(String target, String sessionId, String user, Str gen.write("text", text); } if (lower != null) { - gen.write("lower", DateTools.dateToString(lower, Resolution.MINUTE)); + gen.write("lower", roundMinute(lower, 0)); } if (upper != null) { - gen.write("upper", DateTools.dateToString(upper, Resolution.MINUTE)); + gen.write("upper", roundMinute(upper, 59999)); } if (parameters != null && !parameters.isEmpty()) { writeParameters(gen, parameters); @@ -665,10 +662,10 @@ String searchDatasets(String sessionId, String user, String text, Date lower, Da gen.write("text", text); } if (lower != null) { - gen.write("lower", DateTools.dateToString(lower, Resolution.MINUTE)); + gen.write("lower", roundMinute(lower, 0)); } if (upper != null) { - gen.write("upper", DateTools.dateToString(upper, Resolution.MINUTE)); + gen.write("upper", roundMinute(upper, 59999)); } if (parameters != null && !parameters.isEmpty()) { writeParameters(gen, parameters); @@ -711,8 +708,8 @@ private void writeParameters(JsonGenerator gen, List paramet if (parameter.getStringValue() != null) { gen.write("stringValue", parameter.getStringValue()); } else if (parameter.getLowerDateValue() != null && parameter.getUpperDateValue() != null) { - gen.write("lowerDateValue", DateTools.dateToString(parameter.getLowerDateValue(), Resolution.MINUTE)); - gen.write("upperDateValue", DateTools.dateToString(parameter.getUpperDateValue(), Resolution.MINUTE)); + gen.write("lowerDateValue", roundMinute(parameter.getLowerDateValue(), 0)); + gen.write("upperDateValue", roundMinute(parameter.getUpperDateValue(), 59999)); } else if (parameter.getLowerNumericValue() != null && parameter.getUpperNumericValue() != null) { gen.write("lowerNumericValue", parameter.getLowerNumericValue()); gen.write("upperNumericValue", parameter.getUpperNumericValue()); @@ -851,10 +848,10 @@ String searchDatafiles(String sessionId, String user, String text, Date lower, D gen.write("text", text); } if (lower != null) { - gen.write("lower", DateTools.dateToString(lower, Resolution.MINUTE)); + gen.write("lower", roundMinute(lower, 0)); } if (upper != null) { - gen.write("upper", DateTools.dateToString(upper, Resolution.MINUTE)); + gen.write("upper", roundMinute(upper, 59999)); } if (parameters != null && !parameters.isEmpty()) { writeParameters(gen, parameters); @@ -943,4 +940,13 @@ public String list(String sessionId, String path) throws IcatException { } } + /** + * @param date Date to round down to the minute + * @param offset Number of ms to be added to the returned value + * @return Rounded date, converted to ms with offset applied + */ + private long roundMinute(Date date, long offset) { + return (date.getTime() / 60000) * 60000 + offset; + } + } diff --git a/src/test/java/org/icatproject/icat/TestIcatClient.java b/src/test/java/org/icatproject/icat/TestIcatClient.java index 43de411..6f0a160 100644 --- a/src/test/java/org/icatproject/icat/TestIcatClient.java +++ b/src/test/java/org/icatproject/icat/TestIcatClient.java @@ -106,7 +106,7 @@ public void testSession() throws Exception { @Test public void testInfo() throws Exception { - assertTrue(icat.getVersion().startsWith("5.0.")); + assertTrue(icat.getVersion().startsWith("5.1.")); } } From 7a53bc84ee41878de1b7bc4efba2f8ff19cbed32 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Thu, 7 Sep 2023 13:12:27 +0000 Subject: [PATCH 8/9] Replace javax -> jakarta in Session --- src/main/java/org/icatproject/icat/client/Session.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/icatproject/icat/client/Session.java b/src/main/java/org/icatproject/icat/client/Session.java index ce97ed6..c5d1832 100644 --- a/src/main/java/org/icatproject/icat/client/Session.java +++ b/src/main/java/org/icatproject/icat/client/Session.java @@ -6,7 +6,7 @@ import java.util.List; import java.util.Map; -import javax.json.JsonArray; +import jakarta.json.JsonArray; /** * A RESTful ICAT session. From ce846892d7a87a6f8c837a282804925c70864848 Mon Sep 17 00:00:00 2001 From: Patrick Austin Date: Fri, 8 Sep 2023 16:11:47 +0000 Subject: [PATCH 9/9] 6.1.0 release notes --- src/site/xhtml/release-notes.xhtml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/site/xhtml/release-notes.xhtml b/src/site/xhtml/release-notes.xhtml index 0b337ec..1552819 100644 --- a/src/site/xhtml/release-notes.xhtml +++ b/src/site/xhtml/release-notes.xhtml @@ -6,6 +6,9 @@

ICAT Client Release Notes

+

6.1.0

+

Support for new free text search endpoints, and deprecation of old endpoints.

+

6.0.0

Upgrade from JavaEE to JakartaEE 10. Requires Java 11+.