From 54950dc75543d7c4abde26319af237f1dd0844f4 Mon Sep 17 00:00:00 2001 From: Stefan Meyer Date: Wed, 18 Oct 2023 22:31:43 +0200 Subject: [PATCH] search: libs updates; removed some deprecations --- Services/WebServices/RPC/lib/pom.xml | 82 +++++++++------- .../services/lucene/index/IndexHolder.java | 76 +++------------ .../transform/ContentObjectTransformer.java | 51 +++------- .../lucene/search/SearchResultWriter.java | 35 +------ .../search/highlight/HitHighlighter.java | 95 +++++++------------ .../java/de/ilias/services/rpc/RPCServer.java | 6 +- .../lib/src/main/resources/log4j2.properties | 2 +- 7 files changed, 114 insertions(+), 233 deletions(-) diff --git a/Services/WebServices/RPC/lib/pom.xml b/Services/WebServices/RPC/lib/pom.xml index caee5baadaf5..e58f688c0c42 100644 --- a/Services/WebServices/RPC/lib/pom.xml +++ b/Services/WebServices/RPC/lib/pom.xml @@ -9,6 +9,14 @@ ilServer + + org.apache.maven.plugins + maven-resources-plugin + 3.3.1 + + UTF-8 + + org.apache.maven.plugins maven-shade-plugin @@ -20,6 +28,7 @@ shade + false false @@ -33,6 +42,8 @@ META-INF/*.DSA META-INF/*.RSA META-INF/*.MF + META-INF/AL2.0 + META-INF/LGPL2.1 META-INF/versions/9/module-info.class license/** @@ -56,77 +67,83 @@ + + + commons-httpclient + commons-httpclient + 3.1 + org.apache.logging.log4j log4j-core - 2.20.0 + 2.21.0 org.apache.logging.log4j log4j-api - 2.20.0 + 2.21.0 org.apache.logging.log4j log4j-jcl - 2.20.0 + 2.21.0 org.apache.logging.log4j log4j-slf4j-impl - 2.20.0 + 2.21.0 + + + + org.apache.logging.log4j + log4j-1.2-api + 2.21.0 org.apache.commons commons-configuration2 - 2.8.0 - - - - commons-httpclient - commons-httpclient - 3.1 + 2.9.0 org.apache.lucene lucene-core - 9.5.0 + 9.8.0 org.apache.lucene lucene-highlighter - 9.5.0 + 9.8.0 org.apache.lucene lucene-join - 9.5.0 + 9.8.0 org.apache.lucene lucene-memory - 9.5.0 + 9.8.0 org.apache.lucene lucene-queries - 9.5.0 + 9.8.0 org.apache.lucene lucene-queryparser - 9.5.0 + 9.8.0 org.mariadb.jdbc mariadb-java-client - 3.0.6 + 3.2.0 compile @@ -150,67 +167,60 @@ 2.0.6.1 compile - org.apache.tika tika-core - 2.7.0 - - - org.apache.tika - tika-parser-html-module - 2.7.0 + 2.9.0 org.apache.tika tika-parser-html-module - 2.7.0 + 2.9.0 org.apache.tika tika-parser-microsoft-module - 2.7.0 + 2.9.0 org.apache.tika tika-parser-miscoffice-module - 2.7.0 + 2.9.0 org.apache.tika tika-parser-pdf-module - 2.7.0 + 2.9.0 org.apache.tika tika-parser-text-module - 2.7.0 + 2.9.0 org.apache.tika tika-parser-xml-module - 2.7.0 + 2.9.0 - org.apache.xmlgraphics fop-core - 2.8 + 2.9 org.apache.xmlgraphics fop-util - 2.8 + 2.9 - - jar ilServer https://www.ilias.de + true + UTF-8 17 17 diff --git a/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/index/IndexHolder.java b/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/index/IndexHolder.java index f382ec4a09ea..05b36002b983 100644 --- a/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/index/IndexHolder.java +++ b/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/index/IndexHolder.java @@ -38,13 +38,13 @@ import java.util.HashMap; /** - * Capsulates the interaction between IndexReader and IndexWriter + * Capsules the interaction between IndexReader and IndexWriter * This class is a singleton for each index path. * * @author Stefan Meyer - * @version $Id$ */ -public class IndexHolder { +public class IndexHolder implements AutoCloseable +{ protected static Logger logger = LogManager.getLogger(IndexHolder.class); @@ -56,9 +56,6 @@ public class IndexHolder { - /** - * @throws IOException - */ private IndexHolder(String clientKey) throws IOException { try { @@ -70,29 +67,16 @@ private IndexHolder(String clientKey) throws IOException { } - /** - * - * @param clientKey - * @return - * @throws IOException - */ - public static synchronized IndexHolder getInstance(String clientKey) throws - IOException { - - String hash = clientKey; - - if(instances.containsKey(hash)) { - return instances.get(hash); + public static synchronized IndexHolder getInstance(String clientKey) throws + IOException { + + if(instances.containsKey(clientKey)) { + return instances.get(clientKey); } - instances.put(hash,new IndexHolder(clientKey)); - return instances.get(hash); + instances.put(clientKey,new IndexHolder(clientKey)); + return instances.get(clientKey); } - /** - * - * @return - * @throws IOException - */ public static synchronized IndexHolder getInstance() throws IOException { return getInstance(LocalSettings.getClientKey()); @@ -106,16 +90,11 @@ public static void deleteIndex() throws ConfigurationException logger.info("Deleted index directory: " + indexPath.getAbsoluteFile()); } - /** - * Delete directory recursive - * @param path - * @return - */ - private static boolean deleteTree(File path) { + private static void deleteTree(File path) { if(!path.exists() || !path.isDirectory()) { - return false; + return; } for(File del : path.listFiles()) { @@ -127,21 +106,17 @@ private static boolean deleteTree(File path) { } } path.delete(); - return true; } - /** - * Close all writers - */ public static synchronized void closeAllWriters() { logger.info("Closing document writers..."); - for(Object key : instances.keySet()) { + for(String key : instances.keySet()) { try { logger.info("Closing writer: " + key); - IndexHolder holder = instances.get((String) key); - IndexDirectoryFactory.getDirectory(ClientSettings.getInstance((String) key).getIndexPath()).close(); + IndexHolder holder = instances.get(key); + IndexDirectoryFactory.getDirectory(ClientSettings.getInstance(key).getIndexPath()).close(); holder.close(); } catch (ConfigurationException | IOException ex) @@ -156,7 +131,6 @@ public static synchronized void closeAllWriters() { /** * @todo obtain lock for index writer - * @throws IOException */ public void init() throws IOException, ConfigurationException { @@ -180,16 +154,10 @@ public void init() throws IOException, ConfigurationException { } - /** - * @return the writer - */ public IndexWriter getWriter() { return writer; } - /** - * @param writer the writer to set - */ public void setWriter(IndexWriter writer) { this.writer = writer; } @@ -209,18 +177,4 @@ public void close() { logger.fatal("Error closing writer." + e); } } - - /* (non-Javadoc) - * @see java.lang.Object#finalize() - */ - @Override - protected void finalize() throws Throwable { - - try { - close(); - } - finally { - super.finalize(); - } - } } diff --git a/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/index/transform/ContentObjectTransformer.java b/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/index/transform/ContentObjectTransformer.java index e414d7b90598..44b233c46335 100644 --- a/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/index/transform/ContentObjectTransformer.java +++ b/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/index/transform/ContentObjectTransformer.java @@ -1,48 +1,23 @@ -/* - +-----------------------------------------------------------------------------+ - | ILIAS open source | - +-----------------------------------------------------------------------------+ - | Copyright (c) 1998-2001 ILIAS open source, University of Cologne | - | | - | This program is free software; you can redistribute it and/or | - | modify it under the terms of the GNU General Public License | - | as published by the Free Software Foundation; either version 2 | - | of the License, or (at your option) any later version. | - | | - | This program is distributed in the hope that it will be useful, | - | but WITHOUT ANY WARRANTY; without even the implied warranty of | - | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | - | GNU General Public License for more details. | - | | - | You should have received a copy of the GNU General Public License | - | along with this program; if not, write to the Free Software | - | Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | - +-----------------------------------------------------------------------------+ -*/ package de.ilias.services.lucene.index.transform; -import java.io.IOException; -import java.io.StringReader; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.xml.sax.InputSource; import org.xml.sax.SAXException; import org.xml.sax.XMLReader; -import org.xml.sax.helpers.XMLReaderFactory; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParserFactory; +import java.io.IOException; +import java.io.StringReader; /** - * - * * @author Stefan Meyer - * @version $Id$ */ public class ContentObjectTransformer implements ContentTransformer { protected Logger logger = LogManager.getLogger(ContentObjectTransformer.class); - - /** * Extract text from page_objects @@ -50,12 +25,12 @@ public class ContentObjectTransformer implements ContentTransformer { */ public String transform(String content) { - XMLReader reader = null; - PageObjectHandler handler = null; + XMLReader reader; + PageObjectHandler handler; StringReader stringReader = new StringReader(content); try { - reader = XMLReaderFactory.createXMLReader(); + reader = SAXParserFactory.newInstance().newSAXParser().getXMLReader(); handler = new PageObjectHandler(); reader.setContentHandler(handler); @@ -66,12 +41,12 @@ public String transform(String content) { } catch (SAXException e) { logger.warn("Cannot parse page_object content." + e); - } - catch (IOException e) { + } catch (IOException e) { logger.warn("Found invalid content." + e); - } - - return ""; + } catch (ParserConfigurationException e) { + logger.error("Creating XMLReader failed: " + e); + } + return ""; } diff --git a/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/search/SearchResultWriter.java b/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/search/SearchResultWriter.java index 6e0741ea4d63..835880ab54e5 100644 --- a/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/search/SearchResultWriter.java +++ b/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/search/SearchResultWriter.java @@ -26,7 +26,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.document.Document; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.jdom2.output.XMLOutputter; @@ -34,25 +33,17 @@ import java.io.IOException; /** - * - * * @author Stefan Meyer - * @version $Id$ */ public class SearchResultWriter { protected Logger logger = LogManager.getLogger(SearchResultWriter.class); - private IndexSearcher searcher = null; - private ScoreDoc[] hits = null; - private SearchHits result = null; + private IndexSearcher searcher; + private ScoreDoc[] hits; + private SearchHits result; private int offset = 0; - /** - * @param hits - * @throws ConfigurationException - * @throws IOException - */ public SearchResultWriter(ScoreDoc[] hits) throws IOException, ConfigurationException { this.hits = hits; @@ -61,12 +52,7 @@ public SearchResultWriter(ScoreDoc[] hits) throws IOException, ConfigurationExce result = new SearchHits(); } - /** - * @throws IOException - * @throws CorruptIndexException - * - */ - public void write() throws CorruptIndexException, IOException { + public void write() throws IOException { result.setTotalHits(hits.length); logger.info("Found " + result.getTotalHits() + " hits!"); @@ -89,7 +75,7 @@ public void write() throws CorruptIndexException, IOException { try { logger.debug("Added object"); object = new SearchObject(); - hitDoc = searcher.doc(hits[i].doc); + hitDoc = searcher.getIndexReader().storedFields().document(hits[i].doc); object.setId(Integer.parseInt(hitDoc.get("objId"))); object.setAbsoluteScore(hits[i].score); result.addObject(object); @@ -100,27 +86,16 @@ public void write() throws CorruptIndexException, IOException { } } - /** - * @return - */ public String toXML() { - org.jdom2.Document doc = new org.jdom2.Document(result.addXML()); XMLOutputter outputter = new XMLOutputter(); return outputter.outputString(doc); - } - /** - * @param offset the offset to set - */ public void setOffset(int offset) { this.offset = offset; } - /** - * @return the offset - */ public int getOffset() { return offset; } diff --git a/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/search/highlight/HitHighlighter.java b/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/search/highlight/HitHighlighter.java index 33115594fe81..a58ded318d55 100644 --- a/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/search/highlight/HitHighlighter.java +++ b/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/lucene/search/highlight/HitHighlighter.java @@ -22,39 +22,29 @@ package de.ilias.services.lucene.search.highlight; -import java.io.IOException; -import java.io.StringReader; -import java.sql.SQLException; - +import de.ilias.services.lucene.index.FieldInfo; +import de.ilias.services.lucene.search.SearchHolder; +import de.ilias.services.lucene.settings.LuceneSettings; +import de.ilias.services.settings.ConfigurationException; +import de.ilias.services.settings.LocalSettings; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.highlight.Fragmenter; -import org.apache.lucene.search.highlight.Highlighter; -import org.apache.lucene.search.highlight.QueryScorer; -import org.apache.lucene.search.highlight.SimpleFragmenter; -import org.apache.lucene.search.highlight.SimpleHTMLFormatter; +import org.apache.lucene.search.highlight.*; -import de.ilias.services.lucene.index.FieldInfo; -import de.ilias.services.lucene.search.SearchHolder; -import de.ilias.services.lucene.settings.LuceneSettings; -import de.ilias.services.settings.ConfigurationException; -import de.ilias.services.settings.LocalSettings; -import org.apache.logging.log4j.Logger; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; +import java.io.IOException; +import java.io.StringReader; +import java.sql.SQLException; /** - * - * * @author Stefan Meyer - * @version $Id$ */ public class HitHighlighter { @@ -77,12 +67,6 @@ public class HitHighlighter { private LuceneSettings luceneSettings; - /** - * @throws IOException - * @throws ConfigurationException - * @throws SQLException - * - */ public HitHighlighter(Query query,ScoreDoc[] hits) throws ConfigurationException, IOException, SQLException { this.query = query; @@ -90,18 +74,12 @@ public HitHighlighter(Query query,ScoreDoc[] hits) throws ConfigurationException init(); } - /** - * @throws IOException - * @throws CorruptIndexException - * - */ public void highlight() throws CorruptIndexException, IOException, InvalidTokenOffsetsException { result = new HighlightHits(); HighlightObject resObject; HighlightItem resItem; - HighlightField resField; - + TokenStream token; String fragment; @@ -113,8 +91,8 @@ public void highlight() throws CorruptIndexException, IOException, InvalidTokenO result.setMaxScore(hits[i].score); } - StringBuffer allContent = new StringBuffer(); - Document hitDoc = searcher.doc(hits[i].doc); + StringBuilder allContent = new StringBuilder(); + Document hitDoc = searcher.getIndexReader().storedFields().document(hits[i].doc); int objId; int subItem; @@ -164,23 +142,23 @@ public void highlight() throws CorruptIndexException, IOException, InvalidTokenO } } // All content - for(int j = 0; j < fields.length; j++) { - - // Do not add metaData Field, since this information is stored redundant in lom* fields - if(fields[j].equals("metaData")) { - continue; - } - - if(fields[j].equals("title") || fields[j].equals("description")) { - continue; - } - - IndexableField[] separatedFields = hitDoc.getFields(fields[j]); - for(int k = 0; k < separatedFields.length; k++) { - allContent.append(separatedFields[k].stringValue()); - allContent.append(" "); - } - } + for (String field : fields) { + + // Do not add metaData Field, since this information is stored redundant in lom* fields + if (field.equals("metaData")) { + continue; + } + + if (field.equals("title") || field.equals("description")) { + continue; + } + + IndexableField[] separatedFields = hitDoc.getFields(field); + for (IndexableField separatedField : separatedFields) { + allContent.append(separatedField.stringValue()); + allContent.append(" "); + } + } //logger.debug("All content" + allContent.toString()); token = new StandardAnalyzer().tokenStream("content", new StringReader(allContent.toString())); fragment = highlighter.getBestFragments( @@ -197,12 +175,6 @@ public void highlight() throws CorruptIndexException, IOException, InvalidTokenO } } - /** - * @throws ConfigurationException - * @throws IOException - * @throws SQLException - * - */ private void init() throws ConfigurationException, IOException, SQLException { // init lucene settings @@ -223,7 +195,7 @@ private void init() throws ConfigurationException, IOException, SQLException { Fragmenter titleFragmenter = new SimpleFragmenter(FRAGMENT_TITLE_SIZE); titleHighlighter.setTextFragmenter(titleFragmenter); - // init fieldinfo + // init field info fieldInfo = FieldInfo.getInstance(LocalSettings.getClientKey()); // init searcher @@ -231,9 +203,6 @@ private void init() throws ConfigurationException, IOException, SQLException { } - /** - * @return - */ public String toXML() { return result.toXML(); diff --git a/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/rpc/RPCServer.java b/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/rpc/RPCServer.java index 078fdf29daea..826952a43649 100644 --- a/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/rpc/RPCServer.java +++ b/Services/WebServices/RPC/lib/src/main/java/de/ilias/services/rpc/RPCServer.java @@ -160,12 +160,10 @@ public void start() throws ConfigurationException { { logger.error("Cannot bind to host: " + getHost() + ", port: " + port + " " + e); throw new ConfigurationException(e.getMessage()); - } - catch(Exception e) { + } catch(Exception e) { logger.error("Cannot bind to host: " + getHost() + ", port: " + port + " " + e); throw new ConfigurationException(e.getMessage()); - } - catch(Throwable e) { + } catch(Throwable e) { logger.error(e); throw new ConfigurationException(e.getMessage()); } diff --git a/Services/WebServices/RPC/lib/src/main/resources/log4j2.properties b/Services/WebServices/RPC/lib/src/main/resources/log4j2.properties index 3e58d7eafb06..2a91110f9856 100644 --- a/Services/WebServices/RPC/lib/src/main/resources/log4j2.properties +++ b/Services/WebServices/RPC/lib/src/main/resources/log4j2.properties @@ -21,7 +21,7 @@ rootLogger.level = ERROR logger.xmlrpc.name = org.apache.xmlrpc logger.xmlrpc.appenderRef.Console.ref = ConsoleMin -logger.xmlrpc.level = INFO +logger.xmlrpc.level = FATAL logger.xmlrpc.additivity = false # de.ilias. Level is set programmatically using ilServer.ini LogLevel and LogFile