From 37cf9047c63eb00903e225c7fae146be4ad07cdc Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:37:22 +0100 Subject: [PATCH 01/10] Show db connection statistics in the log file or the `dbstatistics` endpoint (#815) * Show db statistics in the log file or the `dbstatistics` endpoint * Finding out why github checks are failed - undo hibernate.cfg * Disabled automatic logging * Use scheduled CRON job instead of PostConstruct * hibernate generating property true --------- Co-authored-by: Paurikova2 --- .../main/java/org/dspace/core/Context.java | 12 ++++ .../dspace/core/HibernateDBConnection.java | 62 +++++++++++++++++++ .../ClarinAutoRegistrationController.java | 2 +- .../DBConnectionStatisticsController.java | 39 ++++++++++++ dspace/config/hibernate.cfg.xml | 1 + 5 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 dspace-server-webapp/src/main/java/org/dspace/app/rest/DBConnectionStatisticsController.java diff --git a/dspace-api/src/main/java/org/dspace/core/Context.java b/dspace-api/src/main/java/org/dspace/core/Context.java index 02a3fee09f8a..8eed24348c39 100644 --- a/dspace-api/src/main/java/org/dspace/core/Context.java +++ b/dspace-api/src/main/java/org/dspace/core/Context.java @@ -976,4 +976,16 @@ public Group getAdminGroup() throws SQLException { .getGroupService() .findByName(this, Group.ADMIN) : adminGroup; } + + /** + * Get the Hibernate statistics for this context. + * Only available when using HibernateDBConnection. + * @return the Hibernate statistics as a String + */ + public String getHibernateStatistics() { + if (dbConnection instanceof HibernateDBConnection) { + return ((HibernateDBConnection) dbConnection).getHibernateStatistics(); + } + return "Hibernate statistics are not available for this database connection"; + } } diff --git a/dspace-api/src/main/java/org/dspace/core/HibernateDBConnection.java b/dspace-api/src/main/java/org/dspace/core/HibernateDBConnection.java index b371af80eede..f8c620380d5f 100644 --- a/dspace-api/src/main/java/org/dspace/core/HibernateDBConnection.java +++ b/dspace-api/src/main/java/org/dspace/core/HibernateDBConnection.java @@ -12,6 +12,8 @@ import java.sql.SQLException; import javax.sql.DataSource; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.dspace.authorize.ResourcePolicy; import org.dspace.content.Bitstream; import org.dspace.content.Bundle; @@ -29,9 +31,11 @@ import org.hibernate.engine.spi.SessionFactoryImplementor; import org.hibernate.proxy.HibernateProxyHelper; import org.hibernate.resource.transaction.spi.TransactionStatus; +import org.hibernate.stat.Statistics; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.orm.hibernate5.SessionFactoryUtils; +import org.springframework.scheduling.annotation.Scheduled; /** * Hibernate implementation of the DBConnection. @@ -64,6 +68,8 @@ public class HibernateDBConnection implements DBConnection { private boolean batchModeEnabled = false; private boolean readOnlyEnabled = false; + private static final Logger log = LogManager.getLogger(HibernateDBConnection.class); + /** * Retrieves the current Session from Hibernate (per our settings, Hibernate is configured to create one Session * per thread). If Session doesn't yet exist, it is created. A Transaction is also initialized (or reinintialized) @@ -102,6 +108,13 @@ protected Transaction getTransaction() { return sessionFactory.getCurrentSession().getTransaction(); } + // This method will run every 10 seconds + @Scheduled(fixedRate = 10000) // Fixed rate in milliseconds + public void logConnectionMetrics() { + logHibernateStatistics(); + logDatabaseMetaData(); + } + /** * Check if Hibernate Session is still "alive" / open. An open Session may or may not have an open Transaction * (so isTransactionAlive() may return false even if isSessionAlive() returns true). A Session may be reused for @@ -350,4 +363,53 @@ public void flushSession() throws SQLException { getSession().flush(); } } + + + /** + * Log the Hibernate statistics (e.g. open sessions, closed sessions, transactions, connections obtained) + */ + private void logHibernateStatistics() { + if (sessionFactory != null) { + log.info(getHibernateStatistics()); + } else { + log.warn(getHibernateStatistics()); + } + } + + /** + * Log the database metadata (URL, User, Driver, Product, Version) + */ + private void logDatabaseMetaData() { + try (Session session = sessionFactory.openSession()) { + // Use doReturningWork to safely interact with the JDBC Connection + session.doReturningWork(connection -> { + try { + DatabaseMetaData metaData = connection.getMetaData(); + log.info("Database Metadata - URL: {}, User: {}, Driver: {}, Product: {} {}" + , metaData.getURL(), metaData.getUserName(), metaData.getDriverName(), + metaData.getDatabaseProductName(), metaData.getDatabaseProductVersion()); + } catch (SQLException e) { + log.warn("Failed to retrieve database metadata: {}", e.getMessage()); + } + return null; // Returning null as no specific result is needed + }); + } catch (Exception e) { + log.warn("Failed to log database metadata: {}", e.getMessage()); + } + } + + /** + * Get Hibernate statistics as a string + */ + public String getHibernateStatistics() { + if (sessionFactory != null) { + Statistics stats = sessionFactory.getStatistics(); + return "Hibernate Statistics - Open Sessions: " + stats.getSessionOpenCount() + ", Closed Sessions: " + + stats.getSessionCloseCount() + ", Transactions: " + stats.getTransactionCount() + + ", Connections Obtained: " + stats.getConnectCount(); + } else { + return "SessionFactory is not available for logging Hibernate statistics."; + } + } } + diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinAutoRegistrationController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinAutoRegistrationController.java index af6c01714fa7..eef3b8495699 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinAutoRegistrationController.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/ClarinAutoRegistrationController.java @@ -41,7 +41,7 @@ * * This Shibboleth Authentication process is tested in ClarinShibbolethLoginFilterIT. * - * @author Milan Majchrak (milan.majchrak at dataquest.sk) + * @author Milan Majchrak (dspace at dataquest.sk) */ @RequestMapping(value = "/api/autoregistration") @RestController diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/DBConnectionStatisticsController.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/DBConnectionStatisticsController.java new file mode 100644 index 000000000000..9472a3e12f5d --- /dev/null +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/DBConnectionStatisticsController.java @@ -0,0 +1,39 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.app.rest; + +import javax.servlet.http.HttpServletRequest; + +import org.dspace.app.rest.utils.ContextUtil; +import org.dspace.core.Context; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +/** + * Controller for retrieving database connection statistics + * + * @author Milan Majchrak (dspace at dataquest.sk) + */ +@PreAuthorize("hasAuthority('ADMIN')") +@RequestMapping(value = "/api/dbstatistics") +@RestController +public class DBConnectionStatisticsController { + @RequestMapping(method = RequestMethod.GET) + public ResponseEntity getStatistics(HttpServletRequest request) { + + Context context = ContextUtil.obtainContext(request); + if (context == null) { + return ResponseEntity.status(500).build(); + } + // Return response entity with the statistics + return ResponseEntity.ok().body(context.getHibernateStatistics()); + } +} diff --git a/dspace/config/hibernate.cfg.xml b/dspace/config/hibernate.cfg.xml index 82e4fd738038..5a7de653a8cb 100644 --- a/dspace/config/hibernate.cfg.xml +++ b/dspace/config/hibernate.cfg.xml @@ -28,6 +28,7 @@ org.ehcache.jsr107.EhcacheCachingProvider + true From e29101b7da95c98455895e0090d348870f2bb352 Mon Sep 17 00:00:00 2001 From: jurinecko <95219754+jr-rk@users.noreply.github.com> Date: Tue, 26 Nov 2024 17:10:32 +0100 Subject: [PATCH 02/10] Translation of submission-forms to _cs (#816) * Translation of submission-forms to _cs * Translated bitstream metadata and complex input fields * Translated the rest of submission-froms_cs.xml * Fixed regex... it must contain regex value, not the message. --------- Co-authored-by: Juraj Roka Co-authored-by: milanmajchrak --- dspace/config/submission-forms.xml | 35 +- dspace/config/submission-forms_cs.xml | 779 ++++++++++++-------------- 2 files changed, 389 insertions(+), 425 deletions(-) diff --git a/dspace/config/submission-forms.xml b/dspace/config/submission-forms.xml index f6079e4b8262..b1f0a6b032f6 100644 --- a/dspace/config/submission-forms.xml +++ b/dspace/config/submission-forms.xml @@ -59,11 +59,14 @@ type true - + dropdown - Select the type of content of the item. + Type of the resource: "Corpus" refers to text, speech and multimodal corpora. + "Lexical Conceptual Resource" includes lexica, ontologies, dictionaries, word lists etc. + "language Description" covers language models and grammars. + "Technology / Tool / Service" is used for tools, systems, system components etc. - + Please select a resource type for your submission. @@ -142,7 +145,8 @@ autocomplete Enter the name of the publisher of the previously issued instance of this item, or your home institution. Start typing the publisher and use autocomplete form that will appear if - applicable. End your input by pressing ESC if you don't want to use the preselected value. + applicable. End your input by pressing ESC if you don't want to use the preselected value. + You must enter the name of the publisher. @@ -156,7 +160,8 @@ list Indicate whether you want to hide this item from browse and search. Combine with "Upload cmdi" - for weblicht submissions. + for weblicht submissions. + policy=deny,action=read,grantee-type=user,grantee-id=* @@ -171,7 +176,8 @@ list Indicate whether you will upload cmdi file in the next step. Combine with "hide" for weblicht - submissions. + submissions. + policy=deny,action=read,grantee-type=user,grantee-id=* @@ -188,7 +194,8 @@ autocomplete Enter the names of the authors of this item. Start typing the author's last name and use autocomplete form that will appear if applicable. End your input by pressing ESC if you don't - want to use the preselected value.. + want to use the preselected value. + Please add author(s) - - - local - contact - person - true - - - complex - This is contact person - - - - - - local - sponsor - true - - - complex - This is funding - - - dc - contributor - author + type + true - - autocomplete - Enter the author's name (Family name, Given names). - - + + dropdown + "Corpus" označuje textové, řečové i multimodální korpusy. + "Lexical Conceptual Resource" zahrnuje lexikony, ontologie, slovníky, seznamy slov apod. + "Language Description" zahrnuje jazykové modely a gramatiky. + "Technology / Tool / Service" se používá pro nástroje, systémy, systémové komponenty atd. + + Prosím zvolte typ dat pro váš příspěvek. @@ -140,11 +113,11 @@ relation isreferencedby true - + onebox - If the item has any alternative titles, please enter them here. - - http.* + Odkaz na původní článek, který zmiňuje tento záznam. + + http.* @@ -153,78 +126,111 @@ date issued false - + date - Please give the date of previous publication or public distribution. - You can leave out the day and/or month if they aren't applicable. - - You must enter at least the year. + Uveďte prosím datum vydání příspěvku, např 2014-01-21 nebo alespoň rok. + Musíte uvést datum v platném formátu. dc publisher - false - + true + autocomplete - Enter the name of the publisher of the previously issued instance of this item. - + Uveďte vydavatele předchozího vydání, nebo vaši domovskou instituci. Začnete-li vyplňovat vydavatele, objeví se nápověda. Nechcete-li nápovědu využít, stiskněte ESC. + Musíte uvést vydavatele. - dc - identifier - citation - false - - onebox - Enter the standard citation for the previously issued instance of this item. - + local + hidden + + true + + + list + Uveďte, má-li být záznam skryt ve vyhledávání a procházení. Pro příspěvky pro weblicht kombinujte s "Nahrát cmdi". + + + policy=deny,action=read,grantee-type=user,grantee-id=* + - dc - relation - ispartofseries + local + hasCMDI true - - Technical Report - series - Enter the series and number assigned to this item by your community. + + list + Uveďte, jestli se chystáte v dalším kroku nahrát cmdi soubor. Kombinujte se schováváním záznamů pro weblicht příspěvky. + + policy=deny,action=read,grantee-type=user,grantee-id=* + dc - identifier - - + contributor + author true - - qualdrop_value - If the item has any identification numbers or codes associated with - it, please enter the types and the actual numbers or codes. - + + autocomplete + Uveďte jména autorů tohoto záznamu. Začnete-li vyplňovat příjmení, objeví se nápověda. Nechcete-li nápovědu využít, stiskněte ESC. + Uveďte prosím autora(y) + + + + + + local + contact + person + true + + + complex + Osoba, která bude kontaktována v případě problémů s tímto záznamem. + Vyplňte prosím všechna pole u kontaktní osoby + + + + + local + sponsor + true + + + complex + Uveďte sponzory a zdroje financí podporující vznik práce popsané v tomto příspěvku. + + +
dc - type + description - true - - dropdown - Select the type of content of the item. - - + false + + textarea + Popište nahrávaná data. + Uveďte prosím popis. @@ -233,14 +239,11 @@ language iso false - + corpus,lexicalConceptualResource,languageDescription autocomplete - Select the language of the main content of the item. If the language does not appear in the - list, please select 'Other'. If the content does not really have a language (for example, if it - is a dataset or an image) please select 'N/A'. - - Please choose a language for the resource. + Vyberte jazyky, jichž se data tohoto záznamu týkají. Je možné zvolit více jazyků. Začnete-li psát, objeví se nápověda. Je lepší vyjmenovat všechny dotčené jazyky (pokud jich je větší množství, kontaktujte podporu), než používat iso kód 'mul'. + Prosím zvolte jazyk. @@ -249,38 +252,54 @@ language iso true - + toolService autocomplete - If the tool/service is language dependent, select the appropriate language(s). Otherwise leave the field empty. Multiple languages are possible. Start typing the language and use autocomplete form that will appear. + Pokud je nástroj/služba jazykově závislá, uveďte potřebné jazyky. Jinak můžete nechat nevyplněné. Je možné zvolit více jazyků. Začnete-li psát, objeví se nápověda. + + + dc + subject + + + true + + tag + Uveďte vhodná klíčová slova, nebo fráze a zmáčkněte tlačítko přidat. + Klíčová slova buď přidávejte po jednom, nebo je oddělte čárkou, nebo středníkem. Začnete-li psát, objeví se nápověda. + + Uveďte alespoň jedno klíčové slovo. + srsc + + local size info true - + corpus,languageDescription,lexicalConceptualResource complex - You can state the extent of the submitted data, eg. the number of tokens. + Můžete uvést rozsah nahraných dat, například počet tokenů. + + metashare ResourceInfo#ContentInfo mediaType false - + corpus,lexicalConceptualResource dropdown - Media type of the main content of the item e.g., "text" for - textual corpora or "audio" for audio recordings. - - Media type is required + Zvolte druh média tohoto záznamu, např. "text" pro textový korpus, "audio" pro audio nahrávky. + Uveďte typ média @@ -289,28 +308,24 @@ ResourceInfo#ContentInfo mediaType false - + languageDescription dropdown - Media type of the main content of the item e.g., "text" for - textual corpora or "audio" for audio recordings. - - Media type is required + Zvolte druh média tohoto záznamu, např. "text" pro textový korpus, "audio" pro audio nahrávky. + Uveďte typ média - - metashare ResourceInfo#ContentInfo detailedType false - + toolService dropdown - Choose one of the types + Zvolte jeden z podtypů @@ -319,11 +334,11 @@ ResourceInfo#ContentInfo detailedType false - + languageDescription dropdown - Choose one of the types + Zvolte jeden z podtypů @@ -332,11 +347,11 @@ ResourceInfo#ContentInfo detailedType false - + lexicalConceptualResource dropdown - Choose one of the types + Zvolte jeden z podtypů @@ -345,43 +360,11 @@ ResourceInfo#ResourceComponentType#ToolServiceInfo languageDependent false - + toolService list - Indicate whether the operation of the tool or service is - language dependent or not - - Please indicate whether the tool is language dependent - - - - - local - hasCMDI - true - - list - Are you going to upload cmdi file? - - - policy=deny,action=read,grantee-type=user,grantee-id=* - - - - - - local - hidden - - true - - - list - Should item be harvestable thru OAI-PMH but behave like private? - - - policy=deny,action=read,grantee-type=user,grantee-id=* - + Uveďte zda funkce nástroje či služby závisí na konkrétním jazyku. + Uveďte prosím zda je nástroj jazykově závislý. @@ -390,13 +373,14 @@ bitstream redirectToURL false - + onebox - The actual maximum upload size of the file is 4GB. To upload a file bigger than the - maximum upload size, enter the URL of that large file. The admin must know the URL - of that bitstream file. Then, click on the 'Save' button, and the file will start - to upload. The file path must be an absolute path. + Prosím, zadejte úplnou cestu k souboru, ať už z místního serveru, nebo z webové lokace (HTTP). + Maximální velikost nahrávaného souboru je 4 GB. Pokud chcete nahrát soubor větší než tento limit, zadejte jeho URL. + Ujistěte se, že administrátor je obeznámen s URL souboru datového proudu (bitstream). + Jakmile zadáte cestu k souboru nebo URL, klikněte na tlačítko „Uložit“ pro zahájení nahrávání. + Poznámka: Cesta k souboru musí být absolutní. @@ -405,65 +389,8 @@
- -
- - - dc - subject - - - true - - autocomplete - Enter appropriate subject keyword or phrase and press the Add button. You can repeat it for - multiple keywords or use separators i.e., Enter and comma, which will split it accordingly. - Start typing the keyword and use autocomplete form that will appear. End your input by pressing - ESC if you don't want to use the preselected value. - - Please enter at least one subject related to your submission - srsc - - - - - dc - description - abstract - false - - textarea - Enter the abstract of the item. - - - - - - dc - description - sponsorship - false - - textarea - Enter the names of any sponsors and/or funding codes in the box. - - - - - - dc - description - - false - - textarea - Enter any other description or comments in this box. - - - -
- - - - - - - - - + + + + + + + + + + person @@ -711,36 +638,36 @@ Enter the id of the project - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + isProjectOfPerson @@ -1582,12 +1509,12 @@ type false - + dropdown - Choose one of TEXT, VIDEO, SOUND, IMAGE, 3D. If choosing - TEXT consider adding the resource among other Language Resources. Images are visual resources for users to - look at. Text materials are meant to be read and not looked at. - Please select one of the options. + Vyberte z TEXT, VIDEO, ZVUK, OBRAZ, 3D. Pokud zvolíte TEXT, + zvažte, jestli se nejedná o Language Resource (jazykový zdroj). Obrazy jsou vizuální materiály, na které se + uživatelé mohou dívat. Textové materiály jsou určeny ke čtení a nikoli k dívání. + Zvolte prosím jednu z možností @@ -1596,10 +1523,10 @@ title false - + onebox - Enter the main title of the item in English. - You must enter a main title for this item. + Uveďte anglický název tohoto příspěvku. + Musíte uvést název. @@ -1608,11 +1535,9 @@ demo uri false - + onebox - A url with samples of the resource or, in the case of tools, - of samples of the output. - + URL se vzorky dat, v případě nástrojů předvedení výstupu. http.* @@ -1623,9 +1548,9 @@ relation isreferencedby true - + onebox - Link to original paper that references this dataset. + Odkaz na původní článek, který zmiňuje tento záznam. http.* @@ -1636,12 +1561,10 @@ date issued false - + date - Please give the date when the submission data were issued if any e.g., 2014-01-21 or at least - the year. - - You must enter the date or at least the year in a valid format. + Uveďte prosím datum vydání příspěvku, např 2014-01-21 nebo alespoň rok. + Musíte uvést datum v platném formátu. @@ -1650,11 +1573,9 @@ hidden false - + list - Indicate whether you want to hide this item from browse and search. Combine with "Upload cmdi" - for weblicht submissions. - + Uveďte, má-li být záznam skryt ve vyhledávání a procházení. Pro příspěvky pro weblicht kombinujte s "Nahrát cmdi". policy=deny,action=read,grantee-type=user,grantee-id=* @@ -1667,12 +1588,10 @@ hasMetadata false - + list - Indicate whether you will upload cmdi file in the next step. Combine with "hide" for weblicht - submissions. - + Uveďte, jestli se chystáte v dalším kroku nahrát cmdi soubor. Kombinujte se schováváním záznamů pro weblicht příspěvky. policy=deny,action=read,grantee-type=user,grantee-id=* @@ -1687,13 +1606,10 @@ contributor author true - + clarin-name - Enter the names of the authors of this item. Start typing the author's last name and use - autocomplete form that will appear if applicable. End your input by pressing ESC if you don't - want to use the preselected value. - - Please add author(s) + Uveďte jména autorů tohoto záznamu. Začnete-li vyplňovat příjmení, objeví se nápověda. Nechcete-li nápovědu využít, stiskněte ESC. + Uveďte prosím autora(y) @@ -1702,14 +1618,12 @@ publisher true - + autocomplete - The name of the publisher of the original analog or born - digital object. Use your home institution if this is a born digital object being published now. Start typing the - publisher and use autocomplete form that will appear if applicable. End your input by pressing ESC if you - don't want to use the preselected value. - - You must enter the name of the publisher. + Uveďte vydavatele analogového originálu, případně + vydavatele "born digital" originálu. Pokud se jedná o právě zrozený zdroj, uveďte vaši domovskou instituci. Začnete-li + vyplňovat vydavatele, objeví se nápověda. Nechcete-li nápovědu využít, stiskněte ESC. + Musíte uvést vydavatele. @@ -1718,11 +1632,11 @@ dataProvider false - + autocomplete - This concerns the digital object (not the analog - original). An institution from which the data come. Used e.g. to give proper attribution. Generally - different from publisher. + Týká se digitálního objektu, nikoliv analogového + originálu. Instituce, od které data pocházejí. Např. pro uvedení původu u některých licencí. Obecně se bude + lišit od vydavatele. @@ -1732,10 +1646,10 @@ contact person true - + complex - Person to contact in case of any issues with this submission. - Please fill all the fields for the contact person. + Osoba, která bude kontaktována v případě problémů s tímto záznamem. + Vyplňte prosím všechna pole u kontaktní osoby @@ -1743,9 +1657,9 @@ local sponsor true - + complex - Acknowledge sponsors and funding that supported work described by this submission. + Uveďte sponzory a zdroje financí podporující vznik práce popsané v tomto příspěvku. @@ -1756,11 +1670,11 @@ type false - + autocomplete - The type should be different from what you have - entered in the first step. Examples: photo or painting for IMAGE, book or letter for TEXT, etc. - Type is required + Typ by měl být odlišný od druhu, který jste zvolili v + prvním kroku. Například: fotografie nebo malba pro druh OBRAZ, kniha nebo dopis pro typ TEXT apod. + Vyplňte typ @@ -1770,25 +1684,30 @@ description false - + textarea - Enter a description of the submitted data. - Please give us a description + Popište nahrávaná data. + Uveďte prosím popis. + dc language iso true - + TEXT autocomplete - Select the language of the main content of the item. Multiple languages are possible. Start - typing the language and use autocomplete form that will appear if applicable. Better to list all the languages then to use the 'mul' iso code (if there are too many, contact support). + + Vyberte jazyky, jichž se data tohoto záznamu týkají. Je možné zvolit více jazyků. Začnete-li psát, + objeví se nápověda. Je lepší vyjmenovat všechny dotčené jazyky (pokud jich je větší množství, kontaktujte podporu), + než používat iso kód 'mul'. - The language is required for TEXT resources + Pro TEXTy je jazyk povinný @@ -1797,12 +1716,13 @@ language iso true - + VIDEO,IMAGE,SOUND,3D autocomplete - Optionally, select the language of the main content - of the item. Multiple languages are possible. Start - typing the language and use autocomplete form that will appear if applicable. Better to list all the languages then to use the 'mul' iso code (if there are too many, contact support). + + Volitelné. Vyberte jazyky, jichž se data tohoto záznamu týkají. Je možné zvolit více jazyků. Začnete-li psát, + objeví se nápověda. Je lepší vyjmenovat všechny dotčené jazyky (pokud jich je větší množství, kontaktujte podporu), + než používat iso kód 'mul'. @@ -1814,17 +1734,16 @@ true - + autocomplete - Enter appropriate subject keyword or phrase and press - the Add button. Use keywords to specify also people, places and times (period, era, date range etc) the resource - is about. You can use hierarchical subjects, separate the hierarchy levels with two colons (::). Eg. - People::John Doe, Places::New York, Times::WWII. - You can repeat it for multiple keywords or use separators i.e., comma and semicolon, which will split it accordingly. - Start typing the keyword and use autocomplete form that will appear. End your input by pressing - ESC if you don't want to use the preselected value. + Uveďte vhodná klíčová slova, nebo fráze a zmáčkněte + tlačítko přidat. + Klíčová slova využijte také pro lidi, místa a časy (období, éry, rozsah dat apod.) o kterých záznam je. + Možnost využítvat hierarchické předměty. Oddělte jednotlivé úrovně hierarchie dvěma dvojtečkami (::). + Např. People::Jára Cimrman, Places::Liptákov, Times::počátek 20. století. + Klíčová slova buď přidávejte po jednom, nebo je oddělte čárkou, nebo středníkem. Začnete-li psát, objeví se nápověda. - Please enter at least one subject related to your submission + Uveďte alespoň jedno klíčové slovo. @@ -1833,12 +1752,12 @@ identifier other true - + onebox - The item will get a handle. If the item has any - identification numbers or codes associated with it, please enter the types and the actual numbers or codes. + Pro tento záznam bude vytvořen handle. Pokud má + zdroj přidělený jiný identifikátor, nebo kód, uveďte jej i jeho typ. @@ -1849,9 +1768,30 @@ size info true - + complex - You can state the extent of the submitted data, eg. the number of tokens. + Můžete uvést rozsah nahraných dat, například počet tokenů. + + + + + local + bitstream + redirectToURL + false + + onebox + + Prosím, zadejte úplnou cestu k souboru, ať už z místního serveru, nebo z webové lokace (HTTP). + Maximální velikost nahrávaného souboru je 4 GB. Pokud chcete nahrát soubor větší než tento limit, zadejte jeho URL. + Ujistěte se, že administrátor je obeznámen s URL souboru datového proudu (bitstream). + Jakmile zadáte cestu k souboru nebo URL, klikněte na tlačítko „Uložit“ pro zahájení nahrávání. + Poznámka: Cesta k souboru musí být absolutní. + + + + policy=deny,action=read,grantee-type=user,grantee-id=* + @@ -1870,10 +1810,10 @@ title false - + onebox - Enter the main title of the item in English. - You must enter a main title for this item. + Uveďte anglický název tohoto příspěvku. + Musíte uvést název. @@ -1882,9 +1822,9 @@ demo uri false - + onebox - Course homepage + Stránky kurzu http.* @@ -1895,9 +1835,9 @@ relation isreferencedby true - + onebox - Link to original paper that references this dataset. + Odkaz na původní článek, který zmiňuje tento záznam. http.* @@ -1908,12 +1848,10 @@ date issued false - + date - Please give the date when the submission data were issued if any e.g., 2014-01-21 or at least - the year. - - You must enter the date or at least the year in a valid format. + Uveďte prosím datum vydání příspěvku, např 2014-01-21 nebo alespoň rok. + Musíte uvést datum v platném formátu. @@ -1922,11 +1860,9 @@ hidden false - + list - Indicate whether you want to hide this item from browse and search. Combine with "Upload cmdi" - for weblicht submissions. - + Uveďte, má-li být záznam skryt ve vyhledávání a procházení. Pro příspěvky pro weblicht kombinujte s "Nahrát cmdi". policy=deny,action=read,grantee-type=user,grantee-id=* @@ -1939,12 +1875,10 @@ hasMetadata false - + list - Indicate whether you will upload cmdi file in the next step. Combine with "hide" for weblicht - submissions. - + Uveďte, jestli se chystáte v dalším kroku nahrát cmdi soubor. Kombinujte se schováváním záznamů pro weblicht příspěvky. policy=deny,action=read,grantee-type=user,grantee-id=* @@ -1954,25 +1888,22 @@
- + dc contributor author true - + clarin-name - Enter the names of the authors of this item. Start typing the author's last name and use - autocomplete form that will appear if applicable. End your input by pressing ESC if you don't - want to use the preselected value. - - Please add author(s) + Uveďte jména autorů tohoto záznamu. Začnete-li vyplňovat příjmení, objeví se nápověda. Nechcete-li nápovědu využít, stiskněte ESC. + Uveďte prosím autora(y) @@ -1981,14 +1912,12 @@ publisher true - + autocomplete - The name of the publisher of the original analog or born - digital object. Use your home institution if this is a born digital object being published now. Start typing the - publisher and use autocomplete form that will appear if applicable. End your input by pressing ESC if you - don't want to use the preselected value. - - You must enter the name of the publisher. + Uveďte vydavatele analogového originálu, případně + vydavatele "born digital" originálu. Pokud se jedná o právě zrozený zdroj, uveďte vaši domovskou instituci. Začnete-li + vyplňovat vydavatele, objeví se nápověda. Nechcete-li nápovědu využít, stiskněte ESC. + Musíte uvést vydavatele. @@ -1997,10 +1926,10 @@ contact person true - + complex - Person to contact in case of any issues with this submission. - Please fill all the fields for the contact person. + Osoba, která bude kontaktována v případě problémů s tímto záznamem. + Vyplňte prosím všechna pole u kontaktní osoby @@ -2008,9 +1937,9 @@ local sponsor true - + complex - Acknowledge sponsors and funding that supported work described by this submission. + Uveďte sponzory a zdroje financí podporující vznik práce popsané v tomto příspěvku.
@@ -2022,12 +1951,12 @@ type false - + dropdown teachingMaterials - This is here to autofill a value. The value should not be changed. - Please select a resource type for your submission. + Toto pole je zde pro automatické vyplnění hodnoty. Hodnota by neměla být měněna. + Prosím zvolte typ dat pro váš příspěvek. @@ -2036,10 +1965,10 @@ description false - + textarea - Enter a description of the submitted data. - Please give us a description + Popište nahrávaná data. + Uveďte prosím popis. @@ -2048,12 +1977,13 @@ language iso true - + autocomplete - Select the language of the main content of the item. Multiple languages are possible. Start - typing the language and use autocomplete form that will appear if applicable. Better to list all the languages then to use the 'mul' iso code (if there are too many, contact support). + + Vyberte jazyky, jichž se data tohoto záznamu týkají. Je možné zvolit více jazyků. Začnete-li psát, objeví se nápověda. + Je lepší vyjmenovat všechny dotčené jazyky (pokud jich je větší množství, kontaktujte podporu), než používat iso kód 'mul'. - Please choose a language for the resource. + Prosím zvolte jazyk. @@ -2063,14 +1993,13 @@ true - + autocomplete - Enter appropriate subject keyword or phrase and press the Add button. You can repeat it for - multiple keywords or use separators i.e., comma and semicolon, which will split it accordingly. - Start typing the keyword and use autocomplete form that will appear. End your input by pressing - ESC if you don't want to use the preselected value. + + Uveďte vhodná klíčová slova, nebo fráze a zmáčkněte tlačítko přidat. + Klíčová slova buď přidávejte po jednom, nebo je oddělte čárkou, nebo středníkem. Začnete-li psát, objeví se nápověda. - Please enter at least one subject related to your submission + Uveďte alespoň jedno klíčové slovo. @@ -2079,19 +2008,41 @@ identifier other true - + onebox - The item will get a handle. If the item has any - identification numbers or codes associated with it, please enter the types and the actual numbers or codes. + + Pro tento záznam bude vytvořen handle. Pokud má + zdroj přidělený jiný identifikátor, nebo kód, uveďte jej i jeho typ. + + + local + bitstream + redirectToURL + false + + onebox + + Prosím, zadejte úplnou cestu k souboru, ať už z místního serveru, nebo z webové lokace (HTTP). + Maximální velikost nahrávaného souboru je 4 GB. Pokud chcete nahrát soubor větší než tento limit, zadejte jeho URL. + Ujistěte se, že administrátor je obeznámen s URL souboru datového proudu (bitstream). + Jakmile zadáte cestu k souboru nebo URL, klikněte na tlačítko „Uložit“ pro zahájení nahrávání. + Poznámka: Cesta k souboru musí být absolutní. + + + + policy=deny,action=read,grantee-type=user,grantee-id=* + + + -
+ local @@ -2114,7 +2065,10 @@ true autocomplete - URL příbuzného záznamu, který je tímto záznamem nahrazen. Pokud je příbuzný záznam v tomto repozitáři, začněte psát jeho název, nebo handle a vyberte záznam z nabídky. + + URL příbuzného záznamu, který je tímto záznamem nahrazen. + Pokud je příbuzný záznam v tomto repozitáři, začněte psát jeho název, nebo handle a vyberte záznam z nabídky. + @@ -2126,7 +2080,7 @@ relation isreplacedby true - + autocomplete Příbuzný záznam, který nahrazuje tento. @@ -2156,7 +2110,7 @@ - Yes + Ano true @@ -2750,11 +2704,11 @@ VIDEO - SOUND + ZVUK SOUND - IMAGE + OBRAZ IMAGE @@ -3191,17 +3145,17 @@ - Yes + Ano true - No + Ne false - Hidden + Ano hidden @@ -3219,29 +3173,28 @@ Uncomment the example row of the complex input type definition to see this input in the submission UI. --> - - - - + + + + - - - - - + + + + + - + - + - + \ No newline at end of file From dadd1fccb953581c93f9d1974e6d6afdbc4e297c Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Wed, 27 Nov 2024 08:08:12 +0100 Subject: [PATCH 03/10] Updated cfg to pretify the author suggestions (#819) --- dspace/config/clarin-dspace.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/dspace/config/clarin-dspace.cfg b/dspace/config/clarin-dspace.cfg index 708e5c9b1c9b..ebb3fa782c51 100644 --- a/dspace/config/clarin-dspace.cfg +++ b/dspace/config/clarin-dspace.cfg @@ -298,4 +298,5 @@ autocomplete.custom.separator.solr-subject_ac = \\|\\|\\| autocomplete.custom.separator.solr-publisher_ac = \\|\\|\\| autocomplete.custom.separator.solr-dataProvider_ac = \\|\\|\\| autocomplete.custom.separator.solr-dctype_ac = \\|\\|\\| +autocomplete.custom.separator.solr-author_ac = \\|\\|\\| autocomplete.custom.allowed = solr-author_ac,solr-publisher_ac,solr-dataProvider_ac,solr-dctype_ac,solr-subject_ac,solr-handle_title_ac,json_static-iso_langs.json From 0a0466e397b675d0ca054755411cce609f6411d4 Mon Sep 17 00:00:00 2001 From: Paurikova2 <107862249+Paurikova2@users.noreply.github.com> Date: Wed, 27 Nov 2024 13:08:43 +0100 Subject: [PATCH 04/10] crosswalk-embargo (#821) * added fn for embargo * using of res policy end_date and added comments * fix string format problem with %s * integration tests are falling down * checkstyle violations * removed findHandle duplicity * added deleted line * checkstyle violations --- .../VersionedHandleIdentifierProviderIT.java | 3 - .../org/dspace/utils/SpecialItemService.java | 157 ++++++++++++++++++ .../resources/DSpaceResourceResolver.java | 3 +- .../resources/functions/GetAvailableFn.java | 31 ++++ .../functions/StringXSLFunction.java | 1 - .../oai/metadataFormats/datacite_openaire.xsl | 17 +- 6 files changed, 194 insertions(+), 18 deletions(-) create mode 100644 dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/GetAvailableFn.java diff --git a/dspace-api/src/test/java/org/dspace/identifier/VersionedHandleIdentifierProviderIT.java b/dspace-api/src/test/java/org/dspace/identifier/VersionedHandleIdentifierProviderIT.java index 57acf1f1c453..a28a5a4c7508 100644 --- a/dspace-api/src/test/java/org/dspace/identifier/VersionedHandleIdentifierProviderIT.java +++ b/dspace-api/src/test/java/org/dspace/identifier/VersionedHandleIdentifierProviderIT.java @@ -62,9 +62,6 @@ public void setUp() throws Exception { @Override public void destroy() throws Exception { super.destroy(); - // After this test has finished running, refresh application context and - // set the expected 'default' versioned handle provider back to ensure other tests don't fail - DSpaceServicesFactory.getInstance().getServiceManager().getApplicationContext().refresh(); } private void registerProvider(Class type) { diff --git a/dspace-oai/src/main/java/org/dspace/utils/SpecialItemService.java b/dspace-oai/src/main/java/org/dspace/utils/SpecialItemService.java index d62cbc481fcb..c801ccc156a6 100644 --- a/dspace-oai/src/main/java/org/dspace/utils/SpecialItemService.java +++ b/dspace-oai/src/main/java/org/dspace/utils/SpecialItemService.java @@ -11,6 +11,10 @@ import java.io.InputStreamReader; import java.io.Reader; +import java.sql.SQLException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.List; import java.util.Objects; import javax.xml.parsers.DocumentBuilder; @@ -18,17 +22,27 @@ import javax.xml.parsers.ParserConfigurationException; import org.dspace.app.util.DCInput; +import org.dspace.authorize.ResourcePolicy; +import org.dspace.authorize.factory.AuthorizeServiceFactory; +import org.dspace.authorize.service.ResourcePolicyService; import org.dspace.content.Bitstream; +import org.dspace.content.Bundle; import org.dspace.content.DSpaceObject; import org.dspace.content.Item; +import org.dspace.content.MetadataField; import org.dspace.content.MetadataValue; +import org.dspace.content.factory.ClarinServiceFactory; import org.dspace.content.factory.ContentServiceFactory; import org.dspace.content.service.BitstreamService; import org.dspace.content.service.ItemService; +import org.dspace.content.service.MetadataFieldService; +import org.dspace.content.service.clarin.ClarinItemService; import org.dspace.core.Constants; import org.dspace.core.Context; import org.dspace.handle.factory.HandleServiceFactory; import org.dspace.handle.service.HandleService; +import org.dspace.xoai.exceptions.InvalidMetadataFieldException; +import org.dspace.xoai.services.impl.DSpaceFieldResolver; import org.springframework.stereotype.Component; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -267,6 +281,149 @@ public static Node getAuthor(String mdValue) { } } + /** + * Retrieves the earliest available date for an item identified by the given identifier URI. + * This method checks for any embargo date first and then retrieves the "dc.date.available" + * metadata value as a fallback if no embargo date is found. + * + * @param identifierUri The identifier URI of the item whose available date is to be retrieved. + * @return A string representation of the earliest available date, or null if no date is found or an error occurs. + */ + public static String getAvailable(String identifierUri) { + Context context = new Context(); + // Find the metadata field for "dc.identifier.uri" + String mtdField = "dc.identifier.uri"; + MetadataField metadataField = findMetadataField(context, mtdField); + if (Objects.isNull(metadataField)) { + log.error(String.format("Metadata field for %s not found.", mtdField)); + return null; + } + + // Retrieve the item using the handle + ClarinItemService clarinItemService = ClarinServiceFactory.getInstance().getClarinItemService(); + Item item; + try { + List itemList = clarinItemService.findByHandle(context, metadataField, identifierUri); + item = itemList.isEmpty() ? null : itemList.get(0); + } catch (SQLException e) { + log.error("Error retrieving item by handle.", e); + return null; + } + if (Objects.isNull(item)) { + log.error(String.format("Item for handle %s doesn't exist!", identifierUri)); + return null; + } + + // Check if there is an embargo or get the earliest available date + Date startDate = getEmbargoDate(context, item); + if (Objects.isNull(startDate)) { + startDate = getAvailableDate(context, item); + } + return (Objects.nonNull(startDate)) ? startDate.toString() : null; + } + + /** + * Finds the metadata field corresponding to the provided string. + * + * @param context The DSpace context + * @param mtd The metadata field string + * @return The MetadataField object, or null if not found. + */ + private static MetadataField findMetadataField(Context context, String mtd) { + MetadataFieldService metadataFieldService = ContentServiceFactory.getInstance().getMetadataFieldService(); + try { + return metadataFieldService.findByString(context, mtd, '.'); + } catch (SQLException e) { + log.error(String.format("Error finding metadata field %s.", mtd), e); + return null; + } + } + + /** + * Retrieves the embargo start date for the given item bitstreams. + * If an embargo has ended, the end date is returned. + * + * @param context The DSpace context + * @param item The item whose embargo date is to be retrieved. + * @return The start or end date of the embargo, or null if no embargo exists. + */ + private static Date getEmbargoDate(Context context, Item item) { + ResourcePolicyService resPolicyService = AuthorizeServiceFactory.getInstance().getResourcePolicyService(); + Date startDate = null; + for (Bundle bundle : item.getBundles()) { + for (Bitstream bitstream : bundle.getBitstreams()) { + List resPolList; + try { + resPolList = resPolicyService.find(context, bitstream, Constants.READ); + } catch (SQLException e) { + log.error(String.format("Error during finding resource policies READ for bitstream %s", + bitstream.getID().toString())); + return null; + } + for (ResourcePolicy resPol : resPolList) { + Date date = resPol.getStartDate(); + // If the embargo has already ended, use the date of its end. + if (Objects.nonNull(date) && Objects.nonNull(resPol.getEndDate())) { + date = resPol.getEndDate(); + } + if (Objects.isNull(startDate) || (Objects.nonNull(date) && date.compareTo(startDate) > 0)) { + startDate = date; + } + } + } + } + return startDate; + } + + /** + * Retrieves the available date for the given item by checking the "dc.date.available" metadata. + * + * @param context The DSpace context + * @param item The item whose available date is to be retrieved. + * @return The available date, or null if no available date is found. + */ + private static Date getAvailableDate(Context context, Item item) { + DSpaceFieldResolver dSpaceFieldResolver = new DSpaceFieldResolver(); + List metadataValueList = item.getMetadata(); + String mtdField = "dc.date.available"; + int fieldID; + try { + fieldID = dSpaceFieldResolver.getFieldID(context, mtdField); + } catch (SQLException | InvalidMetadataFieldException e) { + log.error(String.format("Error during finding ID of metadata field %s.", mtdField)); + return null; + } + Date startDate = null; + for (MetadataValue mtd : metadataValueList) { + if (mtd.getMetadataField().getID() == fieldID) { + Date availableDate = parseDate(mtd.getValue()); + if (Objects.isNull(startDate) || (Objects.nonNull(availableDate) + && availableDate.compareTo(startDate) > 0)) { + startDate = availableDate; + } + } + } + return startDate; + } + + /** + * Parses a date string in the format "yyyy-MM-dd" into a Date object. + * + * @param dateString The date string to be parsed. + * @return A Date object representing the parsed date, or null if parsing fails. + */ + private static Date parseDate(String dateString) { + String format = "yyyy-MM-dd"; + SimpleDateFormat dateFormat = new SimpleDateFormat(format); // Example format + dateFormat.setLenient(false); // Set lenient to false to avoid parsing incorrect dates + try { + return dateFormat.parse(dateString); // Attempt to parse the date + } catch (ParseException e) { + log.warn(String.format("Date %s cannot be parsed using the format %s.", dateString, format)); + return null; + } + } + public static boolean hasOwnMetadata(List metadataValues) { if (metadataValues.size() == 1 && metadataValues.get(0).getValue().equalsIgnoreCase("true")) { return true; diff --git a/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/DSpaceResourceResolver.java b/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/DSpaceResourceResolver.java index 9d4790b9ff47..c0e540b9576c 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/DSpaceResourceResolver.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/DSpaceResourceResolver.java @@ -26,6 +26,7 @@ import org.dspace.xoai.services.impl.resources.functions.BibtexifyFn; import org.dspace.xoai.services.impl.resources.functions.FormatFn; import org.dspace.xoai.services.impl.resources.functions.GetAuthorFn; +import org.dspace.xoai.services.impl.resources.functions.GetAvailableFn; import org.dspace.xoai.services.impl.resources.functions.GetContactFn; import org.dspace.xoai.services.impl.resources.functions.GetFundingFn; import org.dspace.xoai.services.impl.resources.functions.GetLangForCodeFn; @@ -54,7 +55,7 @@ public class DSpaceResourceResolver implements ResourceResolver { new UriToLicenseFn(), new LogMissingMsgFn(), new UriToRestrictionsFn(), new ShortestIdFn(), new GetContactFn(), new GetAuthorFn(), new GetFundingFn(), new GetLangForCodeFn(), new GetPropertyFn(), new GetSizeFn(), new GetUploadedMetadataFn(), new LogMissingFn(), - new BibtexifyFn(), new FormatFn() + new BibtexifyFn(), new FormatFn(), new GetAvailableFn() ); SaxonTransformerFactory saxonTransformerFactory = (SaxonTransformerFactory) transformerFactory; diff --git a/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/GetAvailableFn.java b/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/GetAvailableFn.java new file mode 100644 index 000000000000..f7843abed51c --- /dev/null +++ b/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/GetAvailableFn.java @@ -0,0 +1,31 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.xoai.services.impl.resources.functions; + +import org.dspace.utils.SpecialItemService; + +/** + * The GetAvailableFn class extends the StringXSLFunction to provide a custom function + * that retrieves the availability status of an item based on its identifier. + * It uses the SpecialItemService to fetch the available information. + * This function is intended to be used in XSL transformations where the + * "getAvailable" function is called with an item's identifier as a parameter. + * + * @author Michaela Paurikova(michaela.paurikova at dataquest.sk) + */ +public class GetAvailableFn extends StringXSLFunction { + @Override + protected String getFnName() { + return "getAvailable"; + } + + @Override + protected String getStringResult(String param) { + return SpecialItemService.getAvailable(param); + } +} diff --git a/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/StringXSLFunction.java b/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/StringXSLFunction.java index 163a9eb49ca1..ed260c8b2d4a 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/StringXSLFunction.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/services/impl/resources/functions/StringXSLFunction.java @@ -74,7 +74,6 @@ final public XdmValue call(XdmValue[] xdmValues) throws SaxonApiException { log.warn("Empty value in call of function of StringXslFunction type"); val = ""; } - return new XdmAtomicValue(checks(getStringResult(val))); } diff --git a/dspace/config/crosswalks/oai/metadataFormats/datacite_openaire.xsl b/dspace/config/crosswalks/oai/metadataFormats/datacite_openaire.xsl index d6823c6f2c12..64cfda7f20ee 100644 --- a/dspace/config/crosswalks/oai/metadataFormats/datacite_openaire.xsl +++ b/dspace/config/crosswalks/oai/metadataFormats/datacite_openaire.xsl @@ -141,19 +141,10 @@ - - - - - - - - - - - - - + + + + From 9911fcc1380dbbbb205696561b3e17f5324afc0d Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Thu, 28 Nov 2024 08:26:41 +0100 Subject: [PATCH 05/10] For now the complex input field is without autocomplete for the size and contact person (#823) --- dspace/config/submission-forms.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dspace/config/submission-forms.xml b/dspace/config/submission-forms.xml index b1f0a6b032f6..19aa68c31d78 100644 --- a/dspace/config/submission-forms.xml +++ b/dspace/config/submission-forms.xml @@ -3222,7 +3222,7 @@ - + @@ -3240,7 +3240,7 @@ - From da0590498fe8bda2e469c2afd4f18966d6bb993c Mon Sep 17 00:00:00 2001 From: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> Date: Thu, 28 Nov 2024 08:27:37 +0100 Subject: [PATCH 06/10] Send the custom type bind `field` to the FE configuration (#822) --- .../java/org/dspace/app/util/DCInput.java | 15 +++++++- .../org/dspace/app/util/DCInputsReader.java | 34 ++++++++++++------- .../converter/SubmissionFormConverter.java | 1 + .../rest/model/SubmissionFormFieldRest.java | 14 ++++++++ 4 files changed, 50 insertions(+), 14 deletions(-) diff --git a/dspace-api/src/main/java/org/dspace/app/util/DCInput.java b/dspace-api/src/main/java/org/dspace/app/util/DCInput.java index bc15392a3818..2ea8a8866016 100644 --- a/dspace-api/src/main/java/org/dspace/app/util/DCInput.java +++ b/dspace-api/src/main/java/org/dspace/app/util/DCInput.java @@ -169,6 +169,11 @@ public class DCInput { */ private String autocompleteCustom = null; + /** + * the custom field for the type bind + */ + private String typeBindField = null; + /** * the dropdown input type could have defined a default value */ @@ -259,7 +264,7 @@ public DCInput(Map fieldMap, Map> listMap, typeBind = new ArrayList(); String typeBindDef = fieldMap.get("type-bind"); this.insertToTypeBind(typeBindDef); - String typeBindField = fieldMap.get(DCInputsReader.TYPE_BIND_FIELD_ATTRIBUTE); + typeBindField = fieldMap.get(DCInputsReader.TYPE_BIND_FIELD_ATTRIBUTE); this.insertToTypeBind(typeBindField); @@ -741,6 +746,14 @@ public void setAutocompleteCustom(String autocompleteCustom) { this.autocompleteCustom = autocompleteCustom; } + public String getTypeBindField() { + return typeBindField; + } + + public void setTypeBindField(String typeBindField) { + this.typeBindField = typeBindField; + } + /** * Class representing a Map of the ComplexDefinition object * Class is copied from UFAL/CLARIN-DSPACE (https://github.com/ufal/clarin-dspace) and modified by diff --git a/dspace-api/src/main/java/org/dspace/app/util/DCInputsReader.java b/dspace-api/src/main/java/org/dspace/app/util/DCInputsReader.java index c6e3701ef003..f327a647db3e 100644 --- a/dspace-api/src/main/java/org/dspace/app/util/DCInputsReader.java +++ b/dspace-api/src/main/java/org/dspace/app/util/DCInputsReader.java @@ -501,6 +501,11 @@ private void processField(String formName, Node n, Map field) handleInputTypeTagName(formName, field, nestedNode, nestedValue); } } + } else if (StringUtils.equals(tagName, "type-bind")) { + String customField = getAttribute(nd, TYPE_BIND_FIELD_ATTRIBUTE); + if (customField != null) { + field.put(TYPE_BIND_FIELD_ATTRIBUTE, customField); + } } } } @@ -547,20 +552,23 @@ private void handleInputTypeTagName(String formName, Map field, } else { field.put(PAIR_TYPE_NAME, pairTypeName); } - } else if (value.equals("complex")) { - String definitionName = getAttribute(nd, COMPLEX_DEFINITION_REF); - if (definitionName == null) { - throw new SAXException("Form " + formName - + ", field " + field.get("dc-element") - + "." + field.get("dc-qualifier") - + " has no linked definition"); - } else { - field.put(COMPLEX_DEFINITION_REF, definitionName); + } else { + if (value.equals("complex")) { + String definitionName = getAttribute(nd, COMPLEX_DEFINITION_REF); + if (definitionName == null) { + throw new SAXException("Form " + formName + + ", field " + field.get("dc-element") + + "." + field.get("dc-qualifier") + + " has no linked definition"); + } else { + field.put(COMPLEX_DEFINITION_REF, definitionName); + } } - } else if (value.equals("autocomplete")) { - String definitionName = getAttribute(nd, AUTOCOMPLETE_CUSTOM); - if (definitionName != null) { - field.put(AUTOCOMPLETE_CUSTOM, definitionName); + if (value.equals("autocomplete")) { + String definitionName = getAttribute(nd, AUTOCOMPLETE_CUSTOM); + if (definitionName != null) { + field.put(AUTOCOMPLETE_CUSTOM, definitionName); + } } } } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SubmissionFormConverter.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SubmissionFormConverter.java index 8021e4e0d771..5f62f7cf8ae6 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SubmissionFormConverter.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/SubmissionFormConverter.java @@ -163,6 +163,7 @@ private SubmissionFormFieldRest getField(DCInput dcinput, String formName) { if (dcinput.isMetadataField()) { inputField.setSelectableMetadata(selectableMetadata); inputField.setTypeBind(dcinput.getTypeBindList()); + inputField.setTypeBindField(dcinput.getTypeBindField()); inputField.setComplexDefinition(dcinput.getComplexDefinitionJSONString()); inputField.setAutocompleteCustom(dcinput.getAutocompleteCustom()); } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/SubmissionFormFieldRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/SubmissionFormFieldRest.java index efafa5927e8a..ee49f29da876 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/SubmissionFormFieldRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/SubmissionFormFieldRest.java @@ -98,6 +98,12 @@ public class SubmissionFormFieldRest { */ private String autocompleteCustom; + /** + * The custom field to type bind. It is used to check that the custom type bind field is defined when + * it is defined in the configuration property `submit.type-bind.field` + */ + private String typeBindField; + /** * Getter for {@link #selectableMetadata} @@ -313,4 +319,12 @@ public String getAutocompleteCustom() { public void setAutocompleteCustom(String autocompleteCustom) { this.autocompleteCustom = autocompleteCustom; } + + public String getTypeBindField() { + return typeBindField; + } + + public void setTypeBindField(String typeBindField) { + this.typeBindField = typeBindField; + } } From 3aec3688428119804385a0d8dd584f07e151b906 Mon Sep 17 00:00:00 2001 From: Paurikova2 <107862249+Paurikova2@users.noreply.github.com> Date: Thu, 28 Nov 2024 10:28:24 +0100 Subject: [PATCH 07/10] fix date converting to string (#825) * fix date converting to string * made const from format * checkstyle --- .../org/dspace/utils/SpecialItemService.java | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/dspace-oai/src/main/java/org/dspace/utils/SpecialItemService.java b/dspace-oai/src/main/java/org/dspace/utils/SpecialItemService.java index c801ccc156a6..6facd49b6c28 100644 --- a/dspace-oai/src/main/java/org/dspace/utils/SpecialItemService.java +++ b/dspace-oai/src/main/java/org/dspace/utils/SpecialItemService.java @@ -61,6 +61,7 @@ */ public class SpecialItemService { private SpecialItemService() {} + private static final String FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; /** log4j logger */ private static final org.apache.logging.log4j.Logger log = org.apache.logging.log4j .LogManager.getLogger(SpecialItemService.class); @@ -319,7 +320,7 @@ public static String getAvailable(String identifierUri) { if (Objects.isNull(startDate)) { startDate = getAvailableDate(context, item); } - return (Objects.nonNull(startDate)) ? startDate.toString() : null; + return (Objects.nonNull(startDate)) ? parseDateToString(startDate) : null; } /** @@ -396,7 +397,7 @@ private static Date getAvailableDate(Context context, Item item) { Date startDate = null; for (MetadataValue mtd : metadataValueList) { if (mtd.getMetadataField().getID() == fieldID) { - Date availableDate = parseDate(mtd.getValue()); + Date availableDate = parseStringToDate(mtd.getValue()); if (Objects.isNull(startDate) || (Objects.nonNull(availableDate) && availableDate.compareTo(startDate) > 0)) { startDate = availableDate; @@ -407,19 +408,28 @@ private static Date getAvailableDate(Context context, Item item) { } /** - * Parses a date string in the format "yyyy-MM-dd" into a Date object. + * Converts date object to string formatted in the pattern. * - * @param dateString The date string to be parsed. + * @param date The date + * @return A string representation of the provided date + */ + private static String parseDateToString(Date date) { + SimpleDateFormat dateFormat = new SimpleDateFormat(FORMAT); + return dateFormat.format(date); + } + + /** + * Parses a date string in the format into a Date object. + * + * @param dateString date string to be parsed. * @return A Date object representing the parsed date, or null if parsing fails. */ - private static Date parseDate(String dateString) { - String format = "yyyy-MM-dd"; - SimpleDateFormat dateFormat = new SimpleDateFormat(format); // Example format - dateFormat.setLenient(false); // Set lenient to false to avoid parsing incorrect dates + private static Date parseStringToDate(String dateString) { + SimpleDateFormat dateFormat = new SimpleDateFormat(FORMAT); try { - return dateFormat.parse(dateString); // Attempt to parse the date + return dateFormat.parse(dateString); } catch (ParseException e) { - log.warn(String.format("Date %s cannot be parsed using the format %s.", dateString, format)); + log.warn(String.format("Date %s cannot be parsed using the format %s.", dateString, FORMAT)); return null; } } From 005c939d87c1cbcd32a91c45d0c56fca2432fb2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Ko=C5=A1arko?= Date: Fri, 29 Nov 2024 09:12:03 +0100 Subject: [PATCH 08/10] cherry-pick clarin v7 into dtq dev (#820) * cherry-picked DataCite related changes from customer/uk * Add a script that adds a file from url to an item intended for large file workflows * Add ways to influence the bitstream name * add more options to specify an item * Expose resourceId (#1134) A BE part of #1127 - this exposes the resourceId so it can be used in the handle mgmt table * fixes ufal/clarin-dspace#1135 - findEpersonByNetId should stop searching when it finds an eperson - moved the `return eperson` inside the for cycle (after eperson non null check). - removed the eperson param (both callers were passing in `null`) --- .../org/dspace/administer/FileDownloader.java | 229 ++++++++++++++++++ .../FileDownloaderConfiguration.java | 73 ++++++ .../clarin/ClarinShibAuthentication.java | 11 +- .../identifier/doi/DataCiteConnector.java | 3 +- .../config/spring/api/scripts.xml | 5 + .../dspace/administer/FileDownloaderIT.java | 110 +++++++++ .../app/rest/converter/HandleConverter.java | 7 + .../org/dspace/app/rest/model/HandleRest.java | 13 + .../clarin/ClarinShibbolethLoginFilter.java | 2 +- dspace/config/clarin-dspace.cfg | 6 +- dspace/config/crosswalks/DIM2DataCite.xsl | 2 +- dspace/config/spring/rest/scripts.xml | 5 + 12 files changed, 455 insertions(+), 11 deletions(-) create mode 100644 dspace-api/src/main/java/org/dspace/administer/FileDownloader.java create mode 100644 dspace-api/src/main/java/org/dspace/administer/FileDownloaderConfiguration.java create mode 100644 dspace-api/src/test/java/org/dspace/administer/FileDownloaderIT.java diff --git a/dspace-api/src/main/java/org/dspace/administer/FileDownloader.java b/dspace-api/src/main/java/org/dspace/administer/FileDownloader.java new file mode 100644 index 000000000000..fb592627adef --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/administer/FileDownloader.java @@ -0,0 +1,229 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.administer; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.sql.SQLException; +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; + +import org.apache.commons.cli.ParseException; +import org.dspace.authorize.AuthorizeException; +import org.dspace.content.Bitstream; +import org.dspace.content.BitstreamFormat; +import org.dspace.content.Bundle; +import org.dspace.content.DSpaceObject; +import org.dspace.content.Item; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.BitstreamFormatService; +import org.dspace.content.service.BitstreamService; +import org.dspace.content.service.ItemService; +import org.dspace.content.service.WorkspaceItemService; +import org.dspace.core.Context; +import org.dspace.eperson.EPerson; +import org.dspace.eperson.factory.EPersonServiceFactory; +import org.dspace.eperson.service.EPersonService; +import org.dspace.identifier.IdentifierNotFoundException; +import org.dspace.identifier.IdentifierNotResolvableException; +import org.dspace.identifier.factory.IdentifierServiceFactory; +import org.dspace.identifier.service.IdentifierService; +import org.dspace.scripts.DSpaceRunnable; +import org.dspace.scripts.configuration.ScriptConfiguration; +import org.dspace.utils.DSpace; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +public class FileDownloader extends DSpaceRunnable { + + private static final Logger log = LoggerFactory.getLogger(FileDownloader.class); + private boolean help = false; + private UUID itemUUID; + private int workspaceID; + private String pid; + private URI uri; + private String epersonMail; + private String bitstreamName; + private EPersonService epersonService; + private ItemService itemService; + private WorkspaceItemService workspaceItemService; + private IdentifierService identifierService; + private BitstreamService bitstreamService; + private BitstreamFormatService bitstreamFormatService; + private final HttpClient httpClient = HttpClient.newBuilder() + .followRedirects(HttpClient.Redirect.NORMAL) + .build(); + + /** + * This method will return the Configuration that the implementing DSpaceRunnable uses + * + * @return The {@link ScriptConfiguration} that this implementing DspaceRunnable uses + */ + @Override + public FileDownloaderConfiguration getScriptConfiguration() { + return new DSpace().getServiceManager().getServiceByName("file-downloader", + FileDownloaderConfiguration.class); + } + + /** + * This method has to be included in every script and handles the setup of the script by parsing the CommandLine + * and setting the variables + * + * @throws ParseException If something goes wrong + */ + @Override + public void setup() throws ParseException { + log.debug("Setting up {}", FileDownloader.class.getName()); + if (commandLine.hasOption("h")) { + help = true; + return; + } + + if (!commandLine.hasOption("u")) { + throw new ParseException("No URL option has been provided"); + } + + if (!commandLine.hasOption("i") && !commandLine.hasOption("w") && !commandLine.hasOption("p")) { + throw new ParseException("No item id option has been provided"); + } + + if (getEpersonIdentifier() == null && !commandLine.hasOption("e")) { + throw new ParseException("No eperson option has been provided"); + } + + + this.epersonService = EPersonServiceFactory.getInstance().getEPersonService(); + this.itemService = ContentServiceFactory.getInstance().getItemService(); + this.workspaceItemService = ContentServiceFactory.getInstance().getWorkspaceItemService(); + this.bitstreamService = ContentServiceFactory.getInstance().getBitstreamService(); + this.bitstreamFormatService = ContentServiceFactory.getInstance().getBitstreamFormatService(); + this.identifierService = IdentifierServiceFactory.getInstance().getIdentifierService(); + + try { + uri = new URI(commandLine.getOptionValue("u")); + } catch (URISyntaxException e) { + throw new ParseException("The provided URL is not a valid URL"); + } + + if (commandLine.hasOption("i")) { + itemUUID = UUID.fromString(commandLine.getOptionValue("i")); + } else if (commandLine.hasOption("w")) { + workspaceID = Integer.parseInt(commandLine.getOptionValue("w")); + } else if (commandLine.hasOption("p")) { + pid = commandLine.getOptionValue("p"); + } + + epersonMail = commandLine.getOptionValue("e"); + + if (commandLine.hasOption("n")) { + bitstreamName = commandLine.getOptionValue("n"); + } + } + + /** + * This method has to be included in every script and this will be the main execution block for the script that'll + * contain all the logic needed + * + * @throws Exception If something goes wrong + */ + @Override + public void internalRun() throws Exception { + log.debug("Running {}", FileDownloader.class.getName()); + if (help) { + printHelp(); + return; + } + + Context context = new Context(); + context.setCurrentUser(getEperson(context)); + + //find the item by the given id + Item item = findItem(context); + if (item == null) { + throw new IllegalArgumentException("No item found for the given ID"); + } + + HttpRequest request = HttpRequest.newBuilder() + .uri(uri) + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); + + if (response.statusCode() >= 400) { + throw new IllegalArgumentException("The provided URL returned a status code of " + response.statusCode()); + } + + //use the provided value, the content-disposition header, the last part of the uri + if (bitstreamName == null) { + bitstreamName = response.headers().firstValue("Content-Disposition") + .filter(value -> value.contains("filename=")).flatMap(value -> Stream.of(value.split(";")) + .filter(v -> v.contains("filename=")) + .findFirst() + .map(fvalue -> fvalue.replaceFirst("filename=", "").replaceAll("\"", ""))) + .orElse(uri.getPath().substring(uri.getPath().lastIndexOf('/') + 1)); + } + + try (InputStream is = response.body()) { + saveFileToItem(context, item, is, bitstreamName); + } + + context.commit(); + } + + private Item findItem(Context context) throws SQLException { + if (itemUUID != null) { + return itemService.find(context, itemUUID); + } else if (workspaceID != 0) { + return workspaceItemService.find(context, workspaceID).getItem(); + } else { + try { + DSpaceObject dso = identifierService.resolve(context, pid); + if (dso instanceof Item) { + return (Item) dso; + } else { + throw new IllegalArgumentException("The provided identifier does not resolve to an item"); + } + } catch (IdentifierNotFoundException | IdentifierNotResolvableException e) { + throw new IllegalArgumentException(e); + } + } + } + + private void saveFileToItem(Context context, Item item, InputStream is, String name) + throws SQLException, AuthorizeException, IOException { + log.debug("Saving file to item {}", item.getID()); + List originals = item.getBundles("ORIGINAL"); + Bitstream b; + if (originals.isEmpty()) { + b = itemService.createSingleBitstream(context, is, item); + } else { + Bundle bundle = originals.get(0); + b = bitstreamService.create(context, bundle, is); + } + b.setName(context, name); + //now guess format of the bitstream + BitstreamFormat bf = bitstreamFormatService.guessFormat(context, b); + b.setFormat(context, bf); + } + + private EPerson getEperson(Context context) throws SQLException { + if (getEpersonIdentifier() != null) { + return epersonService.find(context, getEpersonIdentifier()); + } else { + return epersonService.findByEmail(context, epersonMail); + } + } +} + diff --git a/dspace-api/src/main/java/org/dspace/administer/FileDownloaderConfiguration.java b/dspace-api/src/main/java/org/dspace/administer/FileDownloaderConfiguration.java new file mode 100644 index 000000000000..848b2d99f7c0 --- /dev/null +++ b/dspace-api/src/main/java/org/dspace/administer/FileDownloaderConfiguration.java @@ -0,0 +1,73 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.administer; + +import org.apache.commons.cli.OptionGroup; +import org.apache.commons.cli.Options; +import org.dspace.scripts.configuration.ScriptConfiguration; + +public class FileDownloaderConfiguration extends ScriptConfiguration { + + private Class dspaceRunnableClass; + + /** + * Generic getter for the dspaceRunnableClass + * + * @return the dspaceRunnableClass value of this ScriptConfiguration + */ + @Override + public Class getDspaceRunnableClass() { + return dspaceRunnableClass; + } + + /** + * Generic setter for the dspaceRunnableClass + * + * @param dspaceRunnableClass The dspaceRunnableClass to be set on this IndexDiscoveryScriptConfiguration + */ + @Override + public void setDspaceRunnableClass(Class dspaceRunnableClass) { + this.dspaceRunnableClass = dspaceRunnableClass; + } + + /** + * The getter for the options of the Script + * + * @return the options value of this ScriptConfiguration + */ + @Override + public Options getOptions() { + if (options == null) { + + Options options = new Options(); + OptionGroup ids = new OptionGroup(); + + options.addOption("h", "help", false, "help"); + + options.addOption("u", "url", true, "source url"); + options.getOption("u").setRequired(true); + + options.addOption("i", "uuid", true, "item uuid"); + options.addOption("w", "wsid", true, "workspace id"); + options.addOption("p", "pid", true, "item pid (e.g. handle or doi)"); + ids.addOption(options.getOption("i")); + ids.addOption(options.getOption("w")); + ids.addOption(options.getOption("p")); + ids.setRequired(true); + + options.addOption("e", "eperson", true, "eperson email"); + options.getOption("e").setRequired(false); + + options.addOption("n", "name", true, "name of the file/bitstream"); + options.getOption("n").setRequired(false); + + super.options = options; + } + return options; + } +} diff --git a/dspace-api/src/main/java/org/dspace/authenticate/clarin/ClarinShibAuthentication.java b/dspace-api/src/main/java/org/dspace/authenticate/clarin/ClarinShibAuthentication.java index 53adf9d79b8c..ba5d8cd65bfa 100644 --- a/dspace-api/src/main/java/org/dspace/authenticate/clarin/ClarinShibAuthentication.java +++ b/dspace-api/src/main/java/org/dspace/authenticate/clarin/ClarinShibAuthentication.java @@ -565,7 +565,7 @@ protected EPerson findEPerson(Context context, HttpServletRequest request, Strin // 1) First, look for a netid header. if (netidHeaders != null) { - eperson = findEpersonByNetId(netidHeaders, shibheaders, eperson, ePersonService, context, true); + eperson = findEpersonByNetId(netidHeaders, shibheaders, ePersonService, context, true); if (eperson != null) { foundNetID = true; } @@ -1318,7 +1318,7 @@ public String getEmailAcceptedOrNull(String email) { /** * Find an EPerson by a NetID header. The method will go through all the netid headers and try to find a user. */ - public static EPerson findEpersonByNetId(String[] netidHeaders, ShibHeaders shibheaders, EPerson eperson, + public static EPerson findEpersonByNetId(String[] netidHeaders, ShibHeaders shibheaders, EPersonService ePersonService, Context context, boolean logAllowed) throws SQLException { // Go through all the netid headers and try to find a user. It could be e.g., `eppn`, `persistent-id`,.. @@ -1329,19 +1329,20 @@ public static EPerson findEpersonByNetId(String[] netidHeaders, ShibHeaders shib continue; } - eperson = ePersonService.findByNetid(context, netid); + EPerson eperson = ePersonService.findByNetid(context, netid); if (eperson == null && logAllowed) { log.info( "Unable to identify EPerson based upon Shibboleth netid header: '" + netidHeader + "'='" + netid + "'."); - } else { + } else if (eperson != null) { log.debug( "Identified EPerson based upon Shibboleth netid header: '" + netidHeader + "'='" + netid + "'" + "."); + return eperson; } } - return eperson; + return null; } /** diff --git a/dspace-api/src/main/java/org/dspace/identifier/doi/DataCiteConnector.java b/dspace-api/src/main/java/org/dspace/identifier/doi/DataCiteConnector.java index 57136d6143bb..62e8e46a49dd 100644 --- a/dspace-api/src/main/java/org/dspace/identifier/doi/DataCiteConnector.java +++ b/dspace-api/src/main/java/org/dspace/identifier/doi/DataCiteConnector.java @@ -438,10 +438,11 @@ public void reserveDOI(Context context, DSpaceObject dso, String doi) return; } // 400 -> invalid XML + case (422): case (400): { log.warn("DataCite was unable to understand the XML we send."); log.warn("DataCite Metadata API returned a http status code " - + "400: " + resp.getContent()); + + resp.getStatusCode() + ": " + resp.getContent()); Format format = Format.getCompactFormat(); format.setEncoding("UTF-8"); XMLOutputter xout = new XMLOutputter(format); diff --git a/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml b/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml index e388065b68fd..738e11f7b432 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml +++ b/dspace-api/src/test/data/dspaceFolder/config/spring/api/scripts.xml @@ -91,4 +91,9 @@ + + + + + diff --git a/dspace-api/src/test/java/org/dspace/administer/FileDownloaderIT.java b/dspace-api/src/test/java/org/dspace/administer/FileDownloaderIT.java new file mode 100644 index 000000000000..ee75fddc57e4 --- /dev/null +++ b/dspace-api/src/test/java/org/dspace/administer/FileDownloaderIT.java @@ -0,0 +1,110 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE and NOTICE files at the root of the source + * tree and available online at + * + * http://www.dspace.org/license/ + */ +package org.dspace.administer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; + +import java.util.List; + +import org.dspace.AbstractIntegrationTestWithDatabase; +import org.dspace.builder.CollectionBuilder; +import org.dspace.builder.CommunityBuilder; +import org.dspace.builder.ItemBuilder; +import org.dspace.content.Bitstream; +import org.dspace.content.Collection; +import org.dspace.content.Community; +import org.dspace.content.Item; +import org.dspace.content.factory.ContentServiceFactory; +import org.dspace.content.service.BitstreamService; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.mockserver.junit.MockServerRule; + + +public class FileDownloaderIT extends AbstractIntegrationTestWithDatabase { + + @Rule + public MockServerRule mockServerRule = new MockServerRule(this); + + private Item item; + + //Prepare a community and a collection before the test + @Before + @Override + public void setUp() throws Exception { + super.setUp(); + context.setCurrentUser(admin); + Community community = CommunityBuilder.createCommunity(context).build(); + Collection collection = CollectionBuilder.createCollection(context, community).build(); + item = ItemBuilder.createItem(context, collection).withTitle("FileDownloaderIT Item").build(); + + mockServerRule.getClient().when(request() + .withMethod("GET") + .withPath("/test400") + ).respond( + response() + .withStatusCode(400) + .withBody("test") + ); + + mockServerRule.getClient().when(request() + .withMethod("GET") + .withPath("/test") + ).respond( + response() + .withStatusCode(200) + .withHeader("Content-Disposition", "attachment; filename=\"test.txt\"") + .withBody("test") + ); + } + + //Test that when an error occurs no bitstream is actually added to the item + @Test() + public void testDownloadFileError() throws Exception { + + + BitstreamService bitstreamService = ContentServiceFactory.getInstance().getBitstreamService(); + int oldBitCount = bitstreamService.countTotal(context); + + int port = mockServerRule.getPort(); + String[] args = new String[]{"file-downloader", "-i", item.getID().toString(), + "-u", String.format("http://localhost:%s/test400", port), "-e", "admin@email.com"}; + try { + runDSpaceScript(args); + } catch (IllegalArgumentException e) { + assertEquals(0, item.getBundles().size()); + int newBitCount = bitstreamService.countTotal(context); + assertEquals(oldBitCount, newBitCount); + return; + } + assertEquals("Not expecting to get here", 0, 1); + } + + + //Test that FileDownlaoder actually adds the bitstream to the item + @Test + public void testDownloadFile() throws Exception { + + int port = mockServerRule.getPort(); + String[] args = new String[] {"file-downloader", "-i", item.getID().toString(), + "-u", String.format("http://localhost:%s/test", port), "-e", "admin@email.com"}; + runDSpaceScript(args); + + + assertEquals(1, item.getBundles().size()); + List bs = item.getBundles().get(0).getBitstreams(); + assertEquals(1, bs.size()); + assertNotNull("Expecting name to be defined", bs.get(0).getName()); + + } + +} \ No newline at end of file diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/HandleConverter.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/HandleConverter.java index dbc40a21cb60..ceb6409429a1 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/HandleConverter.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/converter/HandleConverter.java @@ -7,6 +7,8 @@ */ package org.dspace.app.rest.converter; +import java.util.Objects; + import org.dspace.app.rest.model.HandleRest; import org.dspace.app.rest.projection.Projection; import org.dspace.handle.Handle; @@ -35,6 +37,11 @@ public HandleRest convert(Handle modelObject, Projection projection) { handleRest.setHandle(modelObject.getHandle()); handleRest.setResourceTypeID(modelObject.getResourceTypeId()); handleRest.setUrl(modelObject.getUrl()); + if (Objects.nonNull(modelObject.getDSpaceObject())) { + handleRest.setResourceId(modelObject.getDSpaceObject().getID()); + } else { + handleRest.setResourceId(null); + } return handleRest; } diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/HandleRest.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/HandleRest.java index 6bb989eff09d..c40c9faa7017 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/HandleRest.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/model/HandleRest.java @@ -7,6 +7,8 @@ */ package org.dspace.app.rest.model; +import java.util.UUID; + import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import org.dspace.app.rest.RestResourceController; @@ -29,6 +31,8 @@ public class HandleRest extends BaseObjectRest { private String url; + private UUID resourceId; + public String getHandle() { return handle; } @@ -41,6 +45,10 @@ public String getUrl() { return url; } + public UUID getResourceId() { + return resourceId; + } + public void setHandle(String handle) { this.handle = handle; } @@ -53,6 +61,11 @@ public void setUrl(String url) { this.url = url; } + public void setResourceId(UUID resourceId) { + this.resourceId = resourceId; + + } + @Override public String getCategory() { return CATEGORY; diff --git a/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/clarin/ClarinShibbolethLoginFilter.java b/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/clarin/ClarinShibbolethLoginFilter.java index 774cdcd80850..053ddd6915ee 100644 --- a/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/clarin/ClarinShibbolethLoginFilter.java +++ b/dspace-server-webapp/src/main/java/org/dspace/app/rest/security/clarin/ClarinShibbolethLoginFilter.java @@ -167,7 +167,7 @@ public Authentication attemptAuthentication(HttpServletRequest req, EPerson ePerson = null; try { - ePerson = ClarinShibAuthentication.findEpersonByNetId(shib_headers.getNetIdHeaders(), shib_headers, ePerson, + ePerson = ClarinShibAuthentication.findEpersonByNetId(shib_headers.getNetIdHeaders(), shib_headers, ePersonService, context, false); } catch (SQLException e) { // It is logged in the ClarinShibAuthentication class. diff --git a/dspace/config/clarin-dspace.cfg b/dspace/config/clarin-dspace.cfg index ebb3fa782c51..14b972d7d295 100644 --- a/dspace/config/clarin-dspace.cfg +++ b/dspace/config/clarin-dspace.cfg @@ -209,13 +209,13 @@ identifier.doi.namespaceseparator = dspace/ ## crosswalk.dissemination.DataCite.stylesheet = crosswalks/DIM2DataCite.xsl crosswalk.dissemination.DataCite.schemaLocation = \ - http://datacite.org/schema/kernel-3 \ - http://schema.datacite.org/meta/kernel-3/metadata.xsd + http://datacite.org/schema/kernel-4 \ + https://schema.datacite.org/meta/kernel-4.5/metadata.xsd crosswalk.dissemination.DataCite.preferList = false crosswalk.dissemination.DataCite.publisher = My University #crosswalk.dissemination.DataCite.dataManager = # defaults to publisher #crosswalk.dissemination.DataCite.hostingInstitution = # defaults to publisher -crosswalk.dissemination.DataCite.namespace = http://datacite.org/schema/kernel-3 +crosswalk.dissemination.DataCite.namespace = http://datacite.org/schema/kernel-4 # consumer to update metadata of DOIs event.consumer.doi.class = org.dspace.identifier.doi.DOIConsumer diff --git a/dspace/config/crosswalks/DIM2DataCite.xsl b/dspace/config/crosswalks/DIM2DataCite.xsl index 51fef2275c40..fe6622a1f5fb 100644 --- a/dspace/config/crosswalks/DIM2DataCite.xsl +++ b/dspace/config/crosswalks/DIM2DataCite.xsl @@ -48,7 +48,7 @@ --> + xsi:schemaLocation="http://datacite.org/schema/kernel-4 https://schema.datacite.org/meta/kernel-4.5/metadata.xsd"> From 3486d2da1c304f1d1c374cd100fae42d21f01698 Mon Sep 17 00:00:00 2001 From: jurinecko <95219754+jr-rk@users.noreply.github.com> Date: Fri, 29 Nov 2024 11:08:36 +0100 Subject: [PATCH 10/10] UFAL/Matomo statistics with dimension (#813) * Updated the version of matomo dependency and tried to change request from Custom Variables to Dimension * Added a custom dimension with item's handle URL * Send custom dimension also in oai tracker * Use only IPv4 address, the Matomo tracker has a problem with IPv6 * Do not change custom dimension when the Item is null * First custom dimension should have ID '1'. * Use a valid URL for Matomo tracker in the IT * Configure handle custom dimension ID in the clarin-dspace.cfg * Refactored ipv4 method to be more readable - return null --------- Co-authored-by: Juraj Roka Co-authored-by: milanmajchrak Co-authored-by: milanmajchrak <90026355+milanmajchrak@users.noreply.github.com> --- dspace-api/pom.xml | 4 +- .../clarin/ClarinMatomoBitstreamTracker.java | 7 ++- .../clarin/ClarinMatomoTracker.java | 52 +++++++++++-------- .../test/data/dspaceFolder/config/local.cfg | 4 ++ dspace/config/clarin-dspace.cfg | 1 + 5 files changed, 44 insertions(+), 24 deletions(-) diff --git a/dspace-api/pom.xml b/dspace-api/pom.xml index c7b93f6ac377..6ac36d129ade 100644 --- a/dspace-api/pom.xml +++ b/dspace-api/pom.xml @@ -339,8 +339,8 @@ org.piwik.java.tracking - matomo-java-tracker - 2.0 + matomo-java-tracker-java11 + 3.4.0 org.apache.logging.log4j diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java index 838a2650586a..5b90ab7fc740 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoBitstreamTracker.java @@ -9,6 +9,7 @@ import java.sql.SQLException; import java.text.MessageFormat; +import java.util.LinkedHashMap; import java.util.List; import java.util.Objects; import javax.servlet.http.HttpServletRequest; @@ -69,7 +70,6 @@ public ClarinMatomoBitstreamTracker() { @Override protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, HttpServletRequest request) { super.preTrack(context, matomoRequest, item, request); - matomoRequest.setSiteId(siteId); log.debug("Logging to site " + matomoRequest.getSiteId()); String itemIdentifier = getItemIdentifier(item); @@ -82,6 +82,11 @@ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, } try { matomoRequest.setPageCustomVariable(new CustomVariable("source", "bitstream"), 1); + // Add the Item handle into the request as a custom dimension + LinkedHashMap handleDimension = new LinkedHashMap<>(); + handleDimension.put(configurationService.getLongProperty("matomo.custom.dimension.handle.id", + 1L), item.getHandle()); + matomoRequest.setDimensions(handleDimension); } catch (MatomoException e) { log.error(e); } diff --git a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java index 360b4efe8e93..34615bc2ed24 100644 --- a/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java +++ b/dspace-api/src/main/java/org/dspace/app/statistics/clarin/ClarinMatomoTracker.java @@ -7,14 +7,14 @@ */ package org.dspace.app.statistics.clarin; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Calendar; import java.util.Objects; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; +import java.util.concurrent.CompletableFuture; import javax.servlet.http.HttpServletRequest; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpResponse; import org.apache.logging.log4j.Logger; import org.dspace.content.Item; import org.dspace.content.factory.ClarinServiceFactory; @@ -23,6 +23,7 @@ import org.dspace.services.factory.DSpaceServicesFactory; import org.matomo.java.tracking.MatomoException; import org.matomo.java.tracking.MatomoRequest; +import org.matomo.java.tracking.parameters.AcceptLanguage; /** * The statistics Tracker for Matomo. This class prepare and send the track GET request to the `/matomo.php` @@ -99,13 +100,13 @@ protected MatomoRequest createMatomoRequest(HttpServletRequest request, String p */ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, HttpServletRequest request) { if (StringUtils.isNotBlank(request.getHeader("referer"))) { - matomoRequest.setHeaderUserAgent(request.getHeader("referer")); + matomoRequest.setReferrerUrl(request.getHeader("referer")); } if (StringUtils.isNotBlank(request.getHeader("user-agent"))) { matomoRequest.setHeaderUserAgent(request.getHeader("user-agent")); } if (StringUtils.isNotBlank(request.getHeader("accept-language"))) { - matomoRequest.setHeaderUserAgent(request.getHeader("accept-language")); + matomoRequest.setHeaderAcceptLanguage(AcceptLanguage.fromHeader(request.getHeader("accept-language"))); } // Creating a calendar using getInstance method @@ -134,18 +135,13 @@ protected void preTrack(Context context, MatomoRequest matomoRequest, Item item, * @param matomoRequest prepared MatomoRequest for sending */ public void sendTrackingRequest(MatomoRequest matomoRequest) { - try { - Future response = tracker.sendRequestAsync(matomoRequest); - // usually not needed: - HttpResponse httpResponse = response.get(); - int statusCode = httpResponse.getStatusLine().getStatusCode(); - if (statusCode > 399) { - // problem - log.error("Matomo tracker error the response has status code: " + statusCode); + CompletableFuture completableFuture = tracker.sendRequestAsync(matomoRequest); + + completableFuture.whenComplete((result, exception) -> { + if (exception != null) { + log.error("Matomo tracker error - the response exception message: {}", exception.getMessage()); } - } catch (ExecutionException | InterruptedException e) { - e.printStackTrace(); - } + }); } protected String getFullURL(HttpServletRequest request) { @@ -164,21 +160,35 @@ protected String getFullURL(HttpServletRequest request) { } /** - * Get IpAddress of the current user which throws this statistic event + * Get IpAddress of the current user which throws this statistic event. Return only the first valid IPv4 address + * because the Matomo tracker has a problem with IPv6 addresses. * * @param request current request - * @return + * @return only the first valid IPv4 address */ protected String getIpAddress(HttpServletRequest request) { - String ip = ""; String header = request.getHeader("X-Forwarded-For"); if (header == null) { header = request.getRemoteAddr(); } if (header != null) { String[] ips = header.split(", "); - ip = ips.length > 0 ? ips[0] : ""; + for (String candidateIp : ips) { + // Validate if it's an IPv4 address + if (isIPv4Address(candidateIp)) { + return candidateIp; + } + } + } + return null; + } + + private boolean isIPv4Address(String ip) { + try { + InetAddress inetAddress = InetAddress.getByName(ip); + return inetAddress.getHostAddress().equals(ip) && inetAddress instanceof java.net.Inet4Address; + } catch (UnknownHostException e) { + return false; // Not a valid IP address } - return ip; } } diff --git a/dspace-api/src/test/data/dspaceFolder/config/local.cfg b/dspace-api/src/test/data/dspaceFolder/config/local.cfg index 6b31cfc98123..14ff9e3a72a3 100644 --- a/dspace-api/src/test/data/dspaceFolder/config/local.cfg +++ b/dspace-api/src/test/data/dspaceFolder/config/local.cfg @@ -312,6 +312,10 @@ webui.supported.locales = en # When title is something like "Type-bind test" the type-bind field will popped up submit.type-bind.field = dc.type,dc.identifier.citation=>dc.title +# The configuration for the Matomo tracker must have a valid URL, as it will throw an exception if it does not. +matomo.tracker.host.url = http://localhost:8135/matomo.php + autocomplete.custom.separator.solr-subject_ac = \\|\\|\\| autocomplete.custom.separator.solr-title_ac = \\|\\|\\| autocomplete.custom.allowed = solr-author_ac,solr-publisher_ac,solr-dataProvider_ac,solr-dctype_ac,solr-subject_ac,solr-handle_title_ac,json_static-iso_langs.json,solr-title_ac + diff --git a/dspace/config/clarin-dspace.cfg b/dspace/config/clarin-dspace.cfg index 14b972d7d295..b6dbcef3ca11 100644 --- a/dspace/config/clarin-dspace.cfg +++ b/dspace/config/clarin-dspace.cfg @@ -155,6 +155,7 @@ matomo.site.id = 1 matomo.tracker.bitstream.site_id = 1 matomo.tracker.oai.site_id = 1 matomo.tracker.host.url = http://url:port/matomo.php +matomo.custom.dimension.handle.id = 1 statistics.cache-server.uri = http://cache-server.none #### Statistic usage reports ####