Skip to content

Commit

Permalink
remove discofeed related fetching from obsolete discojuice servers (#508
Browse files Browse the repository at this point in the history 
)

* remove discofeed related fetching from obsolete discojuice servers

* check style, turned off in testing, turned of by default

* fix failing test

---------

Co-authored-by: jm <jm@maz>
  • Loading branch information
@vidiecan
vidiecan and jm authored Feb 9, 2024
1 parent a9d00d4 commit 3f7ca7e
Show file tree
Hide file tree
Showing 5 changed files with 44 additions and 74 deletions.
5 changes: 1 addition & 4 deletions dspace-api/src/test/data/dspaceFolder/config/local.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,12 +220,9 @@ featured.service.teitok.description = A web-based platform for viewing, creating

##### Shibboleth #####
# Turn off the discofeed, it is allowed by default
shibboleth.discofeed.allowed = true
shibboleth.discofeed.allowed = false
# File where is DiscoJuiceFeed response
shibboleth.discofeed.url = TEST:/org/dspace/app/rest/discofeedResponse.json

# Configuration properties for DiscoJuice
discojuice.feeds = edugain, dfn, cesnet, surfnet2, haka, kalmar
# CRON job refresh time definition - default is refresh in every 2 hours.
discojuice.refresh = 0 */2 * * * ?
# Comma separated list of entityIDs; we try to guess country on these
Expand Down
88 changes: 26 additions & 62 deletions ...server-webapp/src/main/java/org/dspace/app/rest/ClarinDiscoJuiceFeedsDownloadService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
*/
package org.dspace.app.rest;

import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.commons.lang.StringUtils.isNotBlank;

import java.io.BufferedInputStream;
Expand Down Expand Up @@ -55,10 +54,9 @@ public class ClarinDiscoJuiceFeedsDownloadService implements InitializingBean {

protected static Logger log = org.apache.logging.log4j.LogManager.getLogger(
ClarinDiscoJuiceFeedsDownloadService.class);
private static final String DISCOJUICE_URL = "https://static.discojuice.org/feeds/";

/**
* contains entityIDs of idps we wish to set the country to something different than discojuice feeds suggests
* contains entityIDs of idps we wish to set the country to something different then discovery feeds suggests
**/
private Set<String> rewriteCountries;
protected static DatabaseReader locationService;
Expand Down Expand Up @@ -94,84 +92,50 @@ public void afterPropertiesSet() throws Exception {
}

for (String country : propRewriteCountries) {
country = country.trim();
rewriteCountries.add(country);
rewriteCountries.add(country.trim());
}
}

public String createFeedsContent() {
log.debug("Going to create feeds content.");
String[] feedsConfig = configurationService.getArrayProperty("discojuice.feeds");
String shibbolethDiscoFeedUrl = configurationService.getProperty("shibboleth.discofeed.url");
log.debug("Starting to create feeds content.");

if (StringUtils.isEmpty(shibbolethDiscoFeedUrl)) {
throw new RuntimeException("Cannot load the property `shibboleth.discofeed.url` from the configuration " +
"file, maybe it is not set in the configuration file");
}
String shibbolethDiscoFeedUrl = configurationService.getProperty("shibboleth.discofeed.url");

if (ArrayUtils.isEmpty(feedsConfig)) {
throw new RuntimeException("Cannot load the property `discojuice.feeds` from the configuration " +
if (StringUtils.isBlank(shibbolethDiscoFeedUrl)) {
throw new IllegalStateException(
"Cannot load the property `shibboleth.discofeed.url` from the configuration " +
"file, maybe it is not set in the configuration file");
}

String old_value = System.getProperty("jsse.enableSNIExtension");
String origSniVal = System.getProperty("jsse.enableSNIExtension");
System.setProperty("jsse.enableSNIExtension", "false");
try {

final Map<String, JSONObject> shibDiscoEntities = toMap(shrink(
ClarinDiscoJuiceFeedsDownloadService.downloadJSON(shibbolethDiscoFeedUrl)));

//true is the default http://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html
old_value = (old_value == null) ? "true" : old_value;
System.setProperty("jsse.enableSNIExtension", old_value);
final Map<String, JSONObject> shibDiscoEntities = toMap(shrink(
ClarinDiscoJuiceFeedsDownloadService.downloadJSON(shibbolethDiscoFeedUrl)));

String feedsContent = "";
Set<String> processedEntities = new HashSet<>();
//loop through disco cdn feeds
for (String feed : feedsConfig) {
Map<String, JSONObject> feedMap = toMap(
ClarinDiscoJuiceFeedsDownloadService.downloadJSON(DISCOJUICE_URL + feed.trim()));
//loop through entities in one feed
for (Map.Entry<String, JSONObject> entry: feedMap.entrySet()) {
String entityID = entry.getKey();
JSONObject cdnEntity = entry.getValue();
//keep only entities from shibboleth, add only once, but copy geo, icon, country
if (shibDiscoEntities.containsKey(entityID) && !processedEntities.contains(entityID)) {
JSONObject geo = (JSONObject) cdnEntity.get("geo");
String icon = (String) cdnEntity.get("icon");
String country = (String) cdnEntity.get("country");
JSONObject shibEntity = shibDiscoEntities.get(entityID);
if (geo != null) {
shibEntity.put("geo", geo);
}
if (icon != null) {
shibEntity.put("icon", icon);
}
if (country != null) {
shibEntity.put("country", country);
}
processedEntities.add(entityID);
// iterate through the entities to update countries as needed
shibDiscoEntities.forEach((entityId, shibEntity) -> {
if (rewriteCountries.contains(entityId) || StringUtils.isBlank((String) shibEntity.get("country"))) {
String oldCountry = (String) shibEntity.remove("country");
String newCountry = guessCountry(shibEntity);
shibEntity.put("country", newCountry);
log.debug("Changed country for {} from {} to {}", entityId, oldCountry, newCountry);
}
}
}
});

//loop through shib entities, we show these...
for (JSONObject shibEntity : shibDiscoEntities.values()) {
//rewrite or guess countries
if (rewriteCountries.contains(shibEntity.get("entityID")) || isBlank((String)shibEntity.get("country"))) {
String old_country = (String)shibEntity.remove("country");
String new_country = guessCountry(shibEntity);
shibEntity.put("country", new_country);
log.debug(String.format("For %s changed country from %s to %s", shibEntity.get("entityID"),
old_country, new_country));
if (shibDiscoEntities.isEmpty()) {
return null;
}
}

if (shibDiscoEntities.isEmpty()) {
return null;
} else {
JSONArray ret = new JSONArray();
ret.addAll(shibDiscoEntities.values());
return ret.toJSONString();

} finally {
// true is the default http://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html
origSniVal = (origSniVal == null) ? "true" : origSniVal;
System.setProperty("jsse.enableSNIExtension", origSniVal);
}
}

Expand Down
7 changes: 4 additions & 3 deletions ...server-webapp/src/main/java/org/dspace/app/rest/ClarinDiscoJuiceFeedsUpdateScheduler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,17 +49,18 @@ public void afterPropertiesSet() throws Exception {
*/
@Scheduled(cron = "${discojuice.refresh:-}")
public void cronJobSch() {
boolean isAllowed = configurationService.getBooleanProperty("shibboleth.discofeed.allowed", true);
// 2024/02 - unless explicitly turned on, do not use discofeed
boolean isAllowed = configurationService.getBooleanProperty("shibboleth.discofeed.allowed", false);
if (!isAllowed) {
return;
}

log.debug("CRON Job - going to download the discojuice feeds.");
log.debug("CRON Job - going to download the discovery feeds.");
String newFeedsContent = clarinDiscoJuiceFeedsDownloadService.createFeedsContent();
if (isNotBlank(newFeedsContent)) {
feedsContent = newFeedsContent;
} else {
log.error("Failed to obtain discojuice feeds!");
log.error("Failed to obtain additional discovery feeds!");
}
}

Expand Down
16 changes: 13 additions & 3 deletions ...ce-server-webapp/src/test/java/org/dspace/app/rest/ClarinDiscoJuiceFeedsControllerIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;


/**
* Test class for the controller ClarinDiscoJuiceFeedsController
*
Expand All @@ -27,13 +28,21 @@ public class ClarinDiscoJuiceFeedsControllerIT extends AbstractControllerIntegra
@Autowired
ConfigurationService configurationService;

@Autowired
ClarinDiscoJuiceFeedsUpdateScheduler clarinDiscoJuiceFeedsUpdateScheduler;

@Test
public void getDiscoFeeds() throws Exception {
String authTokenAdmin = getAuthToken(eperson.getEmail(), password);

String configKey = "shibboleth.discofeed.allowed";
boolean origVal = configurationService.getBooleanProperty(configKey);
configurationService.setProperty(configKey, true);
clarinDiscoJuiceFeedsUpdateScheduler.afterPropertiesSet();

// Expected response created from the test file: `discofeedResponse.json`
// Wrapped to the `callback` string = `dj_md_1`
String responseString = "dj_md_1([{\"country\":\"CZ\",\"keywords\":[\"Identity Provider for employees and " +
String expStr = "dj_md_1([{\"country\":\"CZ\",\"keywords\":[\"Identity Provider for employees and " +
"readers of the Archiepiscopal Gymnasium in Kromeriz - Library\",\"Identity Provider pro zamstnance " +
"a tene knihovny Arcibiskupskho gymnzia v Kromi\",\"Arcibiskupsk gymnzium v Kromi - " +
"Knihovna\"],\"entityID\":\"https:\\/\\/agkm.cz\\/idp\\/shibboleth\",\"title\":\"Archiepiscopal " +
Expand All @@ -47,12 +56,13 @@ public void getDiscoFeeds() throws Exception {
"\"Studijn a vdeck knihovna v Hradci Krlov\"],\"entityID\":\"https:\\/\\/aleph.svkhk.cz\\" +
"/idp\\/shibboleth\",\"title\":\"The Research Library in Hradec Krlov\"}])";


// Load bitstream from the item.
// Request with callback
getClient(authTokenAdmin).perform(get("/api/discojuice/feeds?callback=dj_md_1"))
.andExpect(status().isOk())
.andExpect(content().contentType(APPLICATION_JAVASCRIPT_UTF8))
.andExpect(content().string(responseString));
.andExpect(content().string(expStr));

configurationService.setProperty(configKey, origVal);
}
}
2 changes: 0 additions & 2 deletions dspace/config/clarin-dspace.cfg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,6 @@ featured.service.teitok.description = A web-based platform for viewing, creating
# File where is DiscoJuiceFeed response
shibboleth.discofeed.url = https://lindat.mff.cuni.cz/Shibboleth.sso/DiscoFeed

# Configuration properties for DiscoJuice
discojuice.feeds = edugain, dfn, cesnet, surfnet2, haka, kalmar
# CRON job refresh time definition - default is refresh in every 2 hours.
discojuice.refresh = 0 0 */2 * * ?
# Comma separated list of entityIDs; we try to guess country on these
Expand Down

0 comments on commit 3f7ca7e

Please sign in to comment.