From 4c4050506fcb0111f9215a64121aaef30f119173 Mon Sep 17 00:00:00 2001 From: glassjoseph Date: Mon, 13 Dec 2021 20:37:30 -0500 Subject: [PATCH] Update resource Language - Fix SNACLookupCache --- pom.xml | 2 +- .../exporters/SNACLookupCache.java | 52 ++++++++++++------- .../exporters/SNACResourceItem.java | 29 +++++------ .../openrefine/snac/commands/CommandTest.java | 6 +-- 4 files changed, 48 insertions(+), 41 deletions(-) diff --git a/pom.xml b/pom.xml index 0afe7d7..7532774 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ org.snaccooperative datamodel - 1.2.1 + 1.2.2 org.apache.httpcomponents diff --git a/src/org/snaccooperative/exporters/SNACLookupCache.java b/src/org/snaccooperative/exporters/SNACLookupCache.java index ea4cc1c..945a352 100644 --- a/src/org/snaccooperative/exporters/SNACLookupCache.java +++ b/src/org/snaccooperative/exporters/SNACLookupCache.java @@ -10,26 +10,28 @@ import org.slf4j.LoggerFactory; import org.snaccooperative.commands.SNACAPIClient; import org.snaccooperative.commands.SNACAPIResponse; +import org.snaccooperative.data.Language; +import org.snaccooperative.data.Term; public class SNACLookupCache { static final Logger logger = LoggerFactory.getLogger("SNACLookupCache"); protected SNACAPIClient _client; - protected HashMap _languageCodes; + protected HashMap _languageCodes; public SNACLookupCache() { - this._client = new SNACAPIClient("http://api.snaccooperative.org/"); - this._languageCodes = new HashMap(); + this._client = new SNACAPIClient("https://api.snaccooperative.org/"); + this._languageCodes = new HashMap(); } /** * Helps determine whether a given ISO language code exists on the SNAC database * * @param lang (ISO language code) - * @return lang_term or null (ISO language code found in API Request) + * @return Language or null (ISO language code found in API Request) */ - private String lookupLanguageCode(String lang) { + private Language lookupLanguageCode(String lang) { try { // Insert API request calls for lang (if exists: insert into language dict, if not: return // None) @@ -42,36 +44,46 @@ private String lookupLanguageCode(String lang) { SNACAPIResponse lookupResponse = _client.post(apiQuery); JSONParser jp = new JSONParser(); - JSONArray json_result = - (JSONArray) ((JSONObject) jp.parse(lookupResponse.getAPIResponse())).get("results"); + JSONArray json_result = (JSONArray) ((JSONObject) jp.parse(lookupResponse.getAPIResponse())).get("results"); if (json_result.size() <= 0) { return null; } else { + Language fetchedLanguage = new Language(); + Term langTerm = new Term(); + JSONObject json_val = (JSONObject) json_result.get(0); - // String lang_id = (String) json_val.get("id"); - // String lang_desc = (String) json_val.get("description"); - String lang_term = (String) json_val.get("term"); - return lang_term; + int term_id = Integer.parseInt((String) json_val.get("id")); + String description = (String) json_val.get("description"); + String term_string = (String) json_val.get("term"); + fetchedLanguage.setOperation("insert"); + + langTerm.setID(term_id); + langTerm.setTerm(term_string); + langTerm.setDescription(description); + + fetchedLanguage.setLanguage(langTerm); + // fetchedLanguage.setScript(script_term); + return fetchedLanguage; } } catch (ParseException e) { return null; } } - public String getLanguageCode(String key) { - String langCode = _languageCodes.get(key); + public Language getLanguageCode(String key) { + Language language = _languageCodes.get(key); - if (langCode != null) { - return langCode; + if (language != null) { + return language; } - langCode = lookupLanguageCode(key); + language = lookupLanguageCode(key); - if (langCode != null) { - logger.info("getLanguageCode(): looked up [" + key + "] => [" + langCode + "]"); - _languageCodes.put(key, langCode); - return langCode; + if (language != null) { + logger.info("getLanguageCode(): looked up [" + key + "] => [" + language.toString() + "]"); + _languageCodes.put(key, language); + return language; } logger.warn("getLanguageCode(): no cache or lookup results for [" + key + "]"); diff --git a/src/org/snaccooperative/exporters/SNACResourceItem.java b/src/org/snaccooperative/exporters/SNACResourceItem.java index 18596fb..1d552b1 100644 --- a/src/org/snaccooperative/exporters/SNACResourceItem.java +++ b/src/org/snaccooperative/exporters/SNACResourceItem.java @@ -40,6 +40,9 @@ public SNACResourceItem( // Insert by default res.setOperation("insert"); + // things to accumulate + List languages = new LinkedList(); + for (Map.Entry entry : schema.getColumnMappings().entrySet()) { String csvColumn = entry.getKey(); String snacField = entry.getValue().toLowerCase(); @@ -100,16 +103,9 @@ public SNACResourceItem( continue; case "language": - // reduced to just adding language codes, as script does not seem to be used right now - String langCode = cache.getLanguageCode(cellValue); - - if (langCode != null) { - Language lang = new Language(); - Term langTerm = new Term(); - - langTerm.setType(cellValue); - lang.setLanguage(langTerm); - res.addLanguage(lang); + Language lang = cache.getLanguageCode(cellValue); + if (lang != null) { + languages.add(lang); } continue; @@ -134,6 +130,8 @@ public SNACResourceItem( } } + // add accumulated languages + res.setLanguages(languages); this._resource = res; } @@ -194,21 +192,16 @@ public String getPreviewText() { String _resourceLanguages = ""; if (languageList.size() > 0) { - List valid_lang = new LinkedList(); for (int i = 0; i < languageList.size(); i++) { if (languageList.get(i).getLanguage() == null) { continue; } - String lang_var = languageList.get(i).getLanguage().getType(); + String lang_var = languageList.get(i).getLanguage().getDescription(); if (lang_var.equals("")) { continue; } - valid_lang.add(_cache.getLanguageCode(lang_var) + " (" + lang_var + ")"); + _resourceLanguages += lang_var + ", "; } - for (int j = 0; j < valid_lang.size() - 1; j++) { - _resourceLanguages += valid_lang.get(j) + ", "; - } - _resourceLanguages += valid_lang.get(valid_lang.size() - 1) + "\n"; } resourceFields.put(snacText, _resourceLanguages); @@ -216,6 +209,7 @@ public String getPreviewText() { case "holding repository snac id": snacText = "Repository ID"; + if (_resource.getRepository() != null) { // TODO: handle missing repository. Could check for cell value type and show error in Issues tab. int repo_id = _resource.getRepository().getID(); @@ -225,6 +219,7 @@ public String getPreviewText() { repo_str = Integer.toString(repo_id); } resourceFields.put(snacText, repo_str); + } } } diff --git a/tests/src/org/openrefine/snac/commands/CommandTest.java b/tests/src/org/openrefine/snac/commands/CommandTest.java index 6cb3e72..cb29032 100644 --- a/tests/src/org/openrefine/snac/commands/CommandTest.java +++ b/tests/src/org/openrefine/snac/commands/CommandTest.java @@ -187,7 +187,7 @@ public class CommandTest extends RefineTest{ // @Test // public void testRecentlyPublished() throws Exception{ // DefaultHttpClient client = new DefaultHttpClient(); - // HttpPost post = new HttpPost("http://api.snaccooperative.org"); + // HttpPost post = new HttpPost("https://api.snaccooperative.org"); // post.setEntity(new StringEntity("{\"command\":\"recently_published\"}", "UTF-8")); // HttpResponse response = client.execute(post); // String result = EntityUtils.toString(response.getEntity()); @@ -207,7 +207,7 @@ public class CommandTest extends RefineTest{ // @Test // public void testStringToJSONDownload1() throws Exception{ // DefaultHttpClient client = new DefaultHttpClient(); - // HttpPost post = new HttpPost("http://api.snaccooperative.org"); + // HttpPost post = new HttpPost("https://api.snaccooperative.org"); // post.setEntity(new StringEntity("{\"command\": \"search\", \"term\": \"Washington\",\"count\": 10,\"start\": 0,\"entity_type\": \"person\"}","UTF-8")); // HttpResponse response = client.execute(post); // String result = EntityUtils.toString(response.getEntity()); @@ -220,7 +220,7 @@ public class CommandTest extends RefineTest{ // @Test // public void testStringToJSONDownload2() throws Exception{ // DefaultHttpClient client = new DefaultHttpClient(); - // HttpPost post = new HttpPost("http://api.snaccooperative.org"); + // HttpPost post = new HttpPost("https://api.snaccooperative.org"); // post.setEntity(new StringEntity("","UTF-8")); // HttpResponse response = client.execute(post); // String result = EntityUtils.toString(response.getEntity());