Skip to content

Commit

Permalink
Update resource Language
Browse files Browse the repository at this point in the history 
- Fix SNACLookupCache
  • Loading branch information
@glassjoseph
glassjoseph committed Dec 14, 2021
1 parent 2f936b0 commit 4c40505
Show file tree
Hide file tree
Showing 4 changed files with 48 additions and 41 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@
<dependency>
<groupId>org.snaccooperative</groupId>
<artifactId>datamodel</artifactId>
<version>1.2.1</version>
<version>1.2.2</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
Expand Down
52 changes: 32 additions & 20 deletions src/org/snaccooperative/exporters/SNACLookupCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,26 +10,28 @@
import org.slf4j.LoggerFactory;
import org.snaccooperative.commands.SNACAPIClient;
import org.snaccooperative.commands.SNACAPIResponse;
import org.snaccooperative.data.Language;
import org.snaccooperative.data.Term;

public class SNACLookupCache {

static final Logger logger = LoggerFactory.getLogger("SNACLookupCache");

protected SNACAPIClient _client;
protected HashMap<String, String> _languageCodes;
protected HashMap<String, Language> _languageCodes;

public SNACLookupCache() {
this._client = new SNACAPIClient("http://api.snaccooperative.org/");
this._languageCodes = new HashMap<String, String>();
this._client = new SNACAPIClient("https://api.snaccooperative.org/");
this._languageCodes = new HashMap<String, Language>();
}

/**
* Helps determine whether a given ISO language code exists on the SNAC database
*
* @param lang (ISO language code)
* @return lang_term or null (ISO language code found in API Request)
* @return Language or null (ISO language code found in API Request)
*/
private String lookupLanguageCode(String lang) {
private Language lookupLanguageCode(String lang) {
try {
// Insert API request calls for lang (if exists: insert into language dict, if not: return
// None)
Expand All @@ -42,36 +44,46 @@ private String lookupLanguageCode(String lang) {
SNACAPIResponse lookupResponse = _client.post(apiQuery);

JSONParser jp = new JSONParser();
JSONArray json_result =
(JSONArray) ((JSONObject) jp.parse(lookupResponse.getAPIResponse())).get("results");
JSONArray json_result = (JSONArray) ((JSONObject) jp.parse(lookupResponse.getAPIResponse())).get("results");

if (json_result.size() <= 0) {
return null;
} else {
Language fetchedLanguage = new Language();
Term langTerm = new Term();

JSONObject json_val = (JSONObject) json_result.get(0);
// String lang_id = (String) json_val.get("id");
// String lang_desc = (String) json_val.get("description");
String lang_term = (String) json_val.get("term");
return lang_term;
int term_id = Integer.parseInt((String) json_val.get("id"));
String description = (String) json_val.get("description");
String term_string = (String) json_val.get("term");
fetchedLanguage.setOperation("insert");

langTerm.setID(term_id);
langTerm.setTerm(term_string);
langTerm.setDescription(description);

fetchedLanguage.setLanguage(langTerm);
// fetchedLanguage.setScript(script_term);
return fetchedLanguage;
}
} catch (ParseException e) {
return null;
}
}

public String getLanguageCode(String key) {
String langCode = _languageCodes.get(key);
public Language getLanguageCode(String key) {
Language language = _languageCodes.get(key);

if (langCode != null) {
return langCode;
if (language != null) {
return language;
}

langCode = lookupLanguageCode(key);
language = lookupLanguageCode(key);

if (langCode != null) {
logger.info("getLanguageCode(): looked up [" + key + "] => [" + langCode + "]");
_languageCodes.put(key, langCode);
return langCode;
if (language != null) {
logger.info("getLanguageCode(): looked up [" + key + "] => [" + language.toString() + "]");
_languageCodes.put(key, language);
return language;
}

logger.warn("getLanguageCode(): no cache or lookup results for [" + key + "]");
Expand Down
29 changes: 12 additions & 17 deletions src/org/snaccooperative/exporters/SNACResourceItem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ public SNACResourceItem(
// Insert by default
res.setOperation("insert");

// things to accumulate
List<Language> languages = new LinkedList<Language>();

for (Map.Entry<String, String> entry : schema.getColumnMappings().entrySet()) {
String csvColumn = entry.getKey();
String snacField = entry.getValue().toLowerCase();
Expand Down Expand Up @@ -100,16 +103,9 @@ public SNACResourceItem(
continue;

case "language":
// reduced to just adding language codes, as script does not seem to be used right now
String langCode = cache.getLanguageCode(cellValue);

if (langCode != null) {
Language lang = new Language();
Term langTerm = new Term();

langTerm.setType(cellValue);
lang.setLanguage(langTerm);
res.addLanguage(lang);
Language lang = cache.getLanguageCode(cellValue);
if (lang != null) {
languages.add(lang);
}

continue;
Expand All @@ -134,6 +130,8 @@ public SNACResourceItem(
}
}

// add accumulated languages
res.setLanguages(languages);
this._resource = res;
}

Expand Down Expand Up @@ -194,28 +192,24 @@ public String getPreviewText() {
String _resourceLanguages = "";

if (languageList.size() > 0) {
List<String> valid_lang = new LinkedList<String>();
for (int i = 0; i < languageList.size(); i++) {
if (languageList.get(i).getLanguage() == null) {
continue;
}
String lang_var = languageList.get(i).getLanguage().getType();
String lang_var = languageList.get(i).getLanguage().getDescription();
if (lang_var.equals("")) {
continue;
}
valid_lang.add(_cache.getLanguageCode(lang_var) + " (" + lang_var + ")");
_resourceLanguages += lang_var + ", ";
}
for (int j = 0; j < valid_lang.size() - 1; j++) {
_resourceLanguages += valid_lang.get(j) + ", ";
}
_resourceLanguages += valid_lang.get(valid_lang.size() - 1) + "\n";
}

resourceFields.put(snacText, _resourceLanguages);
break;

case "holding repository snac id":
snacText = "Repository ID";
if (_resource.getRepository() != null) {

// TODO: handle missing repository. Could check for cell value type and show error in Issues tab.
int repo_id = _resource.getRepository().getID();
Expand All @@ -225,6 +219,7 @@ public String getPreviewText() {
repo_str = Integer.toString(repo_id);
}
resourceFields.put(snacText, repo_str);
}
}
}

Expand Down
6 changes: 3 additions & 3 deletions tests/src/org/openrefine/snac/commands/CommandTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ public class CommandTest extends RefineTest{
// @Test
// public void testRecentlyPublished() throws Exception{
// DefaultHttpClient client = new DefaultHttpClient();
// HttpPost post = new HttpPost("http://api.snaccooperative.org");
// HttpPost post = new HttpPost("https://api.snaccooperative.org");
// post.setEntity(new StringEntity("{\"command\":\"recently_published\"}", "UTF-8"));
// HttpResponse response = client.execute(post);
// String result = EntityUtils.toString(response.getEntity());
Expand All @@ -207,7 +207,7 @@ public class CommandTest extends RefineTest{
// @Test
// public void testStringToJSONDownload1() throws Exception{
// DefaultHttpClient client = new DefaultHttpClient();
// HttpPost post = new HttpPost("http://api.snaccooperative.org");
// HttpPost post = new HttpPost("https://api.snaccooperative.org");
// post.setEntity(new StringEntity("{\"command\": \"search\", \"term\": \"Washington\",\"count\": 10,\"start\": 0,\"entity_type\": \"person\"}","UTF-8"));
// HttpResponse response = client.execute(post);
// String result = EntityUtils.toString(response.getEntity());
Expand All @@ -220,7 +220,7 @@ public class CommandTest extends RefineTest{
// @Test
// public void testStringToJSONDownload2() throws Exception{
// DefaultHttpClient client = new DefaultHttpClient();
// HttpPost post = new HttpPost("http://api.snaccooperative.org");
// HttpPost post = new HttpPost("https://api.snaccooperative.org");
// post.setEntity(new StringEntity("","UTF-8"));
// HttpResponse response = client.execute(post);
// String result = EntityUtils.toString(response.getEntity());
Expand Down

0 comments on commit 4c40505

Please sign in to comment.