Skip to content

Commit

Permalink
Moved all the handling of custom dataverse_json metadata out of OaiHa…
Browse files Browse the repository at this point in the history 
…ndler and GetRecord

parser. (#8372)
  • Loading branch information
@landreev
landreev committed Jun 6, 2022
1 parent 6d6a49b commit bf413e7
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 192 deletions.
104 changes: 9 additions & 95 deletions src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@

public class FastGetRecord {

private static final String DATAVERSE_EXTENDED_METADATA = "dataverse_json";
private static final String XML_METADATA_TAG = "metadata";
private static final String XML_METADATA_TAG_OPEN = "<"+XML_METADATA_TAG+">";
private static final String XML_METADATA_TAG_CLOSE = "</"+XML_METADATA_TAG+">";
Expand Down Expand Up @@ -222,13 +221,7 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref
//metadataOut.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>"); /* ? */

metadataFlag = true;
} else if (line.matches(".*<"+XML_METADATA_TAG+" [^>]*>.*")) {
if (metadataPrefix.equals(DATAVERSE_EXTENDED_METADATA)) {
oaiResponseHeader = oaiResponseHeader.concat(line);
metadataWritten = true;
metadataFlag = true;
}
}
}
}

//System.out.println(line);
Expand Down Expand Up @@ -380,19 +373,12 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref
try {
StringReader reader = new StringReader(oaiResponseHeader);
xmlr = xmlInputFactory.createXMLStreamReader(reader);
processOAIheader(xmlr, metadataPrefix.equals(DATAVERSE_EXTENDED_METADATA));
processOAIheader(xmlr);

} catch (XMLStreamException ex) {
//Logger.getLogger("global").log(Level.SEVERE, null, ex);
if (this.errorMessage == null) {
this.errorMessage = "Malformed GetRecord response; baseURL=" + baseURL + ", identifier=" + identifier + ", metadataPrefix=" + metadataPrefix;
}

// delete the temp metadata file; we won't need it:
if (savedMetadataFile != null) {
//savedMetadataFile.delete();
}

}

try {
Expand All @@ -414,14 +400,8 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref

if (!(metadataWritten) && !(this.isDeleted())) {
this.errorMessage = "Failed to parse GetRecord response; baseURL=" + baseURL + ", identifier=" + identifier + ", metadataPrefix=" + metadataPrefix;
//savedMetadataFile.delete();
}

if (this.isDeleted()) {
//savedMetadataFile.delete();
}


} else {
this.errorMessage = "GetRecord request failed. HTTP error code "+responseCode;
}
Expand All @@ -445,16 +425,16 @@ private static String getRequestURL(String baseURL,
return requestURL.toString();
}

private void processOAIheader (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException {
private void processOAIheader (XMLStreamReader xmlr) throws XMLStreamException, IOException {

// is this really a GetRecord response?
xmlr.nextTag();
xmlr.require(XMLStreamConstants.START_ELEMENT, null, "OAI-PMH");
processOAIPMH(xmlr, extensionMode);
processOAIPMH(xmlr);

}

private void processOAIPMH (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException {
private void processOAIPMH (XMLStreamReader xmlr) throws XMLStreamException, IOException {

for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
if (event == XMLStreamConstants.START_ELEMENT) {
Expand All @@ -477,19 +457,19 @@ else if (xmlr.getLocalName().equals("error")) {

}
else if (xmlr.getLocalName().equals("GetRecord")) {
processGetRecordSection(xmlr, extensionMode);
processGetRecordSection(xmlr);
}
} else if (event == XMLStreamConstants.END_ELEMENT) {
if (xmlr.getLocalName().equals("OAI-PMH")) return;
}
}
}

private void processGetRecordSection (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException {
private void processGetRecordSection (XMLStreamReader xmlr) throws XMLStreamException, IOException {
for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
if (event == XMLStreamConstants.START_ELEMENT) {
if (xmlr.getLocalName().equals("record")) {
processRecord(xmlr, extensionMode);
processRecord(xmlr);
}
} else if (event == XMLStreamConstants.END_ELEMENT) {
if (xmlr.getLocalName().equals("GetRecord")) return;
Expand All @@ -498,19 +478,14 @@ private void processGetRecordSection (XMLStreamReader xmlr, boolean extensionMod

}

private void processRecord (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException {
private void processRecord (XMLStreamReader xmlr) throws XMLStreamException, IOException {
for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
if (event == XMLStreamConstants.START_ELEMENT) {
if (xmlr.getLocalName().equals("header")) {
if ("deleted".equals( xmlr.getAttributeValue(null, "status"))) {
this.recordDeleted = true;
}
processHeader(xmlr);
} else if (xmlr.getLocalName().equals("metadata")) {
if (extensionMode) {
String extendedMetadataApiUrl = xmlr.getAttributeValue(null, "directApiCall");
processMetadataExtended(extendedMetadataApiUrl);
}
}
} else if (event == XMLStreamConstants.END_ELEMENT) {
if (xmlr.getLocalName().equals("record")) return;
Expand All @@ -532,67 +507,6 @@ else if (xmlr.getLocalName().equals("setSpec")) {/*do nothing*/}
}
}

private void processMetadataExtended (String extendedApiUrl) throws IOException {
InputStream in = null;
int responseCode = 0;
HttpURLConnection con = null;



try {
URL url = new URL(extendedApiUrl.replaceAll("&amp;", "&")); // is this necessary?

con = (HttpURLConnection) url.openConnection();
con.setRequestProperty("User-Agent", "DataverseHarvester/3.0");
responseCode = con.getResponseCode();
} catch (MalformedURLException mue) {
throw new IOException ("Bad API URL: "+extendedApiUrl);
} catch (FileNotFoundException e) {
responseCode = HttpURLConnection.HTTP_UNAVAILABLE;
}




if (responseCode == 200) {
in = con.getInputStream();
// TODO:
/* we should probably still support gzip/compress encoding here - ?
String contentEncoding = con.getHeaderField("Content-Encoding");
// support for the standard compress/gzip/deflate compression
// schemes:
if ("compress".equals(contentEncoding)) {
ZipInputStream zis = new ZipInputStream(con.getInputStream());
zis.getNextEntry();
in = zis;
} else if ("gzip".equals(contentEncoding)) {
in = new GZIPInputStream(con.getInputStream());
} else if ("deflate".equals(contentEncoding)) {
in = new InflaterInputStream(con.getInputStream());
} ...
*/
FileOutputStream tempOut = new FileOutputStream(savedMetadataFile);

int bufsize;
byte[] buffer = new byte[4 * 8192];

while ((bufsize = in.read(buffer)) != -1) {
tempOut.write(buffer, 0, bufsize);
tempOut.flush();
}

in.close();
tempOut.close();
return;
}

throw new IOException("Failed to download extended metadata.");

}


// (from Gustavo's ddiServiceBean -- L.A.)
//
/* We had to add this method because the ref getElementText has a bug where it
Expand Down
Loading

0 comments on commit bf413e7

Please sign in to comment.