Skip to content

Commit

Permalink
big file/memory issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@nkons
nkons committed Oct 29, 2013
1 parent 30e1597 commit 38a04eb
Showing 1 changed file with 18 additions and 18 deletions.
36 changes: 18 additions & 18 deletions src/main/java/gr/seab/r2rml/beans/Generator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ public void createTriples(MappingDocument mappingDocument) {
InputStream isMap = FileManager.get().open(reifiedModelFileName);
resultModel = ModelFactory.createDefaultModel();
try {
resultModel.read(isMap, null, properties.getProperty("jena.destinationFileSyntax"));
resultModel.read(isMap, null, "N-TRIPLE");
} catch (Exception e) {
log.error("Error reading last run model. Cannot proceed with incremental, going for a full run."); // Please change property default.incremental in file r2rml.properties to false.
resultModel.setNsPrefixes(mappingDocument.getPrefixes());
Expand Down Expand Up @@ -525,17 +525,16 @@ public void createTriples(MappingDocument mappingDocument) {
if ((!incremental && writeReifiedModel) || incremental) {
log.info("Generating clean model.");
Model cleanModel = ModelFactory.createDefaultModel();
//TODO disable reasoning/rules on cleanModel
cleanModel.setNsPrefixes(resultModel.getNsPrefixMap());
ArrayList<Statement> cleanStatements = new ArrayList<Statement>();
//ArrayList<Statement> cleanStatements = new ArrayList<Statement>();

RSIterator rsIter = resultModel.listReifiedStatements();
long addedStatements = 0;
while (rsIter.hasNext()) {
ReifiedStatement rstmt = rsIter.next();
//Statement st = rstmt.getStatement();
//cleanModel.add(st);
cleanStatements.add(rstmt.getStatement());
cleanModel.add(rstmt.getStatement());
//cleanStatements.add(rstmt.getStatement());
addedStatements++;
if (verbose && addedStatements % 10000 == 0) log.info("At " + addedStatements);
}
Expand All @@ -548,17 +547,18 @@ public void createTriples(MappingDocument mappingDocument) {
StmtIterator stmtIter = resultModel.listStatements();
while (stmtIter.hasNext()) {
Statement st = stmtIter.nextStatement();
cleanStatements.add(st);
//cleanModel.add(st);
//cleanStatements.add(st);
cleanModel.add(st);
addedStatements++;
if (verbose && addedStatements % 10000 == 0) log.info("At " + addedStatements);
}
stmtIter.close();
}
log.info("Adding " + cleanStatements.size() + " statements to clean model.");
cleanModel.add(cleanStatements);
cleanStatements.clear(); //free some memory
log.info("Writing clean model to " + destinationFileName + ". Clean model has " + cleanModel.listStatements().toList().size() + " statements.");
//log.info("Adding " + cleanStatements.size() + " statements to clean model.");
//cleanModel.add(cleanStatements);
//cleanStatements.clear(); //free some memory
log.info("Writing clean model to " + destinationFileName);
//log.info("Clean model has " + cleanModel.listStatements().toList().size() + " statements.");

//Could as well be an empty model, this is why we check if it actually has any triples
//if (cleanModel.listStatements().hasNext()) {
Expand All @@ -568,7 +568,7 @@ public void createTriples(MappingDocument mappingDocument) {
long t0 = c0.getTimeInMillis();
BufferedWriter out = new BufferedWriter(new FileWriter(destinationFileName));
cleanModel.write(out, properties.getProperty("jena.destinationFileSyntax"));
out.flush();
out.close();
Calendar c1 = Calendar.getInstance();
long t1 = c1.getTimeInMillis();
log.info("Writing clean model to disk took " + (t1 - t0) + " milliseconds.");
Expand All @@ -577,6 +577,7 @@ public void createTriples(MappingDocument mappingDocument) {
} catch (IOException e) {
e.printStackTrace();
}
log.info("Clean model has " + cleanModel.listStatements().toList().size() + " statements.");
cleanModel.close();
} else {
log.info("Nothing to write.");
Expand All @@ -590,7 +591,7 @@ public void createTriples(MappingDocument mappingDocument) {
long t0 = c0.getTimeInMillis();
BufferedWriter out = new BufferedWriter(new FileWriter(destinationFileName));
resultModel.write(out, properties.getProperty("jena.destinationFileSyntax"));
out.flush();
out.close();
Calendar c1 = Calendar.getInstance();
long t1 = c1.getTimeInMillis();
log.info("Writing model to disk took " + (t1 - t0) + " milliseconds.");
Expand All @@ -610,15 +611,13 @@ public void createTriples(MappingDocument mappingDocument) {
}

if (writeReifiedModel) {

log.info("Writing reified model to " + reifiedModelFileName + ".");
log.info("Reified model has " + resultModel.listStatements().toList().size() + " statements.");
try {
Calendar c0 = Calendar.getInstance();
long t0 = c0.getTimeInMillis();
BufferedWriter out = new BufferedWriter(new FileWriter(reifiedModelFileName));
resultModel.write(out, properties.getProperty("jena.destinationFileSyntax"));
out.flush();
resultModel.write(out, "N-TRIPLE"); //properties.getProperty("jena.destinationFileSyntax"));
out.close();
Calendar c1 = Calendar.getInstance();
long t1 = c1.getTimeInMillis();
log.info("Writing reified model to disk took " + (t1 - t0) + " milliseconds.");
Expand All @@ -627,6 +626,7 @@ public void createTriples(MappingDocument mappingDocument) {
} catch (IOException e) {
e.printStackTrace();
}
log.info("Reified model has " + resultModel.listStatements().toList().size() + " statements.");
} else {
log.info("Not Writing reified model.");
}
Expand Down Expand Up @@ -755,7 +755,7 @@ public void createTriples(MappingDocument mappingDocument) {

BufferedWriter out = new BufferedWriter(new FileWriter(properties.getProperty("default.log")));
logModel.write(out, properties.getProperty("mapping.file.type"));
out.flush();
out.close();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
Expand Down

0 comments on commit 38a04eb

Please sign in to comment.