Skip to content

Commit

Permalink
Merge pull request #9 from ebremer/develop
Browse files Browse the repository at this point in the history 
Minor fixes
  • Loading branch information
@ebremer
ebremer authored May 25, 2023
2 parents c6fcf54 + a030465 commit bb159a3
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 56 deletions.
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.ebremer</groupId>
<artifactId>BeakGraph</artifactId>
<version>0.3.1</version>
<version>0.3.2</version>
<packaging>jar</packaging>
<build>
<plugins>
Expand All @@ -29,7 +29,7 @@
<exec.mainClass>com.ebremer.beakgraph.BeakGraph</exec.mainClass>
<jena.version>4.8.0</jena.version>
<arrow.version>12.0.0</arrow.version>
<rocrate4j.version>0.3.0</rocrate4j.version>
<rocrate4j.version>0.3.1</rocrate4j.version>
</properties>
<dependencies>
<dependency>
Expand Down
102 changes: 51 additions & 51 deletions src/main/java/com/ebremer/beakgraph/rdf/BeakReader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,61 +50,60 @@ public final class BeakReader implements AutoCloseable {
private int numtriples = 0;

public BeakReader(URI uri) throws FileNotFoundException, IOException {
ROCrateReader reader = new ROCrateReader(uri);
byPredicate = new HashMap<>();
root = new RootAllocator();
manifest = reader.getManifest();
ParameterizedSparqlString pss = new ParameterizedSparqlString(
"""
select ?file where {
?s a bg:BeakGraph; so:hasPart ?file .
?file a bg:PredicateVector .
}
""");
pss.setNsPrefix("bg", BG.NS);
pss.setNsPrefix("rdfs", RDFS.uri);
pss.setNsPrefix("so", SchemaDO.NS);
QueryExecution qe = QueryExecutionFactory.create(pss.toString(), manifest);
ResultSet rs = qe.execSelect();
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String x = qs.get("file").asResource().getURI();
try {
SeekableByteChannel xxx = reader.getSeekableByteChannel(x);
ArrowFileReader afr = new ArrowFileReader(xxx, root);
VectorSchemaRoot za = afr.getVectorSchemaRoot();
afr.loadNextBatch();
StructVector v = (StructVector) za.getVector(0);
String p = v.getName();
String dt = p.substring(0, 1);
numtriples = numtriples + v.getValueCount();
p = p.substring(1);
if (!byPredicate.containsKey(p)) {
byPredicate.put(p, new PAR(p));
try (ROCrateReader reader = new ROCrateReader(uri)) {
byPredicate = new HashMap<>();
root = new RootAllocator();
manifest = reader.getManifest();
ParameterizedSparqlString pss = new ParameterizedSparqlString(
"""
select ?file where {
?s a bg:BeakGraph; so:hasPart ?file .
?file a bg:PredicateVector .
}
""");
pss.setNsPrefix("bg", BG.NS);
pss.setNsPrefix("rdfs", RDFS.uri);
pss.setNsPrefix("so", SchemaDO.NS);
QueryExecution qe = QueryExecutionFactory.create(pss.toString(), manifest);
ResultSet rs = qe.execSelect();
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String x = qs.get("file").asResource().getURI();
try {
SeekableByteChannel xxx = reader.getSeekableByteChannel(x);
ArrowFileReader afr = new ArrowFileReader(xxx, root);
VectorSchemaRoot za = afr.getVectorSchemaRoot();
afr.loadNextBatch();
StructVector v = (StructVector) za.getVector(0);
String p = v.getName();
String dt = p.substring(0, 1);
numtriples = numtriples + v.getValueCount();
p = p.substring(1);
if (!byPredicate.containsKey(p)) {
byPredicate.put(p, new PAR(p));
}
PAR par = byPredicate.get(p);
par.put(dt, v);
} catch (FileNotFoundException ex) {
Logger.getLogger(ROCrateReader.class.getName()).log(Level.SEVERE, null, ex);
} catch (IOException ex) {
Logger.getLogger(ROCrateReader.class.getName()).log(Level.SEVERE, null, ex);
}
PAR par = byPredicate.get(p);
par.put(dt, v);
} catch (FileNotFoundException ex) {
Logger.getLogger(ROCrateReader.class.getName()).log(Level.SEVERE, null, ex);
} catch (IOException ex) {
Logger.getLogger(ROCrateReader.class.getName()).log(Level.SEVERE, null, ex);
}
}
DictionaryEncoding dictionaryEncoding = new DictionaryEncoding(0, true, new ArrowType.Int(32, true));
String cha = uri.toString();
if (cha.startsWith("file:/")) {
if (!cha.startsWith("file://")) {
cha = "file:///"+cha.substring("file:/".length());
DictionaryEncoding dictionaryEncoding = new DictionaryEncoding(0, true, new ArrowType.Int(32, true));
String cha = uri.toString();
if (cha.startsWith("file:/")) {
if (!cha.startsWith("file://")) {
cha = "file:///"+cha.substring("file:/".length());
}
}
SeekableByteChannel d = reader.getSeekableByteChannel(cha+"/halcyon/dictionary.arrow");
ArrowFileReader afr = new ArrowFileReader(d, root);
VectorSchemaRoot za = afr.getVectorSchemaRoot();
afr.loadNextBatch();
dictionary = new Dictionary(za.getVector(0), dictionaryEncoding);
nodeTable = new NodeTable(dictionary);
}
SeekableByteChannel d = reader.getSeekableByteChannel(cha+"/halcyon/dictionary");
ArrowFileReader afr = new ArrowFileReader(d, root);
VectorSchemaRoot za = afr.getVectorSchemaRoot();
afr.loadNextBatch();
dictionary = new Dictionary(za.getVector(0), dictionaryEncoding);
nodeTable = new NodeTable(dictionary);
reader.close();
// DisplayAll();
}

public Model getManifest() {
Expand All @@ -117,6 +116,7 @@ public void close() {
v.close();
});
nodeTable.close();
dictionary.getVector().close();
root.close();
}

Expand Down
6 changes: 3 additions & 3 deletions src/main/java/com/ebremer/beakgraph/rdf/BeakWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ public Resource WriteDictionaryToFile(String base, ROCrate.ROCrateBuilder roc) {
VectorSchemaRoot root = new VectorSchemaRoot(List.of(v.getField()), List.of(v));
) {
try {
OutputStream zos = roc.getDestination().GetOutputStream(base+"/dictionary", CompressionMethod.STORE);
OutputStream zos = roc.getDestination().GetOutputStream(base+"/dictionary.arrow", CompressionMethod.STORE);
CountingOutputStream cos = new CountingOutputStream(zos);
ArrowFileWriter writer = new ArrowFileWriter(root, null, Channels.newChannel(cos));
writer.start();
Expand All @@ -174,7 +174,7 @@ public Resource WriteDictionaryToFile(String base, ROCrate.ROCrateBuilder roc) {
fh.setUncompressedSize(numbytes);
}
roc
.Add(target, base, "dictionary", CompressionMethod.STORE, true)
.Add(target, base, "dictionary.arrow", CompressionMethod.STORE, true)
.addProperty(SchemaDO.encodingFormat, "application/vnd.apache.arrow.file")
.addLiteral(SchemaDO.contentSize, numbytes)
.addProperty(RDF.type, SchemaDO.MediaObject)
Expand Down Expand Up @@ -207,7 +207,7 @@ public Resource WriteDataToFile(String base, ROCrate.ROCrateBuilder roc) {
fh.setUncompressedSize(numbytes);
}
roc
.Add(target, base, MD5(v.getField().getName().getBytes()), CompressionMethod.STORE, true)
.Add(target, base, MD5(v.getField().getName().getBytes())+".arrow", CompressionMethod.STORE, true)
.addProperty(SchemaDO.name, v.getField().getName().substring(1))
.addProperty(BG.property, ResourceFactory.createResource(v.getName().substring(1,v.getName().length())))
.addProperty(SchemaDO.encodingFormat, "application/vnd.apache.arrow.file")
Expand Down

0 comments on commit bb159a3

Please sign in to comment.