Skip to content

Commit

Permalink
Issue #144 identifying single/multi valued properties
Browse files Browse the repository at this point in the history
  • Loading branch information
@jyotsnaraveendran
jyotsnaraveendran committed Apr 30, 2018
1 parent e8c96e5 commit d50e4c0
Show file tree
Hide file tree
Showing 15 changed files with 804 additions and 312 deletions.
8 changes: 4 additions & 4 deletions java/converters/rdf2Graph/src/main/java/io/opensaber/utils/converters/RDF2Graph.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ private static void updateGraph(Value subjectValue, IRI property, Value objectVa
Literal literal = (Literal)objectValue;
String datatype = literal.getDatatype().toString();
logger.info("TYPE saved is "+datatype);
/*VertexProperty vp = s.property(property.toString());
VertexProperty vp = s.property(property.toString());
if(vp.isPresent()){
Object value = vp.value();
List valueList = new ArrayList();
Expand All @@ -72,11 +72,11 @@ private static void updateGraph(Value subjectValue, IRI property, Value objectVa
String valueStr = (String)value;
valueList.add(valueStr);
}
s.property(property.toString(), valueList);
s.property(property.toString(), valueList).property("@type",datatype);

}else{*/
}else{
s.property(property.toString(), literal.getLabel()).property("@type",datatype);
//}
}
} else if (objectValue instanceof IRI) {
IRI objectIRI = (IRI)objectValue;
Vertex o = getExistingVertexOrAdd(objectIRI.toString(), graph);
Expand Down
10 changes: 10 additions & 0 deletions java/registry/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<jacoco.version>0.8.0</jacoco.version>
<runSuite>**/RegistryDaoImplTest.class</runSuite>
</properties>

<dependencies>
Expand Down Expand Up @@ -296,6 +297,15 @@
</systemPropertyVariables>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<includes>
<include>${runSuite}</include>
</includes>
</configuration>
</plugin>
</plugins>
</build>

Expand Down
3 changes: 2 additions & 1 deletion java/registry/src/main/java/io/opensaber/registry/config/GenericConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,8 @@ public AuditRecord auditRecord() {
@Bean
public SchemaConfigurator schemaConfiguration() throws IOException{
String fieldConfigFileName = environment.getProperty(Constants.FIELD_CONFIG_SCEHEMA_FILE);
return new SchemaConfigurator(fieldConfigFileName);
String validationConfigFile = environment.getProperty(Constants.SHEX_PROPERTY_NAME);
return new SchemaConfigurator(fieldConfigFileName, validationConfigFile);
}

@Bean
Expand Down
159 changes: 127 additions & 32 deletions java/registry/src/main/java/io/opensaber/registry/dao/impl/RegistryDaoImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.apache.tinkerpop.gremlin.structure.*;
import org.apache.tinkerpop.gremlin.structure.io.IoCore;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import org.javatuples.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -146,6 +147,13 @@ private void connectRootToEntity(GraphTraversalSource dbTraversalSource, String
Vertex rootVertex = rootGts.next();
Vertex entityVertex = entityGts.next();
rootVertex.addEdge(property, entityVertex);
AuditRecord record = appContext.getBean(AuditRecord.class);
record
.subject(rootVertex.label())
.predicate(property)
.oldObject(null)
.newObject(entityVertex.label())
.record(databaseProvider);
}

/**
Expand Down Expand Up @@ -233,33 +241,49 @@ private String addOrUpdateVerticesAndEdges(GraphTraversalSource dbTraversalSourc
private void addOrUpdateVertexAndEdge(Vertex v, Vertex dbVertex, GraphTraversalSource dbGraph, String methodOrigin)
throws NoSuchElementException, EncryptionException, AuditFailedException, RecordNotFoundException{
Iterator<Edge> edges = v.edges(Direction.OUT);
Iterator<Edge> edgeList = v.edges(Direction.OUT);
Stack<Pair<Vertex, Vertex>> parsedVertices = new Stack<>();
List<Edge> dbEdgesForVertex = ImmutableList.copyOf(dbVertex.edges(Direction.OUT));

List<Edge> edgeVertexMatchList = new ArrayList<Edge>();
while(edgeList.hasNext()) {
Edge e = edgeList.next();
Vertex ver = e.inVertex();
String edgeLabel = e.label();
Optional<Edge> edgeVertexAlreadyExists =
dbEdgesForVertex.stream().filter(ed -> ed.label().equalsIgnoreCase(edgeLabel) && ed.inVertex().label().equalsIgnoreCase(ver.label())).findFirst();
if(edgeVertexAlreadyExists.isPresent()){
edgeVertexMatchList.add(edgeVertexAlreadyExists.get());
}
}
logger.info("Matching list size:"+edgeVertexMatchList.size());

while(edges.hasNext()) {
Edge e = edges.next();
Vertex ver = e.inVertex();

System.out.println("Vertex being encountered"+ver.label());
String edgeLabel = e.label();
GraphTraversal<Vertex, Vertex> gt = dbGraph.clone().V().hasLabel(ver.label());
Optional<Edge> edgeAlreadyExists =
dbEdgesForVertex.stream().filter(ed -> ed.label().equalsIgnoreCase(e.label())).findFirst();
Optional<Edge> edgeVertexAlreadyExists =
dbEdgesForVertex.stream().filter(ed -> ed.label().equalsIgnoreCase(edgeLabel) && ed.inVertex().label().equalsIgnoreCase(ver.label())).findFirst();
removeEdge(dbVertex, e, edgeAlreadyExists, edgeVertexMatchList, methodOrigin);
if (gt.hasNext()) {
Vertex existingV = gt.next();
logger.info(String.format("Vertex with label %s already exists. Updating properties for the vertex", existingV.label()));
copyProperties(ver, existingV,methodOrigin);
Optional<Edge> edgeAlreadyExists =
dbEdgesForVertex.stream().filter(ed -> ed.label().equalsIgnoreCase(e.label())).findFirst();
if(!edgeAlreadyExists.isPresent()) {
logger.info(String.format("Adding edge with label %s for the vertex label %s.", e.label(), existingV.label()));
dbVertex.addEdge(e.label(), existingV);

AuditRecord record = appContext.getBean(AuditRecord.class);
record
.subject(dbVertex.label())
.predicate(e.label())
.oldObject(null)
.newObject(existingV.label())
.record(databaseProvider);
}
parsedVertices.push(new Pair<>(ver, existingV));
Vertex existingV = gt.next();
logger.info(String.format("Vertex with label %s already exists. Updating properties for the vertex", existingV.label()));
copyProperties(ver, existingV,methodOrigin);
if(!edgeVertexAlreadyExists.isPresent()){
Edge edgeAdded = dbVertex.addEdge(edgeLabel, existingV);
edgeVertexMatchList.add(edgeAdded);
AuditRecord record = appContext.getBean(AuditRecord.class);
record
.subject(dbVertex.label())
.predicate(e.label())
.oldObject(null)
.newObject(existingV.label())
.record(databaseProvider);
}
parsedVertices.push(new Pair<>(ver, existingV));
} else {
if(methodOrigin.equalsIgnoreCase("update") && !isIRI(ver.label()) && featureToggling){
throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND);
Expand All @@ -269,8 +293,9 @@ private void addOrUpdateVertexAndEdge(Vertex v, Vertex dbVertex, GraphTraversalS
logger.info(String.format("Adding vertex with label %s and adding properties", newV.label()));
copyProperties(ver, newV, methodOrigin);
logger.info(String.format("Adding edge with label %s for the vertex label %s.", e.label(), newV.label()));
dbVertex.addEdge(e.label(), newV);

Edge edgeAdded = dbVertex.addEdge(edgeLabel, newV);
edgeVertexMatchList.add(edgeAdded);
AuditRecord record = appContext.getBean(AuditRecord.class);
record
.subject(dbVertex.label())
Expand All @@ -288,6 +313,81 @@ private void addOrUpdateVertexAndEdge(Vertex v, Vertex dbVertex, GraphTraversalS


}

private void removeEdge(Vertex dbVertex, Edge e, Optional<Edge> edgeAlreadyExists,List<Edge> edgeVertexMatchList, String methodOrigin)
throws AuditFailedException, RecordNotFoundException{

Graph graphFromStore = databaseProvider.getGraphStore();
GraphTraversalSource traversalSource = graphFromStore.traversal();
GraphTraversal<Vertex, Vertex> dbHasLabel = traversalSource.clone().V().hasLabel(dbVertex.label());
if (!dbHasLabel.hasNext()) {
throw new RecordNotFoundException(Constants.ENTITY_NOT_FOUND);
}
boolean isSingleValued = schemaConfigurator.isSingleValued(e.label());
if(dbHasLabel.hasNext()){
Vertex dbSourceVertex = dbHasLabel.next();
if (graphFromStore.features().graph().supportsTransactions()) {
org.apache.tinkerpop.gremlin.structure.Transaction tx = graphFromStore.tx();
tx.onReadWrite(org.apache.tinkerpop.gremlin.structure.Transaction.READ_WRITE_BEHAVIOR.AUTO);
removeEdge(isSingleValued, dbSourceVertex, e, edgeAlreadyExists, edgeVertexMatchList, methodOrigin);
tx.commit();
}else{
removeEdge(isSingleValued, dbSourceVertex, e, edgeAlreadyExists, edgeVertexMatchList, methodOrigin);
}
}
}

private void removeEdge(boolean isSingleValued, Vertex dbSourceVertex, Edge e, Optional<Edge> edgeAlreadyExists,List<Edge> edgeVertexMatchList, String methodOrigin)
throws AuditFailedException{
if((edgeAlreadyExists.isPresent() && methodOrigin.equalsIgnoreCase("update")) || isSingleValued){
Iterator<Edge> edgeIter = dbSourceVertex.edges(Direction.OUT, e.label());
while(edgeIter.hasNext()){
Edge edge = edgeIter.next();
Optional<Edge> existingEdgeVertex =
edgeVertexMatchList.stream().filter(ed -> ed.label().equalsIgnoreCase(edge.label()) && ed.inVertex().label().equalsIgnoreCase(edge.inVertex().label())).findFirst();
if(!existingEdgeVertex.isPresent()){
deleteEdgeAndNodeRecursively(dbSourceVertex, edge, null);
}
}
}
}

private void deleteEdgeAndNodeRecursively(Vertex v, Edge dbEdgeToBeRemoved, Vertex dbVertexToBeDeleted) throws AuditFailedException{
logger.info("Deleting edge and node:"+dbEdgeToBeRemoved.label());
if(dbVertexToBeDeleted == null){
dbVertexToBeDeleted = dbEdgeToBeRemoved.inVertex();
}
Iterator<Edge> inEdgeIter = dbVertexToBeDeleted.edges(Direction.IN);
Iterator<Edge> outEdgeIter = dbVertexToBeDeleted.edges(Direction.OUT);
if((inEdgeIter.hasNext() && IteratorUtils.count(inEdgeIter) > 1) || outEdgeIter.hasNext()){
dbEdgeToBeRemoved.remove();
AuditRecord record = appContext.getBean(AuditRecord.class);
String tailOfdbVertex=v.label().substring(v.label().lastIndexOf("/") + 1).trim();
String auditVertexlabel= registryContext+tailOfdbVertex;
record
.subject(auditVertexlabel)
.predicate(dbEdgeToBeRemoved.label())
.oldObject(dbVertexToBeDeleted.label())
.newObject(null)
.record(databaseProvider);
}else{
dbVertexToBeDeleted.remove();
dbEdgeToBeRemoved.remove();
AuditRecord record = appContext.getBean(AuditRecord.class);
String tailOfdbVertex=v.label().substring(v.label().lastIndexOf("/") + 1).trim();
String auditVertexlabel= registryContext+tailOfdbVertex;
record
.subject(auditVertexlabel)
.predicate(dbEdgeToBeRemoved.label())
.oldObject(dbVertexToBeDeleted.label())
.newObject(null)
.record(databaseProvider);
}

}




/**
* Blank nodes are no longer supported. If the input data has a blank node, which is identified
Expand Down Expand Up @@ -406,10 +506,9 @@ private boolean isaMetaProperty(String key) {
private void setProperty(Vertex v, String key, Object newValue, String methodOrigin) throws AuditFailedException {
VertexProperty vp = v.property(key);
Object oldValue = vp.isPresent() ? vp.value() : null;
if(RDFUtil.isSingleValued(key)){
if(schemaConfigurator.isSingleValued(key)){
v.property(key, newValue);
}/*else if(featureToggling){
if(vp.isPresent()){
}else if(vp.isPresent()){
Object value = vp.value();
List valueList = new ArrayList();
if(value instanceof List){
Expand All @@ -419,11 +518,7 @@ private void setProperty(Vertex v, String key, Object newValue, String methodOri
valueList.add(valueStr);
}
v.property(key, valueList);
}else{
v.property(key, newValue);
}
}*/else{
}else{
v.property(key, newValue);
}
if (!isaMetaProperty(key) && !Objects.equals(oldValue, newValue)) {
Expand Down Expand Up @@ -531,9 +626,9 @@ public boolean deleteEntity (Graph entity, String rootLabel) throws RecordNotFou
}
return false;
}
}*/

private void deleteEdgeAndNode(Vertex v, GraphTraversal<Vertex, Vertex> gts) throws AuditFailedException{
/*private void deleteEdgeAndNode(Vertex v, GraphTraversal<Vertex, Vertex> gts) throws AuditFailedException{
List<Edge> dbEdgesForVertex = ImmutableList.copyOf(v.edges(Direction.OUT));
if(gts.hasNext()){
Vertex vertex = gts.next();
Expand All @@ -557,7 +652,7 @@ private void deleteEdgeAndNode(Vertex v, GraphTraversal<Vertex, Vertex> gts) thr
}
AuditRecord record = appContext.getBean(AuditRecord.class);
String tailOfdbVertex=v.label().substring(v.label().lastIndexOf("/") + 1).trim();
String auditVertexlabel= registrySystemContext+tailOfdbVertex;
String auditVertexlabel= registryContext+tailOfdbVertex;
record
.subject(auditVertexlabel)
.predicate(dbEdge.label())
Expand Down
4 changes: 1 addition & 3 deletions java/registry/src/main/java/io/opensaber/registry/util/RDFUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,6 @@ public static void updateRdfModelNodeId(Model rdfModel, RDFNode object, String l
rdfModel.add(updatedStatements.toArray(new Statement[0]));
}

public static boolean isSingleValued(String property){
return false;
}

public static List<Resource> getRootLabels(Model rdfModel){
List<Resource> rootLabelList = new ArrayList<Resource>();
Expand All @@ -193,5 +190,6 @@ public static List<String> getTypeForSubject(Model rdfModel, String label){
}
return typeIRIs;
}


}
28 changes: 14 additions & 14 deletions java/registry/src/main/resources/good1.shex
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ sample:isAYear xsd:gYear

sample:SchoolShape CLOSED{
a [sample:School] ;
sample:udiseNumber @sample:isAString ;
sample:academicCalendarYearStart @sample:isAString ;
sample:academicCalendarYearEnd @sample:isAString ;
sample:area [sample:AreaTypeCode-RURAL sample:AreaTypeCode-URBAN] ;
sample:schoolName @sample:isAString ;
sample:udiseNumber @sample:isAString {0,1};
sample:academicCalendarYearStart @sample:isAString {0,1};
sample:academicCalendarYearEnd @sample:isAString {0,1};
sample:area [sample:AreaTypeCode-RURAL sample:AreaTypeCode-URBAN] {0,1};
sample:schoolName @sample:isAString {0,1};
sample:address @sample:AddressShape ;
sample:revenueBlock @sample:isAString {0,1} ;
sample:assemblyConstituency @sample:isAString {0,1} ;
Expand All @@ -21,15 +21,15 @@ sample:SchoolShape CLOSED{
sample:AddressShape CLOSED{
a [sample:IndianUrbanPostalAddress sample:IndianRuralPostalAddress] ;
(
sample:mohalla @sample:isAString ;
sample:wardNumber @sample:isAString ;
sample:municipality @sample:isAString ;
sample:city @sample:isAString
sample:mohalla @sample:isAString {0,1};
sample:wardNumber @sample:isAString {0,1};
sample:municipality @sample:isAString {0,1};
sample:city @sample:isAString {0,1}
|
sample:habitation @sample:isAString ;
sample:villageName @sample:isAString ;
sample:villagePanchayat @sample:isAString ;
sample:habitation @sample:isAString {0,1};
sample:villageName @sample:isAString {0,1};
sample:villagePanchayat @sample:isAString {0,1};
);
sample:pinCode @sample:isAString ;
sample:district @sample:isAString
sample:pinCode @sample:isAString {0,1};
sample:district @sample:isAString {0,1}
}
28 changes: 28 additions & 0 deletions java/registry/src/test/java/io/opensaber/registry/RegistryTestSuite.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package io.opensaber.registry;

import org.junit.runner.RunWith;
import org.junit.runners.Suite;
import org.junit.runners.Suite.SuiteClasses;

import io.opensaber.registry.controller.RegistryControllerTest;
import io.opensaber.registry.dao.impl.EncryptionDaoImplTest;
import io.opensaber.registry.dao.impl.RegistryDaoImplTest;
import io.opensaber.registry.service.impl.RegistryEncryptionServiceImplTest;
import io.opensaber.registry.service.impl.RegistryServiceImplTest;
import junit.framework.Test;
import junit.framework.TestSuite;

@SuiteClasses({RegistryDaoImplTest.class, RegistryControllerTest.class, RegistryServiceImplTest.class,
EncryptionDaoImplTest.class, RegistryEncryptionServiceImplTest.class})
@RunWith(Suite.class)
public class RegistryTestSuite {

public static Test suite() {
TestSuite suite = new TestSuite(RegistryTestSuite.class.getName());
//$JUnit-BEGIN$

//$JUnit-END$
return suite;
}

}
5 changes: 5 additions & 0 deletions java/registry/src/test/java/io/opensaber/registry/controller/RegistryTestBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,11 @@ public Model getNewValidRdf(String fileName, String contextConstant){
Model model = ShaclexValidator.parse(jsonld, FORMAT);
return model;
}

public Model getNewValidRdf(String fileName){
setJsonld(fileName);
return ShaclexValidator.parse(jsonld, FORMAT);
}

public Model getNewValidRdf(String fileName, String contextConstant, String rootNodeLabel){
setJsonld(fileName);
Expand Down
Loading

0 comments on commit d50e4c0

Please sign in to comment.