Extract QAnswer queries to files (#347)

* create SPARQL insert queries using commons methods

* create SPARQL insert queries using commons methods

* increment versions

* change handling of QAnswer API response

* look for question transltaion in the triplestore

* add QB DeepPavlovWrapper to component reactor pom

* add ability to retrieve answer sparql by index

QAnswer provides a confidence score, but that has no relevance for
selecting the best query candidate. Instead, only order is relevant for
this Service. AnnotationOfAnswerSPARQL now also contains an index, s.t.
the nth query candidate can be retrieved from the triple store.

* use new naming for answer data type annotation

* use existing question translation from triplestore

* add index to generated insert query

* add index for annotated answer SPARQL

* increment versions

* use predefined datatype URIs

.gitignore

@@ -148,5 +148,6 @@ dmypy.json
 
 .factorypath
 
-# LOCAL APPLEICATION PROPERTIES
-application-local.properties
+# LOCAL FILES
+application-local.properties
+.run_tests

pom.xml

@@ -154,6 +154,8 @@
         <module>qanary-component-KG2KG-TranslateAnnotationsOfInstance</module>
 
         <module>qanary-component-QB-DateOfDeathDBpedia</module>
+
+        <module>qanary-component-QB-DeepPavlovWrapper</module>
 
       </modules>
 

qanary-component-QB-BirthDataWikidata/pom.xml

@@ -5,7 +5,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>eu.wdaqua.qanary.component</groupId>
 	<artifactId>qanary-component-QB-BirthDataWikidata</artifactId>
-	<version>3.4.0</version>
+	<version>3.4.1</version>
 	<parent>
 		<groupId>eu.wdaqua.qanary</groupId>
 		<artifactId>qa.qanarycomponent-parent</artifactId>

qanary-component-QB-BirthDataWikidata/src/main/java/eu/wdaqua/component/qb/birthdata/wikidata/BirthDataQueryBuilder.java

@@ -243,6 +243,7 @@ private QanaryMessage processForFirstNameAndLastName(QanaryMessage myQanaryMessa
 																												// confidence
 																												// is
 																												// expressed
+				bindings.add("index", ResourceFactory.createTypedLiteral(Integer.toString(i), XSDDatatype.XSDint));
 				bindings.add("application", ResourceFactory.createResource("urn:qanary:" + this.applicationName));
 
 				// get the template of the INSERT query

qanary-component-QB-DateOfDeathDBpedia/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>eu.wdaqua.qanary.component</groupId>
     <artifactId>qanary-component-QB-DateOfDeathDBpedia</artifactId>
-    <version>0.1.3</version>
+    <version>0.1.4</version>
     <parent> 
         <groupId>eu.wdaqua.qanary</groupId> 
         <artifactId>qa.qanarycomponent-parent</artifactId>

qanary-component-QB-DateOfDeathDBpedia/src/main/java/eu/wdaqua/qanary/component/dateofdeathrealpersons/qb/QueryBuilderDateOfDeathDBpedia.java

@@ -49,7 +49,6 @@ public class QueryBuilderDateOfDeathDBpedia extends QanaryComponent {
     private static final String QUERY_FILE_DBPEDIA_QUERY = "/queries/dbpedia_query.rq";
 
     // used from the Qanary commons queries, query to create a new annotation and store the computed query
-    private static final String QUERY_FILE_STORE_COMPUTED_ANNOTATIONS = "/queries/insert_one_AnnotationOfAnswerSPARQL.rq";
     private static final Logger logger = LoggerFactory.getLogger(QueryBuilderDateOfDeathDBpedia.class);
     private final String SUPPORTED_PREFIX = "What is the date of death of ";
     private final String applicationName;
@@ -60,7 +59,6 @@ public QueryBuilderDateOfDeathDBpedia(@Value("${spring.application.name}") final
         // here if the files are available and do contain content // do files exist?
         QanaryTripleStoreConnector.guardNonEmptyFileFromResources(QUERY_FILE_FETCH_REQUIRED_ANNOTATIONS);
         QanaryTripleStoreConnector.guardNonEmptyFileFromResources(QUERY_FILE_DBPEDIA_QUERY);
-        QanaryTripleStoreConnector.guardNonEmptyFileFromResources(QUERY_FILE_STORE_COMPUTED_ANNOTATIONS);
     }
 
     /**
@@ -155,9 +153,9 @@ public List<String> fetchEntitiesAndCreateQueries(QanaryQuestion qanaryQuestion,
      */
     public List<String> createQueries(QanaryQuestion qanaryQuestion, List<String> queries) throws Exception {
         List<String> createdQueries = new ArrayList<>();
-        for (String entity : queries
-        ) {
-            createdQueries.add(getInsertQuery(qanaryQuestion, entity));
+
+        for (int i = 0; i > queries.size(); i++) {
+            createdQueries.add(getInsertQuery(qanaryQuestion, queries.get(i), i));
         }
         return createdQueries;
     }
@@ -195,7 +193,7 @@ public String getDbpediaQuery(String dbpediaResource) throws Exception {
     }
 
     // binds query variables with concrete values and returns the Insert-query
-    public String getInsertQuery(QanaryQuestion<String> myQanaryQuestion, String createdDbpediaQuery)
+    public String getInsertQuery(QanaryQuestion<String> myQanaryQuestion, String createdDbpediaQuery, int index)
             throws SparqlQueryFailed, URISyntaxException, QanaryExceptionNoOrMultipleQuestions, IOException {
 
         QuerySolutionMap bindingsForInsert = new QuerySolutionMap();
@@ -204,8 +202,9 @@ public String getInsertQuery(QanaryQuestion<String> myQanaryQuestion, String cre
         bindingsForInsert.add("application", ResourceFactory.createResource("urn:qanary:" + this.applicationName));
         bindingsForInsert.add("selectQueryThatShouldComputeTheAnswer", ResourceFactory.createTypedLiteral(createdDbpediaQuery, XSDDatatype.XSDdate));
         bindingsForInsert.add("score", ResourceFactory.createTypedLiteral("1.0", XSDDatatype.XSDfloat));
+        bindingsForInsert.add("index", ResourceFactory.createTypedLiteral(Integer.toString(index), XSDDatatype.XSDint));
 
-        return QanaryTripleStoreConnector.readFileFromResourcesWithMap(QUERY_FILE_STORE_COMPUTED_ANNOTATIONS, bindingsForInsert);
+        return QanaryTripleStoreConnector.insertAnnotationOfAnswerSPARQL(bindingsForInsert);
     }
 
 }

qanary-component-QB-DeepPavlovWrapper/pom.xml

@@ -5,7 +5,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>eu.wdaqua.qanary.component</groupId>
 	<artifactId>qanary-component-QB-DeepPavlovWrapper</artifactId>
-	<version>0.1.1</version>
+	<version>0.2.0</version>
 
 	<parent>
 		<groupId>eu.wdaqua.qanary</groupId>

qanary-component-QB-DeepPavlovWrapper/src/main/java/eu/wdaqua/qanary/component/deeppavlovwrapper/qb/DeepPavlovWrapper.java

@@ -159,7 +159,8 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) {
         // STEP 3: store computed knowledge about the given question into the Qanary triplestore 
         // (the global process memory)
         // --------------------------------------------------------------------
-        String sparql = getSparqlInsertQuery(myQanaryQuestion, result);
+        int index = 0; // only one query expected
+        String sparql = getSparqlInsertQuery(myQanaryQuestion, result, index);
         myQanaryUtils.getQanaryTripleStoreConnector().update(sparql);
       }
     } catch (Exception e) {
@@ -239,7 +240,7 @@ protected String prepareResultQueryForSparqlInsert(String resultQuery) {
      * @throws SparqlQueryFailed
      * @throws IOException
      */
-    protected String getSparqlInsertQuery(QanaryQuestion<String> myQanaryQuestion, DeepPavlovResult result) throws QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed, IOException {
+    protected String getSparqlInsertQuery(QanaryQuestion<String> myQanaryQuestion, DeepPavlovResult result, int index) throws QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed, IOException {
 
       String query = prepareResultQueryForSparqlInsert(result.getSparql());
 
@@ -250,6 +251,7 @@ protected String getSparqlInsertQuery(QanaryQuestion<String> myQanaryQuestion, D
       bindings.add("targetQuestion", ResourceFactory.createResource(myQanaryQuestion.getUri().toASCIIString()));
       bindings.add("selectQueryThatShouldComputeTheAnswer", ResourceFactory.createStringLiteral(query));
       bindings.add("confidence", ResourceFactory.createTypedLiteral(result.getConfidence()));
+      bindings.add("index", ResourceFactory.createTypedLiteral(index)); 
       bindings.add("application", ResourceFactory.createResource("urn:qanary:" + this.applicationName));
 
       // get the template of the INSERT query

qanary-component-QB-GAnswerWrapper/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>eu.wdaqua.qanary.component</groupId>
     <artifactId>qanary-component-QB-GAnswerWrapper</artifactId>
-    <version>3.2.2</version>
+    <version>3.2.3</version>
 
     <parent>
       <groupId>eu.wdaqua.qanary</groupId>

qanary-component-QB-GAnswerWrapper/src/main/java/eu/wdaqua/qanary/component/ganswerwrapper/qb/GAnswerQueryBuilder.java

@@ -138,7 +138,8 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) throws Exception {
         }
 
         // STEP 3: add information to Qanary triplestore
-        String sparql = getSparqlInsertQuery(myQanaryQuestion, result);
+        int index = 0; // only one query expected
+        String sparql = getSparqlInsertQuery(myQanaryQuestion, result, index);
         myQanaryUtils.getQanaryTripleStoreConnector().update(sparql);
 
         return myQanaryMessage;
@@ -197,7 +198,7 @@ private String cleanStringForSparqlQuery(String myString) {
      * @throws SparqlQueryFailed
      * @throws IOException
      */
-    protected String getSparqlInsertQuery(QanaryQuestion<String> myQanaryQuestion, GAnswerResult result) throws QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed, IOException {
+    protected String getSparqlInsertQuery(QanaryQuestion<String> myQanaryQuestion, GAnswerResult result, int index) throws QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed, IOException {
 
         String answerSparql = cleanStringForSparqlQuery(result.getSparql());
 
@@ -208,6 +209,7 @@ protected String getSparqlInsertQuery(QanaryQuestion<String> myQanaryQuestion, G
         bindings.add("targetQuestion", ResourceFactory.createResource(myQanaryQuestion.getUri().toASCIIString()));
         bindings.add("selectQueryThatShouldComputeTheAnswer", ResourceFactory.createStringLiteral(answerSparql));
         bindings.add("confidence", ResourceFactory.createTypedLiteral(result.getConfidence()));
+        bindings.add("index", ResourceFactory.createTypedLiteral(index));
         bindings.add("application", ResourceFactory.createResource("urn:qanary:" + this.applicationName));
 
         // get the template of the INSERT query

qanary-component-QB-PlatypusWrapper/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>eu.wdaqua.qanary.component</groupId>
     <artifactId>qanary-component-QB-PlatypusWrapper</artifactId>
-    <version>3.3.4</version>
+    <version>3.3.5</version>
 
     <parent>
         <groupId>eu.wdaqua.qanary</groupId>

qanary-component-QB-PlatypusWrapper/src/main/java/eu/wdaqua/qanary/component/platypuswrapper/qb/PlatypusQueryBuilder.java

@@ -150,7 +150,8 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) throws Exception {
         // STEP 3: add information to Qanary triplestore
         //String sparql = getSparqlInsertQuery(myQanaryQuestion, result);
         //myQanaryUtils.getQanaryTripleStoreConnector().update(sparql);
-        QuerySolutionMap bindings = populateBindings(myQanaryQuestion, result);
+        int index = 0; // only one query expected
+        QuerySolutionMap bindings = populateBindings(myQanaryQuestion, result, index);
         String insertAnswerSPARQL = getSparqlInsertQueryForAnswerSPARQL(bindings);
         String insertTypedLiteral = getSparqlInsertQueryForTypedLiteral(bindings);
         myQanaryUtils.getQanaryTripleStoreConnector().update(insertAnswerSPARQL);
@@ -257,7 +258,7 @@ protected String getSparqlInsertQueryForTypedLiteral(QuerySolutionMap bindings)
         return sparql;
     }
 
-    protected QuerySolutionMap populateBindings(QanaryQuestion<String> myQanaryQuestion, PlatypusResult result) throws QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed {
+    protected QuerySolutionMap populateBindings(QanaryQuestion<String> myQanaryQuestion, PlatypusResult result, int index) throws QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed {
 
         String answerSparql = cleanStringForSparqlQuery(result.getSparql());
 
@@ -268,6 +269,7 @@ protected QuerySolutionMap populateBindings(QanaryQuestion<String> myQanaryQuest
         bindings.add("targetQuestion", ResourceFactory.createResource(myQanaryQuestion.getUri().toASCIIString()));
         bindings.add("selectQueryThatShouldComputeTheAnswer", ResourceFactory.createStringLiteral(answerSparql));
         bindings.add("confidence", ResourceFactory.createTypedLiteral(result.getConfidence()));
+        bindings.add("index", ResourceFactory.createTypedLiteral(index));
         bindings.add("application", ResourceFactory.createResource("urn:qanary:" + this.applicationName));
         // TODO: add result value with correct datatype
         // TODO: how does this handle a *list* of values? 

qanary-component-QB-QAnswer/pom.xml

@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>eu.wdaqua.qanary.component</groupId>
     <artifactId>qanary-component-QB-QAnswer</artifactId>
-    <version>4.1.1</version>
+    <version>4.2.0</version>
 
     <parent>
         <groupId>eu.wdaqua.qanary</groupId>

qanary-component-QB-QAnswer/src/main/java/eu/wdaqua/qanary/component/qanswer/qb/QAnswerQueryBuilderAndSparqlResultFetcher.java

@@ -18,13 +18,16 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Component;
 import org.springframework.web.client.RestTemplate;
 import org.springframework.web.util.UriComponentsBuilder;
 
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
 import eu.wdaqua.qanary.commons.QanaryExceptionNoOrMultipleQuestions;
 import eu.wdaqua.qanary.commons.QanaryMessage;
 import eu.wdaqua.qanary.commons.QanaryQuestion;
@@ -35,7 +38,6 @@
 import eu.wdaqua.qanary.component.qanswer.qb.messages.QAnswerResult;
 import eu.wdaqua.qanary.component.qanswer.qb.messages.QAnswerResult.QAnswerQueryCandidate;
 import eu.wdaqua.qanary.exceptions.SparqlQueryFailed;
-import net.minidev.json.JSONObject;
 
 @Component
 /**
@@ -135,14 +137,31 @@ public QanaryMessage process(QanaryMessage myQanaryMessage) throws Exception {
         // STEP 1: get the required data from the Qanary triplestore (the global process
         // memory)
         QanaryQuestion<String> myQanaryQuestion = this.getQanaryQuestion(myQanaryMessage);
-        String questionString = myQanaryQuestion.getTextualRepresentation();
-        List<NamedEntity> retrievedNamedEntities = getNamedEntitiesOfQuestion(myQanaryQuestion,
-                myQanaryQuestion.getInGraph());
 
+        String questionString = "";
+        try {
+          questionString = myQanaryQuestion.getTextualRepresentation(lang);
+          logger.info("Using specific textual representation for language {}: {}", lang, questionString);
+        } catch (Exception e) {
+          logger.warn("Could not retrieve specific textual representation for language {}:\n{}", e.getMessage());
+        }
+        // only if no language-specific text could be found
+        if (questionString.length() == 0){
+            try {
+                questionString = myQanaryQuestion.getTextualRepresentation();
+                logger.info("Using default textual representation {}", questionString);
+            } catch (Exception e) {
+                logger.warn("Could not retrieve textual representation:\n{}", e.getMessage());
+                // stop processing of the question, as it will not work without a question text
+                return myQanaryMessage;
+            }
+        }
 
         // STEP 2: compute new information about the question
 
         // enriching of query, based on recognized resources
+        List<NamedEntity> retrievedNamedEntities = getNamedEntitiesOfQuestion(myQanaryQuestion,
+                myQanaryQuestion.getInGraph());
         String questionStringWithResources = computeQuestionStringWithReplacedResources(
                 questionString, retrievedNamedEntities, threshold);
         // fetching SPARQL query candidates from the QAnswer API
@@ -185,11 +204,13 @@ protected QAnswerResult requestQAnswerWebService(URI qanaryApiUri, String questi
                 .queryParam("user", "{user}") //
                 .encode().toUriString();
 
-        HttpEntity<JSONObject> response = myRestTemplate.getForEntity(urlTemplate, JSONObject.class, parameters);
-        logger.info("QAnswer JSON result for question '{}': {}", questionString,
-                response.getBody().getAsString("question"));
+        ResponseEntity<String> stringResponse = myRestTemplate.getForEntity(urlTemplate, String.class, parameters);
+        logger.info("QAnswer String result for question '{}': {}", questionString,
+                stringResponse.getBody());
+
+        JsonObject jsonResponse = JsonParser.parseString(stringResponse.getBody()).getAsJsonObject();
 
-        return new QAnswerResult(response.getBody(), questionString, qanaryApiUri, lang, knowledgeBaseId, user);
+        return new QAnswerResult(jsonResponse, questionString, qanaryApiUri, lang, knowledgeBaseId, user);
     }
 
     /**
@@ -320,6 +341,7 @@ public List<String> getSparqlInsertQueriesForQueryCandidates(
             bindings.add("targetQuestion", ResourceFactory.createResource(questionUri.toASCIIString()));
             bindings.add("selectQueryThatShouldComputeTheAnswer", ResourceFactory.createStringLiteral(queryCandidate.getQueryString()));
             bindings.add("confidence", ResourceFactory.createTypedLiteral(queryCandidate.getScore()));
+            bindings.add("index", ResourceFactory.createTypedLiteral(queryCandidate.getIndex()));
             bindings.add("application", ResourceFactory.createResource("urn:qanary:" + this.applicationName));
 
             // get the template of the INSERT query
@@ -333,41 +355,21 @@ public List<String> getSparqlInsertQueriesForQueryCandidates(
     }
 
     public String getSparqlInsertQueryForImprovedQuestion(
-            URI graph, URI questionUri, QAnswerResult result) throws QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed {
+            URI graph, URI questionUri, QAnswerResult result) throws QanaryExceptionNoOrMultipleQuestions, URISyntaxException, SparqlQueryFailed, IOException {
 
+        logger.warn("get query for Improved Question");
         // the computed answer's SPARQL query needs to be cleaned
         String improvedQuestion = cleanStringForSparqlQuery(result.getQuestion());
 
-        String sparql = "" //
-                + "PREFIX qa: <http://www.wdaqua.eu/qa#> \n" //
-                + "PREFIX oa: <http://www.w3.org/ns/openannotation/core/> \n" //
-                + "PREFIX xsd: <http://www.w3.org/2001/XMLSchema#> \n" //
-                + "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> \n" //
-                + "INSERT { \n" //
-                + "GRAPH ?graph { \n" //
-                // improved question
-                + "  ?annotationImprovedQuestion  a 	qa:AnnotationOfImprovedQuestion ; \n" //
-                + " 		oa:hasTarget    ?question ; \n" //
-                + " 		oa:hasBody      ?improvedQuestion ; \n" //
-                + " 		oa:annotatedBy  ?service ; \n" //
-                + " 		oa:annotatedAt  ?time ; \n" //
-                + " 		qa:score        ?score . \n" //
-                //
-                + "  ?improvedQuestion a    qa:ImprovedQuestion ; \n " //
-                + "         rdf:value 		?improvedQuestionText . \n " //
-                + "  }\n" // end: GRAPH
-                + "}\n" // end: insert
-                + "WHERE { \n" //
-                + "  BIND (IRI(str(RAND())) AS ?annotationImprovedQuestion) . \n" //
-                + "  BIND (IRI(str(RAND())) AS ?improvedQuestion) . \n" //
-                //
-                + "  BIND (now() AS ?time) . \n" //
-                + "  BIND (<" + graph.toASCIIString() + "> AS ?graph) . \n" //
-                + "  BIND (<" + questionUri.toASCIIString() + "> AS ?question) . \n" //
-                + "  BIND (<urn:qanary:" + this.applicationName + "> AS ?service ) . \n" //
-                + "  BIND (\"\"\"" + improvedQuestion + "\"\"\"^^xsd:string  AS ?improvedQuestionText ) . \n" //
-                //
-                + "} \n"; // end: where
+        // bind: graph, question, service, improvedQuestionText
+        QuerySolutionMap bindings = new QuerySolutionMap();
+        bindings.add("graph", ResourceFactory.createResource(graph.toASCIIString()));
+        bindings.add("question", ResourceFactory.createResource(questionUri.toASCIIString()));
+        bindings.add("application", ResourceFactory.createResource("urn:qanary:" + this.applicationName));
+        bindings.add("improvedQuestionText", ResourceFactory.createStringLiteral(improvedQuestion));
+
+        String sparql = QanaryTripleStoreConnector.insertAnnotationOfImprovedQuestion(bindings);
+        logger.warn("sparql: {}", sparql);
 
         return sparql;
     }