From 335ce355e3e0e3de51064ad791b16b78443b3a4f Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Wed, 7 Apr 2021 12:11:46 +0200 Subject: [PATCH 01/95] Add dummy endpoint for partial source path auto-completion --- .../scala/org/silkframework/config/Task.scala | 2 + .../transform/AutoCompletionApi.scala | 265 +++++------------- .../autoCompletion/Completions.scala | 183 ++++++++++++ ...rtialSourcePathAutoCompletionRequest.scala | 35 +++ .../conf/transform.routes | 1 + .../workflowApi/workflow/WorkflowInfo.scala | 2 +- .../projectApi/ProjectTaskApi.scala | 2 +- .../controllers/workspace/WorkspaceApi.scala | 2 +- .../workspaceApi/search/SearchApiModel.scala | 4 +- 9 files changed, 290 insertions(+), 206 deletions(-) create mode 100644 silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/Completions.scala create mode 100644 silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala diff --git a/silk-core/src/main/scala/org/silkframework/config/Task.scala b/silk-core/src/main/scala/org/silkframework/config/Task.scala index 20f5168d0c..6ca075b79d 100644 --- a/silk-core/src/main/scala/org/silkframework/config/Task.scala +++ b/silk-core/src/main/scala/org/silkframework/config/Task.scala @@ -53,6 +53,8 @@ trait Task[+TaskType <: TaskSpec] { metaData.formattedLabel(id, maxLength) } + def fullTaskLabel: String = taskLabel(Int.MaxValue) + override def equals(obj: scala.Any): Boolean = obj match { case task: Task[_] => id == task.id && diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 7b32097053..7290116cdf 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -1,36 +1,32 @@ package controllers.transform -import controllers.core.util.JsonUtils - -import java.net.URLDecoder -import java.util.logging.Logger +import controllers.core.util.ControllerUtilsTrait import controllers.core.{RequestUserContextAction, UserContextAction} import controllers.transform.AutoCompletionApi.Categories -import controllers.transform.autoCompletion.TargetPropertyAutoCompleteRequest - -import javax.inject.Inject +import controllers.transform.autoCompletion._ import org.silkframework.config.Prefixes -import org.silkframework.entity.{CustomValueType, ValueType, ValueTypeAnnotation} -import org.silkframework.entity.paths._ -import org.silkframework.rule.TransformSpec +import org.silkframework.entity.paths.{PathOperator, _} +import org.silkframework.entity.{ValueType, ValueTypeAnnotation} import org.silkframework.rule.vocab.ObjectPropertyType +import org.silkframework.rule.{TransformRule, TransformSpec} import org.silkframework.runtime.activity.UserContext import org.silkframework.runtime.plugin.{PluginDescription, PluginRegistry} import org.silkframework.runtime.validation.NotFoundException import org.silkframework.serialization.json.JsonHelpers -import org.silkframework.util.StringUtils -import org.silkframework.workspace.activity.transform.{TransformPathsCache, VocabularyCache, VocabularyCacheValue} +import org.silkframework.workspace.activity.transform.{TransformPathsCache, VocabularyCacheValue} import org.silkframework.workspace.{ProjectTask, WorkspaceFactory} -import play.api.libs.json.{JsArray, JsObject, JsString, JsValue, Json} +import play.api.libs.json.{JsValue, Json} import play.api.mvc._ +import java.util.logging.Logger +import javax.inject.Inject +import scala.collection.immutable import scala.language.implicitConversions -import scala.util.Try /** * Generates auto completions for mapping paths and types. */ -class AutoCompletionApi @Inject() () extends InjectedController { +class AutoCompletionApi @Inject() () extends InjectedController with ControllerUtilsTrait { val log: Logger = Logger.getLogger(this.getClass.getName) /** @@ -41,20 +37,57 @@ class AutoCompletionApi @Inject() () extends InjectedController { implicit val prefixes: Prefixes = project.config.prefixes val task = project.task[TransformSpec](taskName) var completions = Completions() - task.nestedRuleAndSourcePath(ruleName) match { - case Some((_, sourcePath)) => - val simpleSourcePath = sourcePath.filter(op => op.isInstanceOf[ForwardOperator] || op.isInstanceOf[BackwardOperator]) - val forwardOnlySourcePath = forwardOnlyPath(simpleSourcePath) - val allPaths = pathsCacheCompletions(task, simpleSourcePath) - val isRdfInput = task.activity[TransformPathsCache].value().isRdfInput(task) - // FIXME: No only generate relative "forward" paths, but also generate paths that would be accessible by following backward paths. - val relativeForwardPaths = relativePaths(simpleSourcePath, forwardOnlySourcePath, allPaths, isRdfInput) - // Add known paths - completions += relativeForwardPaths - // Return filtered result - Ok(completions.filterAndSort(term, maxResults, sortEmptyTermResult = false).toJson) + withRule(task, ruleName) { case (_, sourcePath) => + val simpleSourcePath = simplePath(sourcePath) + val forwardOnlySourcePath = forwardOnlyPath(simpleSourcePath) + val allPaths = pathsCacheCompletions(task, simpleSourcePath) + val isRdfInput = task.activity[TransformPathsCache].value().isRdfInput(task) + // FIXME: No only generate relative "forward" paths, but also generate paths that would be accessible by following backward paths. + val relativeForwardPaths = relativePaths(simpleSourcePath, forwardOnlySourcePath, allPaths, isRdfInput) + // Add known paths + completions += relativeForwardPaths + // Return filtered result + Ok(completions.filterAndSort(term, maxResults, sortEmptyTermResult = false).toJson) + } + } + + private def simplePath(sourcePath: List[PathOperator]) = { + sourcePath.filter(op => op.isInstanceOf[ForwardOperator] || op.isInstanceOf[BackwardOperator]) + } + + /** A more fine-grained auto-completion of a source path that suggests auto-completion in parts of a path. */ + def partialSourcePath(projectId: String, + transformTaskId: String, + ruleId: String): Action[JsValue] = RequestUserContextAction(parse.json) { implicit request => + implicit userContext => + val (project, transformTask) = projectAndTask[TransformSpec](projectId, transformTaskId) + validateJson[PartialSourcePathAutoCompletionRequest] { autoCompletionRequest => + withRule(transformTask, ruleId) { case (_, sourcePath) => + val simpleSourcePath = simplePath(sourcePath) + // TODO + val from = math.min(autoCompletionRequest.cursorPosition, 1) + val to = math.max(autoCompletionRequest.cursorPosition, autoCompletionRequest.inputString.length - 1) + val response = PartialSourcePathAutoCompletionResponse( + autoCompletionRequest.inputString, + autoCompletionRequest.cursorPosition, + Some((from, to)), + Completions(Seq( + Completion("dummy", label = Some("dummy value"), description = Some("dummy value description"), category = Categories.partialSourcePaths, isCompletion = true) + )).toCompletionsBase + ) + Ok(Json.toJson(response)) + } + } + } + + private def withRule[T](transformTask: ProjectTask[TransformSpec], + ruleId: String) + (block: ((TransformRule, List[PathOperator])) => T): T = { + transformTask.nestedRuleAndSourcePath(ruleId) match { + case Some(value) => + block(value) case None => - throw new NotFoundException("Requesting auto-completion for non-existent rule " + ruleName + " in transformation task " + taskName + "!") + throw new NotFoundException("Requesting auto-completion for non-existent rule " + ruleId + " in transformation task " + transformTask.fullTaskLabel + "!") } } @@ -84,7 +117,7 @@ class AutoCompletionApi @Inject() () extends InjectedController { } // Normalize this path by eliminating backward operators - private def forwardOnlyPath(simpleSourcePath: List[PathOperator]) = { + private def forwardOnlyPath(simpleSourcePath: List[PathOperator]): List[PathOperator] = { // Remove BackwardOperators var pathStack = List.empty[PathOperator] for (op <- simpleSourcePath) { @@ -187,25 +220,6 @@ class AutoCompletionApi @Inject() () extends InjectedController { ) } - /** - * Retrieves completions for prefixes. - * - * @return The completions, sorted alphabetically - */ - private def prefixCompletions(prefixes: Prefixes): Completions = { - Completions( - for(prefix <- prefixes.prefixMap.keys.toSeq.sorted) yield { - Completion( - value = prefix + ":", - label = Some(prefix + ":"), - description = None, - category = Categories.prefixes, - isCompletion = true - ) - } - ) - } - private def pathsCacheCompletions(task: ProjectTask[TransformSpec], sourcePath: List[PathOperator]) (implicit userContext: UserContext): Completions = { if (Option(task.activity[TransformPathsCache].value).isDefined) { @@ -277,162 +291,9 @@ class AutoCompletionApi @Inject() () extends InjectedController { propertyCompletions.distinct } - // Characters that are removed before comparing (in addition to whitespaces) - private val ignoredCharacters = Set('/', '\\') - - /** - * Normalizes a term. - */ - private def normalizeTerm(term: String): String = { - term.toLowerCase.filterNot(c => c.isWhitespace || ignoredCharacters.contains(c)) - } private implicit def createCompletion(completions: Seq[Completion]): Completions = Completions(completions) - /** - * A list of auto completions. - */ - case class Completions(values: Seq[Completion] = Seq.empty) { - - /** - * Adds another list of completions to this one and returns the result. - */ - def +(completions: Completions): Completions = { - Completions(values ++ completions.values) - } - - /** - * Filters and ranks all completions using a search term. - */ - def filterAndSort(term: String, - maxResults: Int, - sortEmptyTermResult: Boolean = true, - multiWordFilter: Boolean = false): Completions = { - if (term.trim.isEmpty) { - // If the term is empty, return some completions anyway - val sortedValues = if(sortEmptyTermResult) values.sortBy(_.labelOrGenerated.length) else values - Completions(sortedValues.take(maxResults)) - } else { - // Filter all completions that match the search term and sort them by score - val fm = filterMethod(term, multiWordFilter) - val scoredValues = for(value <- values; score <- fm(value)) yield (value, score) - val sortedValues = scoredValues.sortBy(-_._2).map(_._1) - Completions(sortedValues.take(maxResults)) - } - } - - // Choose the filter / ranking method - private def filterMethod(term: String, - multiWordFilter: Boolean): (Completion => Option[Double]) = { - if(multiWordFilter) { - val searchWords = StringUtils.extractSearchTerms(term) - val termMinLength = if(searchWords.length > 0) searchWords.map(_.length).min.toDouble else 1.0 - completion: Completion => completion.matchesMultiWordQuery(searchWords, termMinLength) - } else { - val normalizedTerm = normalizeTerm(term) - completion: Completion => completion.matches(normalizedTerm) - } - } - - def toJson: JsValue = { - JsArray(values.map(_.toJson)) - } - - } - - /** - * A single completion. - * - * @param value The value to be filled if the user selects this completion. - * @param confidence The confidence of this completion. - * @param label A user readable label if available - * @param description A user readable description if available - * @param category The category to be shown in the autocompletion - * @param isCompletion True, if this is a valid completion. False, if this is a (error) message. - * @param extra Some extra values depending on the category - */ - case class Completion(value: String, - confidence: Double = Double.MinValue, - label: Option[String], - description: Option[String], - category: String, - isCompletion: Boolean, - extra: Option[JsValue] = None) { - - /** - * Returns the label if present or generates a label from the value if no label is set. - */ - lazy val labelOrGenerated: String = label match { - case Some(existingLabel) => - existingLabel - case None => - val lastPart = value.substring(value.lastIndexWhere(c => c == '#' || c == '/' || c == ':') + 1).filterNot(_ == '>') - Try(URLDecoder.decode(lastPart, "UTF8")).getOrElse(lastPart) - } - - /** - * Checks if a term matches this completion. - * - * @param normalizedTerm the term normalized using normalizeTerm(term) - * @return None, if the term does not match at all. - * Some(matchScore), if the terms match. - */ - def matches(normalizedTerm: String): Option[Double] = { - val values = Set(value, labelOrGenerated) ++ description - val scores = values.flatMap(rank(normalizedTerm)) - if(scores.isEmpty) - None - else - Some(scores.max) - } - - /** Match against a multi word query, rank matches higher that have more matches in the label, then value and then description. */ - def matchesMultiWordQuery(lowerCaseTerms: Array[String], - termMinLength: Double): Option[Double] = { - val lowerCaseValue = value.toLowerCase - val lowerCaseLabel = label.getOrElse("").toLowerCase - val lowerCaseDescription = description.getOrElse("").toLowerCase - val searchIn = s"$lowerCaseValue $lowerCaseLabel $lowerCaseDescription" - val matches = StringUtils.matchesSearchTerm(lowerCaseTerms, searchIn) - if(matches) { - var score = 0.0 - val labelMatchCount = StringUtils.matchCount(lowerCaseTerms, lowerCaseLabel) - val labelLengthBonus = termMinLength / lowerCaseLabel.size - score += (0.5 + labelLengthBonus) * labelMatchCount - score += 0.2 * StringUtils.matchCount(lowerCaseTerms, lowerCaseValue) - score += 0.1 * StringUtils.matchCount(lowerCaseTerms, lowerCaseDescription) - Some(score) - } else { - None - } - } - - /** - * Ranks a term, the higher the result the higher the ranking. - */ - private def rank(normalizedTerm: String)(value: String): Option[Double] = { - val normalizedValue = normalizeTerm(value) - if(normalizedValue.contains(normalizedTerm)) { - Some(normalizedTerm.length.toDouble / normalizedValue.length) - } else { - None - } - } - - def toJson: JsValue = { - val genericObject = Json.obj( - "value" -> value, - "label" -> labelOrGenerated, - "description" -> description, - "category" -> category, - "isCompletion" -> isCompletion - ) - extra match { - case Some(ex) => genericObject ++ JsObject(Seq("extra" -> ex)) - case None => genericObject - } - } - } } object AutoCompletionApi { @@ -446,6 +307,8 @@ object AutoCompletionApi { val sourcePaths = "Source Paths" + val partialSourcePaths = "Partial Source Paths" + val vocabularyTypes = "Vocabulary Types" val vocabularyProperties = "Vocabulary Properties" diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/Completions.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/Completions.scala new file mode 100644 index 0000000000..2b8f56a7d4 --- /dev/null +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/Completions.scala @@ -0,0 +1,183 @@ +package controllers.transform.autoCompletion + +import org.silkframework.util.StringUtils +import play.api.libs.json._ + +import java.net.URLDecoder +import scala.util.Try + +/** The base properties of an auto-completion result. */ +case class CompletionsBase(completions: Seq[CompletionBase]) + +object CompletionsBase { + implicit val completionsBaseFormat: Format[CompletionsBase] = Json.format[CompletionsBase] +} + +/** + * A list of auto completions. + */ +case class Completions(values: Seq[Completion] = Seq.empty) { + + def toCompletionsBase: CompletionsBase = CompletionsBase(values.map(_.toCompletionBase)) + + /** + * Adds another list of completions to this one and returns the result. + */ + def +(completions: Completions): Completions = { + Completions(values ++ completions.values) + } + + /** + * Filters and ranks all completions using a search term. + */ + def filterAndSort(term: String, + maxResults: Int, + sortEmptyTermResult: Boolean = true, + multiWordFilter: Boolean = false): Completions = { + if (term.trim.isEmpty) { + // If the term is empty, return some completions anyway + val sortedValues = if(sortEmptyTermResult) values.sortBy(_.labelOrGenerated.length) else values + Completions(sortedValues.take(maxResults)) + } else { + // Filter all completions that match the search term and sort them by score + val fm = filterMethod(term, multiWordFilter) + val scoredValues = for(value <- values; score <- fm(value)) yield (value, score) + val sortedValues = scoredValues.sortBy(-_._2).map(_._1) + Completions(sortedValues.take(maxResults)) + } + } + + // Choose the filter / ranking method + private def filterMethod(term: String, + multiWordFilter: Boolean): (Completion => Option[Double]) = { + if(multiWordFilter) { + val searchWords = StringUtils.extractSearchTerms(term) + val termMinLength = if(searchWords.length > 0) searchWords.map(_.length).min.toDouble else 1.0 + completion: Completion => completion.matchesMultiWordQuery(searchWords, termMinLength) + } else { + val normalizedTerm = Completions.normalizeTerm(term) + completion: Completion => completion.matches(normalizedTerm) + } + } + + def toJson: JsValue = { + JsArray(values.map(_.toJson)) + } +} + +/** The base properties for auto-completion results. */ +case class CompletionBase(value: String, + label: Option[String], + description: Option[String]) + +object CompletionBase { + implicit val completionBaseFormat: Format[CompletionBase] = Json.format[CompletionBase] +} + +/** + * A single completion. + * + * @param value The value to be filled if the user selects this completion. + * @param confidence The confidence of this completion. + * @param label A user readable label if available + * @param description A user readable description if available + * @param category The category to be shown in the autocompletion + * @param isCompletion True, if this is a valid completion. False, if this is a (error) message. + * @param extra Some extra values depending on the category + */ +case class Completion(value: String, + confidence: Double = Double.MinValue, + label: Option[String], + description: Option[String], + category: String, + isCompletion: Boolean, + extra: Option[JsValue] = None) { + + /** + * Returns the label if present or generates a label from the value if no label is set. + */ + lazy val labelOrGenerated: String = label match { + case Some(existingLabel) => + existingLabel + case None => + val lastPart = value.substring(value.lastIndexWhere(c => c == '#' || c == '/' || c == ':') + 1).filterNot(_ == '>') + Try(URLDecoder.decode(lastPart, "UTF8")).getOrElse(lastPart) + } + + /** + * Checks if a term matches this completion. + * + * @param normalizedTerm the term normalized using normalizeTerm(term) + * @return None, if the term does not match at all. + * Some(matchScore), if the terms match. + */ + def matches(normalizedTerm: String): Option[Double] = { + val values = Set(value, labelOrGenerated) ++ description + val scores = values.flatMap(rank(normalizedTerm)) + if(scores.isEmpty) + None + else + Some(scores.max) + } + + /** Match against a multi word query, rank matches higher that have more matches in the label, then value and then description. */ + def matchesMultiWordQuery(lowerCaseTerms: Array[String], + termMinLength: Double): Option[Double] = { + val lowerCaseValue = value.toLowerCase + val lowerCaseLabel = label.getOrElse("").toLowerCase + val lowerCaseDescription = description.getOrElse("").toLowerCase + val searchIn = s"$lowerCaseValue $lowerCaseLabel $lowerCaseDescription" + val matches = StringUtils.matchesSearchTerm(lowerCaseTerms, searchIn) + if(matches) { + var score = 0.0 + val labelMatchCount = StringUtils.matchCount(lowerCaseTerms, lowerCaseLabel) + val labelLengthBonus = termMinLength / lowerCaseLabel.size + score += (0.5 + labelLengthBonus) * labelMatchCount + score += 0.2 * StringUtils.matchCount(lowerCaseTerms, lowerCaseValue) + score += 0.1 * StringUtils.matchCount(lowerCaseTerms, lowerCaseDescription) + Some(score) + } else { + None + } + } + + /** + * Ranks a term, the higher the result the higher the ranking. + */ + private def rank(normalizedTerm: String)(value: String): Option[Double] = { + val normalizedValue = Completions.normalizeTerm(value) + if(normalizedValue.contains(normalizedTerm)) { + Some(normalizedTerm.length.toDouble / normalizedValue.length) + } else { + None + } + } + + def toJson: JsValue = { + val genericObject = Json.obj( + "value" -> value, + "label" -> labelOrGenerated, + "description" -> description, + "category" -> category, + "isCompletion" -> isCompletion + ) + extra match { + case Some(ex) => genericObject ++ JsObject(Seq("extra" -> ex)) + case None => genericObject + } + } + + def toCompletionBase: CompletionBase = CompletionBase(value, label, description) +} + +object Completions { + // Characters that are removed before comparing (in addition to whitespaces) + private val ignoredCharacters = Set('/', '\\') + + /** + * Normalizes a term. + */ + def normalizeTerm(term: String): String = { + term.toLowerCase.filterNot(c => c.isWhitespace || ignoredCharacters.contains(c)) + } +} \ No newline at end of file diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala new file mode 100644 index 0000000000..c4699fe6b7 --- /dev/null +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala @@ -0,0 +1,35 @@ +package controllers.transform.autoCompletion + +import play.api.libs.json.{Format, Json} + +/** + * Request payload for partial source path auto-completion, i.e. suggest replacements for only parts of a more complex source path. + * + * @param inputString The currently entered source path string. + * @param cursorPosition The cursor position inside the source path string. + * @param maxSuggestions The max. number of suggestions to return. + */ +case class PartialSourcePathAutoCompletionRequest(inputString: String, + cursorPosition: Int, + maxSuggestions: Option[Int]) + +object PartialSourcePathAutoCompletionRequest { + implicit val partialSourcePathAutoCompletionRequestFormat: Format[PartialSourcePathAutoCompletionRequest] = Json.format[PartialSourcePathAutoCompletionRequest] +} + +/** + * The response for a partial source path auto-completion request. + * @param inputString The input string from the request for validation. + * @param cursorPosition The cursor position from the request for validation. + * @param replacementInterval An optional interval if there has been found a part of the source path that can be replaced. + * @param replacementResults The auto-completion results. + */ +case class PartialSourcePathAutoCompletionResponse(inputString: String, + cursorPosition: Int, + replacementInterval: Option[(Int, Int)], + replacementResults: CompletionsBase) + +object PartialSourcePathAutoCompletionResponse { + implicit val partialSourcePathAutoCompletionResponseFormat: Format[PartialSourcePathAutoCompletionResponse] = Json.format[PartialSourcePathAutoCompletionResponse] +} + diff --git a/silk-workbench/silk-workbench-rules/conf/transform.routes b/silk-workbench/silk-workbench-rules/conf/transform.routes index 8eaa8d086d..7d3dfab407 100644 --- a/silk-workbench/silk-workbench-rules/conf/transform.routes +++ b/silk-workbench/silk-workbench-rules/conf/transform.routes @@ -43,6 +43,7 @@ GET /tasks/:project/:task/rule/:rule/completions/targetTypes GET /tasks/:project/:task/rule/:rule/completions/targetProperties controllers.transform.AutoCompletionApi.targetProperties(project: String, task: String, rule: String, term ?= "", maxResults: Int ?= 30, fullUris: Boolean ?= false) POST /tasks/:project/:task/rule/:rule/completions/targetProperties controllers.transform.AutoCompletionApi.targetProperties(project: String, task: String, rule: String, term ?= "", maxResults: Int ?= 30, fullUris: Boolean ?= false) GET /tasks/:project/:task/rule/:rule/completions/valueTypes controllers.transform.AutoCompletionApi.valueTypes(project: String, task: String, rule: String, term ?= "", maxResults: Int ?= 30) +POST /tasks/:project/:task/rule/:rule/completions/partialSourcePaths controllers.transform.AutoCompletionApi.partialSourcePath(project: String, task: String, rule: String) GET /tasks/:project/:task/targetVocabulary/vocabularies controllers.transform.TargetVocabularyApi.vocabularyInfos(project: String, task: String) GET /tasks/:project/:task/targetVocabulary/type controllers.transform.TargetVocabularyApi.getTypeInfo(project: String, task: String, uri: String) diff --git a/silk-workbench/silk-workbench-workflow/app/controllers/workflowApi/workflow/WorkflowInfo.scala b/silk-workbench/silk-workbench-workflow/app/controllers/workflowApi/workflow/WorkflowInfo.scala index 7c4e6f63e7..f546edc653 100644 --- a/silk-workbench/silk-workbench-workflow/app/controllers/workflowApi/workflow/WorkflowInfo.scala +++ b/silk-workbench/silk-workbench-workflow/app/controllers/workflowApi/workflow/WorkflowInfo.scala @@ -32,7 +32,7 @@ object WorkflowInfo { val variableDatasets = workflow.variableDatasets(project) WorkflowInfo( workflow.id, - workflow.taskLabel(Int.MaxValue), + workflow.fullTaskLabel, project.name, project.config.metaData.formattedLabel(project.name, Int.MaxValue), variableDatasets.dataSources, diff --git a/silk-workbench/silk-workbench-workspace/app/controllers/projectApi/ProjectTaskApi.scala b/silk-workbench/silk-workbench-workspace/app/controllers/projectApi/ProjectTaskApi.scala index f9251f26e6..b61627cefe 100644 --- a/silk-workbench/silk-workbench-workspace/app/controllers/projectApi/ProjectTaskApi.scala +++ b/silk-workbench/silk-workbench-workspace/app/controllers/projectApi/ProjectTaskApi.scala @@ -28,7 +28,7 @@ class ProjectTaskApi @Inject()() extends InjectedController with ControllerUtils val relatedItems = relatedTasks map { task => val itemType = ItemType.itemType(task) val itemLinks = ItemType.itemTypeLinks(itemType, projectId, task.id, Some(task.data)) - RelatedItem(task.id, task.taskLabel(Int.MaxValue), task.metaData.description, itemType.label, itemLinks) + RelatedItem(task.id, task.fullTaskLabel, task.metaData.description, itemType.label, itemLinks) } val filteredItems = filterRelatedItems(relatedItems, textQuery) val total = relatedItems.size diff --git a/silk-workbench/silk-workbench-workspace/app/controllers/workspace/WorkspaceApi.scala b/silk-workbench/silk-workbench-workspace/app/controllers/workspace/WorkspaceApi.scala index 126066d78d..9b6af67c8a 100644 --- a/silk-workbench/silk-workbench-workspace/app/controllers/workspace/WorkspaceApi.scala +++ b/silk-workbench/silk-workbench-workspace/app/controllers/workspace/WorkspaceApi.scala @@ -243,7 +243,7 @@ class WorkspaceApi @Inject() (accessMonitor: WorkbenchAccessMonitor, pluginApiC val dependentTasks: Seq[TaskLinkInfo] = project.allTasks .filter(_.referencedResources.map(_.name).contains(resourceName)) .map { task => - TaskLinkInfo(task.id, task.taskLabel(Int.MaxValue), pluginApiCache.taskTypeByClass(task.taskType)) + TaskLinkInfo(task.id, task.fullTaskLabel, pluginApiCache.taskTypeByClass(task.taskType)) } Ok(Json.toJson(dependentTasks)) } diff --git a/silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/search/SearchApiModel.scala b/silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/search/SearchApiModel.scala index 840847315c..1dc1c019b8 100644 --- a/silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/search/SearchApiModel.scala +++ b/silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/search/SearchApiModel.scala @@ -137,7 +137,7 @@ object SearchApiModel { task: ProjectTask[_ <: TaskSpec], matchTaskProperties: Boolean, matchProject: Boolean): Boolean = { - val taskLabel = task.taskLabel(Int.MaxValue) + val taskLabel = task.fullTaskLabel val description = task.metaData.description.getOrElse("") val searchInProperties = if(matchTaskProperties) task.data.properties(task.project.config.prefixes).map(p => p._2).mkString(" ") else "" val searchInProject = if(matchProject) label(task.project) else "" @@ -417,7 +417,7 @@ object SearchApiModel { } private def label(task: ProjectTask[_ <: TaskSpec]): String = { - task.taskLabel(Int.MaxValue) + task.fullTaskLabel } private def label(projectOrTask: ProjectOrTask): String = { From 57c8803e33c84fed262d1a5b9610a27b9874adbc Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Thu, 8 Apr 2021 15:22:03 +0200 Subject: [PATCH 02/95] Support single hop paths for partial auto-completion --- .../transform/AutoCompletionApi.scala | 29 ++++++--- ...rtialSourcePathAutoCompletionRequest.scala | 10 +++- .../PartialAutoCompletionApiTest.scala | 55 ++++++++++++++++++ ...4fce3d1b_Partialauto-completionproject.zip | Bin 0 -> 5420 bytes 4 files changed, 85 insertions(+), 9 deletions(-) create mode 100644 silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala create mode 100644 silk-workbench/silk-workbench-rules/test/resources/diProjects/423a27b9-c6e6-45e5-84d2-26d94fce3d1b_Partialauto-completionproject.zip diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 7290116cdf..75940b6fc4 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -20,7 +20,6 @@ import play.api.mvc._ import java.util.logging.Logger import javax.inject.Inject -import scala.collection.immutable import scala.language.implicitConversions /** @@ -55,25 +54,39 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU sourcePath.filter(op => op.isInstanceOf[ForwardOperator] || op.isInstanceOf[BackwardOperator]) } + final val DEFAULT_AUTO_COMPLETE_RESULTS = 30 + /** A more fine-grained auto-completion of a source path that suggests auto-completion in parts of a path. */ def partialSourcePath(projectId: String, transformTaskId: String, ruleId: String): Action[JsValue] = RequestUserContextAction(parse.json) { implicit request => implicit userContext => val (project, transformTask) = projectAndTask[TransformSpec](projectId, transformTaskId) + implicit val prefixes: Prefixes = project.config.prefixes validateJson[PartialSourcePathAutoCompletionRequest] { autoCompletionRequest => withRule(transformTask, ruleId) { case (_, sourcePath) => val simpleSourcePath = simplePath(sourcePath) - // TODO - val from = math.min(autoCompletionRequest.cursorPosition, 1) - val to = math.max(autoCompletionRequest.cursorPosition, autoCompletionRequest.inputString.length - 1) + val forwardOnlySourcePath = forwardOnlyPath(simpleSourcePath) + val allPaths = pathsCacheCompletions(transformTask, simpleSourcePath) + val isRdfInput = transformTask.activity[TransformPathsCache].value().isRdfInput(transformTask) + val relativeForwardPaths = relativePaths(simpleSourcePath, forwardOnlySourcePath, allPaths, isRdfInput) + // Add known paths + val completions: Completions = relativeForwardPaths + val from = 0 // TODO: What part to replace + val length = autoCompletionRequest.inputString.length + // Return filtered result + val filteredResults = completions. + filterAndSort( + autoCompletionRequest.inputString, + autoCompletionRequest.maxSuggestions.getOrElse(DEFAULT_AUTO_COMPLETE_RESULTS), + sortEmptyTermResult = false, + multiWordFilter = true + ) val response = PartialSourcePathAutoCompletionResponse( autoCompletionRequest.inputString, autoCompletionRequest.cursorPosition, - Some((from, to)), - Completions(Seq( - Completion("dummy", label = Some("dummy value"), description = Some("dummy value description"), category = Categories.partialSourcePaths, isCompletion = true) - )).toCompletionsBase + Some(ReplacementInterval(from, length)), + filteredResults.toCompletionsBase ) Ok(Json.toJson(response)) } diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala index c4699fe6b7..4f2f57613a 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala @@ -26,10 +26,18 @@ object PartialSourcePathAutoCompletionRequest { */ case class PartialSourcePathAutoCompletionResponse(inputString: String, cursorPosition: Int, - replacementInterval: Option[(Int, Int)], + replacementInterval: Option[ReplacementInterval], replacementResults: CompletionsBase) +/** The part of a string to replace. + * + * @param from The start index of the string to be replaced. + * @param length The length in characters that should be replaced. + */ +case class ReplacementInterval(from: Int, length: Int) + object PartialSourcePathAutoCompletionResponse { + implicit val ReplacementIntervalFormat: Format[ReplacementInterval] = Json.format[ReplacementInterval] implicit val partialSourcePathAutoCompletionResponseFormat: Format[PartialSourcePathAutoCompletionResponse] = Json.format[PartialSourcePathAutoCompletionResponse] } diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala new file mode 100644 index 0000000000..1f559fc19b --- /dev/null +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -0,0 +1,55 @@ +package controllers.transform + +import controllers.transform.autoCompletion.{PartialSourcePathAutoCompletionRequest, PartialSourcePathAutoCompletionResponse, ReplacementInterval} +import helper.IntegrationTestTrait +import org.scalatest.{FlatSpec, MustMatchers} +import org.silkframework.serialization.json.JsonHelpers +import org.silkframework.workspace.SingleProjectWorkspaceProviderTestTrait +import play.api.libs.json.Json +import test.Routes + +class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with SingleProjectWorkspaceProviderTestTrait with IntegrationTestTrait { + + behavior of "Partial source path auto-complete endpoint" + + private val rdfTransform = "17fef5a5-a920-4665-92f5-cc729900e8f1_TransformRDF" + private val jsonTransform = "2a997fb4-1bc7-4344-882e-868193568e87_TransformJSON" + + /** + * Returns the path of the XML zip project that should be loaded before the test suite starts. + */ + override def projectPathInClasspath: String = "diProjects/423a27b9-c6e6-45e5-84d2-26d94fce3d1b_Partialauto-completionproject.zip" + + override def workspaceProviderId: String = "inMemory" + + protected override def routes: Option[Class[Routes]] = Some(classOf[test.Routes]) + + it should "auto-complete all paths when input is an empty string" in { + val result = partialSourcePathAutoCompleteRequest(jsonTransform) + result.copy(replacementResults = null) mustBe PartialSourcePathAutoCompletionResponse("", 0, Some(ReplacementInterval(0, 0)), null) + suggestedValues(result) mustBe Seq("department", "department/id", "department/tags", + "department/tags/evenMoreNested", "department/tags/evenMoreNested/value", "department/tags/tagId", "id", "name", + "phoneNumbers", "phoneNumbers/number", "phoneNumbers/type") + } + + it should "auto-complete with multi-word text filter if there is a one hop path entered" in { + val inputText = "depart id" + val result = partialSourcePathAutoCompleteRequest(jsonTransform, inputText = inputText, cursorPosition = 2) + result.copy(replacementResults = null) mustBe PartialSourcePathAutoCompletionResponse(inputText, 2, Some(ReplacementInterval(0, inputText.length)), null) + suggestedValues(result) mustBe Seq("department/id", "department/tags/tagId") + } + + private def suggestedValues(result: PartialSourcePathAutoCompletionResponse): Seq[String] = { + result.replacementResults.completions.map(_.value) + } + + private def partialSourcePathAutoCompleteRequest(transformId: String, + ruleId: String = "root", + inputText: String = "", + cursorPosition: Int = 0, + maxSuggestions: Option[Int] = None): PartialSourcePathAutoCompletionResponse = { + val partialUrl = controllers.transform.routes.AutoCompletionApi.partialSourcePath(projectId, transformId, ruleId).url + val response = client.url(s"$baseUrl$partialUrl").post(Json.toJson(PartialSourcePathAutoCompletionRequest(inputText, cursorPosition, maxSuggestions))) + JsonHelpers.fromJsonValidated[PartialSourcePathAutoCompletionResponse](checkResponse(response).json) + } +} diff --git a/silk-workbench/silk-workbench-rules/test/resources/diProjects/423a27b9-c6e6-45e5-84d2-26d94fce3d1b_Partialauto-completionproject.zip b/silk-workbench/silk-workbench-rules/test/resources/diProjects/423a27b9-c6e6-45e5-84d2-26d94fce3d1b_Partialauto-completionproject.zip new file mode 100644 index 0000000000000000000000000000000000000000..48c5ea1132e8e9a5022c2caca6f90bf060482937 GIT binary patch literal 5420 zcmd5<2{hD;9v;Muie%qqCo^W>LR6NruUW2SX3T`f5EIfJA+jY*)@VYAl%mO1iEz_G z$ySzVPYTI0MDk|tyYF&4ug-bry63*%oH^$|=gc|(?|k2H`96Cqfi;o<5C{ag8RKu~Bh8iGUlARq{g4<1D!k&!-d;$C|~Fx`(pB^;y&LP&uD zK~yr`FOU`#9OzFb(V?V38pUs)es};ij@dzrfk__w$XMG_X=SxdGDgE3iiYP}WyxLX z<`nFSVmbm3FBBpyFi2?L&r452zUGIQ^cVyooy<1RW*xIsLdWleXmc8K5KV47fw=e> zWPF8Ub~b=he&?O9jGD4Z)ECwD9lbJ=9VW4n_Zua~#{+J&)4?Q*v-S%5NSLG4=_XsD zEU8PV;_B%4)3sY6x>_~Tx|7-=(r~J&>0E*3{+=X;<<}OT}l+^dsH%S@TTU*@2{N@7{d5 z$Dd`WK=)1;+T#1C?=4e53EnV$YkvUJihT6iy*tIJCJZnyFsc#R+xE6=hJIyG3qJnh zw0C)|wPr%B_IknJwAY8y?eZO--)xyn@)Spkw0%*S(|WjWX*ArUKCI&2^L6%C zK+se&)7qX70I1^!0PMdjrQ7OA8B7idJQz$Ohd_hK!6AXP5PdqG8fQ1KHpWDZF{>wJ ze9P>1Hl#=mRAlmQUDmLJaj4~2cI2a6Yc~FlBF=hDr@rL0qPmz zS0&R?Qdde<6R83YViSt@5}aWmML6C45PCt~?r)I&B|;VwQHCh~OkR3lGLQW%@Y0c`rX~xotc(?b0dmOnriM z@RaHNW)^u$q|2{{4j+(nS#*5U?i0wPQ1`iD%uOyq&0HG4PoewYMWH{J$g7Txv?kd4 zo5Ok+@(;Vo9Z#9kw~kRUQRGLnJN;e#%Dl^#>NPSjeb)zTO*x0#p`rN|fV5M6VC1zZ z6koP$+5XageuX@K&qcq)ro`7Lt;Yf7;!4wvcoE-BW}e97z%ySwfiC!m@^IzpGvIZ^T1{((5k<+(3AaAfhRzMkWlSBm)=J|?BU4O z5x|E)CxnpcP&@+RgMg7?5HyyEgrKlQ9E1p?ARtI0298C+kT?_$yVw4=Hn()Lv*qot z5@xGyGfd2Ec5&@1hWv1cmNTvRe6*RFpT_l~^J#c`q^@Wvt`Zb$3m@SMwed`Yw>^ zqZ#8(nc+tdmxv7YlufX!&toS+z`7Q&so!%&>RI`(Zisn)o}e65VX-E6nrJ^1l(D*k zQiw1DmW0G`4-|?5L17Sh2muSnLntIP9ErdY@K_(%?}9p-n*aMA{j|RcCK>f?QOaEz zcy)waV+l^Utu46mj27a=Sou0OM&rP{$i>c3QNp9Gto$iPzE${IA(Xt@BpzbBz6hLy zJ0L4EOAZ8U>9PAf8zb8!x>uY+vkKt zQ1e=sU&!0KX(`!)+Xu%>T1(52n#CJ2hbr#pW~I4bb1rjWsM^f65_0u9$<3c~f@aIV zxLz&~aR9YRocYXZ-X4GZaC@F$$8ob)IW|Mr7F;_wTE%%sL|d34Qx*(GpT)!^c@|U~ zMXtM6ZP5EdE|6aE`P2S^$mAsnsjKI2t=Hwak}XHO*uatOyPWkARO>#{7QM7HJI*UB zL(Ov*Jl-`HvNSo@2hUlX8}qb$oQx}0M^MlZ5($gI<6$r|jsoB7{GY#b1%>`T*gRof>{>-*U}AMsQZ^q;>Rx9y zC5B_l_Fc)+WP$?vEvG-aB*@PqPTp(t**)6xU?=rfw3)?qnMzBug%CjQai=WpUF}TN zW?3IXW5hM#S32=(A7Z*hQ(T+|sJgk0&XPI@Hl$b}ou^;s*Juu(oN=(s*O+rmd8)to zvrT8ex+&ZDv|O=)PO!?QlAa0W0uc`sQlHh%D1YAIJSO~gcITK8ACFg!9lp&4T-N%y z-|t_Ygnypb;Dc0h2#?i^PBy7|Fv+@M*V=js0bxEh6A#UPr?b1=YvT7@+zox3e%16c zIj`*V>>Kq$pB|30K1ZS5FlSix9=f|UV}80gO~RqLdz}?n0Hm0}c;a5UJ6d9a{5(i` z{=n#U+03B@oq<3VKa9FYUtt`??OjsMm$kqP)#>IQUQI$lo2VT@kPNA6En~Z$ex2tz zjwKxhTh$*hp_lT;c6^MyV?`_ZEPi_liv>(Vmq40+FiJ)5Zih(?g$Gr7A|8(H4^hQz!&n#-a|Q&PkKOj_$D#8-dG`H+Mkfx;@=pc zzhuZ}p$%VQ2n{YruNWYW*x8-qCb|wPvo>)IF?#agkO3a)j?MF9lspymK5p_P*K;IH zcuqyGwn3e9&ul~F{tN!prH)?$fWrl;zDIV7#+ULl`q;CP8S~6-E*l*b1}fdsrcyRN z(ipu|~EI*R3=GB}C)wWZ1&S#YjvS9YEULPJc zxN2{H%S@}YAin>aBm(vf-Y6b$=1V9z&#KRoFUBsj{ZL`uTQ5b;_C;#Z+Vl_GE1HF1 zoNV$7SVn-I@unb=_r>5G8ONtMYTny~w>% z@V;lgNrv1(1cD~CdzR9;H1RCq_Xjia|A5F9liA9(kGy?H8>g|vfVn$Cwl2C-#YU2a z_jgJA=>D2MRxmmHAQ|NxU^jDns88;2m_=#2&$TU}L;ccen^+Im#|uUuNid?`NlZ0* zG6ho=^+u?MK0l$oCShr)Rk`Mawz{IKscS(W({DG>W$rHVZWAlsUk&{en+)Ej$&@*j ztjNf(F5m3Nk35@JV8UtyjSSMN@-3zYBUKzjOg`$nC1$ALiVI12OpPHm{cg>=x|j0IiI2h;J?M4FkdMA;5O`I4R?HKck_es|TaCX#Vz)$pA)cOEx3 z(hRE@?ZJtzHluRI;)js>bV@JpG*8J z8gy%;G27os>8)j{V97!$gVjxyA4P*VUszCSQF_zzW@vbx(%lj0`E|pYKnASgU`Nyk zN0|~@QoZti9B~pf@1g%V$a&~ zdPmaK=x!W$2#tRmLOy;;z;d)+nbdgE`p>nhLEGNy&#zvYaClL0xmvkkSK-zFH5h)2 zVx`UJrK#l-<z6H4h1^@_iKQ{%ru4)7C*S`Qtf8Ox` literal 0 HcmV?d00001 From cf3054ab18ea00b5823faf363027fe427b7193a1 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Tue, 13 Apr 2021 17:49:59 +0200 Subject: [PATCH 03/95] Add initial functional partial source path completion --- .../entity/paths/PathParser.scala | 63 ++++++++++++++-- .../entity/paths/UntypedPath.scala | 4 + .../entity/paths/PathParserTest.scala | 35 +++++++++ .../transform/AutoCompletionApi.scala | 16 +++- ...rtialSourcePathAutoCompletionRequest.scala | 1 + ...artialSourcePathAutocompletionHelper.scala | 75 +++++++++++++++++++ .../PartialAutoCompletionApiTest.scala | 29 +++++-- ...alSourcePathAutocompletionHelperTest.scala | 20 +++++ 8 files changed, 227 insertions(+), 16 deletions(-) create mode 100644 silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala create mode 100644 silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala create mode 100644 silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala diff --git a/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala b/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala index 4750926000..e1da22acf3 100644 --- a/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala +++ b/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala @@ -30,12 +30,7 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { if(pathStr.isEmpty) { UntypedPath(Nil) } else { - // Complete path if a simplified syntax is used - val completePath = pathStr.head match { - case '?' => pathStr // Path includes a variable - case '/' | '\\' => "?a" + pathStr // Variable has been left out - case _ => "?a/" + pathStr // Variable and leading '/' have been left out - } + val completePath = normalized(pathStr) // Parse path parseAll(path, new CharSequenceReader(completePath)) match { case Success(parsedPath, _) => parsedPath @@ -44,7 +39,50 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { } } - private def path = variable ~ rep(forwardOperator | backwardOperator | filterOperator) ^^ { + /** + * Returns the part of the path that could be parsed until a parse error and the error if one occurred. + * @param pathStr The input path string. + */ + def parseUntilError(pathStr: String): PartialParseResult = { + val completePath = normalized(pathStr) + // Added characters because of normalization. Need to be removed when reporting the actual error offset. + val addedOffset = completePath.length - pathStr.length + val inputSequence = new CharSequenceReader(completePath) + var partialPathOps: Vector[PathOperator] = Vector.empty + var errorPosition: Option[PartialParseError] = None + val variableResult = parse(variable, inputSequence) // Ignore variable + var parseOffset = variableResult.next.offset + while(errorPosition.isEmpty && parseOffset < completePath.length) { + parse(ops, inputSequence.drop(parseOffset)) match { + case Success(pathOperator, next) => { + partialPathOps :+= pathOperator + parseOffset = next.offset + } + case error: NoSuccess => + errorPosition = Some(PartialParseError( + // Subtract 1 because next is positioned after the character that lead to the parse error. + error.next.offset - addedOffset - 1, + error.msg + )) + } + } + PartialParseResult(UntypedPath(partialPathOps.toList), errorPosition) + } + + // Normalizes the path syntax in case a simplified syntax has been used + private def normalized(pathStr: String): String = { + pathStr.head match { + case '?' => pathStr // Path includes a variable + case '/' | '\\' => "?a" + pathStr // Variable has been left out + case _ => "?a/" + pathStr // Variable and leading '/' have been left out + } + } + + private def ops = forwardOperator | backwardOperator | filterOperator ^^ { + case operator => operator + } + + private def path = variable ~ rep(ops) ^^ { case variable ~ operators => UntypedPath(operators) } @@ -84,3 +122,14 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { // A comparison operator private def compOperator = ">" | "<" | ">=" | "<=" | "=" | "!=" } + +/** + * A partial path parse result. + * + * @param partialPath The (valid) partial path that has been parsed until the parse error. + * @param error An optional parse error when not all of the input string could be parsed. + */ +case class PartialParseResult(partialPath: UntypedPath, error: Option[PartialParseError]) + +/** Offset and error message of the parse error. The offset defines the position before the character that lead to the parse error. */ +case class PartialParseError(offset: Int, message: String) \ No newline at end of file diff --git a/silk-core/src/main/scala/org/silkframework/entity/paths/UntypedPath.scala b/silk-core/src/main/scala/org/silkframework/entity/paths/UntypedPath.scala index fcae618281..bd2b532172 100644 --- a/silk-core/src/main/scala/org/silkframework/entity/paths/UntypedPath.scala +++ b/silk-core/src/main/scala/org/silkframework/entity/paths/UntypedPath.scala @@ -119,4 +119,8 @@ object UntypedPath { path } } + + def partialParse(pathStr: String)(implicit prefixes: Prefixes = Prefixes.empty): PartialParseResult = { + new PathParser(prefixes).parseUntilError(pathStr) + } } diff --git a/silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala b/silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala new file mode 100644 index 0000000000..a7b7418d3d --- /dev/null +++ b/silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala @@ -0,0 +1,35 @@ +package org.silkframework.entity.paths + +import org.scalatest.{FlatSpec, MustMatchers} +import org.silkframework.config.Prefixes + +class PathParserTest extends FlatSpec with MustMatchers { + behavior of "path parser" + + val parser = new PathParser(Prefixes.default) + + it should "parse the full path and not return any errors for valid paths" in { + testValidPath("/a/b[d = 5]\\c") + testValidPath("""\[ != "check value"]/abc""") + } + + it should "parse invalid paths and return the parsed part and the position where it failed" in { + testInvalidPath("/a/b/c/not valid/d/e", "/a/b/c/not", 10) + testInvalidPath(" /alreadyInvalid/a", "", 0) + testInvalidPath("""\/a[b :+ 1]/c""", "\\/a", 17) + testInvalidPath("""/a\b/c/""", """/a\b/c""",6) + } + + private def testValidPath(inputString: String): Unit = { + val result = parser.parseUntilError(inputString) + result mustBe PartialParseResult(UntypedPath.parse(inputString), None) + } + + private def testInvalidPath(inputString: String, expectedParsedString: String, expectedErrorOffset: Int): Unit = { + val result = parser.parseUntilError(inputString) + result.copy(error = result.error.map(e => e.copy(message = ""))) mustBe PartialParseResult( + UntypedPath.parse(expectedParsedString), + Some(PartialParseError(expectedErrorOffset, "")) + ) + } +} diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 75940b6fc4..7720b654a9 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -69,15 +69,22 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU val forwardOnlySourcePath = forwardOnlyPath(simpleSourcePath) val allPaths = pathsCacheCompletions(transformTask, simpleSourcePath) val isRdfInput = transformTask.activity[TransformPathsCache].value().isRdfInput(transformTask) - val relativeForwardPaths = relativePaths(simpleSourcePath, forwardOnlySourcePath, allPaths, isRdfInput) + var relativeForwardPaths = relativePaths(simpleSourcePath, forwardOnlySourcePath, allPaths, isRdfInput) + val openWorld = false // TODO: set openWorld correctly + val pathToReplace = PartialSourcePathAutocompletionHelper.pathToReplace(autoCompletionRequest, openWorld) + if(!openWorld && pathToReplace.from > 0) { + val pathBeforeReplacement = UntypedPath.parse(autoCompletionRequest.inputString.take(pathToReplace.from)) + val simplePathBeforeReplacement = simplePath(pathBeforeReplacement.operators) + relativeForwardPaths = relativePaths(simplePathBeforeReplacement, forwardOnlyPath(simplePathBeforeReplacement), relativeForwardPaths, isRdfInput) + } // Add known paths val completions: Completions = relativeForwardPaths - val from = 0 // TODO: What part to replace - val length = autoCompletionRequest.inputString.length + val from = pathToReplace.from + val length = pathToReplace.length // Return filtered result val filteredResults = completions. filterAndSort( - autoCompletionRequest.inputString, + pathToReplace.query.map(_.mkString(" ")).getOrElse(""), autoCompletionRequest.maxSuggestions.getOrElse(DEFAULT_AUTO_COMPLETE_RESULTS), sortEmptyTermResult = false, multiWordFilter = true @@ -86,6 +93,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU autoCompletionRequest.inputString, autoCompletionRequest.cursorPosition, Some(ReplacementInterval(from, length)), + pathToReplace.query.map(_.mkString(" ")).getOrElse(""), filteredResults.toCompletionsBase ) Ok(Json.toJson(response)) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala index 4f2f57613a..3a1556569d 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala @@ -27,6 +27,7 @@ object PartialSourcePathAutoCompletionRequest { case class PartialSourcePathAutoCompletionResponse(inputString: String, cursorPosition: Int, replacementInterval: Option[ReplacementInterval], + extractedQuery: String, replacementResults: CompletionsBase) /** The part of a string to replace. diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala new file mode 100644 index 0000000000..5556a89221 --- /dev/null +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala @@ -0,0 +1,75 @@ +package controllers.transform.autoCompletion + +import org.silkframework.config.Prefixes +import org.silkframework.entity.paths.{DirectionalPathOperator, PathOperator, PathParser, UntypedPath} +import org.silkframework.util.StringUtils + +object PartialSourcePathAutocompletionHelper { + /** + * Returns the part of the path that should be replaced and the extracted query words that can be used for search. + * @param request The partial source path auto-completion request payload. + * @param openWorld If true, this is a + * @param prefixes + */ + def pathToReplace(request: PartialSourcePathAutoCompletionRequest, + openWorld: Boolean) + (implicit prefixes: Prefixes): PathToReplace = { + val input = request.inputString + val unfilteredQuery = Some(Seq.empty) + if(input.isEmpty) { + return PathToReplace(0, 0, unfilteredQuery) + } + val cursorPosition = request.cursorPosition + val pathUntilCursor = input.take(cursorPosition) + // If the cursor is placed in the middle of an operator expression, this contains the remaining characters after the cursor. + val remainingCharsInOperator = input.substring(request.cursorPosition).takeWhile(c => !Set('/', '\\', '[').contains(c)) + val partialResult = UntypedPath.partialParse(pathUntilCursor) + partialResult.error match { + case Some(error) => + // TODO: Handle error cases, most likely inside filter expressions + ??? + case None if partialResult.partialPath.operators.nonEmpty => + // No parse problem, use the last path segment (must be forward or backward path op) for auto-completion + val lastPathOp = partialResult.partialPath.operators.last + val extractedTextQuery = extractTextPart(lastPathOp) + val fullQuery = extractedTextQuery.map(q => extractQuery(q + remainingCharsInOperator)) + if(extractedTextQuery.isEmpty) { + // This is the end of a valid filter expression, do not replace or suggest anything besides default completions + PathToReplace(pathUntilCursor.length, 0, None) + } else if(lastPathOp.serialize.length >= pathUntilCursor.length) { + // The path op is the complete input path + PathToReplace(0, pathUntilCursor.length + remainingCharsInOperator.length, fullQuery) + } else { + // Replace the last path operator of the input path + val lastOpLength = lastPathOp.serialize.length + PathToReplace(cursorPosition - lastOpLength, lastOpLength + remainingCharsInOperator.length, fullQuery) + } + case None => + // Should never come so far + PathToReplace(0, 0, unfilteredQuery) + } + } + + private def extractTextPart(pathOp: PathOperator): Option[String] = { + pathOp match { + case op: DirectionalPathOperator => + Some(op.property.uri) + case _ => + // This is the end of a complete filter expression, suggest nothing to replace it with. + None + } + } + + private def extractQuery(input: String): Seq[String] = { + StringUtils.extractSearchTerms(input) + } +} + +/** + * The part of the input path that should be replaced. + * @param from The start index of the substring that should be replaced. + * @param length The length in characters of the string that should be replaced. + * @param query Extracted query as multi-word sequence, from the position of the cursor. + * If it is None this means that no query should be asked to find suggestions, i.e. only suggest operator or nothing. + */ +case class PathToReplace(from: Int, length: Int, query: Option[Seq[String]]) \ No newline at end of file diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index 1f559fc19b..6a0cd933ca 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -15,6 +15,10 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl private val rdfTransform = "17fef5a5-a920-4665-92f5-cc729900e8f1_TransformRDF" private val jsonTransform = "2a997fb4-1bc7-4344-882e-868193568e87_TransformJSON" + val allJsonPaths = Seq("department", "department/id", "department/tags", + "department/tags/evenMoreNested", "department/tags/evenMoreNested/value", "department/tags/tagId", "id", "name", + "phoneNumbers", "phoneNumbers/number", "phoneNumbers/type") + /** * Returns the path of the XML zip project that should be loaded before the test suite starts. */ @@ -26,19 +30,34 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl it should "auto-complete all paths when input is an empty string" in { val result = partialSourcePathAutoCompleteRequest(jsonTransform) - result.copy(replacementResults = null) mustBe PartialSourcePathAutoCompletionResponse("", 0, Some(ReplacementInterval(0, 0)), null) - suggestedValues(result) mustBe Seq("department", "department/id", "department/tags", - "department/tags/evenMoreNested", "department/tags/evenMoreNested/value", "department/tags/tagId", "id", "name", - "phoneNumbers", "phoneNumbers/number", "phoneNumbers/type") + result.copy(replacementResults = null) mustBe PartialSourcePathAutoCompletionResponse("", 0, Some(ReplacementInterval(0, 0)), "", null) + suggestedValues(result) mustBe allJsonPaths } it should "auto-complete with multi-word text filter if there is a one hop path entered" in { val inputText = "depart id" val result = partialSourcePathAutoCompleteRequest(jsonTransform, inputText = inputText, cursorPosition = 2) - result.copy(replacementResults = null) mustBe PartialSourcePathAutoCompletionResponse(inputText, 2, Some(ReplacementInterval(0, inputText.length)), null) + result.copy(replacementResults = null) mustBe PartialSourcePathAutoCompletionResponse(inputText, 2, Some(ReplacementInterval(0, inputText.length)), inputText,null) suggestedValues(result) mustBe Seq("department/id", "department/tags/tagId") } + it should "auto-complete JSON paths at any level" in { + val level1EndInput = "department/id" + val level1Prefix = "department/" + // Return all relative paths that match "id" for any cursor position after the first slash + for(cursorPosition <- level1EndInput.length - 1 to level1EndInput.length) { // TODO: Change to -2 + jsonSuggestions(level1EndInput, cursorPosition) mustBe Seq("id", "tags/tagId") + } + } + + private def jsonSuggestions(inputText: String, cursorPosition: Int): Seq[String] = { + suggestedValues(partialSourcePathAutoCompleteRequest(jsonTransform, inputText = inputText, cursorPosition = cursorPosition)) + } + + private def rdfSuggestions(inputText: String, cursorPosition: Int): Set[String] = { + suggestedValues(partialSourcePathAutoCompleteRequest(rdfTransform, inputText = inputText, cursorPosition = cursorPosition)).toSet + } + private def suggestedValues(result: PartialSourcePathAutoCompletionResponse): Seq[String] = { result.replacementResults.completions.map(_.value) } diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala new file mode 100644 index 0000000000..47a887d4bb --- /dev/null +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala @@ -0,0 +1,20 @@ +package controllers.transform.autoCompletion + +import org.scalatest.{FlatSpec, MustMatchers} +import org.silkframework.config.Prefixes + +class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatchers { + behavior of "PartialSourcePathAutocompletionHelper" + + def replace(inputString: String, + cursorPosition: Int, + openWorld: Boolean = false): PathToReplace = { + PartialSourcePathAutocompletionHelper.pathToReplace(PartialSourcePathAutoCompletionRequest(inputString, cursorPosition, None), openWorld)(Prefixes.empty) + } + + it should "correctly find out which part of a path to replace" in { + val input = "a1/b1/c1" + replace(input, 4) mustBe PathToReplace(2, 3, Some(Seq("b1"))) + replace(input, 5) mustBe PathToReplace(2, 3, Some(Seq("b1"))) + } +} From 0f05629cebc42fd4c6b1e61a13a4ffa0d9f44ecc Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Wed, 14 Apr 2021 19:16:04 +0200 Subject: [PATCH 04/95] Handle partial auto-complete use cases of complex paths and invalid paths --- .../entity/paths/PathParser.scala | 39 ++++--- .../entity/paths/PathParserTest.scala | 18 ++-- .../transform/AutoCompletionApi.scala | 9 +- ...rtialSourcePathAutoCompletionRequest.scala | 16 ++- ...artialSourcePathAutocompletionHelper.scala | 100 ++++++++++++------ .../PartialAutoCompletionApiTest.scala | 4 +- ...alSourcePathAutocompletionHelperTest.scala | 24 ++++- 7 files changed, 145 insertions(+), 65 deletions(-) diff --git a/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala b/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala index e1da22acf3..98fa4cf510 100644 --- a/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala +++ b/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala @@ -49,24 +49,37 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { val addedOffset = completePath.length - pathStr.length val inputSequence = new CharSequenceReader(completePath) var partialPathOps: Vector[PathOperator] = Vector.empty - var errorPosition: Option[PartialParseError] = None + var partialParseError: Option[PartialParseError] = None val variableResult = parse(variable, inputSequence) // Ignore variable var parseOffset = variableResult.next.offset - while(errorPosition.isEmpty && parseOffset < completePath.length) { - parse(ops, inputSequence.drop(parseOffset)) match { - case Success(pathOperator, next) => { - partialPathOps :+= pathOperator - parseOffset = next.offset - } - case error: NoSuccess => - errorPosition = Some(PartialParseError( + def originalParseOffset = math.max(0, parseOffset - addedOffset) + while(partialParseError.isEmpty && parseOffset < completePath.length) { + try { + parse(ops, inputSequence.drop(parseOffset)) match { + case Success(pathOperator, next) => { + partialPathOps :+= pathOperator + parseOffset = next.offset + } + case error: NoSuccess => // Subtract 1 because next is positioned after the character that lead to the parse error. - error.next.offset - addedOffset - 1, - error.msg + val originalErrorOffset = math.max(error.next.offset - addedOffset - 1, 0) + partialParseError = Some(PartialParseError( + originalErrorOffset, + error.msg, + pathStr.substring(originalParseOffset, originalErrorOffset + 1) // + 1 since we want to have the character where it failed + )) + } + } catch { + case validationException: ValidationException => + // Can happen e.g. when a qualified name used an invalid/unknown prefix name + partialParseError = Some(PartialParseError( + originalParseOffset, + validationException.getMessage, + "" )) } } - PartialParseResult(UntypedPath(partialPathOps.toList), errorPosition) + PartialParseResult(UntypedPath(partialPathOps.toList), partialParseError) } // Normalizes the path syntax in case a simplified syntax has been used @@ -132,4 +145,4 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { case class PartialParseResult(partialPath: UntypedPath, error: Option[PartialParseError]) /** Offset and error message of the parse error. The offset defines the position before the character that lead to the parse error. */ -case class PartialParseError(offset: Int, message: String) \ No newline at end of file +case class PartialParseError(offset: Int, message: String, inputLeadingToError: String) \ No newline at end of file diff --git a/silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala b/silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala index a7b7418d3d..7c020ae675 100644 --- a/silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala +++ b/silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala @@ -7,6 +7,7 @@ class PathParserTest extends FlatSpec with MustMatchers { behavior of "path parser" val parser = new PathParser(Prefixes.default) + private val SPACE = " " it should "parse the full path and not return any errors for valid paths" in { testValidPath("/a/b[d = 5]\\c") @@ -14,10 +15,12 @@ class PathParserTest extends FlatSpec with MustMatchers { } it should "parse invalid paths and return the parsed part and the position where it failed" in { - testInvalidPath("/a/b/c/not valid/d/e", "/a/b/c/not", 10) - testInvalidPath(" /alreadyInvalid/a", "", 0) - testInvalidPath("""\/a[b :+ 1]/c""", "\\/a", 17) - testInvalidPath("""/a\b/c/""", """/a\b/c""",6) + testInvalidPath("/a/b/c/not valid/d/e", "/a/b/c/not", 10, SPACE) + testInvalidPath(s"$SPACE/alreadyInvalid/a", "", 0, SPACE) + testInvalidPath("""\/a[b :+ 1]/c""", "\\/a", 17, "[b ") + testInvalidPath("""/a\b/c/""", """/a\b/c""",6, "/") + testInvalidPath("""invalidNs:broken""", "",0, "") + testInvalidPath("""<'""", "",0, "<") } private def testValidPath(inputString: String): Unit = { @@ -25,11 +28,14 @@ class PathParserTest extends FlatSpec with MustMatchers { result mustBe PartialParseResult(UntypedPath.parse(inputString), None) } - private def testInvalidPath(inputString: String, expectedParsedString: String, expectedErrorOffset: Int): Unit = { + private def testInvalidPath(inputString: String, + expectedParsedString: String, + expectedErrorOffset: Int, + expectedInputOnError: String): Unit = { val result = parser.parseUntilError(inputString) result.copy(error = result.error.map(e => e.copy(message = ""))) mustBe PartialParseResult( UntypedPath.parse(expectedParsedString), - Some(PartialParseError(expectedErrorOffset, "")) + Some(PartialParseError(expectedErrorOffset, "", expectedInputOnError)) ) } } diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 7720b654a9..b4e44d50f2 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -70,9 +70,8 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU val allPaths = pathsCacheCompletions(transformTask, simpleSourcePath) val isRdfInput = transformTask.activity[TransformPathsCache].value().isRdfInput(transformTask) var relativeForwardPaths = relativePaths(simpleSourcePath, forwardOnlySourcePath, allPaths, isRdfInput) - val openWorld = false // TODO: set openWorld correctly - val pathToReplace = PartialSourcePathAutocompletionHelper.pathToReplace(autoCompletionRequest, openWorld) - if(!openWorld && pathToReplace.from > 0) { + val pathToReplace = PartialSourcePathAutocompletionHelper.pathToReplace(autoCompletionRequest, isRdfInput) + if(!isRdfInput && pathToReplace.from > 0) { val pathBeforeReplacement = UntypedPath.parse(autoCompletionRequest.inputString.take(pathToReplace.from)) val simplePathBeforeReplacement = simplePath(pathBeforeReplacement.operators) relativeForwardPaths = relativePaths(simplePathBeforeReplacement, forwardOnlyPath(simplePathBeforeReplacement), relativeForwardPaths, isRdfInput) @@ -84,7 +83,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU // Return filtered result val filteredResults = completions. filterAndSort( - pathToReplace.query.map(_.mkString(" ")).getOrElse(""), + pathToReplace.query.getOrElse(""), autoCompletionRequest.maxSuggestions.getOrElse(DEFAULT_AUTO_COMPLETE_RESULTS), sortEmptyTermResult = false, multiWordFilter = true @@ -93,7 +92,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU autoCompletionRequest.inputString, autoCompletionRequest.cursorPosition, Some(ReplacementInterval(from, length)), - pathToReplace.query.map(_.mkString(" ")).getOrElse(""), + pathToReplace.query.getOrElse(""), filteredResults.toCompletionsBase ) Ok(Json.toJson(response)) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala index 3a1556569d..81ae0fbda3 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala @@ -11,7 +11,21 @@ import play.api.libs.json.{Format, Json} */ case class PartialSourcePathAutoCompletionRequest(inputString: String, cursorPosition: Int, - maxSuggestions: Option[Int]) + maxSuggestions: Option[Int]) { + /** The path until the cursor position. */ + lazy val pathUntilCursor: String = inputString.take(cursorPosition) + + /** The remaining characters from the cursor position to the end of the current path operator. */ + lazy val remainingStringInOperator: String = { + // TODO: This does not consider being in a literal string, i.e. "...^..." where /, \ and [ would be allowed + val operatorStartChars = Set('/', '\\', '[') + inputString + .substring(cursorPosition) + .takeWhile { char => + !operatorStartChars.contains(char) + } + } +} object PartialSourcePathAutoCompletionRequest { implicit val partialSourcePathAutoCompletionRequestFormat: Format[PartialSourcePathAutoCompletionRequest] = Json.format[PartialSourcePathAutoCompletionRequest] diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala index 5556a89221..a064651bf9 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala @@ -1,53 +1,75 @@ package controllers.transform.autoCompletion import org.silkframework.config.Prefixes -import org.silkframework.entity.paths.{DirectionalPathOperator, PathOperator, PathParser, UntypedPath} +import org.silkframework.entity.paths.{DirectionalPathOperator, PartialParseResult, PathOperator, PathParser, UntypedPath} import org.silkframework.util.StringUtils object PartialSourcePathAutocompletionHelper { /** * Returns the part of the path that should be replaced and the extracted query words that can be used for search. * @param request The partial source path auto-completion request payload. - * @param openWorld If true, this is a - * @param prefixes + * @param subPathOnly If true, only a sub-part of the path is replaced, else a path suffix */ def pathToReplace(request: PartialSourcePathAutoCompletionRequest, - openWorld: Boolean) + subPathOnly: Boolean) (implicit prefixes: Prefixes): PathToReplace = { - val input = request.inputString - val unfilteredQuery = Some(Seq.empty) - if(input.isEmpty) { + val unfilteredQuery: Option[String] = Some("") + if(request.inputString.isEmpty) { return PathToReplace(0, 0, unfilteredQuery) } - val cursorPosition = request.cursorPosition - val pathUntilCursor = input.take(cursorPosition) - // If the cursor is placed in the middle of an operator expression, this contains the remaining characters after the cursor. - val remainingCharsInOperator = input.substring(request.cursorPosition).takeWhile(c => !Set('/', '\\', '[').contains(c)) - val partialResult = UntypedPath.partialParse(pathUntilCursor) - partialResult.error match { + val partialResult = UntypedPath.partialParse(request.pathUntilCursor) + val replacement = partialResult.error match { case Some(error) => - // TODO: Handle error cases, most likely inside filter expressions - ??? - case None if partialResult.partialPath.operators.nonEmpty => - // No parse problem, use the last path segment (must be forward or backward path op) for auto-completion - val lastPathOp = partialResult.partialPath.operators.last - val extractedTextQuery = extractTextPart(lastPathOp) - val fullQuery = extractedTextQuery.map(q => extractQuery(q + remainingCharsInOperator)) - if(extractedTextQuery.isEmpty) { - // This is the end of a valid filter expression, do not replace or suggest anything besides default completions - PathToReplace(pathUntilCursor.length, 0, None) - } else if(lastPathOp.serialize.length >= pathUntilCursor.length) { - // The path op is the complete input path - PathToReplace(0, pathUntilCursor.length + remainingCharsInOperator.length, fullQuery) + val errorOffsetCharacter = request.inputString.substring(error.offset, error.offset + 1) + val parseStartCharacter = if(error.inputLeadingToError.isEmpty) errorOffsetCharacter else error.inputLeadingToError.take(1) + if(error.inputLeadingToError.startsWith("[")) { + // Error happened inside of a filter + // TODO: Find out where in the filter we are and try to auto-complete the correct thing + PathToReplace(0, 0, unfilteredQuery) + } else if(parseStartCharacter == "/" || parseStartCharacter == "\\") { + // It tried to parse a forward or backward path and failed, replace path and use path value as query + val operatorValue = request.inputString.substring(error.offset + 1, request.cursorPosition) + request.remainingStringInOperator + PathToReplace(error.offset, operatorValue.length + 1, Some(extractQuery(operatorValue))) } else { - // Replace the last path operator of the input path - val lastOpLength = lastPathOp.serialize.length - PathToReplace(cursorPosition - lastOpLength, lastOpLength + remainingCharsInOperator.length, fullQuery) + // The parser parsed part of a forward or backward path as a path op and then failed on an invalid char, e.g. "/with space" + // parses "with" as forward op and then fails parsing the space. + assert(partialResult.partialPath.operators.nonEmpty, "Could not detect sub-path to be replaced.") + handleMiddleOfPathOp(partialResult, request) } + case None if partialResult.partialPath.operators.nonEmpty => + // No parse problem, use the last path segment (must be forward or backward path op) for auto-completion + handleMiddleOfPathOp(partialResult, request) case None => // Should never come so far PathToReplace(0, 0, unfilteredQuery) } + handleSubPathOnly(request, replacement, subPathOnly) + } + + private def handleSubPathOnly(request: PartialSourcePathAutoCompletionRequest, pathToReplace: PathToReplace, subPathOnly: Boolean): PathToReplace = { + if(subPathOnly || pathToReplace.query.isEmpty) { + pathToReplace + } else { + pathToReplace.copy(length = request.inputString.length - pathToReplace.from) + } + } + + private def handleMiddleOfPathOp(partialResult: PartialParseResult, + request: PartialSourcePathAutoCompletionRequest): PathToReplace = { + val lastPathOp = partialResult.partialPath.operators.last + val extractedTextQuery = extractTextPart(lastPathOp) + val fullQuery = extractedTextQuery.map(q => extractQuery(q + request.remainingStringInOperator)) + if(extractedTextQuery.isEmpty) { + // This is the end of a valid filter expression, do not replace or suggest anything besides default completions + PathToReplace(request.pathUntilCursor.length, 0, None) + } else if(lastPathOp.serialize.length >= request.pathUntilCursor.length) { + // The path op is the complete input path + PathToReplace(0, request.pathUntilCursor.length + request.remainingStringInOperator.length, fullQuery) + } else { + // Replace the last path operator of the input path + val lastOpLength = lastPathOp.serialize.length + PathToReplace(request.cursorPosition - lastOpLength, lastOpLength + request.remainingStringInOperator.length, fullQuery) + } } private def extractTextPart(pathOp: PathOperator): Option[String] = { @@ -60,8 +82,20 @@ object PartialSourcePathAutocompletionHelper { } } - private def extractQuery(input: String): Seq[String] = { - StringUtils.extractSearchTerms(input) + /** Grammar taken from the Turtle EBNF */ + private val PN_CHARS_BASE = """[A-Za-z\x{00C0}-\x{00D6}\x{00D8}-\x{00F6}\x{00F8}-\x{02FF}\x{0370}-\x{037D}\x{037F}-\x{1FFF}\x{200C}-\x{200D}\x{2070}-\x{218F}\x{2C00}-\x{2FEF}\x{3001}-\x{D7FF}\x{F900}-\x{FDCF}\x{FDF0}-\x{FFFD}\x{10000}-\x{EFFFF}]""" + private val PN_CHARS_U = s"""$PN_CHARS_BASE|_""" + private val PN_CHARS = s"""$PN_CHARS_U|-|[0-9]|\\x{00B7}|[\\x{0300}-\\x{036F}]|[\\x{203F}-\\x{2040}]""" + private val prefixRegex = s"""$PN_CHARS_BASE(($PN_CHARS|\\.)*$PN_CHARS)?""" + private val startsWithPrefix = s"""^$prefixRegex:""".r + + private def extractQuery(input: String): String = { + var inputToProcess: String = input + if(!input.contains("<") && input.contains(":") && !input.contains(" ") && startsWithPrefix.findFirstMatchIn(input).isDefined) { + // heuristic to detect qualified names + inputToProcess = input.drop(input.indexOf(":") + 1) + } + StringUtils.extractSearchTerms(inputToProcess).mkString(" ") } } @@ -69,7 +103,7 @@ object PartialSourcePathAutocompletionHelper { * The part of the input path that should be replaced. * @param from The start index of the substring that should be replaced. * @param length The length in characters of the string that should be replaced. - * @param query Extracted query as multi-word sequence, from the position of the cursor. + * @param query Extracted query from the characters around the position of the cursor. * If it is None this means that no query should be asked to find suggestions, i.e. only suggest operator or nothing. */ -case class PathToReplace(from: Int, length: Int, query: Option[Seq[String]]) \ No newline at end of file +case class PathToReplace(from: Int, length: Int, query: Option[String]) \ No newline at end of file diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index 6a0cd933ca..98ad383519 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -41,11 +41,11 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl suggestedValues(result) mustBe Seq("department/id", "department/tags/tagId") } - it should "auto-complete JSON paths at any level" in { + it should "auto-complete JSON (also XML) paths at any level" in { val level1EndInput = "department/id" val level1Prefix = "department/" // Return all relative paths that match "id" for any cursor position after the first slash - for(cursorPosition <- level1EndInput.length - 1 to level1EndInput.length) { // TODO: Change to -2 + for(cursorPosition <- level1EndInput.length - 2 to level1EndInput.length) { jsonSuggestions(level1EndInput, cursorPosition) mustBe Seq("id", "tags/tagId") } } diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala index 47a887d4bb..1af2f613ef 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala @@ -8,13 +8,27 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche def replace(inputString: String, cursorPosition: Int, - openWorld: Boolean = false): PathToReplace = { - PartialSourcePathAutocompletionHelper.pathToReplace(PartialSourcePathAutoCompletionRequest(inputString, cursorPosition, None), openWorld)(Prefixes.empty) + subPathOnly: Boolean = false): PathToReplace = { + PartialSourcePathAutocompletionHelper.pathToReplace(PartialSourcePathAutoCompletionRequest(inputString, cursorPosition, None), subPathOnly)(Prefixes.empty) } - it should "correctly find out which part of a path to replace" in { + it should "correctly find out which part of a path to replace in simple forward paths for the sub path the cursor is in" in { val input = "a1/b1/c1" - replace(input, 4) mustBe PathToReplace(2, 3, Some(Seq("b1"))) - replace(input, 5) mustBe PathToReplace(2, 3, Some(Seq("b1"))) + replace(input, 4, subPathOnly = true) mustBe PathToReplace(2, 3, Some("b1")) + replace(input, 5, subPathOnly = true) mustBe PathToReplace(2, 3, Some("b1")) + } + + it should "correctly find out which part of a path to replace in simple forward paths for the path prefix the cursor is in " in { + val input = "a1/b1/c1" + replace(input, 4) mustBe PathToReplace(2, 6, Some("b1")) + replace(input, 5) mustBe PathToReplace(2, 6, Some("b1")) + } + + it should "correctly find out what part to replace in mixed forward and backward paths" in { + val input = """\/qns:propper/""" + val initialCursorPosition = "\\/".length + for(cursorPosition <- initialCursorPosition to (initialCursorPosition + 11)) { + replace(input, cursorPosition, subPathOnly = true) mustBe PathToReplace(initialCursorPosition - 1, "/qns:propper".length, Some("propper")) + } } } From 225dbb15218cf8c541b790a5dade454f54cea451 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Thu, 15 Apr 2021 13:07:16 +0200 Subject: [PATCH 05/95] Add source path syntax validation endpoint --- .../workspaceApi/ValidationApi.scala | 26 +++++++++++++++ .../SourcePathValidationRequest.scala | 27 +++++++++++++++ .../conf/workspaceApi.routes | 4 +++ .../workspaceApi/ValidationApiTest.scala | 33 +++++++++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/ValidationApi.scala create mode 100644 silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/validation/SourcePathValidationRequest.scala create mode 100644 silk-workbench/silk-workbench-workspace/test/controllers/workspaceApi/ValidationApiTest.scala diff --git a/silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/ValidationApi.scala b/silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/ValidationApi.scala new file mode 100644 index 0000000000..4cf4195609 --- /dev/null +++ b/silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/ValidationApi.scala @@ -0,0 +1,26 @@ +package controllers.workspaceApi + +import controllers.core.RequestUserContextAction +import controllers.core.util.ControllerUtilsTrait +import controllers.workspaceApi.validation.{SourcePathValidationRequest, SourcePathValidationResponse} +import org.silkframework.config.Prefixes +import org.silkframework.entity.paths.UntypedPath +import org.silkframework.runtime.activity.UserContext +import play.api.libs.json.{JsValue, Json} +import play.api.mvc.{Action, InjectedController} + +import javax.inject.Inject + +/** API to validate different aspects of workspace artifacts. */ +class ValidationApi @Inject() () extends InjectedController with ControllerUtilsTrait { + /** Validates the syntax of a Silk source path expression and returns parse error details. + * Also validate prefix names that they have a valid prefix. */ + def validateSourcePath(projectId: String): Action[JsValue] = RequestUserContextAction(parse.json) { implicit request => implicit userContext: UserContext => + implicit val prefixes: Prefixes = getProject(projectId).config.prefixes + validateJson[SourcePathValidationRequest] { request => + val parseError = UntypedPath.partialParse(request.pathExpression).error + val response = SourcePathValidationResponse(valid = parseError.isEmpty, parseError = parseError) + Ok(Json.toJson(response)) + } + } +} diff --git a/silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/validation/SourcePathValidationRequest.scala b/silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/validation/SourcePathValidationRequest.scala new file mode 100644 index 0000000000..76f777eb7f --- /dev/null +++ b/silk-workbench/silk-workbench-workspace/app/controllers/workspaceApi/validation/SourcePathValidationRequest.scala @@ -0,0 +1,27 @@ +package controllers.workspaceApi.validation + +import org.silkframework.entity.paths.PartialParseError +import play.api.libs.json.{Format, Json} + +/** + * Request to validate a source path, e.g. used in transform or linking tasks. + * + * @param pathExpression The Silk path expression to be validated. + */ +case class SourcePathValidationRequest(pathExpression: String) + +object SourcePathValidationRequest { + implicit val sourcePathValidationRequestFormat: Format[SourcePathValidationRequest] = Json.format[SourcePathValidationRequest] +} + +/** + * Response for a source path validation request. + * @param valid If the path expression id valid or not. + * @param parseError If not valid, this contains the parse error details. + */ +case class SourcePathValidationResponse(valid: Boolean, parseError: Option[PartialParseError]) + +object SourcePathValidationResponse { + implicit val partialParseErrorFormat: Format[PartialParseError] = Json.format[PartialParseError] + implicit val sourcePathValidationResponseFormat: Format[SourcePathValidationResponse] = Json.format[SourcePathValidationResponse] +} \ No newline at end of file diff --git a/silk-workbench/silk-workbench-workspace/conf/workspaceApi.routes b/silk-workbench/silk-workbench-workspace/conf/workspaceApi.routes index 4dc5113057..87cbb6765b 100644 --- a/silk-workbench/silk-workbench-workspace/conf/workspaceApi.routes +++ b/silk-workbench/silk-workbench-workspace/conf/workspaceApi.routes @@ -16,4 +16,8 @@ GET /projectImport/:projectImportId/status GET /reports/list controllers.workspaceApi.ReportsApi.listReports(projectId: Option[String] ?= None, taskId: Option[String] ?= None) GET /reports/report controllers.workspaceApi.ReportsApi.retrieveReport(projectId, taskId, time) +# Validation + +POST /validation/sourcePath/:projectId controllers.workspaceApi.ValidationApi.validateSourcePath(projectId: String) + -> /projects projectsApi.Routes diff --git a/silk-workbench/silk-workbench-workspace/test/controllers/workspaceApi/ValidationApiTest.scala b/silk-workbench/silk-workbench-workspace/test/controllers/workspaceApi/ValidationApiTest.scala new file mode 100644 index 0000000000..f0aad35de9 --- /dev/null +++ b/silk-workbench/silk-workbench-workspace/test/controllers/workspaceApi/ValidationApiTest.scala @@ -0,0 +1,33 @@ +package controllers.workspaceApi + +import controllers.workspaceApi.validation.{SourcePathValidationRequest, SourcePathValidationResponse} +import helper.IntegrationTestTrait +import org.scalatest.{BeforeAndAfterAll, FlatSpec, MustMatchers} +import org.silkframework.entity.paths.PartialParseError +import org.silkframework.serialization.json.JsonHelpers +import play.api.libs.json.Json +import play.api.routing.Router + +class ValidationApiTest extends FlatSpec with IntegrationTestTrait with MustMatchers { + behavior of "Validation API" + + val projectId = "testProject" + + override def workspaceProviderId: String = "inMemory" + + override def routes: Option[Class[_ <: Router]] = Some(classOf[testWorkspace.Routes]) + + it should "validate the syntax of Silk source path expressions" in { + createProject(projectId) + validateSourcePathRequest("""/valid/path[subPath = "filter value"]""") mustBe SourcePathValidationResponse(true, None) + val invalidResult = validateSourcePathRequest("""/invalid/path with spaces at the wrong place""") + invalidResult.valid mustBe false + invalidResult.parseError.get.copy(message = "") mustBe PartialParseError("/invalid/path".length, "", " ") + } + + private def validateSourcePathRequest(pathExpression: String): SourcePathValidationResponse = { + val response = client.url(s"$baseUrl/api/workspace/validation/sourcePath/$projectId") + .post(Json.toJson(SourcePathValidationRequest(pathExpression))) + JsonHelpers.fromJsonValidated[SourcePathValidationResponse](checkResponse(response).json) + } +} From 32ad5c400209ec62fbb3de393ec0fb666842ec32 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Thu, 15 Apr 2021 16:25:06 +0200 Subject: [PATCH 06/95] Support auto-completion of path inside filters --- .../entity/paths/PathParser.scala | 22 +++++-- .../entity/paths/PathParserTest.scala | 31 +++++++--- .../transform/AutoCompletionApi.scala | 48 ++++++++++----- ...artialSourcePathAutocompletionHelper.scala | 60 +++++++++++++++---- .../PartialAutoCompletionApiTest.scala | 23 ++++++- ...alSourcePathAutocompletionHelperTest.scala | 5 ++ .../workspaceApi/ValidationApiTest.scala | 2 +- 7 files changed, 152 insertions(+), 39 deletions(-) diff --git a/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala b/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala index 98fa4cf510..d1c5c3aa12 100644 --- a/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala +++ b/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala @@ -18,6 +18,7 @@ import org.silkframework.config.Prefixes import org.silkframework.runtime.validation.ValidationException import org.silkframework.util.Uri +import scala.util.matching.Regex import scala.util.parsing.combinator.RegexParsers import scala.util.parsing.input.CharSequenceReader @@ -66,7 +67,8 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { partialParseError = Some(PartialParseError( originalErrorOffset, error.msg, - pathStr.substring(originalParseOffset, originalErrorOffset + 1) // + 1 since we want to have the character where it failed + pathStr.substring(originalParseOffset, originalErrorOffset + 1), // + 1 since we want to have the character where it failed + originalParseOffset )) } } catch { @@ -75,7 +77,8 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { partialParseError = Some(PartialParseError( originalParseOffset, validationException.getMessage, - "" + "", + originalParseOffset )) } } @@ -124,7 +127,7 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { } // An identifier that is either a URI enclosed in angle brackets (e.g., ) or a plain identifier (e.g., name or prefix:name) - private def identifier = """<[^>]+>|[^\\/\[\]<>=!" ]+""".r + private def identifier: Regex = PathParser.Regexes.identifier // A language tag according to the Sparql spec private def languageTag = """[a-zA-Z]+('-'[a-zA-Z0-9]+)*""".r @@ -136,6 +139,17 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { private def compOperator = ">" | "<" | ">=" | "<=" | "=" | "!=" } +object PathParser { + object RegexParts { + val uriExpression = "<[^>]+>" + val plainIdentifier = """[^\\/\[\]<>=!" ]+""" + } + object Regexes { + import RegexParts._ + val identifier: Regex = (s"$uriExpression|$plainIdentifier").r + } +} + /** * A partial path parse result. * @@ -145,4 +159,4 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { case class PartialParseResult(partialPath: UntypedPath, error: Option[PartialParseError]) /** Offset and error message of the parse error. The offset defines the position before the character that lead to the parse error. */ -case class PartialParseError(offset: Int, message: String, inputLeadingToError: String) \ No newline at end of file +case class PartialParseError(offset: Int, message: String, inputLeadingToError: String, nextParseOffset: Int) \ No newline at end of file diff --git a/silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala b/silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala index 7c020ae675..dcb44ea5c0 100644 --- a/silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala +++ b/silk-core/src/test/scala/org/silkframework/entity/paths/PathParserTest.scala @@ -2,6 +2,7 @@ package org.silkframework.entity.paths import org.scalatest.{FlatSpec, MustMatchers} import org.silkframework.config.Prefixes +import org.silkframework.util.Uri class PathParserTest extends FlatSpec with MustMatchers { behavior of "path parser" @@ -12,30 +13,42 @@ class PathParserTest extends FlatSpec with MustMatchers { it should "parse the full path and not return any errors for valid paths" in { testValidPath("/a/b[d = 5]\\c") testValidPath("""\[ != "check value"]/abc""") + testValidPath("""abc[@lang ='en']""").operators.drop(1).head mustBe LanguageFilter("=", "en") + testValidPath("""abc[ @lang = 'en']""").operators.drop(1).head mustBe LanguageFilter("=", "en") } it should "parse invalid paths and return the parsed part and the position where it failed" in { - testInvalidPath("/a/b/c/not valid/d/e", "/a/b/c/not", 10, SPACE) - testInvalidPath(s"$SPACE/alreadyInvalid/a", "", 0, SPACE) - testInvalidPath("""\/a[b :+ 1]/c""", "\\/a", 17, "[b ") - testInvalidPath("""/a\b/c/""", """/a\b/c""",6, "/") - testInvalidPath("""invalidNs:broken""", "",0, "") - testInvalidPath("""<'""", "",0, "<") + testInvalidPath(inputString = "/a/b/c/not valid/d/e", expectedParsedString = "/a/b/c/not", expectedErrorOffset = 10, expectedInputOnError = SPACE, expectedNextParseOffset = "/a/b/c/not".length) + testInvalidPath(inputString = s"$SPACE/alreadyInvalid/a", expectedParsedString = "", expectedErrorOffset = 0, expectedInputOnError = SPACE, expectedNextParseOffset = 0) + testInvalidPath(inputString = """\/a[b :+ 1]/c""", expectedParsedString = "\\/a", expectedErrorOffset = 17, expectedInputOnError = "[b ", expectedNextParseOffset = "\\/a".length) + testInvalidPath(inputString = """/a\b/c/""", expectedParsedString = """/a\b/c""",expectedErrorOffset = 6, expectedInputOnError = "/", expectedNextParseOffset = 6) + testInvalidPath(inputString = """invalidNs:broken""", expectedParsedString = "",expectedErrorOffset = 0, expectedInputOnError = "", expectedNextParseOffset = 0) + testInvalidPath(inputString = """<'""", expectedParsedString = "",expectedErrorOffset = 0, expectedInputOnError = "<", expectedNextParseOffset = 0) } - private def testValidPath(inputString: String): Unit = { + it should "partially parse filter expressions correctly" in { + parser.parseUntilError("""a/b[@lang = "en"]""").error must not be defined + val result = parser.parseUntilError("""a/b[@lang = "en"]/error now""") + val error = result.error + error mustBe defined + error.get.offset mustBe """a/b[@lang = "en"]/error""".length + } + + private def testValidPath(inputString: String): UntypedPath = { val result = parser.parseUntilError(inputString) result mustBe PartialParseResult(UntypedPath.parse(inputString), None) + result.partialPath } private def testInvalidPath(inputString: String, expectedParsedString: String, expectedErrorOffset: Int, - expectedInputOnError: String): Unit = { + expectedInputOnError: String, + expectedNextParseOffset: Int): Unit = { val result = parser.parseUntilError(inputString) result.copy(error = result.error.map(e => e.copy(message = ""))) mustBe PartialParseResult( UntypedPath.parse(expectedParsedString), - Some(PartialParseError(expectedErrorOffset, "", expectedInputOnError)) + Some(PartialParseError(expectedErrorOffset, "", expectedInputOnError, expectedNextParseOffset)) ) } } diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index b4e44d50f2..98629a9a96 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -11,7 +11,7 @@ import org.silkframework.rule.vocab.ObjectPropertyType import org.silkframework.rule.{TransformRule, TransformSpec} import org.silkframework.runtime.activity.UserContext import org.silkframework.runtime.plugin.{PluginDescription, PluginRegistry} -import org.silkframework.runtime.validation.NotFoundException +import org.silkframework.runtime.validation.{BadUserInputException, NotFoundException} import org.silkframework.serialization.json.JsonHelpers import org.silkframework.workspace.activity.transform.{TransformPathsCache, VocabularyCacheValue} import org.silkframework.workspace.{ProjectTask, WorkspaceFactory} @@ -64,6 +64,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU val (project, transformTask) = projectAndTask[TransformSpec](projectId, transformTaskId) implicit val prefixes: Prefixes = project.config.prefixes validateJson[PartialSourcePathAutoCompletionRequest] { autoCompletionRequest => + validatePartialSourcePathAutoCompletionRequest(autoCompletionRequest) withRule(transformTask, ruleId) { case (_, sourcePath) => val simpleSourcePath = simplePath(sourcePath) val forwardOnlySourcePath = forwardOnlyPath(simpleSourcePath) @@ -72,9 +73,9 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU var relativeForwardPaths = relativePaths(simpleSourcePath, forwardOnlySourcePath, allPaths, isRdfInput) val pathToReplace = PartialSourcePathAutocompletionHelper.pathToReplace(autoCompletionRequest, isRdfInput) if(!isRdfInput && pathToReplace.from > 0) { - val pathBeforeReplacement = UntypedPath.parse(autoCompletionRequest.inputString.take(pathToReplace.from)) + val pathBeforeReplacement = UntypedPath.partialParse(autoCompletionRequest.inputString.take(pathToReplace.from)).partialPath val simplePathBeforeReplacement = simplePath(pathBeforeReplacement.operators) - relativeForwardPaths = relativePaths(simplePathBeforeReplacement, forwardOnlyPath(simplePathBeforeReplacement), relativeForwardPaths, isRdfInput) + relativeForwardPaths = relativePaths(simplePathBeforeReplacement, forwardOnlyPath(simplePathBeforeReplacement), relativeForwardPaths, isRdfInput, oneHopOnly = pathToReplace.insideFilter) } // Add known paths val completions: Completions = relativeForwardPaths @@ -100,6 +101,15 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU } } + private def validatePartialSourcePathAutoCompletionRequest(request: PartialSourcePathAutoCompletionRequest): Unit = { + var error = "" + if(request.cursorPosition < 0) error = "Cursor position must be >= 0" + if(request.maxSuggestions.nonEmpty && request.maxSuggestions.get <= 0) error = "Max suggestions must be larger zero" + if(error != "") { + throw BadUserInputException(error) + } + } + private def withRule[T](transformTask: ProjectTask[TransformSpec], ruleId: String) (block: ((TransformRule, List[PathOperator])) => T): T = { @@ -116,26 +126,36 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU private def relativePaths(simpleSourcePath: List[PathOperator], forwardOnlySourcePath: List[PathOperator], pathCacheCompletions: Completions, - isRdfInput: Boolean) + isRdfInput: Boolean, + oneHopOnly: Boolean = false) (implicit prefixes: Prefixes): Seq[Completion] = { pathCacheCompletions.values.filter { p => val path = UntypedPath.parse(p.value) - isRdfInput || // FIXME: Currently there are no paths longer 1 in cache, that why return full path - path.operators.startsWith(forwardOnlySourcePath) && path.operators.size > forwardOnlySourcePath.size || - path.operators.startsWith(simpleSourcePath) && path.operators.size > simpleSourcePath.size + val matchesPrefix = isRdfInput || // FIXME: Currently there are no paths longer 1 in cache, that why return full path + path.operators.startsWith(forwardOnlySourcePath) && path.operators.size > forwardOnlySourcePath.size || + path.operators.startsWith(simpleSourcePath) && path.operators.size > simpleSourcePath.size + val truncatedOps = truncatePath(path, simpleSourcePath, forwardOnlySourcePath, isRdfInput) + matchesPrefix && (!oneHopOnly || truncatedOps.size == 1) } map { completion => val path = UntypedPath.parse(completion.value) - val truncatedOps = if (path.operators.startsWith(forwardOnlySourcePath)) { - path.operators.drop(forwardOnlySourcePath.size) - } else if(isRdfInput) { - path.operators - } else { - path.operators.drop(simpleSourcePath.size) - } + val truncatedOps = truncatePath(path, simpleSourcePath, forwardOnlySourcePath, isRdfInput) completion.copy(value = UntypedPath(truncatedOps).serialize()) } } + private def truncatePath(path: UntypedPath, + simpleSourcePath: List[PathOperator], + forwardOnlySourcePath: List[PathOperator], + isRdfInput: Boolean): List[PathOperator] = { + if (path.operators.startsWith(forwardOnlySourcePath)) { + path.operators.drop(forwardOnlySourcePath.size) + } else if(isRdfInput) { + path.operators + } else { + path.operators.drop(simpleSourcePath.size) + } + } + // Normalize this path by eliminating backward operators private def forwardOnlyPath(simpleSourcePath: List[PathOperator]): List[PathOperator] = { // Remove BackwardOperators diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala index a064651bf9..43243c4192 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala @@ -1,7 +1,7 @@ package controllers.transform.autoCompletion import org.silkframework.config.Prefixes -import org.silkframework.entity.paths.{DirectionalPathOperator, PartialParseResult, PathOperator, PathParser, UntypedPath} +import org.silkframework.entity.paths.{DirectionalPathOperator, PartialParseError, PartialParseResult, PathOperator, PathParser, UntypedPath} import org.silkframework.util.StringUtils object PartialSourcePathAutocompletionHelper { @@ -13,6 +13,7 @@ object PartialSourcePathAutocompletionHelper { def pathToReplace(request: PartialSourcePathAutoCompletionRequest, subPathOnly: Boolean) (implicit prefixes: Prefixes): PathToReplace = { + // TODO: Refactor method, clean up val unfilteredQuery: Option[String] = Some("") if(request.inputString.isEmpty) { return PathToReplace(0, 0, unfilteredQuery) @@ -23,9 +24,7 @@ object PartialSourcePathAutocompletionHelper { val errorOffsetCharacter = request.inputString.substring(error.offset, error.offset + 1) val parseStartCharacter = if(error.inputLeadingToError.isEmpty) errorOffsetCharacter else error.inputLeadingToError.take(1) if(error.inputLeadingToError.startsWith("[")) { - // Error happened inside of a filter - // TODO: Find out where in the filter we are and try to auto-complete the correct thing - PathToReplace(0, 0, unfilteredQuery) + handleFilter(request, unfilteredQuery, error) } else if(parseStartCharacter == "/" || parseStartCharacter == "\\") { // It tried to parse a forward or backward path and failed, replace path and use path value as query val operatorValue = request.inputString.substring(error.offset + 1, request.cursorPosition) + request.remainingStringInOperator @@ -46,6 +45,45 @@ object PartialSourcePathAutocompletionHelper { handleSubPathOnly(request, replacement, subPathOnly) } + private def handleFilter(request: PartialSourcePathAutoCompletionRequest, unfilteredQuery: Option[String], error: PartialParseError): PathToReplace = { + // Error happened inside of a filter + // Characters that will usually end an identifier in a filter expression. For some URIs this could lead to false positives, e.g. that contain '='. + val stopChars = Set('!', '=', '<', '>', ']') + val pathFromFilter = request.inputString.substring(error.nextParseOffset) + val insideFilterExpression = pathFromFilter.drop(1).trim + if (insideFilterExpression.startsWith("@lang")) { + // Not sure what to propose inside a lang filter + PathToReplace(0, 0, None) + } else { + var identifier = "" + if (insideFilterExpression.startsWith("<")) { + // URI following + val uriEndingIdx = insideFilterExpression.indexOf('>') + if (uriEndingIdx > 0) { + identifier = insideFilterExpression.take(uriEndingIdx + 1) + } else { + // URI is not closed, just take everything until either an operator or filter closing + identifier = insideFilterExpression.take(1) + insideFilterExpression.drop(1).takeWhile(c => !stopChars.contains(c)) + } + } else { + identifier = insideFilterExpression.takeWhile(c => !stopChars.contains(c)) + } + if(identifier.nonEmpty) { + val pathFromFilterToCursor = request.inputString.substring(error.nextParseOffset, request.cursorPosition) + if(pathFromFilterToCursor.contains(identifier)) { + // The cursor is behind the identifier / URI TODO: auto-complete comparison operator + PathToReplace(0, 0, Some("TODO")) + } else { + // Suggest to replace the identifier + PathToReplace(error.nextParseOffset + 1, error.nextParseOffset + 1 + identifier.stripSuffix(" ").length, Some(extractQuery(identifier)), insideFilter = true) + } + } else { + // Not sure what to replace TODO: Handle case of empty filter expression + PathToReplace(request.cursorPosition, 0, None, insideFilter = true) + } + } + } + private def handleSubPathOnly(request: PartialSourcePathAutoCompletionRequest, pathToReplace: PathToReplace, subPathOnly: Boolean): PathToReplace = { if(subPathOnly || pathToReplace.query.isEmpty) { pathToReplace @@ -90,7 +128,7 @@ object PartialSourcePathAutocompletionHelper { private val startsWithPrefix = s"""^$prefixRegex:""".r private def extractQuery(input: String): String = { - var inputToProcess: String = input + var inputToProcess: String = input.trim if(!input.contains("<") && input.contains(":") && !input.contains(" ") && startsWithPrefix.findFirstMatchIn(input).isDefined) { // heuristic to detect qualified names inputToProcess = input.drop(input.indexOf(":") + 1) @@ -101,9 +139,11 @@ object PartialSourcePathAutocompletionHelper { /** * The part of the input path that should be replaced. - * @param from The start index of the substring that should be replaced. - * @param length The length in characters of the string that should be replaced. - * @param query Extracted query from the characters around the position of the cursor. - * If it is None this means that no query should be asked to find suggestions, i.e. only suggest operator or nothing. + * + * @param from The start index of the substring that should be replaced. + * @param length The length in characters of the string that should be replaced. + * @param query Extracted query from the characters around the position of the cursor. + * If it is None this means that no query should be asked to find suggestions, i.e. only suggest operator or nothing. + * @param insideFilter If the path to be replaced is inside a filter expression */ -case class PathToReplace(from: Int, length: Int, query: Option[String]) \ No newline at end of file +case class PathToReplace(from: Int, length: Int, query: Option[String], insideFilter: Boolean = false) \ No newline at end of file diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index 98ad383519..4e93073cdb 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -43,11 +43,32 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl it should "auto-complete JSON (also XML) paths at any level" in { val level1EndInput = "department/id" - val level1Prefix = "department/" // Return all relative paths that match "id" for any cursor position after the first slash for(cursorPosition <- level1EndInput.length - 2 to level1EndInput.length) { jsonSuggestions(level1EndInput, cursorPosition) mustBe Seq("id", "tags/tagId") } + val level2EndInput = "department/tags/id" + for(cursorPosition <- level2EndInput.length - 2 to level2EndInput.length) { + jsonSuggestions(level2EndInput, cursorPosition) mustBe Seq("tagId") + } + } + + it should "return a client error on invalid requests" in { + intercept[AssertionError] { + partialSourcePathAutoCompleteRequest(jsonTransform, cursorPosition = -1) + } + intercept[AssertionError] { + partialSourcePathAutoCompleteRequest(jsonTransform, maxSuggestions = Some(0)) + } + } + + it should "auto-complete JSON paths inside filter expressions" in { + val inputWithFilter = """department[id = "department X"]/tags[id = ""]/tagId""" + val filterStartIdx = "department[".length + val secondFilterStartIdx = """department[id = "department X"]/tags[""".length + jsonSuggestions(inputWithFilter, filterStartIdx + 2) mustBe Seq("id") + jsonSuggestions(inputWithFilter, filterStartIdx) mustBe Seq("id") + jsonSuggestions(inputWithFilter, secondFilterStartIdx) mustBe Seq("tagId") } private def jsonSuggestions(inputText: String, cursorPosition: Int): Seq[String] = { diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala index 1af2f613ef..466fab2d8e 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala @@ -31,4 +31,9 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche replace(input, cursorPosition, subPathOnly = true) mustBe PathToReplace(initialCursorPosition - 1, "/qns:propper".length, Some("propper")) } } + + it should "correctly find out what to replace in filter expressions" in { + replace("""a/b[@lang = "en"]/error now""", """a/b[@lang = "en"]/err""".length) mustBe + PathToReplace("a/b[@lang = \"en\"]".length, "/error now".length, Some("error now")) + } } diff --git a/silk-workbench/silk-workbench-workspace/test/controllers/workspaceApi/ValidationApiTest.scala b/silk-workbench/silk-workbench-workspace/test/controllers/workspaceApi/ValidationApiTest.scala index f0aad35de9..8f2385624c 100644 --- a/silk-workbench/silk-workbench-workspace/test/controllers/workspaceApi/ValidationApiTest.scala +++ b/silk-workbench/silk-workbench-workspace/test/controllers/workspaceApi/ValidationApiTest.scala @@ -22,7 +22,7 @@ class ValidationApiTest extends FlatSpec with IntegrationTestTrait with MustMatc validateSourcePathRequest("""/valid/path[subPath = "filter value"]""") mustBe SourcePathValidationResponse(true, None) val invalidResult = validateSourcePathRequest("""/invalid/path with spaces at the wrong place""") invalidResult.valid mustBe false - invalidResult.parseError.get.copy(message = "") mustBe PartialParseError("/invalid/path".length, "", " ") + invalidResult.parseError.get.copy(message = "") mustBe PartialParseError("/invalid/path".length, "", " ", "/invalid/path".length) } private def validateSourcePathRequest(pathExpression: String): SourcePathValidationResponse = { From 4149f7f1dd49bcc698fe06083169f6db0d01a724 Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 15 Apr 2021 23:41:46 +0100 Subject: [PATCH 07/95] Added dropdown component and input component from codemirror to package.json --- silk-react-components/package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/silk-react-components/package.json b/silk-react-components/package.json index f5169a6a4c..56d64320d9 100644 --- a/silk-react-components/package.json +++ b/silk-react-components/package.json @@ -69,8 +69,10 @@ "react": "^16.13.0", "react-app-polyfill": "^1.0.6", "react-beautiful-dnd": "^13.0.0", + "react-codemirror2": "^7.2.1", "react-dev-utils": "10.2.1", "react-dom": "^16.13.0", + "react-dropdown": "^1.9.2", "reset-css": "^5.0.1", "resolve": "^1.17.0", "resolve-url-loader": "^3.1.1", From 51765e9f4ca688e6cf488d8dee490f3e76ea7da2 Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 15 Apr 2021 23:42:45 +0100 Subject: [PATCH 08/95] Swapped Autocomplete component with AutoSuggestion component with codemirror's editor --- .../MappingRule/ValueRule/ValueRuleForm.tsx | 72 ++++++++++++++----- .../src/HierarchicalMapping/store.ts | 1 + 2 files changed, 57 insertions(+), 16 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx b/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx index 5c2911daf4..7550fddafe 100644 --- a/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx +++ b/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx @@ -17,7 +17,7 @@ import { Checkbox, TextField, } from '@gui-elements/legacy-replacements'; -import _ from 'lodash'; +import _, { debounce } from 'lodash'; import ExampleView from '../ExampleView'; import store from '../../../store'; import { convertToUri } from '../../../utils/convertToUri'; @@ -30,6 +30,8 @@ import { MAPPING_RULE_TYPE_COMPLEX, MAPPING_RULE_TYPE_DIRECT, MESSAGES } from '. import EventEmitter from '../../../utils/EventEmitter'; import { wasTouched } from '../../../utils/wasTouched'; import { newValueIsIRI } from '../../../utils/newValueIsIRI'; +import AutoSuggestion from '../../../components/AutoSuggestion/AutoSuggestion' +import {getSuggestion, pathValidation} from '../../../store' const LANGUAGES_LIST = [ 'en', 'de', 'es', 'fr', 'bs', 'bg', 'ca', 'ce', 'zh', 'hr', 'cs', 'da', 'nl', 'eo', 'fi', 'ka', 'el', 'hu', 'ga', 'is', 'it', @@ -86,6 +88,8 @@ export function ValueRuleForm(props: IProps) { const [label, setLabel] = useState("") const [comment, setComment] = useState("") const [targetProperty, setTargetProperty] = useState("") + const [suggestions, setSuggestions] = useState([]); + const [pathExpressIsInvalid, setPathExpressValidity] = React.useState(true); const state = { loading, @@ -252,6 +256,36 @@ export function ValueRuleForm(props: IProps) { return targetPropertyNotEmpty && languageTagSet; } + // Autosuggestion handlers editor change + + //editor onChange handler + const handleEditorParamsChange = debounce( + (autoCompleteRuleId, inputString, cursorPosition,getReplacementIndexes) => { + getSuggestion(autoCompleteRuleId, inputString, cursorPosition).then( + (suggestions) => { + if (suggestions) { + setSuggestions( + suggestions.data.replacementResults.completions + ); + getReplacementIndexes(suggestions.data.replacementInterval) + } + } + ).catch(() =>setPathExpressValidity(false)); + }, + 500 + ); + + const checkPathValidity = debounce((inputString) => { + pathValidation(inputString).then((response) =>{ + setPathExpressValidity(response?.data?.valid ?? false) + }).catch(() =>setPathExpressValidity(false)) + },200) + + + + + + // template rendering const render = () => { const { id, parentId } = props; @@ -276,21 +310,27 @@ export function ValueRuleForm(props: IProps) { if (type === MAPPING_RULE_TYPE_DIRECT) { sourcePropertyInput = ( - item.label ? `${item.label} <${item.value}>` : item.value} - /> + void + ) => + handleEditorParamsChange( + autoCompleteRuleId, + inputString, + cursorPosition, + getReplacementIndexes + ) + } + /> + ); } else if (type === MAPPING_RULE_TYPE_COMPLEX) { sourcePropertyInput = ( diff --git a/silk-react-components/src/HierarchicalMapping/store.ts b/silk-react-components/src/HierarchicalMapping/store.ts index 314fccf9b3..b7329fb9a9 100644 --- a/silk-react-components/src/HierarchicalMapping/store.ts +++ b/silk-react-components/src/HierarchicalMapping/store.ts @@ -15,6 +15,7 @@ import { import EventEmitter from './utils/EventEmitter'; import { isDebugMode } from './utils/isDebugMode'; import React, {useState} from "react"; +import silkApi from '../api/silkRestApi' const silkStore = rxmq.channel('silk.api'); export const errorChannel = rxmq.channel('errors'); From 76388011795dc6cf9a61a4f37b9b0cf73cb532a7 Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 15 Apr 2021 23:43:27 +0100 Subject: [PATCH 09/95] added api request handlers for the autosuggestion component --- .../src/HierarchicalMapping/store.ts | 19 +++++++++++++++++++ silk-react-components/src/api/silkRestApi.ts | 19 +++++++++++++++++++ silk-react-components/yarn.lock | 17 +++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/silk-react-components/src/HierarchicalMapping/store.ts b/silk-react-components/src/HierarchicalMapping/store.ts index b7329fb9a9..da3fbc3337 100644 --- a/silk-react-components/src/HierarchicalMapping/store.ts +++ b/silk-react-components/src/HierarchicalMapping/store.ts @@ -689,6 +689,25 @@ export const prefixesAsync = () => { .map(returned => returned.body); }; + +export const getSuggestion = (ruleId:string,inputString: string, cursorPosition:number) => { + const { baseUrl, transformTask, project } = getApiDetails(); + return silkApi.getSuggestionsForAutoCompletion( + baseUrl, + project, + transformTask, + ruleId, + inputString, + cursorPosition + ); +} + +export const pathValidation = (inputString:string) => { + const {baseUrl, project} = getApiDetails() + return silkApi.validatePathExpression(baseUrl,project,inputString) +} + + const exportFunctions = { getHierarchyAsync, getRuleAsync, diff --git a/silk-react-components/src/api/silkRestApi.ts b/silk-react-components/src/api/silkRestApi.ts index 87e861aa47..66713fb13f 100644 --- a/silk-react-components/src/api/silkRestApi.ts +++ b/silk-react-components/src/api/silkRestApi.ts @@ -235,6 +235,25 @@ const silkApi = { const encodedSearchTerm = searchTerm ? encodeURIComponent(searchTerm) : "" return `${baseUrl}/transform/tasks/${projectId}/${transformTaskId}/rule/${ruleId}/completions/targetProperties?term=${encodedSearchTerm}&maxResults=${maxResults}&fullUris=${fullUris}` }, + + getSuggestionsForAutoCompletion: function(baseUrl: string, projectId:string, transformTaskId:string, ruleId:string, inputString:string, cursorPosition: number) { + const requestUrl = `${baseUrl}/transform/tasks/${projectId}/${transformTaskId}/rule/${ruleId}/completions/partialSourcePaths`; + const promise = superagent + .post(requestUrl) + .set("Content-Type", CONTENT_TYPE_JSON) + .send({ inputString, cursorPosition, maxSuggestions: 50 }); + return this.handleErrorCode(promise) + }, + + validatePathExpression: function(baseUrl:string, projectId:string, pathExpression: string) { + const requestUrl = `${baseUrl}/api/workspace/validation/sourcePath/${projectId}`; + const promise = superagent + .post(requestUrl) + .set("Content-Type", CONTENT_TYPE_JSON) + .send({ pathExpression }); + return this.handleErrorCode(promise); + } + }; export default silkApi diff --git a/silk-react-components/yarn.lock b/silk-react-components/yarn.lock index c5174e4e09..aa77f077a8 100644 --- a/silk-react-components/yarn.lock +++ b/silk-react-components/yarn.lock @@ -4119,6 +4119,11 @@ classnames@2.2.6, classnames@^2.2, classnames@^2.2.4, classnames@^2.2.5, classna resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.2.6.tgz#43935bffdd291f326dad0a205309b38d00f650ce" integrity sha512-JR/iSQOSt+LQIWwrwEzJ9uk0xfN3mTVYMwt1Ir5mUcSN6pU+V4zQFFaJsclJbPuAUQH+yfWef6tm7l1quW3C8Q== +classnames@^2.2.3: + version "2.3.1" + resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.3.1.tgz#dfcfa3891e306ec1dad105d0e88f4417b8535e8e" + integrity sha512-OlQdbZ7gLfGarSqxesMesDa5uz7KFbID8Kpq/SxIoNGDqY8lSYs0D+hhtBXhcdB3rcbXArFr7vlHheLk1voeNA== + clean-stack@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" @@ -10923,6 +10928,11 @@ react-beautiful-dnd@^13.0.0: redux "^4.0.4" use-memo-one "^1.1.1" +react-codemirror2@^7.2.1: + version "7.2.1" + resolved "https://registry.yarnpkg.com/react-codemirror2/-/react-codemirror2-7.2.1.tgz#38dab492fcbe5fb8ebf5630e5bb7922db8d3a10c" + integrity sha512-t7YFmz1AXdlImgHXA9Ja0T6AWuopilub24jRaQdPVbzUJVNKIYuy3uCFZYa7CE5S3UW6SrSa5nAqVQvtzRF9gw== + react-datetime@^2.11.0: version "2.16.3" resolved "https://registry.yarnpkg.com/react-datetime/-/react-datetime-2.16.3.tgz#7f9ac7d4014a939c11c761d0c22d1fb506cb505e" @@ -10973,6 +10983,13 @@ react-dom@^16.13.0: prop-types "^15.6.2" scheduler "^0.19.1" +react-dropdown@^1.9.2: + version "1.9.2" + resolved "https://registry.yarnpkg.com/react-dropdown/-/react-dropdown-1.9.2.tgz#db6cbc90184e3f6dd7eb40f7ddabac4b09dccf15" + integrity sha512-g4eufErTi5P5T5bGK+VmLl//qvAHy79jm6KKx8G2Tl3mG90bpigb+Aw85P+C2JUdAnIIQdv8kP/oHN314GvAfw== + dependencies: + classnames "^2.2.3" + react-error-overlay@^6.0.7: version "6.0.8" resolved "https://registry.yarnpkg.com/react-error-overlay/-/react-error-overlay-6.0.8.tgz#474ed11d04fc6bda3af643447d85e9127ed6b5de" From 78873ac1b9d57e6416b4ddd0e67dc55cad3ade6e Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 15 Apr 2021 23:45:18 +0100 Subject: [PATCH 10/95] Created Autosuggestion component using codemirrors editor --- .../AutoSuggestion/AutoSuggestion.jsx | 101 ++++++++++++++++++ .../AutoSuggestion/AutoSuggestion.scss | 42 ++++++++ .../components/CodeEditor.jsx | 39 +++++++ 3 files changed, 182 insertions(+) create mode 100644 silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.jsx create mode 100644 silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss create mode 100644 silk-react-components/src/HierarchicalMapping/components/CodeEditor.jsx diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.jsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.jsx new file mode 100644 index 0000000000..afbc9cbab9 --- /dev/null +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.jsx @@ -0,0 +1,101 @@ +import React from "react"; +import Dropdown from "react-dropdown"; +import { Icon } from "@eccenca/gui-elements"; + +import { CodeEditor } from "../CodeEditor"; + +require("./AutoSuggestion.scss"); + +const AutoSuggestion = ({ + onEditorParamsChange, + suggestions = [], + checkPathValidity, + pathIsValid, +}) => { + const [value, setValue] = React.useState(""); + const [inputString, setInputString] = React.useState(""); + const [cursorPosition, setCursorPosition] = React.useState(0); + const [coords, setCoords] = React.useState({}); + const [shouldShowDropdown, setShouldShowDropdown] = React.useState(false); + const [inputSelection, setInputSelection] = React.useState(""); + const [replacementIndexes, setReplacementIndexes] = React.useState({}); + + React.useEffect(() => { + setInputString(() => value); + setShouldShowDropdown(true); + checkPathValidity(inputString); + onEditorParamsChange( + inputString, + cursorPosition, + handleReplacementIndexes + ); + }, [cursorPosition, value, inputString]); + + const handleChange = (val) => { + setValue(val); + }; + + const handleCursorChange = (pos, coords) => { + setCursorPosition(pos.ch); + setCoords(() => coords); + }; + + const handleInputEditorSelection = (selectedText) => { + setInputSelection(selectedText); + }; + + const handleReplacementIndexes = (indexes) => { + setReplacementIndexes(() => ({ ...indexes })); + }; + + const handleDropdownChange = (item) => { + const { from, length } = replacementIndexes; + setValue( + (value) => + `${value.substring(0, from)}${item.value}${value.substring( + length + )}` + ); + setShouldShowDropdown(false); + }; + + const handleInputEditorClear = () => { + if (!pathIsValid) { + setValue(""); + } + }; + + return ( +
+
+ +
+ +
+
+
+ {shouldShowDropdown ? ( + + ) : null} +
+
+ ); +}; + +export default AutoSuggestion; diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss new file mode 100644 index 0000000000..847e3fad68 --- /dev/null +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss @@ -0,0 +1,42 @@ + +@import "~@eccenca/gui-elements/src/configuration.default"; +@import "~react-dropdown/style.css"; + +.ecc-auto-suggestion-box { + display: flex; + flex-flow: column nowrap; + margin-bottom: $ecc-size-blockelement-margin-vertical; + + &__editor-box { + display: flex; + align-items: center; + } + + &__dropdown { + width: 100%; + } +} + +.ecc-input-editor { + width: 100%; + height: 3rem; + overflow-y: hidden; + border-bottom: 1px solid $color-primary; +} + + +.editor__icon { + color: $color-white; + border-radius:50%; + font-size: 1rem !important; + &.confirm { + background-color: $palette-light-green-900; + + } + + &.clear { + background-color: $palette-deep-orange-A700; + + } +} + diff --git a/silk-react-components/src/HierarchicalMapping/components/CodeEditor.jsx b/silk-react-components/src/HierarchicalMapping/components/CodeEditor.jsx new file mode 100644 index 0000000000..531bbb2e49 --- /dev/null +++ b/silk-react-components/src/HierarchicalMapping/components/CodeEditor.jsx @@ -0,0 +1,39 @@ +import React from "react"; +import { Controlled as ControlledEditor } from "react-codemirror2"; +import "codemirror/mode/sparql/sparql.js"; +import PropTypes from "prop-types"; + +export function CodeEditor({ + onChange, + onCursorChange, + onSelection, + mode = "sparql", + value, +}) { + return ( +
+ onSelection(editor.getSelection())} + options={{ + mode: mode, + lineWrapping: true, + lineNumbers: false, + theme: "xq-light", + }} + onCursor={(editor, data) => { + onCursorChange(data, editor.cursorCoords(true)); + }} + onBeforeChange={(editor, data, value) => onChange(value)} + /> +
+ ); +} + +CodeEditor.propTypes = { + mode: PropTypes.string, + value: PropTypes.string.isRequired, + onChange: PropTypes.func.isRequired, + onCursorChange: PropTypes.func.isRequired, + onSelection: PropTypes.func, +}; From be67e9a6e00999b152f606a05535fc68e3c37109 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Mon, 19 Apr 2021 16:13:16 +0200 Subject: [PATCH 11/95] Do not strip forward slash when auto-completing paths from the middle on --- .../app/controllers/transform/AutoCompletionApi.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 98629a9a96..23f55d19b2 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -75,7 +75,8 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU if(!isRdfInput && pathToReplace.from > 0) { val pathBeforeReplacement = UntypedPath.partialParse(autoCompletionRequest.inputString.take(pathToReplace.from)).partialPath val simplePathBeforeReplacement = simplePath(pathBeforeReplacement.operators) - relativeForwardPaths = relativePaths(simplePathBeforeReplacement, forwardOnlyPath(simplePathBeforeReplacement), relativeForwardPaths, isRdfInput, oneHopOnly = pathToReplace.insideFilter) + relativeForwardPaths = relativePaths(simplePathBeforeReplacement, forwardOnlyPath(simplePathBeforeReplacement), + relativeForwardPaths, isRdfInput, oneHopOnly = pathToReplace.insideFilter, serializeFull = true) } // Add known paths val completions: Completions = relativeForwardPaths @@ -127,7 +128,8 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU forwardOnlySourcePath: List[PathOperator], pathCacheCompletions: Completions, isRdfInput: Boolean, - oneHopOnly: Boolean = false) + oneHopOnly: Boolean = false, + serializeFull: Boolean = false) (implicit prefixes: Prefixes): Seq[Completion] = { pathCacheCompletions.values.filter { p => val path = UntypedPath.parse(p.value) @@ -139,7 +141,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU } map { completion => val path = UntypedPath.parse(completion.value) val truncatedOps = truncatePath(path, simpleSourcePath, forwardOnlySourcePath, isRdfInput) - completion.copy(value = UntypedPath(truncatedOps).serialize()) + completion.copy(value = UntypedPath(truncatedOps).serialize(stripForwardSlash = !serializeFull)) } } From 670be744e0a3c7029044351a983483d4c8681860 Mon Sep 17 00:00:00 2001 From: darausi Date: Tue, 20 Apr 2021 00:20:36 +0100 Subject: [PATCH 12/95] fixed UI path replacement --- .../components/AutoSuggestion/AutoSuggestion.jsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.jsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.jsx index afbc9cbab9..6d03d385e4 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.jsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.jsx @@ -50,11 +50,10 @@ const AutoSuggestion = ({ const handleDropdownChange = (item) => { const { from, length } = replacementIndexes; + const to = from + length; setValue( (value) => - `${value.substring(0, from)}${item.value}${value.substring( - length - )}` + `${value.substring(0, from)}${item.value}${value.substring(to)}` ); setShouldShowDropdown(false); }; From b54080f2592b26025904734765548e0bf9657916 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Tue, 20 Apr 2021 11:04:19 +0200 Subject: [PATCH 13/95] Add data source characteristics to data sources that define allowed paths and special forward paths --- .../silkframework/dataset/DataSource.scala | 33 +++++++++++++++++++ .../silkframework/dataset/DatasetSpec.scala | 2 ++ .../silkframework/dataset/EmptySource.scala | 2 ++ .../dataset/EntityDatasource.scala | 2 ++ .../dataset/SafeModeDataSource.scala | 2 ++ .../dataset/bulk/BulkDataSource.scala | 2 ++ .../plugins/dataset/CacheDataset.scala | 5 +-- .../silkframework/dataset/MockDataset.scala | 2 ++ .../plugins/dataset/csv/CsvSource.scala | 10 +++++- .../plugins/dataset/text/TextFileSource.scala | 4 ++- .../plugins/dataset/json/JsonSource.scala | 16 +++++++++ .../plugins/dataset/json/JsonTraverser.scala | 8 ++--- .../rdf/access/CombinedSparqlSource.scala | 4 ++- .../dataset/rdf/access/SparqlSource.scala | 13 ++++++++ .../dataset/rdf/datasets/RdfFileDataset.scala | 3 ++ .../dataset/xml/XmlSourceInMemory.scala | 3 +- .../dataset/xml/XmlSourceStreaming.scala | 2 ++ .../plugins/dataset/xml/XmlSourceTrait.scala | 28 ++++++++++++++-- silk-react-components/package.json | 1 + ...{AutoSuggestion.jsx => AutoSuggestion.tsx} | 3 +- .../{CodeEditor.jsx => CodeEditor.tsx} | 0 silk-react-components/yarn.lock | 19 +++++++++++ .../rule/TransformedDataSource.scala | 6 +++- .../local/LocalLinkSpecExecutor.scala | 4 ++- .../PartialAutoCompletionApiTest.scala | 9 +++++ 25 files changed, 167 insertions(+), 16 deletions(-) rename silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/{AutoSuggestion.jsx => AutoSuggestion.tsx} (98%) rename silk-react-components/src/HierarchicalMapping/components/{CodeEditor.jsx => CodeEditor.tsx} (100%) diff --git a/silk-core/src/main/scala/org/silkframework/dataset/DataSource.scala b/silk-core/src/main/scala/org/silkframework/dataset/DataSource.scala index 5c800dda94..5c514f4466 100644 --- a/silk-core/src/main/scala/org/silkframework/dataset/DataSource.scala +++ b/silk-core/src/main/scala/org/silkframework/dataset/DataSource.scala @@ -106,6 +106,8 @@ trait DataSource { * @return - the unique IRI */ protected def genericEntityIRI(identifier: Identifier): String = DataSource.generateEntityUri(underlyingTask.id, identifier) + + def characteristics: DataSourceCharacteristics } object DataSource{ @@ -129,3 +131,34 @@ object DataSource{ } } } + +/** Characteristics of a data source. + * + * @param supportedPathExpressions The characteristics of the supported path expressions. + */ +case class DataSourceCharacteristics(supportedPathExpressions: SupportedPathExpressions = SupportedPathExpressions()) + +/** The kind of path expressions supported by a data source. + * + * @param multiHopPaths If enabled it is possible to define multi-hop paths (e.g. in RDF, JSON, XML). Else only single-hop + * path are supported. + * @param backwardPaths If the data source supports backward paths, i.e. reversing the direction of a property (e.g. in + * RDF, JSON (parent), XML (parent)). + * @param languageFilter If the data source supports language filters, i.e. is able to filter by different language versions + * of property values (only supported in RDF). + * @param propertyFilter If the data source supports (single-hop forward) property filters. + * @param specialPaths The data source specific paths that are supported by a data source, e.g. row ID in CSV. + */ +case class SupportedPathExpressions(multiHopPaths: Boolean = false, + backwardPaths: Boolean = false, + languageFilter: Boolean = false, + propertyFilter: Boolean = false, + specialPaths: Seq[SpecialPathInfo] = Seq.empty) + +/** + * Information about data source specific special paths that have a special semantic. + * + * @param value The path value. + * @param description Description of the semantics of the special path. + */ +case class SpecialPathInfo(value: String, description: Option[String]) \ No newline at end of file diff --git a/silk-core/src/main/scala/org/silkframework/dataset/DatasetSpec.scala b/silk-core/src/main/scala/org/silkframework/dataset/DatasetSpec.scala index bc0d6fb1c6..d9b03f605a 100644 --- a/silk-core/src/main/scala/org/silkframework/dataset/DatasetSpec.scala +++ b/silk-core/src/main/scala/org/silkframework/dataset/DatasetSpec.scala @@ -202,6 +202,8 @@ object DatasetSpec { * @return */ override def underlyingTask: Task[DatasetSpec[Dataset]] = source.underlyingTask + + override def characteristics: DataSourceCharacteristics = source.characteristics } case class EntitySinkWrapper(entitySink: EntitySink, datasetSpec: DatasetSpec[Dataset]) extends EntitySink { diff --git a/silk-core/src/main/scala/org/silkframework/dataset/EmptySource.scala b/silk-core/src/main/scala/org/silkframework/dataset/EmptySource.scala index 4874e6d27b..c05ecd011b 100644 --- a/silk-core/src/main/scala/org/silkframework/dataset/EmptySource.scala +++ b/silk-core/src/main/scala/org/silkframework/dataset/EmptySource.scala @@ -33,4 +33,6 @@ object EmptySource extends DataSource { } override def underlyingTask: Task[DatasetSpec[Dataset]] = PlainTask("empty_dataset", DatasetSpec(EmptyDataset)) + + override def characteristics: DataSourceCharacteristics = DataSourceCharacteristics() } diff --git a/silk-core/src/main/scala/org/silkframework/dataset/EntityDatasource.scala b/silk-core/src/main/scala/org/silkframework/dataset/EntityDatasource.scala index a6a153e608..ea4e27044e 100644 --- a/silk-core/src/main/scala/org/silkframework/dataset/EntityDatasource.scala +++ b/silk-core/src/main/scala/org/silkframework/dataset/EntityDatasource.scala @@ -54,4 +54,6 @@ case class EntityDatasource(underlyingTask: Task[DatasetSpec[Dataset]], entities (implicit userContext: UserContext): IndexedSeq[TypedPath] = { entitySchema.typedPaths } + + override def characteristics: DataSourceCharacteristics = DataSourceCharacteristics() } diff --git a/silk-core/src/main/scala/org/silkframework/dataset/SafeModeDataSource.scala b/silk-core/src/main/scala/org/silkframework/dataset/SafeModeDataSource.scala index 4aff1e6ce1..47108cc00e 100644 --- a/silk-core/src/main/scala/org/silkframework/dataset/SafeModeDataSource.scala +++ b/silk-core/src/main/scala/org/silkframework/dataset/SafeModeDataSource.scala @@ -20,6 +20,8 @@ object SafeModeDataSource extends DataSource { override def retrieve(entitySchema: EntitySchema, limit: Option[Int])(implicit userContext: UserContext): EntityHolder = SafeModeException.throwSafeModeException() override def retrieveByUri(entitySchema: EntitySchema, entities: Seq[Uri])(implicit userContext: UserContext): EntityHolder = SafeModeException.throwSafeModeException() + + override def characteristics: DataSourceCharacteristics = SafeModeException.throwSafeModeException() } object SafeModeSink extends DataSink with LinkSink with EntitySink { diff --git a/silk-core/src/main/scala/org/silkframework/dataset/bulk/BulkDataSource.scala b/silk-core/src/main/scala/org/silkframework/dataset/bulk/BulkDataSource.scala index 42241f39f9..069d891071 100644 --- a/silk-core/src/main/scala/org/silkframework/dataset/bulk/BulkDataSource.scala +++ b/silk-core/src/main/scala/org/silkframework/dataset/bulk/BulkDataSource.scala @@ -124,4 +124,6 @@ class BulkDataSource(bulkContainerName: String, } override def underlyingTask: Task[DatasetSpec[Dataset]] = PlainTask(Identifier.fromAllowed(bulkContainerName), DatasetSpec(EmptyDataset)) //FIXME CMEM-1352 replace with actual task + + override def characteristics: DataSourceCharacteristics = sources.headOption.map(_.source.characteristics).getOrElse(DataSourceCharacteristics()) } diff --git a/silk-core/src/main/scala/org/silkframework/plugins/dataset/CacheDataset.scala b/silk-core/src/main/scala/org/silkframework/plugins/dataset/CacheDataset.scala index e6fb851968..4a563e2cd3 100644 --- a/silk-core/src/main/scala/org/silkframework/plugins/dataset/CacheDataset.scala +++ b/silk-core/src/main/scala/org/silkframework/plugins/dataset/CacheDataset.scala @@ -15,10 +15,9 @@ package org.silkframework.plugins.dataset import java.io.File - import org.silkframework.cache.FileEntityCache import org.silkframework.config.{PlainTask, RuntimeConfig, Task} -import org.silkframework.dataset.{DataSource, Dataset, DatasetSpec} +import org.silkframework.dataset.{DataSource, DataSourceCharacteristics, Dataset, DatasetSpec} import org.silkframework.entity._ import org.silkframework.entity.paths.TypedPath import org.silkframework.execution.EntityHolder @@ -56,5 +55,7 @@ case class CacheDataset(dir: String) extends Dataset { (implicit userContext: UserContext): IndexedSeq[TypedPath] = IndexedSeq.empty override def underlyingTask: Task[DatasetSpec[Dataset]] = PlainTask("cache_source", DatasetSpec(CacheDataset.this)) + + override def characteristics: DataSourceCharacteristics = DataSourceCharacteristics() } } \ No newline at end of file diff --git a/silk-core/src/test/scala/org/silkframework/dataset/MockDataset.scala b/silk-core/src/test/scala/org/silkframework/dataset/MockDataset.scala index af0db6a7e8..43bfa7bc14 100644 --- a/silk-core/src/test/scala/org/silkframework/dataset/MockDataset.scala +++ b/silk-core/src/test/scala/org/silkframework/dataset/MockDataset.scala @@ -49,6 +49,8 @@ case class DummyDataSource(retrieveFn: (EntitySchema, Option[Int]) => Traversabl (implicit userContext: UserContext): Traversable[(String, Double)] = Traversable.empty override def underlyingTask: Task[DatasetSpec[Dataset]] = EmptySource.underlyingTask + + override def characteristics: DataSourceCharacteristics = DataSourceCharacteristics() } case class DummyLinkSink(writeLinkFn: (Link, String) => Unit, diff --git a/silk-plugins/silk-plugins-csv/src/main/scala/org/silkframework/plugins/dataset/csv/CsvSource.scala b/silk-plugins/silk-plugins-csv/src/main/scala/org/silkframework/plugins/dataset/csv/CsvSource.scala index 621614b878..3ae9806a0d 100644 --- a/silk-plugins/silk-plugins-csv/src/main/scala/org/silkframework/plugins/dataset/csv/CsvSource.scala +++ b/silk-plugins/silk-plugins-csv/src/main/scala/org/silkframework/plugins/dataset/csv/CsvSource.scala @@ -146,7 +146,7 @@ class CsvSource(file: Resource, val property = path.operators.head.asInstanceOf[ForwardOperator].property.uri val propertyIndex = propertyList.indexOf(property.toString) if (propertyIndex == -1) { - if(property == "#idx") { + if(property == CsvSource.INDEX_PATH) { IDX_PATH_IDX } else { missingColumns :+= property @@ -390,6 +390,14 @@ class CsvSource(file: Resource, * @return */ override def underlyingTask: Task[DatasetSpec[Dataset]] = PlainTask(Identifier.fromAllowed(file.name), DatasetSpec(EmptyDataset)) //FIXME CMEM-1352 replace with actual task + + override def characteristics: DataSourceCharacteristics = DataSourceCharacteristics( + SupportedPathExpressions() + ) +} + +object CsvSource { + final val INDEX_PATH = "#idx" } case class CsvAutoconfiguredParameters(detectedSeparator: String, codecName: String, linesToSkip: Option[Int]) diff --git a/silk-plugins/silk-plugins-csv/src/main/scala/org/silkframework/plugins/dataset/text/TextFileSource.scala b/silk-plugins/silk-plugins-csv/src/main/scala/org/silkframework/plugins/dataset/text/TextFileSource.scala index c3913d68ba..b1d63cfb35 100644 --- a/silk-plugins/silk-plugins-csv/src/main/scala/org/silkframework/plugins/dataset/text/TextFileSource.scala +++ b/silk-plugins/silk-plugins-csv/src/main/scala/org/silkframework/plugins/dataset/text/TextFileSource.scala @@ -1,7 +1,7 @@ package org.silkframework.plugins.dataset.text import org.silkframework.config.{PlainTask, Task} -import org.silkframework.dataset.{DataSource, Dataset, DatasetSpec, EmptyDataset} +import org.silkframework.dataset.{DataSource, DataSourceCharacteristics, Dataset, DatasetSpec, EmptyDataset} import org.silkframework.entity.{Entity, EntitySchema} import org.silkframework.entity.paths.TypedPath import org.silkframework.execution.EntityHolder @@ -51,4 +51,6 @@ class TextFileSource(ds: TextFileDataset) extends DataSource { } override def underlyingTask: Task[DatasetSpec[Dataset]] = PlainTask(Identifier.fromAllowed(ds.file.name), DatasetSpec(EmptyDataset)) + + override def characteristics: DataSourceCharacteristics = DataSourceCharacteristics() } diff --git a/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonSource.scala b/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonSource.scala index 78eb24b5b2..91b81ba5d6 100644 --- a/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonSource.scala +++ b/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonSource.scala @@ -218,6 +218,18 @@ case class JsonSource(taskId: Identifier, input: JsValue, basePath: String, uriP * @return */ override lazy val underlyingTask: Task[DatasetSpec[Dataset]] = PlainTask(taskId, DatasetSpec(EmptyDataset)) //FIXME CMEM 1352 replace with actual task + + override def characteristics: DataSourceCharacteristics = DataSourceCharacteristics( + SupportedPathExpressions( + multiHopPaths = true, + backwardPaths = true, + propertyFilter = true, + specialPaths = Seq( + SpecialPathInfo(JsonSource.specialPaths.ID, Some("Hash value of the JSON node or value.")), + SpecialPathInfo(JsonSource.specialPaths.TEXT, Some("The string value of a node. This will turn a JSON object into it's string representation.")) + ) + ) + ) } object JsonSource{ @@ -228,4 +240,8 @@ object JsonSource{ apply(Identifier.fromAllowed(file.name), file.read(Json.parse), basePath, uriPattern) } + object specialPaths { + final val TEXT = "#text" + final val ID = "#id" + } } diff --git a/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonTraverser.scala b/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonTraverser.scala index 2c8c046b3d..a9c3c997ee 100644 --- a/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonTraverser.scala +++ b/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonTraverser.scala @@ -120,14 +120,14 @@ case class JsonTraverser(taskId: Identifier, parentOpt: Option[ParentTraverser], path match { case ForwardOperator(prop) :: tail => prop.uri match { - case "#id" => + case JsonSource.specialPaths.ID => Seq(nodeId(value)) - case "#text" => + case JsonSource.specialPaths.TEXT => nodeToValue(value, generateUris) case _ => children(prop).flatMap(child => child.evaluate(tail, generateUris)) } - case BackwardOperator(prop) :: tail => + case BackwardOperator(_) :: tail => parentOpt match { case Some(parent) => parent.traverser.evaluate(tail, generateUris) @@ -138,7 +138,7 @@ case class JsonTraverser(taskId: Identifier, parentOpt: Option[ParentTraverser], evaluatePropertyFilter(path, p, tail, generateUris) case Nil => nodeToValue(value, generateUris) - case l: LanguageFilter => + case _: LanguageFilter => throw new IllegalArgumentException("For JSON, language filters are not applicable.") } } diff --git a/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/access/CombinedSparqlSource.scala b/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/access/CombinedSparqlSource.scala index e63e0141fa..f89dbe5c08 100644 --- a/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/access/CombinedSparqlSource.scala +++ b/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/access/CombinedSparqlSource.scala @@ -1,7 +1,7 @@ package org.silkframework.plugins.dataset.rdf.access import org.silkframework.config.Task -import org.silkframework.dataset.{DataSource, Dataset, DatasetSpec} +import org.silkframework.dataset.{DataSource, DataSourceCharacteristics, Dataset, DatasetSpec} import org.silkframework.entity.paths.TypedPath import org.silkframework.entity.{Entity, EntitySchema} import org.silkframework.execution.EntityHolder @@ -59,4 +59,6 @@ case class CombinedSparqlSource(underlyingTask: Task[DatasetSpec[Dataset]], spar override def retrievePaths(typeUri: Uri, depth: Int, limit: Option[Int]) (implicit userContext: UserContext): IndexedSeq[TypedPath] = IndexedSeq.empty + + override def characteristics: DataSourceCharacteristics = SparqlSource.characteristics } diff --git a/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/access/SparqlSource.scala b/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/access/SparqlSource.scala index 02d838e692..8be0c1060e 100644 --- a/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/access/SparqlSource.scala +++ b/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/access/SparqlSource.scala @@ -182,4 +182,17 @@ class SparqlSource(params: SparqlParams, val sparqlEndpoint: SparqlEndpoint) SparqlSamplePathsCollector(sparqlEndpoint, params.graph, restriction, limit).toIndexedSeq } } + + override def characteristics: DataSourceCharacteristics = SparqlSource.characteristics +} + +object SparqlSource { + final val characteristics: DataSourceCharacteristics = DataSourceCharacteristics( + SupportedPathExpressions( + multiHopPaths = true, + backwardPaths = true, + propertyFilter = true, + languageFilter = true + ) + ) } diff --git a/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/datasets/RdfFileDataset.scala b/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/datasets/RdfFileDataset.scala index f5dec43931..ecb09c82a3 100644 --- a/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/datasets/RdfFileDataset.scala +++ b/silk-plugins/silk-plugins-rdf/src/main/scala/org/silkframework/plugins/dataset/rdf/datasets/RdfFileDataset.scala @@ -109,6 +109,7 @@ case class RdfFileDataset( // restrict the fetched entities to following URIs private def entityRestriction: Seq[Uri] = SparqlParams.splitEntityList(entityList.str).map(Uri(_)) + /** RDF file data source. */ class FileSource(resource: Resource) extends DataSource with PeakDataSource with Serializable with SamplingDataSource with SchemaExtractionSource with SparqlRestrictionDataSource { @@ -190,6 +191,8 @@ case class RdfFileDataset( } private def sparqlSource = new SparqlSource(SparqlParams(graph = graphOpt), endpoint) + + override def characteristics: DataSourceCharacteristics = SparqlSource.characteristics } override def tripleSink(implicit userContext: UserContext): TripleSink = new FormattedEntitySink(file, formatter) diff --git a/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceInMemory.scala b/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceInMemory.scala index 1a8c88f326..e160637827 100644 --- a/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceInMemory.scala +++ b/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceInMemory.scala @@ -1,7 +1,6 @@ package org.silkframework.plugins.dataset.xml import java.util.logging.{Level, Logger} - import org.silkframework.config.{PlainTask, Task} import org.silkframework.dataset._ import org.silkframework.entity._ @@ -110,6 +109,8 @@ class XmlSourceInMemory(file: Resource, basePath: String, uriPattern: String) ex * @return */ override def underlyingTask: Task[DatasetSpec[Dataset]] = PlainTask(Identifier.fromAllowed(file.name), DatasetSpec(EmptyDataset)) //FIXME CMEM-1352 replace with actual task + + override def characteristics: DataSourceCharacteristics = XmlSourceTrait.characteristics } diff --git a/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceStreaming.scala b/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceStreaming.scala index dad9920274..1b1c935b46 100644 --- a/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceStreaming.scala +++ b/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceStreaming.scala @@ -457,4 +457,6 @@ class XmlSourceStreaming(file: Resource, basePath: String, uriPattern: String) e * @return */ override def underlyingTask: Task[DatasetSpec[Dataset]] = PlainTask(Identifier.fromAllowed(file.name), DatasetSpec(EmptyDataset)) //FIXME CMEM-1352 replace with actual task + + override def characteristics: DataSourceCharacteristics = XmlSourceTrait.characteristics } diff --git a/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceTrait.scala b/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceTrait.scala index 21d86a3205..c1fc8ca163 100644 --- a/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceTrait.scala +++ b/silk-plugins/silk-plugins-xml/src/main/scala/org/silkframework/plugins/dataset/xml/XmlSourceTrait.scala @@ -1,6 +1,6 @@ package org.silkframework.plugins.dataset.xml -import org.silkframework.dataset.DataSource +import org.silkframework.dataset.{DataSource, DataSourceCharacteristics, SpecialPathInfo, SupportedPathExpressions} import org.silkframework.entity.paths.TypedPath import org.silkframework.entity.{Entity, EntitySchema} import org.silkframework.execution.EntityHolder @@ -29,5 +29,29 @@ trait XmlSourceTrait { this: DataSource => retrieve(entitySchema).filter(entity => uriSet.contains(entity.uri.toString)) } } - } + +object XmlSourceTrait { + object SpecialXmlPaths { + final val ID = "#id" + final val TAG = "#tag" + final val TEXT = "#text" + final val ALL_CHILDREN = "*" + final val ALL_CHILDREN_RECURSIVE = "**" + } + import SpecialXmlPaths._ + final val characteristics = DataSourceCharacteristics( + SupportedPathExpressions( + multiHopPaths = true, + propertyFilter = true, + specialPaths = Seq( + SpecialPathInfo("\\..", Some("Navigate to parent element.")), + SpecialPathInfo(ID, Some("A document-wide unique ID of the entity.")), + SpecialPathInfo(TAG, Some("The element tag of the entity.")), + SpecialPathInfo(TEXT, Some("The concatenated text inside an element.")), + SpecialPathInfo(ALL_CHILDREN, Some("Selects all direct children of the entity.")), + SpecialPathInfo(ALL_CHILDREN_RECURSIVE, Some("Selects all children nested below the entity at any depth.")) + ) + ) + ) +} \ No newline at end of file diff --git a/silk-react-components/package.json b/silk-react-components/package.json index 56d64320d9..fc9ba38929 100644 --- a/silk-react-components/package.json +++ b/silk-react-components/package.json @@ -40,6 +40,7 @@ "@testing-library/user-event": "^12.2.2", "@types/carbon-components": "^10.15.0", "@types/carbon-components-react": "^7.10.10", + "@types/codemirror": "0.0.109", "babel-loader": "8.1.0", "babel-plugin-named-asset-import": "^0.3.6", "babel-preset-react-app": "^9.1.2", diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.jsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx similarity index 98% rename from silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.jsx rename to silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index afbc9cbab9..ff249fe484 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.jsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -18,7 +18,7 @@ const AutoSuggestion = ({ const [coords, setCoords] = React.useState({}); const [shouldShowDropdown, setShouldShowDropdown] = React.useState(false); const [inputSelection, setInputSelection] = React.useState(""); - const [replacementIndexes, setReplacementIndexes] = React.useState({}); + const [replacementIndexes, setReplacementIndexes] = React.useState({from:0, length: 0}); React.useEffect(() => { setInputString(() => value); @@ -69,7 +69,6 @@ const AutoSuggestion = ({
Date: Tue, 20 Apr 2021 15:53:09 +0200 Subject: [PATCH 14/95] Auto-suggest path operators that are valid --- .../transform/AutoCompletionApi.scala | 47 +++++++++++++++-- .../autoCompletion/Completions.scala | 4 +- ...rtialSourcePathAutoCompletionRequest.scala | 19 +++++-- .../PartialAutoCompletionApiTest.scala | 51 +++++++++++++++---- 4 files changed, 100 insertions(+), 21 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 23f55d19b2..36401d15a1 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -5,6 +5,9 @@ import controllers.core.{RequestUserContextAction, UserContextAction} import controllers.transform.AutoCompletionApi.Categories import controllers.transform.autoCompletion._ import org.silkframework.config.Prefixes +import org.silkframework.dataset.{DataSourceCharacteristics, SupportedPathExpressions} +import org.silkframework.dataset.DatasetSpec.GenericDatasetSpec +import org.silkframework.dataset.rdf.RdfDataset import org.silkframework.entity.paths.{PathOperator, _} import org.silkframework.entity.{ValueType, ValueTypeAnnotation} import org.silkframework.rule.vocab.ObjectPropertyType @@ -70,13 +73,14 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU val forwardOnlySourcePath = forwardOnlyPath(simpleSourcePath) val allPaths = pathsCacheCompletions(transformTask, simpleSourcePath) val isRdfInput = transformTask.activity[TransformPathsCache].value().isRdfInput(transformTask) + val dataSourceCharacteristicsOpt = dataSourceCharacteristics(transformTask) var relativeForwardPaths = relativePaths(simpleSourcePath, forwardOnlySourcePath, allPaths, isRdfInput) val pathToReplace = PartialSourcePathAutocompletionHelper.pathToReplace(autoCompletionRequest, isRdfInput) if(!isRdfInput && pathToReplace.from > 0) { val pathBeforeReplacement = UntypedPath.partialParse(autoCompletionRequest.inputString.take(pathToReplace.from)).partialPath val simplePathBeforeReplacement = simplePath(pathBeforeReplacement.operators) relativeForwardPaths = relativePaths(simplePathBeforeReplacement, forwardOnlyPath(simplePathBeforeReplacement), - relativeForwardPaths, isRdfInput, oneHopOnly = pathToReplace.insideFilter, serializeFull = true) + relativeForwardPaths, isRdfInput, oneHopOnly = pathToReplace.insideFilter, serializeFull = !pathToReplace.insideFilter) } // Add known paths val completions: Completions = relativeForwardPaths @@ -93,15 +97,50 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU val response = PartialSourcePathAutoCompletionResponse( autoCompletionRequest.inputString, autoCompletionRequest.cursorPosition, - Some(ReplacementInterval(from, length)), - pathToReplace.query.getOrElse(""), - filteredResults.toCompletionsBase + replacementResults = Seq( + ReplacementResults( + ReplacementInterval(from, length), + pathToReplace.query.getOrElse(""), + filteredResults.toCompletionsBase + ) + ) ++ operatorCompletions(dataSourceCharacteristicsOpt, pathToReplace, autoCompletionRequest) ) Ok(Json.toJson(response)) } } } + private def operatorCompletions(dataSourceCharacteristicsOpt: Option[DataSourceCharacteristics], + pathToReplace: PathToReplace, + autoCompletionRequest: PartialSourcePathAutoCompletionRequest): Option[ReplacementResults] = { + def completion(predicate: Boolean, value: String, description: String): Option[CompletionBase] = { + if(predicate) Some(CompletionBase(value, description = Some(description))) else None + } + // Propose operators + if (!pathToReplace.insideFilter + && !autoCompletionRequest.charBeforeCursor.contains('/') && !autoCompletionRequest.charBeforeCursor.contains('\\')) { + val supportedPathExpressions = dataSourceCharacteristicsOpt.getOrElse(DataSourceCharacteristics()).supportedPathExpressions + val forwardOp = completion(autoCompletionRequest.cursorPosition > 0 && supportedPathExpressions.multiHopPaths, "/", "Starts a forward path segment") + val backwardOp = completion(supportedPathExpressions.backwardPaths && supportedPathExpressions.multiHopPaths, "\\", "Starts a backward path segment") + val langFilterOp = completion(supportedPathExpressions.languageFilter, "[@lang ", "Starts a language filter expression") + val propertyFilter = completion(supportedPathExpressions.propertyFilter, "[", "Starts a property filter expression") + Some(ReplacementResults( + ReplacementInterval(autoCompletionRequest.cursorPosition, 0), + "", + CompletionsBase(forwardOp.toSeq ++ backwardOp ++ propertyFilter ++ langFilterOp) + )) + } + else { + None + } + } + + private def dataSourceCharacteristics(task: ProjectTask[TransformSpec]) + (implicit userContext: UserContext): Option[DataSourceCharacteristics] = { + task.project.taskOption[GenericDatasetSpec](task.selection.inputId) + .map(_.data.source.characteristics) + } + private def validatePartialSourcePathAutoCompletionRequest(request: PartialSourcePathAutoCompletionRequest): Unit = { var error = "" if(request.cursorPosition < 0) error = "Cursor position must be >= 0" diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/Completions.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/Completions.scala index 2b8f56a7d4..26d8653078 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/Completions.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/Completions.scala @@ -67,8 +67,8 @@ case class Completions(values: Seq[Completion] = Seq.empty) { /** The base properties for auto-completion results. */ case class CompletionBase(value: String, - label: Option[String], - description: Option[String]) + label: Option[String] = None, + description: Option[String] = None) object CompletionBase { implicit val completionBaseFormat: Format[CompletionBase] = Json.format[CompletionBase] diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala index 81ae0fbda3..b4159343b0 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala @@ -14,6 +14,7 @@ case class PartialSourcePathAutoCompletionRequest(inputString: String, maxSuggestions: Option[Int]) { /** The path until the cursor position. */ lazy val pathUntilCursor: String = inputString.take(cursorPosition) + lazy val charBeforeCursor: Option[Char] = pathUntilCursor.reverse.headOption /** The remaining characters from the cursor position to the end of the current path operator. */ lazy val remainingStringInOperator: String = { @@ -35,14 +36,21 @@ object PartialSourcePathAutoCompletionRequest { * The response for a partial source path auto-completion request. * @param inputString The input string from the request for validation. * @param cursorPosition The cursor position from the request for validation. - * @param replacementInterval An optional interval if there has been found a part of the source path that can be replaced. - * @param replacementResults The auto-completion results. */ case class PartialSourcePathAutoCompletionResponse(inputString: String, cursorPosition: Int, - replacementInterval: Option[ReplacementInterval], - extractedQuery: String, - replacementResults: CompletionsBase) + replacementResults: Seq[ReplacementResults]) + +/** + * Suggested replacement for a specific part of the input string. + * + * @param replacementInterval An optional interval if there has been found a part of the source path that can be replaced. + * @param replacements The auto-completion results. + * @param extractedQuery A query that has been extracted from around the cursor position that was used for the fil;tering of results. + */ +case class ReplacementResults(replacementInterval: ReplacementInterval, + extractedQuery: String, + replacements: CompletionsBase) /** The part of a string to replace. * @@ -53,6 +61,7 @@ case class ReplacementInterval(from: Int, length: Int) object PartialSourcePathAutoCompletionResponse { implicit val ReplacementIntervalFormat: Format[ReplacementInterval] = Json.format[ReplacementInterval] + implicit val ReplacementResultsFormat: Format[ReplacementResults] = Json.format[ReplacementResults] implicit val partialSourcePathAutoCompletionResponseFormat: Format[PartialSourcePathAutoCompletionResponse] = Json.format[PartialSourcePathAutoCompletionResponse] } diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index 70a2a0f387..a9a7f0f5be 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -1,6 +1,6 @@ package controllers.transform -import controllers.transform.autoCompletion.{PartialSourcePathAutoCompletionRequest, PartialSourcePathAutoCompletionResponse, ReplacementInterval} +import controllers.transform.autoCompletion._ import helper.IntegrationTestTrait import org.scalatest.{FlatSpec, MustMatchers} import org.silkframework.serialization.json.JsonHelpers @@ -19,6 +19,8 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl "department/tags/evenMoreNested", "department/tags/evenMoreNested/value", "department/tags/tagId", "id", "name", "phoneNumbers", "phoneNumbers/number", "phoneNumbers/type") + val jsonOps = Seq("/", "\\", "[") + /** * Returns the path of the XML zip project that should be loaded before the test suite starts. */ @@ -30,26 +32,35 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl it should "auto-complete all paths when input is an empty string" in { val result = partialSourcePathAutoCompleteRequest(jsonTransform) - result.copy(replacementResults = null) mustBe PartialSourcePathAutoCompletionResponse("", 0, Some(ReplacementInterval(0, 0)), "", null) - suggestedValues(result) mustBe allJsonPaths + resultWithoutCompletions(result) mustBe partialAutoCompleteResult(replacementResult = Seq( + replacementResults(completions = null), + replacementResults(completions = null) + )) + suggestedValues(result) mustBe allJsonPaths ++ jsonOps.drop(1) } it should "auto-complete with multi-word text filter if there is a one hop path entered" in { val inputText = "depart id" - val result = partialSourcePathAutoCompleteRequest(jsonTransform, inputText = inputText, cursorPosition = 2) - result.copy(replacementResults = null) mustBe PartialSourcePathAutoCompletionResponse(inputText, 2, Some(ReplacementInterval(0, inputText.length)), inputText,null) - suggestedValues(result) mustBe Seq("department/id", "department/tags/tagId") + val cursorPosition = 2 + val result = partialSourcePathAutoCompleteRequest(jsonTransform, inputText = inputText, cursorPosition = cursorPosition) + resultWithoutCompletions(result) mustBe partialAutoCompleteResult(inputText, cursorPosition, replacementResult = Seq( + replacementResults(0, inputText.length, inputText, completions = null), + replacementResults(cursorPosition, 0, "", completions = null) + )) + suggestedValues(result) mustBe Seq("department/id", "department/tags/tagId") ++ jsonOps } it should "auto-complete JSON (also XML) paths at any level" in { val level1EndInput = "department/id" // Return all relative paths that match "id" for any cursor position after the first slash for(cursorPosition <- level1EndInput.length - 2 to level1EndInput.length) { - jsonSuggestions(level1EndInput, cursorPosition) mustBe Seq("id", "tags/tagId") + val opsSegments = if(level1EndInput(cursorPosition - 1) != '/') jsonOps else Seq.empty + jsonSuggestions(level1EndInput, cursorPosition) mustBe Seq("/id", "/tags/tagId") ++ opsSegments } val level2EndInput = "department/tags/id" for(cursorPosition <- level2EndInput.length - 2 to level2EndInput.length) { - jsonSuggestions(level2EndInput, cursorPosition) mustBe Seq("tagId") + val opsSegments = if(level2EndInput(cursorPosition - 1) != '/') jsonOps else Seq.empty + jsonSuggestions(level2EndInput, cursorPosition) mustBe Seq("/tagId") ++ opsSegments } } @@ -64,7 +75,7 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl it should "auto-complete JSON paths that have backward paths in the path prefix" in { val inputPath = "department/tags\\../id" - jsonSuggestions(inputPath, inputPath.length) mustBe Seq("id", "tags/tagId") + jsonSuggestions(inputPath, inputPath.length) mustBe Seq("/id", "/tags/tagId") ++ jsonOps } it should "auto-complete JSON paths inside filter expressions" in { @@ -80,6 +91,26 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl jsonSuggestions(inputWithFilter.take(secondFilterStartIdx) + "" + inputWithFilter.drop(secondFilterStartIdx + 2), secondFilterStartIdx) mustBe Seq("evenMoreNested", "tagId") } + private def partialAutoCompleteResult(inputString: String = "", + cursorPosition: Int = 0, + replacementResult: Seq[ReplacementResults]): PartialSourcePathAutoCompletionResponse = { + PartialSourcePathAutoCompletionResponse(inputString, cursorPosition, replacementResult) + } + + private def replacementResults(from: Int = 0, + to: Int = 0, + query: String = "", + completions: CompletionsBase + ): ReplacementResults = ReplacementResults( + ReplacementInterval(from, to), + query, + completions + ) + + private def resultWithoutCompletions(result: PartialSourcePathAutoCompletionResponse): PartialSourcePathAutoCompletionResponse = { + result.copy(replacementResults = result.replacementResults.map(_.copy(replacements = null))) + } + private def jsonSuggestions(inputText: String, cursorPosition: Int): Seq[String] = { suggestedValues(partialSourcePathAutoCompleteRequest(jsonTransform, inputText = inputText, cursorPosition = cursorPosition)) } @@ -89,7 +120,7 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl } private def suggestedValues(result: PartialSourcePathAutoCompletionResponse): Seq[String] = { - result.replacementResults.completions.map(_.value) + result.replacementResults.flatMap(_.replacements.completions.map(_.value)) } private def partialSourcePathAutoCompleteRequest(transformId: String, From 614bb4b135f0d1c900ea7d7db61e786dcd3ddc08 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Tue, 20 Apr 2021 16:19:53 +0200 Subject: [PATCH 15/95] Simplify partial auto-complete result object --- .../app/controllers/transform/AutoCompletionApi.scala | 9 ++++----- .../PartialSourcePathAutoCompletionRequest.scala | 2 +- .../transform/PartialAutoCompletionApiTest.scala | 6 ++++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 36401d15a1..2edfee58e3 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -5,9 +5,8 @@ import controllers.core.{RequestUserContextAction, UserContextAction} import controllers.transform.AutoCompletionApi.Categories import controllers.transform.autoCompletion._ import org.silkframework.config.Prefixes -import org.silkframework.dataset.{DataSourceCharacteristics, SupportedPathExpressions} +import org.silkframework.dataset.DataSourceCharacteristics import org.silkframework.dataset.DatasetSpec.GenericDatasetSpec -import org.silkframework.dataset.rdf.RdfDataset import org.silkframework.entity.paths.{PathOperator, _} import org.silkframework.entity.{ValueType, ValueTypeAnnotation} import org.silkframework.rule.vocab.ObjectPropertyType @@ -53,7 +52,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU } } - private def simplePath(sourcePath: List[PathOperator]) = { + private def simplePath(sourcePath: List[PathOperator]): List[PathOperator] = { sourcePath.filter(op => op.isInstanceOf[ForwardOperator] || op.isInstanceOf[BackwardOperator]) } @@ -101,7 +100,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU ReplacementResults( ReplacementInterval(from, length), pathToReplace.query.getOrElse(""), - filteredResults.toCompletionsBase + filteredResults.toCompletionsBase.completions ) ) ++ operatorCompletions(dataSourceCharacteristicsOpt, pathToReplace, autoCompletionRequest) ) @@ -127,7 +126,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU Some(ReplacementResults( ReplacementInterval(autoCompletionRequest.cursorPosition, 0), "", - CompletionsBase(forwardOp.toSeq ++ backwardOp ++ propertyFilter ++ langFilterOp) + forwardOp.toSeq ++ backwardOp ++ propertyFilter ++ langFilterOp )) } else { diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala index b4159343b0..ecd90ec059 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala @@ -50,7 +50,7 @@ case class PartialSourcePathAutoCompletionResponse(inputString: String, */ case class ReplacementResults(replacementInterval: ReplacementInterval, extractedQuery: String, - replacements: CompletionsBase) + replacements: Seq[CompletionBase]) /** The part of a string to replace. * diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index a9a7f0f5be..6fb53aba97 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -37,6 +37,8 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl replacementResults(completions = null) )) suggestedValues(result) mustBe allJsonPaths ++ jsonOps.drop(1) + // Path ops don't count for maxSuggestions + suggestedValues(partialSourcePathAutoCompleteRequest(jsonTransform, maxSuggestions = Some(1))) mustBe allJsonPaths.take(1) ++ jsonOps.drop(1) } it should "auto-complete with multi-word text filter if there is a one hop path entered" in { @@ -100,7 +102,7 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl private def replacementResults(from: Int = 0, to: Int = 0, query: String = "", - completions: CompletionsBase + completions: Seq[CompletionBase] ): ReplacementResults = ReplacementResults( ReplacementInterval(from, to), query, @@ -120,7 +122,7 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl } private def suggestedValues(result: PartialSourcePathAutoCompletionResponse): Seq[String] = { - result.replacementResults.flatMap(_.replacements.completions.map(_.value)) + result.replacementResults.flatMap(_.replacements.map(_.value)) } private def partialSourcePathAutoCompleteRequest(transformId: String, From 4bc5c38eb21eefe066d36109d49e2d2f74bef309 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Wed, 21 Apr 2021 09:31:58 +0200 Subject: [PATCH 16/95] Suggest special paths --- .../entity/paths/PathParser.scala | 11 +++-- .../plugins/dataset/json/JsonSource.scala | 1 + .../transform/AutoCompletionApi.scala | 49 +++++++++++++++---- .../PartialAutoCompletionApiTest.scala | 24 +++++---- 4 files changed, 61 insertions(+), 24 deletions(-) diff --git a/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala b/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala index d1c5c3aa12..b686ccef77 100644 --- a/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala +++ b/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala @@ -45,6 +45,9 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { * @param pathStr The input path string. */ def parseUntilError(pathStr: String): PartialParseResult = { + if(pathStr.isEmpty) { + return PartialParseResult(UntypedPath(List.empty), None) + } val completePath = normalized(pathStr) // Added characters because of normalization. Need to be removed when reporting the actual error offset. val addedOffset = completePath.length - pathStr.length @@ -67,7 +70,7 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { partialParseError = Some(PartialParseError( originalErrorOffset, error.msg, - pathStr.substring(originalParseOffset, originalErrorOffset + 1), // + 1 since we want to have the character where it failed + pathStr.substring(originalParseOffset, math.min(originalErrorOffset + 1, pathStr.length)), // + 1 since we want to have the character where it failed originalParseOffset )) } @@ -87,9 +90,9 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { // Normalizes the path syntax in case a simplified syntax has been used private def normalized(pathStr: String): String = { - pathStr.head match { - case '?' => pathStr // Path includes a variable - case '/' | '\\' => "?a" + pathStr // Variable has been left out + pathStr.headOption match { + case Some('?') => pathStr // Path includes a variable + case Some('/') | Some('\\') => "?a" + pathStr // Variable has been left out case _ => "?a/" + pathStr // Variable and leading '/' have been left out } } diff --git a/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonSource.scala b/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonSource.scala index 91b81ba5d6..4095c258a1 100644 --- a/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonSource.scala +++ b/silk-plugins/silk-plugins-json/src/main/scala/org/silkframework/plugins/dataset/json/JsonSource.scala @@ -243,5 +243,6 @@ object JsonSource{ object specialPaths { final val TEXT = "#text" final val ID = "#id" + final val all = Seq(ID, TEXT) } } diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 2edfee58e3..06d3d0f861 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -81,18 +81,13 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU relativeForwardPaths = relativePaths(simplePathBeforeReplacement, forwardOnlyPath(simplePathBeforeReplacement), relativeForwardPaths, isRdfInput, oneHopOnly = pathToReplace.insideFilter, serializeFull = !pathToReplace.insideFilter) } + val dataSourceSpecialPathCompletions = specialPathCompletions(dataSourceCharacteristicsOpt, pathToReplace) // Add known paths - val completions: Completions = relativeForwardPaths + val completions: Completions = relativeForwardPaths ++ dataSourceSpecialPathCompletions val from = pathToReplace.from val length = pathToReplace.length // Return filtered result - val filteredResults = completions. - filterAndSort( - pathToReplace.query.getOrElse(""), - autoCompletionRequest.maxSuggestions.getOrElse(DEFAULT_AUTO_COMPLETE_RESULTS), - sortEmptyTermResult = false, - multiWordFilter = true - ) + val filteredResults = filterResults(autoCompletionRequest, pathToReplace, completions) val response = PartialSourcePathAutoCompletionResponse( autoCompletionRequest.inputString, autoCompletionRequest.cursorPosition, @@ -109,6 +104,40 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU } } + private def specialPathCompletions(dataSourceCharacteristicsOpt: Option[DataSourceCharacteristics], + pathToReplace: PathToReplace): Seq[Completion] = { + dataSourceCharacteristicsOpt.toSeq.flatMap { characteristics => + val pathOps = Seq("/", "\\", "[") + + def pathWithoutOperator(specialPath: String): Boolean = pathOps.forall(op => !specialPath.startsWith(op)) + + characteristics.supportedPathExpressions.specialPaths + // No backward or filter paths allowed inside filters + .filter(p => !pathToReplace.insideFilter || !pathOps.drop(1).forall(disallowedOp => p.value.startsWith(disallowedOp))) + .map { p => + val pathSegment = if (pathToReplace.from > 0 && !pathToReplace.insideFilter && pathWithoutOperator(p.value)) { + "/" + p.value + } else if ((pathToReplace.from == 0 || pathToReplace.insideFilter) && p.value.startsWith("/")) { + p.value.stripPrefix("/") + } else { + p.value + } + Completion(pathSegment, label = None, description = p.description, category = Categories.sourcePaths, isCompletion = true) + } + } + } + + // Filter results based on text query and limit number of results + private def filterResults(autoCompletionRequest: PartialSourcePathAutoCompletionRequest, pathToReplace: PathToReplace, completions: Completions): Completions = { + completions. + filterAndSort( + pathToReplace.query.getOrElse(""), + autoCompletionRequest.maxSuggestions.getOrElse(DEFAULT_AUTO_COMPLETE_RESULTS), + sortEmptyTermResult = false, + multiWordFilter = true + ) + } + private def operatorCompletions(dataSourceCharacteristicsOpt: Option[DataSourceCharacteristics], pathToReplace: PathToReplace, autoCompletionRequest: PartialSourcePathAutoCompletionRequest): Option[ReplacementResults] = { @@ -121,8 +150,8 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU val supportedPathExpressions = dataSourceCharacteristicsOpt.getOrElse(DataSourceCharacteristics()).supportedPathExpressions val forwardOp = completion(autoCompletionRequest.cursorPosition > 0 && supportedPathExpressions.multiHopPaths, "/", "Starts a forward path segment") val backwardOp = completion(supportedPathExpressions.backwardPaths && supportedPathExpressions.multiHopPaths, "\\", "Starts a backward path segment") - val langFilterOp = completion(supportedPathExpressions.languageFilter, "[@lang ", "Starts a language filter expression") - val propertyFilter = completion(supportedPathExpressions.propertyFilter, "[", "Starts a property filter expression") + val langFilterOp = completion(autoCompletionRequest.cursorPosition > 0 && supportedPathExpressions.languageFilter, "[@lang ", "Starts a language filter expression") + val propertyFilter = completion(autoCompletionRequest.cursorPosition > 0 && supportedPathExpressions.propertyFilter, "[", "Starts a property filter expression") Some(ReplacementResults( ReplacementInterval(autoCompletionRequest.cursorPosition, 0), "", diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index 6fb53aba97..9414fdb6dd 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -3,6 +3,7 @@ package controllers.transform import controllers.transform.autoCompletion._ import helper.IntegrationTestTrait import org.scalatest.{FlatSpec, MustMatchers} +import org.silkframework.plugins.dataset.json.JsonSource import org.silkframework.serialization.json.JsonHelpers import org.silkframework.workspace.SingleProjectWorkspaceProviderTestTrait import play.api.libs.json.Json @@ -19,6 +20,8 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl "department/tags/evenMoreNested", "department/tags/evenMoreNested/value", "department/tags/tagId", "id", "name", "phoneNumbers", "phoneNumbers/number", "phoneNumbers/type") + private val jsonSpecialPaths = JsonSource.specialPaths.all + private val jsonSpecialPathsFull = jsonSpecialPaths.map(p => s"/$p") val jsonOps = Seq("/", "\\", "[") /** @@ -36,9 +39,10 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl replacementResults(completions = null), replacementResults(completions = null) )) - suggestedValues(result) mustBe allJsonPaths ++ jsonOps.drop(1) + // Do not propose filter or forward operators at the beginning of a path. Filters are currently invalid in first position (FIXME: CMEM-226). + suggestedValues(result) mustBe allJsonPaths ++ jsonSpecialPaths ++ Seq("\\") // Path ops don't count for maxSuggestions - suggestedValues(partialSourcePathAutoCompleteRequest(jsonTransform, maxSuggestions = Some(1))) mustBe allJsonPaths.take(1) ++ jsonOps.drop(1) + suggestedValues(partialSourcePathAutoCompleteRequest(jsonTransform, maxSuggestions = Some(1))) mustBe allJsonPaths.take(1) ++ Seq("\\") } it should "auto-complete with multi-word text filter if there is a one hop path entered" in { @@ -56,13 +60,13 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl val level1EndInput = "department/id" // Return all relative paths that match "id" for any cursor position after the first slash for(cursorPosition <- level1EndInput.length - 2 to level1EndInput.length) { - val opsSegments = if(level1EndInput(cursorPosition - 1) != '/') jsonOps else Seq.empty - jsonSuggestions(level1EndInput, cursorPosition) mustBe Seq("/id", "/tags/tagId") ++ opsSegments + val opsSegments = if(level1EndInput(cursorPosition - 1) != '/') jsonOps else Seq.empty + jsonSuggestions(level1EndInput, cursorPosition) mustBe Seq("/id", "/tags/tagId") ++ jsonSpecialPathsFull.filter(_.contains("id")) ++ opsSegments } val level2EndInput = "department/tags/id" for(cursorPosition <- level2EndInput.length - 2 to level2EndInput.length) { val opsSegments = if(level2EndInput(cursorPosition - 1) != '/') jsonOps else Seq.empty - jsonSuggestions(level2EndInput, cursorPosition) mustBe Seq("/tagId") ++ opsSegments + jsonSuggestions(level2EndInput, cursorPosition) mustBe Seq("/tagId") ++ jsonSpecialPathsFull.filter(_.contains("id")) ++ opsSegments } } @@ -77,20 +81,20 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl it should "auto-complete JSON paths that have backward paths in the path prefix" in { val inputPath = "department/tags\\../id" - jsonSuggestions(inputPath, inputPath.length) mustBe Seq("/id", "/tags/tagId") ++ jsonOps + jsonSuggestions(inputPath, inputPath.length) mustBe Seq("/id", "/tags/tagId") ++ jsonSpecialPathsFull.filter(_.contains("id")) ++ jsonOps } it should "auto-complete JSON paths inside filter expressions" in { val inputWithFilter = """department[id = "department X"]/tags[id = ""]/tagId""" val filterStartIdx = "department[".length val secondFilterStartIdx = """department[id = "department X"]/tags[""".length - jsonSuggestions(inputWithFilter, filterStartIdx + 2) mustBe Seq("id") - jsonSuggestions(inputWithFilter, filterStartIdx) mustBe Seq("id") - jsonSuggestions(inputWithFilter, secondFilterStartIdx) mustBe Seq("tagId") + jsonSuggestions(inputWithFilter, filterStartIdx + 2) mustBe Seq("id") ++ jsonSpecialPaths.filter(_.contains("id")) + jsonSuggestions(inputWithFilter, filterStartIdx) mustBe Seq("id") ++ jsonSpecialPaths.filter(_.contains("id")) + jsonSuggestions(inputWithFilter, secondFilterStartIdx) mustBe Seq("tagId") ++ jsonSpecialPaths.filter(_.contains("id")) // Multi word query jsonSuggestions(inputWithFilter.take(secondFilterStartIdx) + "ta id" + inputWithFilter.drop(secondFilterStartIdx + 2), secondFilterStartIdx) mustBe Seq("tagId") // Empty query - jsonSuggestions(inputWithFilter.take(secondFilterStartIdx) + "" + inputWithFilter.drop(secondFilterStartIdx + 2), secondFilterStartIdx) mustBe Seq("evenMoreNested", "tagId") + jsonSuggestions(inputWithFilter.take(secondFilterStartIdx) + "" + inputWithFilter.drop(secondFilterStartIdx + 2), secondFilterStartIdx) mustBe Seq("evenMoreNested", "tagId") ++ jsonSpecialPaths } private def partialAutoCompleteResult(inputString: String = "", From c152b374793675b00ce6c15ef23cfbe2b86ebf69 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Wed, 21 Apr 2021 09:57:01 +0200 Subject: [PATCH 17/95] Fix bugs in partial auto-completion inside filter expressions --- ...PartialSourcePathAutocompletionHelper.scala | 18 ++++++++++++++---- .../PartialAutoCompletionApiTest.scala | 5 +++++ ...ialSourcePathAutocompletionHelperTest.scala | 12 +++++++++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala index 43243c4192..3a1543c992 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala @@ -70,12 +70,15 @@ object PartialSourcePathAutocompletionHelper { } if(identifier.nonEmpty) { val pathFromFilterToCursor = request.inputString.substring(error.nextParseOffset, request.cursorPosition) - if(pathFromFilterToCursor.contains(identifier)) { + if(pathFromFilterToCursor.endsWith(identifier)) { + // The cursor is directly behind the identifier + replaceIdentifierInsideFilter(error, identifier) + } else if(pathFromFilterToCursor.contains(identifier)) { // The cursor is behind the identifier / URI TODO: auto-complete comparison operator PathToReplace(0, 0, Some("TODO")) } else { // Suggest to replace the identifier - PathToReplace(error.nextParseOffset + 1, error.nextParseOffset + 1 + identifier.stripSuffix(" ").length, Some(extractQuery(identifier)), insideFilter = true) + replaceIdentifierInsideFilter(error, identifier) } } else { // Not sure what to replace TODO: Handle case of empty filter expression @@ -84,8 +87,15 @@ object PartialSourcePathAutocompletionHelper { } } - private def handleSubPathOnly(request: PartialSourcePathAutoCompletionRequest, pathToReplace: PathToReplace, subPathOnly: Boolean): PathToReplace = { - if(subPathOnly || pathToReplace.query.isEmpty) { + private def replaceIdentifierInsideFilter(error: PartialParseError, identifier: String): PathToReplace = { + PathToReplace(error.nextParseOffset + 1, identifier.stripSuffix(" ").length, Some(extractQuery(identifier)), insideFilter = true) + } + + /** Replace the complete path prefix if not only the sub-path should be replaced, e.g. for XML and JSON. */ + private def handleSubPathOnly(request: PartialSourcePathAutoCompletionRequest, + pathToReplace: PathToReplace, + subPathOnly: Boolean): PathToReplace = { + if(subPathOnly || pathToReplace.query.isEmpty || pathToReplace.insideFilter) { pathToReplace } else { pathToReplace.copy(length = request.inputString.length - pathToReplace.from) diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index 9414fdb6dd..d673649676 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -97,6 +97,11 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl jsonSuggestions(inputWithFilter.take(secondFilterStartIdx) + "" + inputWithFilter.drop(secondFilterStartIdx + 2), secondFilterStartIdx) mustBe Seq("evenMoreNested", "tagId") ++ jsonSpecialPaths } + it should "auto-complete JSON paths inside started filter expressions" in { + val inputWithFilter = """department[id = "department X"]/tags[id""" + jsonSuggestions(inputWithFilter, inputWithFilter.length) mustBe Seq("tagId") ++ jsonSpecialPaths.filter(_.contains("id")) + } + private def partialAutoCompleteResult(inputString: String = "", cursorPosition: Int = 0, replacementResult: Seq[ReplacementResults]): PartialSourcePathAutoCompletionResponse = { diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala index 466fab2d8e..36547c8def 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala @@ -33,7 +33,17 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche } it should "correctly find out what to replace in filter expressions" in { - replace("""a/b[@lang = "en"]/error now""", """a/b[@lang = "en"]/err""".length) mustBe + val inputString = """a/b[@lang = "en"]/error now""" + val inputString2 = """a/b[c = "val"]/d""" + // Check that expressions with filter containing a lot of whitespace lead to correct results + replace(inputString, inputString.length - 5) mustBe PathToReplace("a/b[@lang = \"en\"]".length, "/error now".length, Some("error now")) + replace(inputString2, "a/b[c".length) mustBe + PathToReplace("a/b[".length, 1, Some("c"), insideFilter = true) + } + + it should "know if the cursor is inside a filter expression" in { + val path = """department[id = "department X"]/tags[id""" + replace(path, path.length).insideFilter mustBe true } } From 0ed52e02e6c8cd93be8a8c1231188a5de9218a78 Mon Sep 17 00:00:00 2001 From: darausi Date: Wed, 21 Apr 2021 09:08:15 +0100 Subject: [PATCH 18/95] updated the dropdown --- silk-react-components/package.json | 3 +- .../AutoSuggestion/AutoSuggestion.scss | 30 ++++- .../AutoSuggestion/AutoSuggestion.tsx | 117 +++++++++++++----- .../components/CodeEditor.tsx | 9 +- .../MappingRule/ValueRule/ValueRuleForm.tsx | 39 +++--- silk-react-components/yarn.lock | 20 +-- 6 files changed, 135 insertions(+), 83 deletions(-) diff --git a/silk-react-components/package.json b/silk-react-components/package.json index fc9ba38929..87422ba70c 100644 --- a/silk-react-components/package.json +++ b/silk-react-components/package.json @@ -48,7 +48,7 @@ "carbon-components": "^10.25.0", "carbon-components-react": "^7.25.0", "classnames": "^2.2.5", - "codemirror": "^5.52.2", + "codemirror": "^5.61.0", "css-loader": "3.4.2", "dialog-polyfill": "0.4.4", "ecc-messagebus": "^3.6.0", @@ -73,7 +73,6 @@ "react-codemirror2": "^7.2.1", "react-dev-utils": "10.2.1", "react-dom": "^16.13.0", - "react-dropdown": "^1.9.2", "reset-css": "^5.0.1", "resolve": "^1.17.0", "resolve-url-loader": "^3.1.1", diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss index 847e3fad68..9789988c38 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss @@ -1,4 +1,3 @@ - @import "~@eccenca/gui-elements/src/configuration.default"; @import "~react-dropdown/style.css"; @@ -6,6 +5,8 @@ display: flex; flex-flow: column nowrap; margin-bottom: $ecc-size-blockelement-margin-vertical; + position: relative; + height: 10rem; &__editor-box { display: flex; @@ -13,7 +14,21 @@ } &__dropdown { - width: 100%; + transition: all 300ms; + background-color:$color-white; + position: absolute; + top: 3rem; + z-index:2; + display: flex; + flex-flow: column wrap; + margin-top: $ecc-size-blockelement-margin-vertical; + max-width: 25rem; + max-height: 15rem; + border: 1px solid $list-container-item-border-color; + border-radius: 0.3rem; + // padding: $input-text-padding; + overflow-x:hidden; + overflow-y: auto; } } @@ -22,21 +37,24 @@ height: 3rem; overflow-y: hidden; border-bottom: 1px solid $color-primary; + position:relative; } - .editor__icon { color: $color-white; - border-radius:50%; + border-radius: 50%; font-size: 1rem !important; &.confirm { background-color: $palette-light-green-900; - } &.clear { background-color: $palette-deep-orange-A700; - } } +.ecc-text-highlighting { + display: inline-block; + border-bottom: 1px dashed $ecc-color-primary; + background-color: red;; +} \ No newline at end of file diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index ec70c72417..9150e996c2 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -1,34 +1,75 @@ import React from "react"; -import Dropdown from "react-dropdown"; +import CodeMirror from "codemirror"; import { Icon } from "@eccenca/gui-elements"; +//custom components import { CodeEditor } from "../CodeEditor"; +import Dropdown from "./Dropdown"; +import { replace } from "lodash"; +//styles require("./AutoSuggestion.scss"); const AutoSuggestion = ({ onEditorParamsChange, - suggestions = [], + data, checkPathValidity, pathIsValid, }) => { const [value, setValue] = React.useState(""); const [inputString, setInputString] = React.useState(""); const [cursorPosition, setCursorPosition] = React.useState(0); - const [coords, setCoords] = React.useState({}); + const [coords, setCoords] = React.useState({ left: 0 }); const [shouldShowDropdown, setShouldShowDropdown] = React.useState(false); - const [inputSelection, setInputSelection] = React.useState(""); - const [replacementIndexes, setReplacementIndexes] = React.useState({from:0, length: 0}); + const [ + replacementIndexesDict, + setReplacementIndexesDict, + ] = React.useState({}); + const [suggestions, setSuggestions] = React.useState< + Array<{ value: string; description?: string; label?: string }> + >([]); + const [ + editorInstance, + setEditorInstance, + ] = React.useState(); + + + React.useEffect(() => { + //perform linting + },[pathIsValid]) + + /** generate suggestions and also populate the replacement indexes dict */ + React.useEffect(() => { + let newSuggestions = []; + let newReplacementIndexesDict = {}; + if(data?.replacementResults?.length === 1 && !(data?.replacementResults?.replacements?.length)){ + setShouldShowDropdown(false) + } + if (data?.replacementResults?.length) { + data.replacementResults.forEach( + ({ replacements, replacementInterval: { from, length } }) => { + newSuggestions = [...newSuggestions, ...replacements]; + replacements.forEach((replacement) => { + newReplacementIndexesDict = { + ...newReplacementIndexesDict, + [replacement.value]: { + from, + length, + }, + }; + }); + } + ); + setSuggestions(() => newSuggestions) + setReplacementIndexesDict(() => newReplacementIndexesDict) + } + }, [data]); React.useEffect(() => { setInputString(() => value); setShouldShowDropdown(true); checkPathValidity(inputString); - onEditorParamsChange( - inputString, - cursorPosition, - handleReplacementIndexes - ); + onEditorParamsChange(inputString, cursorPosition); }, [cursorPosition, value, inputString]); const handleChange = (val) => { @@ -40,22 +81,30 @@ const AutoSuggestion = ({ setCoords(() => coords); }; - const handleInputEditorSelection = (selectedText) => { - setInputSelection(selectedText); - }; - - const handleReplacementIndexes = (indexes) => { - setReplacementIndexes(() => ({ ...indexes })); + const handleTextHighlighting = (focusedSuggestion: string) => { + editorInstance.refresh() + const indexes = replacementIndexesDict[focusedSuggestion]; + if (indexes) { + const { from, length } = indexes; + const to = from + length; + editorInstance.markText({ line: 1, ch: 0}, { line: 1, ch: 10 }, {css:"color: red"}); + } }; - const handleDropdownChange = (item) => { - const { from, length } = replacementIndexes; - const to = from + length; - setValue( - (value) => - `${value.substring(0, from)}${item.value}${value.substring(to)}` - ); - setShouldShowDropdown(false); + const handleDropdownChange = (selectedSuggestion:string) => { + const indexes = replacementIndexesDict[selectedSuggestion]; + if (indexes) { + const { from, length } = indexes; + const to = from + length; + setValue( + (value) => + `${value.substring(0, from)}${selectedSuggestion}${value.substring( + to + )}` + ); + setShouldShowDropdown(false); + editorInstance.setCursor({ line: 1, ch: to }); + } }; const handleInputEditorClear = () => { @@ -68,9 +117,9 @@ const AutoSuggestion = ({
@@ -82,16 +131,20 @@ const AutoSuggestion = ({ />
-
- {shouldShowDropdown ? ( + {shouldShowDropdown ? ( +
- ) : null} -
+
+ ) : null}
); }; diff --git a/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx b/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx index 531bbb2e49..9285173ce8 100644 --- a/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx @@ -4,25 +4,26 @@ import "codemirror/mode/sparql/sparql.js"; import PropTypes from "prop-types"; export function CodeEditor({ + setEditorInstance, onChange, onCursorChange, - onSelection, mode = "sparql", value, }) { return (
setEditorInstance(editor)} value={value} - onSelection={(editor) => onSelection(editor.getSelection())} options={{ mode: mode, - lineWrapping: true, lineNumbers: false, theme: "xq-light", + lint:true, + gutters:["CodeMirror-lint-markers"] }} onCursor={(editor, data) => { - onCursorChange(data, editor.cursorCoords(true)); + onCursorChange(data, editor.cursorCoords(true, "div")); }} onBeforeChange={(editor, data, value) => onChange(value)} /> diff --git a/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx b/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx index 7550fddafe..21b3ff191f 100644 --- a/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx +++ b/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx @@ -260,14 +260,13 @@ export function ValueRuleForm(props: IProps) { //editor onChange handler const handleEditorParamsChange = debounce( - (autoCompleteRuleId, inputString, cursorPosition,getReplacementIndexes) => { + (autoCompleteRuleId, inputString, cursorPosition) => { getSuggestion(autoCompleteRuleId, inputString, cursorPosition).then( (suggestions) => { if (suggestions) { setSuggestions( - suggestions.data.replacementResults.completions + suggestions.data ); - getReplacementIndexes(suggestions.data.replacementInterval) } } ).catch(() =>setPathExpressValidity(false)); @@ -311,26 +310,20 @@ export function ValueRuleForm(props: IProps) { if (type === MAPPING_RULE_TYPE_DIRECT) { sourcePropertyInput = ( void - ) => - handleEditorParamsChange( - autoCompleteRuleId, - inputString, - cursorPosition, - getReplacementIndexes - ) - } - /> - + checkPathValidity={checkPathValidity} + pathIsValid={pathExpressIsInvalid} + data={suggestions} + onEditorParamsChange={( + inputString: string, + cursorPosition: number + ) => + handleEditorParamsChange( + autoCompleteRuleId, + inputString, + cursorPosition + ) + } + /> ); } else if (type === MAPPING_RULE_TYPE_COMPLEX) { sourcePropertyInput = ( diff --git a/silk-react-components/yarn.lock b/silk-react-components/yarn.lock index 6ad1a6a711..9935d3d51a 100644 --- a/silk-react-components/yarn.lock +++ b/silk-react-components/yarn.lock @@ -4138,11 +4138,6 @@ classnames@2.2.6, classnames@^2.2, classnames@^2.2.4, classnames@^2.2.5, classna resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.2.6.tgz#43935bffdd291f326dad0a205309b38d00f650ce" integrity sha512-JR/iSQOSt+LQIWwrwEzJ9uk0xfN3mTVYMwt1Ir5mUcSN6pU+V4zQFFaJsclJbPuAUQH+yfWef6tm7l1quW3C8Q== -classnames@^2.2.3: - version "2.3.1" - resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.3.1.tgz#dfcfa3891e306ec1dad105d0e88f4417b8535e8e" - integrity sha512-OlQdbZ7gLfGarSqxesMesDa5uz7KFbID8Kpq/SxIoNGDqY8lSYs0D+hhtBXhcdB3rcbXArFr7vlHheLk1voeNA== - clean-stack@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" @@ -4235,10 +4230,10 @@ code-point-at@^1.0.0: resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" integrity sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c= -codemirror@^5.52.2: - version "5.58.3" - resolved "https://registry.yarnpkg.com/codemirror/-/codemirror-5.58.3.tgz#3f0689854ecfbed5d4479a98b96148b2c3b79796" - integrity sha512-KBhB+juiyOOgn0AqtRmWyAT3yoElkuvWTI6hsHa9E6GQrl6bk/fdAYcvuqW1/upO9T9rtEtapWdw4XYcNiVDEA== +codemirror@^5.61.0: + version "5.61.0" + resolved "https://registry.yarnpkg.com/codemirror/-/codemirror-5.61.0.tgz#318e5b034a707207948b92ffc2862195e8fdb08e" + integrity sha512-D3wYH90tYY1BsKlUe0oNj2JAhQ9TepkD51auk3N7q+4uz7A/cgJ5JsWHreT0PqieW1QhOuqxQ2reCXV1YXzecg== collect-v8-coverage@^1.0.0: version "1.0.1" @@ -11002,13 +10997,6 @@ react-dom@^16.13.0: prop-types "^15.6.2" scheduler "^0.19.1" -react-dropdown@^1.9.2: - version "1.9.2" - resolved "https://registry.yarnpkg.com/react-dropdown/-/react-dropdown-1.9.2.tgz#db6cbc90184e3f6dd7eb40f7ddabac4b09dccf15" - integrity sha512-g4eufErTi5P5T5bGK+VmLl//qvAHy79jm6KKx8G2Tl3mG90bpigb+Aw85P+C2JUdAnIIQdv8kP/oHN314GvAfw== - dependencies: - classnames "^2.2.3" - react-error-overlay@^6.0.7: version "6.0.8" resolved "https://registry.yarnpkg.com/react-error-overlay/-/react-error-overlay-6.0.8.tgz#474ed11d04fc6bda3af643447d85e9127ed6b5de" From 623f271f33cea87f185faa6351255d049279b8ee Mon Sep 17 00:00:00 2001 From: darausi Date: Wed, 21 Apr 2021 09:46:29 +0100 Subject: [PATCH 19/95] updated the dropdown --- .../AutoSuggestion/AutoSuggestion.tsx | 2 +- .../components/AutoSuggestion/Dropdown.tsx | 66 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 9150e996c2..6df09ef4c3 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -87,7 +87,7 @@ const AutoSuggestion = ({ if (indexes) { const { from, length } = indexes; const to = from + length; - editorInstance.markText({ line: 1, ch: 0}, { line: 1, ch: 10 }, {css:"color: red"}); + editorInstance.markText({ line: 1, ch: from}, { line: 1, ch: to}, {css:"color: red"}); } }; diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx new file mode 100644 index 0000000000..fc0bf3ee92 --- /dev/null +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx @@ -0,0 +1,66 @@ +import React from "react"; +import { + Menu, + MenuItem, + Highlighter, + OverviewItem, + OverviewItemDescription, + OverviewItemLine, + OverflowText, +} from "@gui-elements/index"; + +interface IDropdownProps { + options: Array; + onItemSelectionChange: (item) => void; + onMouseOverItem: (value:string) => void + isOpen: boolean; + query?: string; +} + +const Item = ({ item, query }) => { + return ( + + + + + + + + {item.description ? ( + + {item.description} + + ) : null} + + + ); +}; + +const Dropdown: React.FC = ({ + options, + onItemSelectionChange, + isOpen = true, + query, + onMouseOverItem +}) => { + if (!isOpen) return null; + return ( + + {options.map( + (item: { value: string; description?: string }, index) => ( + onItemSelectionChange(item.value)} + text={} + onMouseEnter={() => onMouseOverItem(item.value)} + > + ) + )} + + ); +}; + +export default Dropdown; From a694ef3531d7653083dadf8641c1ac94e680a3a0 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Wed, 21 Apr 2021 11:13:54 +0200 Subject: [PATCH 20/95] Fix reverse ellipsis for overflow text --- silk-react-components/src/libs/gui-elements | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/silk-react-components/src/libs/gui-elements b/silk-react-components/src/libs/gui-elements index 7b4d953586..2919bd8965 160000 --- a/silk-react-components/src/libs/gui-elements +++ b/silk-react-components/src/libs/gui-elements @@ -1 +1 @@ -Subproject commit 7b4d95358674eea5e39d78dadea8c4c0b46c8c33 +Subproject commit 2919bd896505862613e3d487c557a154ed66799a From 46d588a1744171ea531cd03fb1be728e964f1654 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Wed, 21 Apr 2021 11:54:45 +0200 Subject: [PATCH 21/95] Fix query extraction whenin the middle of a path operator --- .../PartialSourcePathAutoCompletionRequest.scala | 14 ++++++++++---- .../PartialSourcePathAutocompletionHelper.scala | 9 +++++++-- ...PartialSourcePathAutocompletionHelperTest.scala | 8 ++++++++ 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala index ecd90ec059..b5132fa238 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala @@ -13,19 +13,25 @@ case class PartialSourcePathAutoCompletionRequest(inputString: String, cursorPosition: Int, maxSuggestions: Option[Int]) { /** The path until the cursor position. */ - lazy val pathUntilCursor: String = inputString.take(cursorPosition) - lazy val charBeforeCursor: Option[Char] = pathUntilCursor.reverse.headOption + def pathUntilCursor: String = inputString.take(cursorPosition) + def charBeforeCursor: Option[Char] = pathUntilCursor.reverse.headOption + + private val operatorStartChars = Set('/', '\\', '[') /** The remaining characters from the cursor position to the end of the current path operator. */ - lazy val remainingStringInOperator: String = { + def remainingStringInOperator: String = { // TODO: This does not consider being in a literal string, i.e. "...^..." where /, \ and [ would be allowed - val operatorStartChars = Set('/', '\\', '[') inputString .substring(cursorPosition) .takeWhile { char => !operatorStartChars.contains(char) } } + + /** The index of the operator end, i.e. index in the input string from the cursor to the end of the current operator. */ + def indexOfOperatorEnd: Int = { + cursorPosition + remainingStringInOperator.length + } } object PartialSourcePathAutoCompletionRequest { diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala index 3a1543c992..bf2cd38e03 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala @@ -106,7 +106,11 @@ object PartialSourcePathAutocompletionHelper { request: PartialSourcePathAutoCompletionRequest): PathToReplace = { val lastPathOp = partialResult.partialPath.operators.last val extractedTextQuery = extractTextPart(lastPathOp) - val fullQuery = extractedTextQuery.map(q => extractQuery(q + request.remainingStringInOperator)) + val remainingStringInOp = partialResult.error match { + case Some(error) => request.inputString.substring(error.nextParseOffset, request.indexOfOperatorEnd) + case None => request.remainingStringInOperator + } + val fullQuery = extractedTextQuery.map(q => extractQuery(q + remainingStringInOp)) if(extractedTextQuery.isEmpty) { // This is the end of a valid filter expression, do not replace or suggest anything besides default completions PathToReplace(request.pathUntilCursor.length, 0, None) @@ -116,7 +120,8 @@ object PartialSourcePathAutocompletionHelper { } else { // Replace the last path operator of the input path val lastOpLength = lastPathOp.serialize.length - PathToReplace(request.cursorPosition - lastOpLength, lastOpLength + request.remainingStringInOperator.length, fullQuery) + val from = math.max(partialResult.error.map(_.nextParseOffset).getOrElse(request.cursorPosition) - lastOpLength, 0) + PathToReplace(from, lastOpLength + request.remainingStringInOperator.length, fullQuery) } } diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala index 36547c8def..091cd219e7 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala @@ -12,6 +12,12 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche PartialSourcePathAutocompletionHelper.pathToReplace(PartialSourcePathAutoCompletionRequest(inputString, cursorPosition, None), subPathOnly)(Prefixes.empty) } + it should "replace simple multi word input paths" in { + def replaceFull(input: String) = replace(input, input.length) + replaceFull("some test") mustBe PathToReplace(0, 9, Some("some test")) + replace("some test", 3) mustBe PathToReplace(0, 9, Some("some test")) + } + it should "correctly find out which part of a path to replace in simple forward paths for the sub path the cursor is in" in { val input = "a1/b1/c1" replace(input, 4, subPathOnly = true) mustBe PathToReplace(2, 3, Some("b1")) @@ -36,6 +42,8 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche val inputString = """a/b[@lang = "en"]/error now""" val inputString2 = """a/b[c = "val"]/d""" // Check that expressions with filter containing a lot of whitespace lead to correct results + replace(inputString, inputString.length) mustBe + PathToReplace("a/b[@lang = \"en\"]".length, "/error now".length, Some("error now")) replace(inputString, inputString.length - 5) mustBe PathToReplace("a/b[@lang = \"en\"]".length, "/error now".length, Some("error now")) replace(inputString2, "a/b[c".length) mustBe From 37ae3f20734fddbbebfd0f206609dc214f5808bd Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Wed, 21 Apr 2021 13:38:07 +0200 Subject: [PATCH 22/95] Improve handling of URIs --- ...rtialSourcePathAutoCompletionRequest.scala | 32 ++++++++++- ...artialSourcePathAutocompletionHelper.scala | 57 +++++++++++++------ ...alSourcePathAutocompletionHelperTest.scala | 12 ++++ 3 files changed, 83 insertions(+), 18 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala index b5132fa238..576ec88fc2 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala @@ -20,14 +20,42 @@ case class PartialSourcePathAutoCompletionRequest(inputString: String, /** The remaining characters from the cursor position to the end of the current path operator. */ def remainingStringInOperator: String = { - // TODO: This does not consider being in a literal string, i.e. "...^..." where /, \ and [ would be allowed + val positionStatus = cursorPositionStatus inputString .substring(cursorPosition) .takeWhile { char => - !operatorStartChars.contains(char) + positionStatus.update(char) + positionStatus.insideQuotesOrUri || !operatorStartChars.contains(char) } } + class PositionStatus(initialInsideQuotes: Boolean, initialInsideUri: Boolean) { + private var _insideQuotes = initialInsideQuotes + private var _insideUri = initialInsideUri + + def update(char: Char): (Boolean, Boolean) = { + if(char == '"' && !_insideUri) { + _insideQuotes = !_insideQuotes + } else if(char == '<' && !_insideQuotes) { + _insideUri = true + } else if(char == '>' && !_insideQuotes) { + _insideUri = false + } + (_insideQuotes, _insideUri) + } + + def insideQuotes: Boolean = _insideQuotes + def insideUri: Boolean = _insideUri + def insideQuotesOrUri: Boolean = insideQuotes || insideUri + } + + // Checks if the cursor position is inside quotes or URI + private def cursorPositionStatus: PositionStatus = { + val positionStatus = new PositionStatus(false, false) + inputString.take(cursorPosition).foreach(positionStatus.update) + positionStatus + } + /** The index of the operator end, i.e. index in the input string from the cursor to the end of the current operator. */ def indexOfOperatorEnd: Int = { cursorPosition + remainingStringInOperator.length diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala index bf2cd38e03..2fa123adc3 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala @@ -13,7 +13,6 @@ object PartialSourcePathAutocompletionHelper { def pathToReplace(request: PartialSourcePathAutoCompletionRequest, subPathOnly: Boolean) (implicit prefixes: Prefixes): PathToReplace = { - // TODO: Refactor method, clean up val unfilteredQuery: Option[String] = Some("") if(request.inputString.isEmpty) { return PathToReplace(0, 0, unfilteredQuery) @@ -21,23 +20,20 @@ object PartialSourcePathAutocompletionHelper { val partialResult = UntypedPath.partialParse(request.pathUntilCursor) val replacement = partialResult.error match { case Some(error) => - val errorOffsetCharacter = request.inputString.substring(error.offset, error.offset + 1) - val parseStartCharacter = if(error.inputLeadingToError.isEmpty) errorOffsetCharacter else error.inputLeadingToError.take(1) - if(error.inputLeadingToError.startsWith("[")) { - handleFilter(request, unfilteredQuery, error) - } else if(parseStartCharacter == "/" || parseStartCharacter == "\\") { - // It tried to parse a forward or backward path and failed, replace path and use path value as query - val operatorValue = request.inputString.substring(error.offset + 1, request.cursorPosition) + request.remainingStringInOperator - PathToReplace(error.offset, operatorValue.length + 1, Some(extractQuery(operatorValue))) - } else { - // The parser parsed part of a forward or backward path as a path op and then failed on an invalid char, e.g. "/with space" - // parses "with" as forward op and then fails parsing the space. - assert(partialResult.partialPath.operators.nonEmpty, "Could not detect sub-path to be replaced.") - handleMiddleOfPathOp(partialResult, request) - } + handleParseErrorCases(request, unfilteredQuery, partialResult, error) case None if partialResult.partialPath.operators.nonEmpty => // No parse problem, use the last path segment (must be forward or backward path op) for auto-completion handleMiddleOfPathOp(partialResult, request) + case None if partialResult.partialPath.operators.isEmpty => + // Cursor position is 0. + if(operatorChars.contains(request.inputString.head)) { + // Right before another path operator, propose to that a new path op + PathToReplace(0, 0, unfilteredQuery) + } else { + // Replace the remaining string + val query = request.inputString.substring(0, request.indexOfOperatorEnd) + PathToReplace(0, request.indexOfOperatorEnd, Some(extractQuery(query))) + } case None => // Should never come so far PathToReplace(0, 0, unfilteredQuery) @@ -45,6 +41,32 @@ object PartialSourcePathAutocompletionHelper { handleSubPathOnly(request, replacement, subPathOnly) } + val operatorChars = Set('/', '\\', '[') + + // Handles the cases where an error occurred parsing the path until the cursor position + private def handleParseErrorCases(request: PartialSourcePathAutoCompletionRequest, + unfilteredQuery: Option[String], + partialResult: PartialParseResult, error: PartialParseError): PathToReplace = { + val errorOffsetCharacter = request.inputString.substring(error.offset, error.offset + 1) + val parseStartCharacter = if (error.inputLeadingToError.isEmpty) errorOffsetCharacter else error.inputLeadingToError.take(1) + if (error.inputLeadingToError.startsWith("[")) { + handleFilter(request, unfilteredQuery, error) + } else if (parseStartCharacter == "/" || parseStartCharacter == "\\") { + // It tried to parse a forward or backward path and failed, replace path and use path value as query + val operatorValue = request.inputString.substring(error.offset + 1, request.cursorPosition) + request.remainingStringInOperator + PathToReplace(error.offset, operatorValue.length + 1, Some(extractQuery(operatorValue))) + } else if (error.nextParseOffset == 0) { + // It failed to parse the first path op, just take the whole string up to the next path operator as input to replace + val operatorValue = request.inputString.substring(0, request.indexOfOperatorEnd) + PathToReplace(0, operatorValue.length, Some(extractQuery(operatorValue))) + } else { + // The parser parsed part of a forward or backward path as a path op and then failed on an invalid char, e.g. "/with space" + // parses "with" as forward op and then fails parsing the space. + assert(partialResult.partialPath.operators.nonEmpty, "Could not detect sub-path to be replaced.") + handleMiddleOfPathOp(partialResult, request) + } + } + private def handleFilter(request: PartialSourcePathAutoCompletionRequest, unfilteredQuery: Option[String], error: PartialParseError): PathToReplace = { // Error happened inside of a filter // Characters that will usually end an identifier in a filter expression. For some URIs this could lead to false positives, e.g. that contain '='. @@ -144,7 +166,10 @@ object PartialSourcePathAutocompletionHelper { private def extractQuery(input: String): String = { var inputToProcess: String = input.trim - if(!input.contains("<") && input.contains(":") && !input.contains(" ") && startsWithPrefix.findFirstMatchIn(input).isDefined) { + if(input.startsWith("<") && input.endsWith(">")) { + inputToProcess = input.drop(1).dropRight(1) + } else if(!input.contains("<") && input.contains(":") && !input.contains(" ") && startsWithPrefix.findFirstMatchIn(input).isDefined + && !input.startsWith("http") && !input.startsWith("urn")) { // heuristic to detect qualified names inputToProcess = input.drop(input.indexOf(":") + 1) } diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala index 091cd219e7..602d50a13a 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala @@ -54,4 +54,16 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche val path = """department[id = "department X"]/tags[id""" replace(path, path.length).insideFilter mustBe true } + + it should "handle URIs correctly" in { + val pathWithUri = "" + val expectedQuery = Some(pathWithUri.drop(1).dropRight(1)) + replace(pathWithUri, pathWithUri.length) mustBe PathToReplace(0, pathWithUri.length, query = expectedQuery) + replace("/" + pathWithUri, pathWithUri.length - 3) mustBe PathToReplace(0, pathWithUri.length + 1, query = expectedQuery) + replace("/" + pathWithUri, 0, subPathOnly = true) mustBe PathToReplace(0, 0, query = Some("")) + replace("/" + pathWithUri, 0, subPathOnly = false) mustBe PathToReplace(0, pathWithUri.length + 1, query = Some("")) + replace(pathWithUri, pathWithUri.length - 3) mustBe PathToReplace(0, pathWithUri.length, query = expectedQuery) + replace(pathWithUri, 0) mustBe PathToReplace(0, pathWithUri.length, query = expectedQuery) + replace(pathWithUri, 1) mustBe PathToReplace(0, pathWithUri.length, query = expectedQuery) + } } From 2c4d742ae40d428df24a7950487b6cf0aa4e8e44 Mon Sep 17 00:00:00 2001 From: darausi Date: Wed, 21 Apr 2021 12:47:43 +0100 Subject: [PATCH 23/95] added highlighting for specific autocompletion suggestion --- .../AutoSuggestion/AutoSuggestion.scss | 1 - .../AutoSuggestion/AutoSuggestion.tsx | 49 ++++++++++++------- .../components/AutoSuggestion/Dropdown.tsx | 4 +- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss index 9789988c38..982e3fd990 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss @@ -56,5 +56,4 @@ .ecc-text-highlighting { display: inline-block; border-bottom: 1px dashed $ecc-color-primary; - background-color: red;; } \ No newline at end of file diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 6df09ef4c3..ea646ad867 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -21,29 +21,31 @@ const AutoSuggestion = ({ const [cursorPosition, setCursorPosition] = React.useState(0); const [coords, setCoords] = React.useState({ left: 0 }); const [shouldShowDropdown, setShouldShowDropdown] = React.useState(false); - const [ - replacementIndexesDict, - setReplacementIndexesDict, - ] = React.useState({}); + const [replacementIndexesDict, setReplacementIndexesDict] = React.useState( + {} + ); const [suggestions, setSuggestions] = React.useState< Array<{ value: string; description?: string; label?: string }> >([]); + const [markers, setMarkers] = React.useState([]); const [ editorInstance, setEditorInstance, ] = React.useState(); - React.useEffect(() => { //perform linting - },[pathIsValid]) + }, [pathIsValid]); /** generate suggestions and also populate the replacement indexes dict */ React.useEffect(() => { let newSuggestions = []; let newReplacementIndexesDict = {}; - if(data?.replacementResults?.length === 1 && !(data?.replacementResults?.replacements?.length)){ - setShouldShowDropdown(false) + if ( + data?.replacementResults?.length === 1 && + !data?.replacementResults?.replacements?.length + ) { + setShouldShowDropdown(false); } if (data?.replacementResults?.length) { data.replacementResults.forEach( @@ -60,8 +62,8 @@ const AutoSuggestion = ({ }); } ); - setSuggestions(() => newSuggestions) - setReplacementIndexesDict(() => newReplacementIndexesDict) + setSuggestions(() => newSuggestions); + setReplacementIndexesDict(() => newReplacementIndexesDict); } }, [data]); @@ -82,31 +84,44 @@ const AutoSuggestion = ({ }; const handleTextHighlighting = (focusedSuggestion: string) => { - editorInstance.refresh() const indexes = replacementIndexesDict[focusedSuggestion]; if (indexes) { const { from, length } = indexes; const to = from + length; - editorInstance.markText({ line: 1, ch: from}, { line: 1, ch: to}, {css:"color: red"}); + const marker = editorInstance.markText( + { line: 0, ch: from }, + { line: 0, ch: to }, + { className: "ecc-text-highlighting" } + ) + setMarkers((previousMarkers) => [...previousMarkers, marker]) } }; - const handleDropdownChange = (selectedSuggestion:string) => { + + const clearMarkers = () => { + markers.forEach(marker => marker.clear()) + } + + const handleDropdownChange = (selectedSuggestion: string) => { const indexes = replacementIndexesDict[selectedSuggestion]; if (indexes) { const { from, length } = indexes; const to = from + length; setValue( (value) => - `${value.substring(0, from)}${selectedSuggestion}${value.substring( - to - )}` + `${value.substring( + 0, + from + )}${selectedSuggestion}${value.substring(to)}` ); setShouldShowDropdown(false); - editorInstance.setCursor({ line: 1, ch: to }); + editorInstance.setCursor({ line: 0, ch: to }); + clearMarkers() } }; + + const handleInputEditorClear = () => { if (!pathIsValid) { setValue(""); diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx index fc0bf3ee92..860ca8ff80 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx @@ -56,11 +56,11 @@ const Dropdown: React.FC = ({ onClick={() => onItemSelectionChange(item.value)} text={} onMouseEnter={() => onMouseOverItem(item.value)} - > + /> ) )} ); }; -export default Dropdown; +export default React.memo(Dropdown); From cf4bd8095cbec9e44ceb3d6ebdf916905db311d0 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Wed, 21 Apr 2021 14:58:05 +0200 Subject: [PATCH 24/95] Handle edge cases in partial auto-complete - Do not auto-complete at all when cursor is inside quotes - Do not auto-complete path operators when inside URIs --- .../transform/AutoCompletionApi.scala | 59 +++++++++++-------- ...rtialSourcePathAutoCompletionRequest.scala | 15 ++++- ...artialSourcePathAutocompletionHelper.scala | 17 ++++-- .../PartialAutoCompletionApiTest.scala | 29 ++++++++- ...lSourcePathAutoCompletionRequestTest.scala | 16 +++++ ...alSourcePathAutocompletionHelperTest.scala | 11 ++++ 6 files changed, 114 insertions(+), 33 deletions(-) create mode 100644 silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequestTest.scala diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 06d3d0f861..64ff1de74d 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -106,36 +106,45 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU private def specialPathCompletions(dataSourceCharacteristicsOpt: Option[DataSourceCharacteristics], pathToReplace: PathToReplace): Seq[Completion] = { - dataSourceCharacteristicsOpt.toSeq.flatMap { characteristics => - val pathOps = Seq("/", "\\", "[") - - def pathWithoutOperator(specialPath: String): Boolean = pathOps.forall(op => !specialPath.startsWith(op)) - - characteristics.supportedPathExpressions.specialPaths - // No backward or filter paths allowed inside filters - .filter(p => !pathToReplace.insideFilter || !pathOps.drop(1).forall(disallowedOp => p.value.startsWith(disallowedOp))) - .map { p => - val pathSegment = if (pathToReplace.from > 0 && !pathToReplace.insideFilter && pathWithoutOperator(p.value)) { - "/" + p.value - } else if ((pathToReplace.from == 0 || pathToReplace.insideFilter) && p.value.startsWith("/")) { - p.value.stripPrefix("/") - } else { - p.value + if(pathToReplace.insideQuotes) { + Seq.empty + } else { + dataSourceCharacteristicsOpt.toSeq.flatMap { characteristics => + val pathOps = Seq("/", "\\", "[") + + def pathWithoutOperator(specialPath: String): Boolean = pathOps.forall(op => !specialPath.startsWith(op)) + + characteristics.supportedPathExpressions.specialPaths + // No backward or filter paths allowed inside filters + .filter(p => !pathToReplace.insideFilter || !pathOps.drop(1).forall(disallowedOp => p.value.startsWith(disallowedOp))) + .map { p => + val pathSegment = if (pathToReplace.from > 0 && !pathToReplace.insideFilter && pathWithoutOperator(p.value)) { + "/" + p.value + } else if ((pathToReplace.from == 0 || pathToReplace.insideFilter) && p.value.startsWith("/")) { + p.value.stripPrefix("/") + } else { + p.value + } + Completion(pathSegment, label = None, description = p.description, category = Categories.sourcePaths, isCompletion = true) } - Completion(pathSegment, label = None, description = p.description, category = Categories.sourcePaths, isCompletion = true) - } + } } } // Filter results based on text query and limit number of results private def filterResults(autoCompletionRequest: PartialSourcePathAutoCompletionRequest, pathToReplace: PathToReplace, completions: Completions): Completions = { - completions. - filterAndSort( - pathToReplace.query.getOrElse(""), - autoCompletionRequest.maxSuggestions.getOrElse(DEFAULT_AUTO_COMPLETE_RESULTS), - sortEmptyTermResult = false, - multiWordFilter = true - ) + pathToReplace.query match { + case Some(query) => + completions. + filterAndSort( + query, + autoCompletionRequest.maxSuggestions.getOrElse(DEFAULT_AUTO_COMPLETE_RESULTS), + sortEmptyTermResult = false, + multiWordFilter = true + ) + case None => + Seq.empty + } } private def operatorCompletions(dataSourceCharacteristicsOpt: Option[DataSourceCharacteristics], @@ -145,7 +154,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU if(predicate) Some(CompletionBase(value, description = Some(description))) else None } // Propose operators - if (!pathToReplace.insideFilter + if (!pathToReplace.insideFilter && !pathToReplace.insideQuotesOrUri && !autoCompletionRequest.charBeforeCursor.contains('/') && !autoCompletionRequest.charBeforeCursor.contains('\\')) { val supportedPathExpressions = dataSourceCharacteristicsOpt.getOrElse(DataSourceCharacteristics()).supportedPathExpressions val forwardOp = completion(autoCompletionRequest.cursorPosition > 0 && supportedPathExpressions.multiHopPaths, "/", "Starts a forward path segment") diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala index 576ec88fc2..ffdfda345b 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala @@ -29,6 +29,19 @@ case class PartialSourcePathAutoCompletionRequest(inputString: String, } } + /** Index of the path operator before the cursor.*/ + def pathOperatorIdxBeforeCursor: Option[Int] = { + val positionStatus = cursorPositionStatus + val strBackToOperator = inputString.substring(0, cursorPosition).reverse + .takeWhile { char => + // Reverse open and close brackets of URI since we are going backwards + val reversedChar = if(char == '<') '>' else if(char == '>') '<' else char + positionStatus.update(reversedChar) + positionStatus.insideQuotesOrUri || !operatorStartChars.contains(char) + } + Some(cursorPosition - strBackToOperator.length - 1).filter(_ >= 0) + } + class PositionStatus(initialInsideQuotes: Boolean, initialInsideUri: Boolean) { private var _insideQuotes = initialInsideQuotes private var _insideUri = initialInsideUri @@ -50,7 +63,7 @@ case class PartialSourcePathAutoCompletionRequest(inputString: String, } // Checks if the cursor position is inside quotes or URI - private def cursorPositionStatus: PositionStatus = { + def cursorPositionStatus: PositionStatus = { val positionStatus = new PositionStatus(false, false) inputString.take(cursorPosition).foreach(positionStatus.update) positionStatus diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala index 2fa123adc3..adce90ab5e 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala @@ -18,6 +18,10 @@ object PartialSourcePathAutocompletionHelper { return PathToReplace(0, 0, unfilteredQuery) } val partialResult = UntypedPath.partialParse(request.pathUntilCursor) + if(request.cursorPositionStatus.insideQuotes) { + // Do not auto-complete inside quotes + return PathToReplace(request.cursorPosition, 0, None, insideQuotes = true) + } val replacement = partialResult.error match { case Some(error) => handleParseErrorCases(request, unfilteredQuery, partialResult, error) @@ -49,7 +53,9 @@ object PartialSourcePathAutocompletionHelper { partialResult: PartialParseResult, error: PartialParseError): PathToReplace = { val errorOffsetCharacter = request.inputString.substring(error.offset, error.offset + 1) val parseStartCharacter = if (error.inputLeadingToError.isEmpty) errorOffsetCharacter else error.inputLeadingToError.take(1) - if (error.inputLeadingToError.startsWith("[")) { + if(error.offset < request.pathOperatorIdxBeforeCursor.getOrElse(0)) { + PathToReplace(request.cursorPosition, 0, None, insideQuotes = request.cursorPositionStatus.insideQuotes, insideUri = request.cursorPositionStatus.insideUri) + } else if (error.inputLeadingToError.startsWith("[")) { handleFilter(request, unfilteredQuery, error) } else if (parseStartCharacter == "/" || parseStartCharacter == "\\") { // It tried to parse a forward or backward path and failed, replace path and use path value as query @@ -97,14 +103,13 @@ object PartialSourcePathAutocompletionHelper { replaceIdentifierInsideFilter(error, identifier) } else if(pathFromFilterToCursor.contains(identifier)) { // The cursor is behind the identifier / URI TODO: auto-complete comparison operator - PathToReplace(0, 0, Some("TODO")) + PathToReplace(0, 0, None) } else { // Suggest to replace the identifier replaceIdentifierInsideFilter(error, identifier) } } else { - // Not sure what to replace TODO: Handle case of empty filter expression - PathToReplace(request.cursorPosition, 0, None, insideFilter = true) + replaceIdentifierInsideFilter(error, "") } } } @@ -186,4 +191,6 @@ object PartialSourcePathAutocompletionHelper { * If it is None this means that no query should be asked to find suggestions, i.e. only suggest operator or nothing. * @param insideFilter If the path to be replaced is inside a filter expression */ -case class PathToReplace(from: Int, length: Int, query: Option[String], insideFilter: Boolean = false) \ No newline at end of file +case class PathToReplace(from: Int, length: Int, query: Option[String], insideFilter: Boolean = false, insideQuotes: Boolean = false, insideUri: Boolean = false) { + def insideQuotesOrUri: Boolean = insideQuotes || insideUri +} \ No newline at end of file diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index d673649676..dedf09a2c8 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -4,8 +4,10 @@ import controllers.transform.autoCompletion._ import helper.IntegrationTestTrait import org.scalatest.{FlatSpec, MustMatchers} import org.silkframework.plugins.dataset.json.JsonSource +import org.silkframework.rule.TransformSpec import org.silkframework.serialization.json.JsonHelpers import org.silkframework.workspace.SingleProjectWorkspaceProviderTestTrait +import org.silkframework.workspace.activity.transform.TransformPathsCache import play.api.libs.json.Json import test.Routes @@ -24,6 +26,9 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl private val jsonSpecialPathsFull = jsonSpecialPaths.map(p => s"/$p") val jsonOps = Seq("/", "\\", "[") + val allRdfPaths = Seq("", "", "", "rdf:type") + val rdfOps: Seq[String] = jsonOps ++ Seq("[@lang ") + /** * Returns the path of the XML zip project that should be loaded before the test suite starts. */ @@ -102,6 +107,22 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl jsonSuggestions(inputWithFilter, inputWithFilter.length) mustBe Seq("tagId") ++ jsonSpecialPaths.filter(_.contains("id")) } + it should "not suggest anything when inside quotes" in { + val pathWithQuotes = """department[b = "some value"]""" + jsonSuggestions(pathWithQuotes, pathWithQuotes.length - 3) mustBe Seq.empty + } + + it should "suggest all path operators for RDF sources" in { + rdfSuggestions("", 0) mustBe allRdfPaths ++ Seq("\\") + rdfSuggestions("rdf:type/") mustBe allRdfPaths + rdfSuggestions("rdf:type//") mustBe allRdfPaths + rdfSuggestions("rdf:type/\\") mustBe allRdfPaths + } + + it should "suggest URIs based on multi line queries for RDF sources" in { + rdfSuggestions("rdf:type/eccenca ad") mustBe allRdfPaths.filter(_.contains("address")) ++ rdfOps + } + private def partialAutoCompleteResult(inputString: String = "", cursorPosition: Int = 0, replacementResult: Seq[ReplacementResults]): PartialSourcePathAutoCompletionResponse = { @@ -123,13 +144,17 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl } private def jsonSuggestions(inputText: String, cursorPosition: Int): Seq[String] = { + project.task[TransformSpec](jsonTransform).activity[TransformPathsCache].control.waitUntilFinished() suggestedValues(partialSourcePathAutoCompleteRequest(jsonTransform, inputText = inputText, cursorPosition = cursorPosition)) } - private def rdfSuggestions(inputText: String, cursorPosition: Int): Set[String] = { - suggestedValues(partialSourcePathAutoCompleteRequest(rdfTransform, inputText = inputText, cursorPosition = cursorPosition)).toSet + private def rdfSuggestions(inputText: String, cursorPosition: Int): Seq[String] = { + project.task[TransformSpec](rdfTransform).activity[TransformPathsCache].control.waitUntilFinished() + suggestedValues(partialSourcePathAutoCompleteRequest(rdfTransform, inputText = inputText, cursorPosition = cursorPosition)) } + private def rdfSuggestions(inputText: String): Seq[String] = rdfSuggestions(inputText, inputText.length) + private def suggestedValues(result: PartialSourcePathAutoCompletionResponse): Seq[String] = { result.replacementResults.flatMap(_.replacements.map(_.value)) } diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequestTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequestTest.scala new file mode 100644 index 0000000000..bb4a891a56 --- /dev/null +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequestTest.scala @@ -0,0 +1,16 @@ +package controllers.transform.autoCompletion + +import org.scalatest.{FlatSpec, MustMatchers} + +class PartialSourcePathAutoCompletionRequestTest extends FlatSpec with MustMatchers { + behavior of "partial source path auto-completion request" + + it should "compute the index of the last path operator before the cursor position correctly" in { + operatorPositionBeforeCursor("some:uri/", 9) mustBe Some(8) + operatorPositionBeforeCursor("""a/b[c = "value"]""", 12) mustBe Some(3) + } + + private def operatorPositionBeforeCursor(inputString: String, cursorPosition: Int): Option[Int] = { + PartialSourcePathAutoCompletionRequest(inputString, cursorPosition, None).pathOperatorIdxBeforeCursor + } +} diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala index 602d50a13a..641f685dfe 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala @@ -66,4 +66,15 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche replace(pathWithUri, 0) mustBe PathToReplace(0, pathWithUri.length, query = expectedQuery) replace(pathWithUri, 1) mustBe PathToReplace(0, pathWithUri.length, query = expectedQuery) } + + it should "not auto-complete values inside quotes" in { + val pathWithQuotes = """a[b = "some value"]""" + replace(pathWithQuotes, pathWithQuotes.length - 4) mustBe PathToReplace(pathWithQuotes.length - 4, 0, None, insideQuotes = true) + } + + it should "should not suggest anything if path in the beginning is invalid" in { + // some prefix does not exist + val qualifiedNamePath = "some:uri/" + replace(qualifiedNamePath, qualifiedNamePath.length) mustBe PathToReplace(qualifiedNamePath.length, 0, None) + } } From a1b6c5a9a54c7e56cf089a343c905a12e060ce7f Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Wed, 21 Apr 2021 15:14:02 +0200 Subject: [PATCH 25/95] Fix RDF path replacement at hop > 1 --- .../transform/AutoCompletionApi.scala | 16 +++++++++++----- .../transform/PartialAutoCompletionApiTest.scala | 9 +++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 64ff1de74d..841bad10e6 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -75,11 +75,17 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU val dataSourceCharacteristicsOpt = dataSourceCharacteristics(transformTask) var relativeForwardPaths = relativePaths(simpleSourcePath, forwardOnlySourcePath, allPaths, isRdfInput) val pathToReplace = PartialSourcePathAutocompletionHelper.pathToReplace(autoCompletionRequest, isRdfInput) - if(!isRdfInput && pathToReplace.from > 0) { - val pathBeforeReplacement = UntypedPath.partialParse(autoCompletionRequest.inputString.take(pathToReplace.from)).partialPath - val simplePathBeforeReplacement = simplePath(pathBeforeReplacement.operators) - relativeForwardPaths = relativePaths(simplePathBeforeReplacement, forwardOnlyPath(simplePathBeforeReplacement), - relativeForwardPaths, isRdfInput, oneHopOnly = pathToReplace.insideFilter, serializeFull = !pathToReplace.insideFilter) + if(pathToReplace.from > 0) { + if(!isRdfInput) { + // compute relative paths + val pathBeforeReplacement = UntypedPath.partialParse(autoCompletionRequest.inputString.take(pathToReplace.from)).partialPath + val simplePathBeforeReplacement = simplePath(pathBeforeReplacement.operators) + relativeForwardPaths = relativePaths(simplePathBeforeReplacement, forwardOnlyPath(simplePathBeforeReplacement), + relativeForwardPaths, isRdfInput, oneHopOnly = pathToReplace.insideFilter, serializeFull = !pathToReplace.insideFilter) + } else if(isRdfInput && !pathToReplace.insideFilter) { + // add forward operator for RDF paths when not in filter + relativeForwardPaths = relativeForwardPaths.map(c => if(!c.value.startsWith("/") && !c.value.startsWith("\\")) c.copy(value = "/" + c.value) else c) + } } val dataSourceSpecialPathCompletions = specialPathCompletions(dataSourceCharacteristicsOpt, pathToReplace) // Add known paths diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index dedf09a2c8..10df97ea71 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -27,6 +27,7 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl val jsonOps = Seq("/", "\\", "[") val allRdfPaths = Seq("", "", "", "rdf:type") + val allRdfPathsFull = allRdfPaths.map("/" + _) val rdfOps: Seq[String] = jsonOps ++ Seq("[@lang ") /** @@ -114,13 +115,13 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl it should "suggest all path operators for RDF sources" in { rdfSuggestions("", 0) mustBe allRdfPaths ++ Seq("\\") - rdfSuggestions("rdf:type/") mustBe allRdfPaths - rdfSuggestions("rdf:type//") mustBe allRdfPaths - rdfSuggestions("rdf:type/\\") mustBe allRdfPaths + rdfSuggestions("rdf:type/") mustBe allRdfPathsFull + rdfSuggestions("rdf:type//") mustBe allRdfPathsFull + rdfSuggestions("rdf:type/\\") mustBe allRdfPathsFull } it should "suggest URIs based on multi line queries for RDF sources" in { - rdfSuggestions("rdf:type/eccenca ad") mustBe allRdfPaths.filter(_.contains("address")) ++ rdfOps + rdfSuggestions("rdf:type/eccenca ad") mustBe allRdfPathsFull.filter(_.contains("address")) ++ rdfOps } private def partialAutoCompleteResult(inputString: String = "", From f7e270ee645f357c3bfb5c1852b020b9ba16f5e9 Mon Sep 17 00:00:00 2001 From: darausi Date: Wed, 21 Apr 2021 14:27:56 +0100 Subject: [PATCH 26/95] removed valid check icon from autosuggestion component --- .../components/AutoSuggestion/AutoSuggestion.tsx | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 3e1eb56a5d..5f3ad0260c 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -134,14 +134,11 @@ const AutoSuggestion = ({ onCursorChange={handleCursorChange} value={value} /> -
- -
+ {!pathIsValid && ( +
+ +
+ )}
{shouldShowDropdown ? (
Date: Wed, 21 Apr 2021 15:42:57 +0200 Subject: [PATCH 27/95] Sort auto-completion results primarily by number of operators and secondarily alphabetically --- .../controllers/transform/AutoCompletionApi.scala | 12 +++++++++++- .../transform/PartialAutoCompletionApiTest.scala | 6 ++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 841bad10e6..9ca524345d 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -348,7 +348,17 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU (implicit userContext: UserContext): Completions = { if (Option(task.activity[TransformPathsCache].value).isDefined) { val paths = fetchCachedPaths(task, sourcePath) - val serializedPaths = paths.map(_.toUntypedPath.serialize()(task.project.config.prefixes)).sorted.distinct + val serializedPaths = paths + // Sort primarily by path operator length then name + .sortWith { (p1, p2) => + if (p1.operators.length == p2.operators.length) { + p1.serialize() < p2.serialize() + } else { + p1.operators.length < p2.operators.length + } + } + .map(_.toUntypedPath.serialize()(task.project.config.prefixes)) + .distinct for(pathStr <- serializedPaths) yield { Completion( value = pathStr, diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index 10df97ea71..e0f7016143 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -17,10 +17,8 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl private val rdfTransform = "17fef5a5-a920-4665-92f5-cc729900e8f1_TransformRDF" private val jsonTransform = "2a997fb4-1bc7-4344-882e-868193568e87_TransformJSON" - - val allJsonPaths = Seq("department", "department/id", "department/tags", - "department/tags/evenMoreNested", "department/tags/evenMoreNested/value", "department/tags/tagId", "id", "name", - "phoneNumbers", "phoneNumbers/number", "phoneNumbers/type") + val allJsonPaths = Seq("department", "id", "name", "phoneNumbers", "department/id", "department/tags", "phoneNumbers/number", + "phoneNumbers/type", "department/tags/evenMoreNested", "department/tags/tagId", "department/tags/evenMoreNested/value") private val jsonSpecialPaths = JsonSource.specialPaths.all private val jsonSpecialPathsFull = jsonSpecialPaths.map(p => s"/$p") From 796d6f9a499581859062b191f35c5da39137e635 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Wed, 21 Apr 2021 16:14:28 +0200 Subject: [PATCH 28/95] Remove syntax highlighting in auto-suggest component --- .../components/AutoSuggestion/AutoSuggestion.tsx | 1 + silk-react-components/src/libs/gui-elements | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 5f3ad0260c..5cb5939d72 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -129,6 +129,7 @@ const AutoSuggestion = ({
Date: Thu, 22 Apr 2021 01:29:26 +0100 Subject: [PATCH 29/95] added underline error position for invalid paths --- .../AutoSuggestion/AutoSuggestion.scss | 28 ++++++++++++++++--- .../AutoSuggestion/AutoSuggestion.tsx | 21 ++++++++++++-- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss index 982e3fd990..15cc383192 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss @@ -13,6 +13,19 @@ align-items: center; } + &__validation { + position: relative; + height:1rem; + width:1rem; + & > div { + position: absolute; + left: 0; + right:0; + bottom:0; + top:0; + } + } + &__dropdown { transition: all 300ms; background-color:$color-white; @@ -26,7 +39,6 @@ max-height: 15rem; border: 1px solid $list-container-item-border-color; border-radius: 0.3rem; - // padding: $input-text-padding; overflow-x:hidden; overflow-y: auto; } @@ -44,16 +56,24 @@ color: $color-white; border-radius: 50%; font-size: 1rem !important; - &.confirm { - background-color: $palette-light-green-900; + cursor: pointer; + + &.error { + background-color: $palette-deep-orange-A700; } &.clear { - background-color: $palette-deep-orange-A700; + background-color: $palette-grey-300; } } .ecc-text-highlighting { display: inline-block; border-bottom: 1px dashed $ecc-color-primary; +} + +.ecc-text-error-highlighting { + background-image:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAQAAAADCAYAAAC09K7GAAAAAXNSR0IArs4c6QAAAAZiS0dEAP8A/wD/oL2nkwAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB9sJDw4cOCW1/KIAAAAZdEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIEdJTVBXgQ4XAAAAHElEQVQI12NggIL/DAz/GdA5/xkY/qPKMDAwAADLZwf5rvm+LQAAAABJRU5ErkJggg=="); + background-position: left bottom; + background-repeat: repeat-x; } \ No newline at end of file diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 5cb5939d72..0031e12642 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -14,7 +14,9 @@ const AutoSuggestion = ({ onEditorParamsChange, data, checkPathValidity, - pathIsValid, + validationResponse, + pathValidationPending, + suggestionsPending, }) => { const [value, setValue] = React.useState(""); const [inputString, setInputString] = React.useState(""); @@ -32,10 +34,23 @@ const AutoSuggestion = ({ editorInstance, setEditorInstance, ] = React.useState(); + const pathIsValid = validationResponse?.valid ?? true + //handle linting React.useEffect(() => { - //perform linting - }, [pathIsValid]); + const parseError = validationResponse?.parseError + if(parseError){ + clearMarkers() + const {offset:start, inputLeadingToError} = parseError + const end = start + inputLeadingToError?.length + const marker = editorInstance.markText( + { line: 0, ch: start }, + { line: 0, ch: end }, + { className: "ecc-text-error-highlighting" } + ); + setMarkers((previousMarkers) => [...previousMarkers, marker]); + } + },[validationResponse?.parseError]) /** generate suggestions and also populate the replacement indexes dict */ React.useEffect(() => { From f1b7a2923cb6bd8368d9f434374664efdc34778e Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 22 Apr 2021 01:30:33 +0100 Subject: [PATCH 30/95] Fixed UI bug of no autocompletion after selecting forward operator --- .../components/AutoSuggestion/AutoSuggestion.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 0031e12642..96a9811e99 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -1,11 +1,10 @@ import React from "react"; import CodeMirror from "codemirror"; -import { Icon } from "@eccenca/gui-elements"; +import { Icon, Spinner } from "@eccenca/gui-elements"; //custom components import { CodeEditor } from "../CodeEditor"; import Dropdown from "./Dropdown"; -import { replace } from "lodash"; //styles require("./AutoSuggestion.scss"); @@ -58,7 +57,7 @@ const AutoSuggestion = ({ let newReplacementIndexesDict = {}; if ( data?.replacementResults?.length === 1 && - !data?.replacementResults?.replacements?.length + !data?.replacementResults[0]?.replacements?.length ) { setShouldShowDropdown(false); } From 0d5cb9b85f01bcf982171500cade89725e62ad78 Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 22 Apr 2021 01:32:00 +0100 Subject: [PATCH 31/95] Added spinner to autocompletion dropdown --- .../AutoSuggestion/AutoSuggestion.tsx | 26 ++++++++++++------- .../components/AutoSuggestion/Dropdown.tsx | 20 +++++++++++--- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 96a9811e99..a2f3435c7f 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -111,6 +111,7 @@ const AutoSuggestion = ({ } }; + //remove all the underline highlighting const clearMarkers = () => { markers.forEach((marker) => marker.clear()); }; @@ -134,33 +135,38 @@ const AutoSuggestion = ({ }; const handleInputEditorClear = () => { - if (!pathIsValid) { - setValue(""); - } + setValue(""); }; return (
+
+ {pathValidationPending && ( + + )} + {!pathIsValid && !pathValidationPending ? ( + + ) : null} +
- {!pathIsValid && ( -
- -
- )} +
+ +
- {shouldShowDropdown ? ( + {shouldShowDropdown || suggestionsPending ? (
; onItemSelectionChange: (item) => void; - onMouseOverItem: (value:string) => void + onMouseOverItem: (value: string) => void; isOpen: boolean; query?: string; + loading?: boolean; } const Item = ({ item, query }) => { @@ -31,7 +34,9 @@ const Item = ({ item, query }) => { {item.description ? ( - {item.description} + + {item.description} + ) : null} @@ -41,12 +46,21 @@ const Item = ({ item, query }) => { const Dropdown: React.FC = ({ options, + loading, onItemSelectionChange, isOpen = true, query, - onMouseOverItem + onMouseOverItem, }) => { if (!isOpen) return null; + if (loading) + return ( + + Fetching suggestions + + + + ); return ( {options.map( From 2a7e0db3428ee1dfc2f80ecbcb77a6c23318f03a Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 22 Apr 2021 01:34:19 +0100 Subject: [PATCH 32/95] Added spinner for autosuggest editor, clear icon on the right --- .../components/CodeEditor.tsx | 16 ++++++--- .../MappingRule/ValueRule/ValueRuleForm.tsx | 36 +++++++++++-------- silk-react-components/src/libs/gui-elements | 2 +- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx b/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx index 9285173ce8..096da1db76 100644 --- a/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx @@ -13,19 +13,27 @@ export function CodeEditor({ return (
setEditorInstance(editor)} + editorDidMount={(editor) => { + editor.on("beforeChange", (_, change) => { + var newtext = change.text.join("").replace(/\n/g, ""); + change.update(change.from, change.to, [newtext]); + return true; + }); + setEditorInstance(editor); + }} value={value} options={{ mode: mode, lineNumbers: false, theme: "xq-light", - lint:true, - gutters:["CodeMirror-lint-markers"] }} onCursor={(editor, data) => { onCursorChange(data, editor.cursorCoords(true, "div")); }} - onBeforeChange={(editor, data, value) => onChange(value)} + onBeforeChange={(editor, data, value) => { + const trimmedValue = value.replace(/\n/g, ""); + onChange(trimmedValue); + }} />
); diff --git a/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx b/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx index 21b3ff191f..9d6fe3fce2 100644 --- a/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx +++ b/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx @@ -89,7 +89,9 @@ export function ValueRuleForm(props: IProps) { const [comment, setComment] = useState("") const [targetProperty, setTargetProperty] = useState("") const [suggestions, setSuggestions] = useState([]); - const [pathExpressIsInvalid, setPathExpressValidity] = React.useState(true); + const [pathValidationResponse, setPathValidationResponse] = React.useState({}); + const [suggestionsPending, setSuggestionsPending] = React.useState(false); + const [pathValidationPending, setPathValidationPending] = React.useState(false) const state = { loading, @@ -261,24 +263,28 @@ export function ValueRuleForm(props: IProps) { //editor onChange handler const handleEditorParamsChange = debounce( (autoCompleteRuleId, inputString, cursorPosition) => { - getSuggestion(autoCompleteRuleId, inputString, cursorPosition).then( - (suggestions) => { + setSuggestionsPending(true); + getSuggestion(autoCompleteRuleId, inputString, cursorPosition) + .then((suggestions) => { if (suggestions) { - setSuggestions( - suggestions.data - ); + setSuggestions(suggestions.data); } - } - ).catch(() =>setPathExpressValidity(false)); + }) + .catch(() => {}) + .finally(() => setSuggestionsPending(false)); }, - 500 + 200 ); const checkPathValidity = debounce((inputString) => { - pathValidation(inputString).then((response) =>{ - setPathExpressValidity(response?.data?.valid ?? false) - }).catch(() =>setPathExpressValidity(false)) - },200) + setPathValidationPending(true); + pathValidation(inputString) + .then((response) => { + setPathValidationResponse(() => response?.data); + }) + .catch(() => {}) + .finally(() => setPathValidationPending(false)); + }, 200); @@ -311,8 +317,10 @@ export function ValueRuleForm(props: IProps) { sourcePropertyInput = ( Date: Thu, 22 Apr 2021 10:01:02 +0200 Subject: [PATCH 33/95] Mock document.body.createTextRange in JsDom --- .../ValueRule/ValueRuleForm.test.tsx | 1 - silk-react-components/test/setup.js | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/silk-react-components/test/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.test.tsx b/silk-react-components/test/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.test.tsx index 0cd1c1b60d..0366093b56 100644 --- a/silk-react-components/test/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.test.tsx +++ b/silk-react-components/test/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.test.tsx @@ -27,7 +27,6 @@ const selectors = { CONFIRM_BUTTON: 'button.ecc-silk-mapping__ruleseditor__actionrow-save', CANCEL_BUTTON: 'button.ecc-silk-mapping__ruleseditor___actionrow-cancel', }; - const getWrapper = (arg = props) => withMount() jest.mock("../../../../../src/HierarchicalMapping/store", () => { diff --git a/silk-react-components/test/setup.js b/silk-react-components/test/setup.js index 914e2594d5..bb8e15dd8c 100644 --- a/silk-react-components/test/setup.js +++ b/silk-react-components/test/setup.js @@ -8,3 +8,21 @@ const enzyme = require("enzyme"); const Adapter = require("enzyme-adapter-react-16"); enzyme.configure({ adapter: new Adapter() }); +if (window.document) { + window.document.body.createTextRange = function() { + return { + setEnd: function(){}, + setStart: function(){}, + getBoundingClientRect: function(){ + return {right: 0}; + }, + getClientRects: function(){ + return { + length: 0, + left: 0, + right: 0 + } + } + } + } +} \ No newline at end of file From 4308efe7ebc3efbc3f5e3a177da71a28ad4a63d0 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Thu, 22 Apr 2021 10:28:50 +0200 Subject: [PATCH 34/95] Set gui elements correctly --- silk-react-components/src/libs/gui-elements | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/silk-react-components/src/libs/gui-elements b/silk-react-components/src/libs/gui-elements index 7b4d953586..2919bd8965 160000 --- a/silk-react-components/src/libs/gui-elements +++ b/silk-react-components/src/libs/gui-elements @@ -1 +1 @@ -Subproject commit 7b4d95358674eea5e39d78dadea8c4c0b46c8c33 +Subproject commit 2919bd896505862613e3d487c557a154ed66799a From d4412e585b8cdd2eb7bb227608df3232ed2a52b8 Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 22 Apr 2021 11:15:55 +0100 Subject: [PATCH 35/95] adjusted the error icon and clear icon --- .../components/AutoSuggestion/AutoSuggestion.scss | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss index 15cc383192..a99f5c77a0 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss @@ -6,7 +6,7 @@ flex-flow: column nowrap; margin-bottom: $ecc-size-blockelement-margin-vertical; position: relative; - height: 10rem; + height: 5rem; &__editor-box { display: flex; @@ -17,6 +17,7 @@ position: relative; height:1rem; width:1rem; + padding-right: 1.2rem; & > div { position: absolute; left: 0; @@ -57,6 +58,7 @@ border-radius: 50%; font-size: 1rem !important; cursor: pointer; + margin-left: 0.2rem; &.error { background-color: $palette-deep-orange-A700; From e2a47adfd8bc0a7dd2e73066c08f2d1a7d421cc5 Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 22 Apr 2021 11:43:34 +0100 Subject: [PATCH 36/95] Fixed Validation request to fire only when input string changes --- .../AutoSuggestion/AutoSuggestion.tsx | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index a2f3435c7f..596865ff57 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -33,15 +33,16 @@ const AutoSuggestion = ({ editorInstance, setEditorInstance, ] = React.useState(); - const pathIsValid = validationResponse?.valid ?? true + const pathIsValid = validationResponse?.valid ?? true; + const valueRef = React.useRef(""); - //handle linting + //handle linting React.useEffect(() => { - const parseError = validationResponse?.parseError - if(parseError){ - clearMarkers() - const {offset:start, inputLeadingToError} = parseError - const end = start + inputLeadingToError?.length + const parseError = validationResponse?.parseError; + if (parseError) { + clearMarkers(); + const { offset: start, inputLeadingToError } = parseError; + const end = start + inputLeadingToError?.length; const marker = editorInstance.markText( { line: 0, ch: start }, { line: 0, ch: end }, @@ -49,7 +50,7 @@ const AutoSuggestion = ({ ); setMarkers((previousMarkers) => [...previousMarkers, marker]); } - },[validationResponse?.parseError]) + }, [validationResponse?.parseError]); /** generate suggestions and also populate the replacement indexes dict */ React.useEffect(() => { @@ -84,7 +85,11 @@ const AutoSuggestion = ({ React.useEffect(() => { setInputString(() => value); setShouldShowDropdown(true); - checkPathValidity(inputString); + //only change if the input has changed, regardless of the cursor change + if (valueRef.current !== value) { + checkPathValidity(inputString); + valueRef.current = value; + } onEditorParamsChange(inputString, cursorPosition); }, [cursorPosition, value, inputString]); From e4cf538557df0466c73e6ae575ff7fb2fba539de Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 22 Apr 2021 12:02:24 +0100 Subject: [PATCH 37/95] Fixed autosuggestion opening before input is focused --- .../AutoSuggestion/AutoSuggestion.tsx | 21 ++++++++++++------- .../components/CodeEditor.tsx | 3 +++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 596865ff57..c590d5c2f4 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -33,6 +33,8 @@ const AutoSuggestion = ({ editorInstance, setEditorInstance, ] = React.useState(); + const [isFocused, setIsFocused] = React.useState(false); + const pathIsValid = validationResponse?.valid ?? true; const valueRef = React.useRef(""); @@ -83,15 +85,17 @@ const AutoSuggestion = ({ }, [data]); React.useEffect(() => { - setInputString(() => value); - setShouldShowDropdown(true); - //only change if the input has changed, regardless of the cursor change - if (valueRef.current !== value) { - checkPathValidity(inputString); - valueRef.current = value; + if (isFocused) { + setInputString(() => value); + setShouldShowDropdown(true); + //only change if the input has changed, regardless of the cursor change + if (valueRef.current !== value) { + checkPathValidity(inputString); + valueRef.current = value; + } + onEditorParamsChange(inputString, cursorPosition); } - onEditorParamsChange(inputString, cursorPosition); - }, [cursorPosition, value, inputString]); + }, [cursorPosition, value, inputString, isFocused]); const handleChange = (val) => { setValue(val); @@ -160,6 +164,7 @@ const AutoSuggestion = ({ onChange={handleChange} onCursorChange={handleCursorChange} value={value} + onFocusChange={setIsFocused} />
diff --git a/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx b/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx index 096da1db76..fe1d61114c 100644 --- a/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx @@ -9,6 +9,7 @@ export function CodeEditor({ onCursorChange, mode = "sparql", value, + onFocusChange, }) { return (
@@ -22,6 +23,8 @@ export function CodeEditor({ setEditorInstance(editor); }} value={value} + onFocus={() => onFocusChange(true)} + onBlur={() => onFocusChange(false)} options={{ mode: mode, lineNumbers: false, From 6f5cdb22ffacf138ab83d3bcf1383fbc70a49001 Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 22 Apr 2021 12:23:46 +0100 Subject: [PATCH 38/95] Attempting to fix incorrecting highlighting for focused suggestions --- .../components/AutoSuggestion/AutoSuggestion.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index c590d5c2f4..b964a4c5df 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -109,6 +109,7 @@ const AutoSuggestion = ({ const handleTextHighlighting = (focusedSuggestion: string) => { const indexes = replacementIndexesDict[focusedSuggestion]; if (indexes) { + clearMarkers() const { from, length } = indexes; const to = from + length; const marker = editorInstance.markText( From 49a35ae2e69556ff237cf9f120db789144cd3737 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Thu, 22 Apr 2021 16:02:03 +0200 Subject: [PATCH 39/95] Do not propose path ops when inside URI --- .../transform/AutoCompletionApi.scala | 3 +- ...artialSourcePathAutocompletionHelper.scala | 65 +++++++++++-------- .../PartialAutoCompletionApiTest.scala | 7 +- ...alSourcePathAutocompletionHelperTest.scala | 7 +- 4 files changed, 51 insertions(+), 31 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 9ca524345d..80ecf33c9c 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -326,7 +326,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU private def valueTypeCompletion(valueType: PluginDescription[ValueType]): Completion = { val annotation = valueType.pluginClass.getAnnotation(classOf[ValueTypeAnnotation]) - val annotationDescription = + val annotationDescription = { if(annotation != null) { val validValues = annotation.validValues().map(str => s"'$str'").mkString(", ") val invalidValues = annotation.invalidValues().map(str => s"'$str'").mkString(", ") @@ -334,6 +334,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU } else { "" } + } Completion( value = valueType.id, diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala index adce90ab5e..84b779df7e 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala @@ -5,8 +5,10 @@ import org.silkframework.entity.paths.{DirectionalPathOperator, PartialParseErro import org.silkframework.util.StringUtils object PartialSourcePathAutocompletionHelper { + /** * Returns the part of the path that should be replaced and the extracted query words that can be used for search. + * * @param request The partial source path auto-completion request payload. * @param subPathOnly If true, only a sub-part of the path is replaced, else a path suffix */ @@ -14,35 +16,46 @@ object PartialSourcePathAutocompletionHelper { subPathOnly: Boolean) (implicit prefixes: Prefixes): PathToReplace = { val unfilteredQuery: Option[String] = Some("") - if(request.inputString.isEmpty) { - return PathToReplace(0, 0, unfilteredQuery) - } - val partialResult = UntypedPath.partialParse(request.pathUntilCursor) - if(request.cursorPositionStatus.insideQuotes) { + val pathToReplace = if(request.inputString.isEmpty) { + // Empty input, just propose default results + PathToReplace(0, 0, unfilteredQuery) + } else if(request.cursorPositionStatus.insideQuotes) { // Do not auto-complete inside quotes - return PathToReplace(request.cursorPosition, 0, None, insideQuotes = true) - } - val replacement = partialResult.error match { - case Some(error) => - handleParseErrorCases(request, unfilteredQuery, partialResult, error) - case None if partialResult.partialPath.operators.nonEmpty => - // No parse problem, use the last path segment (must be forward or backward path op) for auto-completion - handleMiddleOfPathOp(partialResult, request) - case None if partialResult.partialPath.operators.isEmpty => - // Cursor position is 0. - if(operatorChars.contains(request.inputString.head)) { - // Right before another path operator, propose to that a new path op + PathToReplace(request.cursorPosition, 0, None) + } else { + val partialResult = UntypedPath.partialParse(request.pathUntilCursor) + val replacement = partialResult.error match { + case Some(error) => + handleParseErrorCases(request, unfilteredQuery, partialResult, error) + case None if partialResult.partialPath.operators.nonEmpty => + // No parse problem, use the last path segment (must be forward or backward path op) for auto-completion + handleMiddleOfPathOp(partialResult, request) + case None if partialResult.partialPath.operators.isEmpty => + // Cursor position is 0. + if(operatorChars.contains(request.inputString.head)) { + // Right before another path operator, propose to that a new path op + PathToReplace(0, 0, unfilteredQuery) + } else { + // Replace the remaining string + val query = request.inputString.substring(0, request.indexOfOperatorEnd) + PathToReplace(0, request.indexOfOperatorEnd, Some(extractQuery(query))) + } + case None => + // Should never come so far PathToReplace(0, 0, unfilteredQuery) - } else { - // Replace the remaining string - val query = request.inputString.substring(0, request.indexOfOperatorEnd) - PathToReplace(0, request.indexOfOperatorEnd, Some(extractQuery(query))) - } - case None => - // Should never come so far - PathToReplace(0, 0, unfilteredQuery) + } + handleSubPathOnly(request, replacement, subPathOnly) } - handleSubPathOnly(request, replacement, subPathOnly) + handleCursorStatusFlags(request, pathToReplace) + } + + private def handleCursorStatusFlags(request: PartialSourcePathAutoCompletionRequest, + replacement: PathToReplace): PathToReplace = { + val positionStatus = request.cursorPositionStatus + replacement.copy( + insideUri = positionStatus.insideUri, + insideQuotes = positionStatus.insideQuotes + ) } val operatorChars = Set('/', '\\', '[') diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index e0f7016143..60d5d7fbdd 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -24,7 +24,7 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl private val jsonSpecialPathsFull = jsonSpecialPaths.map(p => s"/$p") val jsonOps = Seq("/", "\\", "[") - val allRdfPaths = Seq("", "", "", "rdf:type") + val allRdfPaths = Seq("rdf:type", "", "", "") val allRdfPathsFull = allRdfPaths.map("/" + _) val rdfOps: Seq[String] = jsonOps ++ Seq("[@lang ") @@ -111,6 +111,11 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl jsonSuggestions(pathWithQuotes, pathWithQuotes.length - 3) mustBe Seq.empty } + it should "not suggest path operators when inside a URI" in { + rdfSuggestions(""" Date: Thu, 22 Apr 2021 16:12:02 +0200 Subject: [PATCH 40/95] Remove invalid import --- .../components/AutoSuggestion/AutoSuggestion.scss | 1 - 1 file changed, 1 deletion(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss index a99f5c77a0..8b049cd4eb 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.scss @@ -1,5 +1,4 @@ @import "~@eccenca/gui-elements/src/configuration.default"; -@import "~react-dropdown/style.css"; .ecc-auto-suggestion-box { display: flex; From d4d38bb9f8a3d40e2cbc6b572ee9a2976bd75ed6 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Thu, 22 Apr 2021 16:53:05 +0200 Subject: [PATCH 41/95] Do not return path suggestions when the only non-path-op suggestion is the value that will be replaced --- .../transform/AutoCompletionApi.scala | 16 +++++++++++++--- .../transform/PartialAutoCompletionApiTest.scala | 11 +++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index 80ecf33c9c..2c63b3d78c 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -93,7 +93,7 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU val from = pathToReplace.from val length = pathToReplace.length // Return filtered result - val filteredResults = filterResults(autoCompletionRequest, pathToReplace, completions) + val filteredResults = filterResults(autoCompletionRequest, pathToReplace, completions, dataSourceSpecialPathCompletions) val response = PartialSourcePathAutoCompletionResponse( autoCompletionRequest.inputString, autoCompletionRequest.cursorPosition, @@ -138,8 +138,12 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU } // Filter results based on text query and limit number of results - private def filterResults(autoCompletionRequest: PartialSourcePathAutoCompletionRequest, pathToReplace: PathToReplace, completions: Completions): Completions = { - pathToReplace.query match { + private def filterResults(autoCompletionRequest: PartialSourcePathAutoCompletionRequest, + pathToReplace: PathToReplace, + completions: Completions, + dataSourceSpecialPathCompletions: Seq[Completion]): Completions = { + val stringToBeReplaced = autoCompletionRequest.inputString.substring(pathToReplace.from, pathToReplace.from + pathToReplace.length) + val filteredCompletions: Completions = pathToReplace.query match { case Some(query) => completions. filterAndSort( @@ -151,6 +155,12 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU case None => Seq.empty } + if(filteredCompletions.values.size == 1 && filteredCompletions.values.head.value == stringToBeReplaced) { + // If only real search result has the exact same value as the string to be replaced, do not suggest anything. + Seq.empty + } else { + filteredCompletions + } } private def operatorCompletions(dataSourceCharacteristicsOpt: Option[DataSourceCharacteristics], diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index 60d5d7fbdd..843685eed6 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -127,6 +127,15 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl rdfSuggestions("rdf:type/eccenca ad") mustBe allRdfPathsFull.filter(_.contains("address")) ++ rdfOps } + it should "not propose the exact same replacement if it is the only result" in { + rdfSuggestions("rdf:type") mustBe rdfOps + rdfSuggestions("") mustBe rdfOps + // The special paths actually match "value" in the comments, that's why they show up here and /value is still proposed + jsonSuggestions("department/tags/evenMoreNested/value") mustBe Seq("/value", "/#id", "/#text") ++ jsonOps + // here it's not the case and only the path ops show up + jsonSuggestions("phoneNumbers/number") mustBe jsonOps + } + private def partialAutoCompleteResult(inputString: String = "", cursorPosition: Int = 0, replacementResult: Seq[ReplacementResults]): PartialSourcePathAutoCompletionResponse = { @@ -152,6 +161,8 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl suggestedValues(partialSourcePathAutoCompleteRequest(jsonTransform, inputText = inputText, cursorPosition = cursorPosition)) } + private def jsonSuggestions(inputText: String): Seq[String] = jsonSuggestions(inputText, inputText.length) + private def rdfSuggestions(inputText: String, cursorPosition: Int): Seq[String] = { project.task[TransformSpec](rdfTransform).activity[TransformPathsCache].control.waitUntilFinished() suggestedValues(partialSourcePathAutoCompleteRequest(rdfTransform, inputText = inputText, cursorPosition = cursorPosition)) From 4d25cc009a885d5fb4f0d42dc44a984dcef9d3c4 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Thu, 22 Apr 2021 17:12:44 +0200 Subject: [PATCH 42/95] Do not propose path ops when inside filter --- ...rtialSourcePathAutoCompletionRequest.scala | 35 ++++++++++++++----- ...artialSourcePathAutocompletionHelper.scala | 3 +- .../PartialAutoCompletionApiTest.scala | 4 +++ ...alSourcePathAutocompletionHelperTest.scala | 2 +- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala index ffdfda345b..7ffad9c244 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala @@ -42,29 +42,48 @@ case class PartialSourcePathAutoCompletionRequest(inputString: String, Some(cursorPosition - strBackToOperator.length - 1).filter(_ >= 0) } - class PositionStatus(initialInsideQuotes: Boolean, initialInsideUri: Boolean) { + class PositionStatus(initialInsideQuotes: Boolean, + initialInsideUri: Boolean, + initialInsideFilter: Boolean) { private var _insideQuotes = initialInsideQuotes private var _insideUri = initialInsideUri + private var _insideFilter = initialInsideFilter def update(char: Char): (Boolean, Boolean) = { - if(char == '"' && !_insideUri) { - _insideQuotes = !_insideQuotes - } else if(char == '<' && !_insideQuotes) { - _insideUri = true - } else if(char == '>' && !_insideQuotes) { - _insideUri = false + // Track quotes status + if (!_insideUri) { + if (char == '"') { + _insideQuotes = !_insideQuotes + } + } + // Track URI status + if (!_insideQuotes) { + if (char == '<') { + _insideUri = true + } else if (char == '>') { + _insideUri = false + } + } + // Track filter status + if (!_insideQuotes && !_insideUri) { + if (char == '[') { + _insideFilter = true + } else if (char == ']') { + _insideFilter = false + } } (_insideQuotes, _insideUri) } def insideQuotes: Boolean = _insideQuotes def insideUri: Boolean = _insideUri + def insideFilter: Boolean = _insideFilter def insideQuotesOrUri: Boolean = insideQuotes || insideUri } // Checks if the cursor position is inside quotes or URI def cursorPositionStatus: PositionStatus = { - val positionStatus = new PositionStatus(false, false) + val positionStatus = new PositionStatus(false, false, false) inputString.take(cursorPosition).foreach(positionStatus.update) positionStatus } diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala index 84b779df7e..ccf01f5608 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala @@ -54,7 +54,8 @@ object PartialSourcePathAutocompletionHelper { val positionStatus = request.cursorPositionStatus replacement.copy( insideUri = positionStatus.insideUri, - insideQuotes = positionStatus.insideQuotes + insideQuotes = positionStatus.insideQuotes, + insideFilter = positionStatus.insideFilter ) } diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index 843685eed6..23532c5ca9 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -136,6 +136,10 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl jsonSuggestions("phoneNumbers/number") mustBe jsonOps } + it should "not propose path ops inside a filter" in { + jsonSuggestions("department/[tags = ") must not contain allOf("/", "\\", "[") + } + private def partialAutoCompleteResult(inputString: String = "", cursorPosition: Int = 0, replacementResult: Seq[ReplacementResults]): PartialSourcePathAutoCompletionResponse = { diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala index 17ec2fce96..beaebe641e 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala @@ -70,7 +70,7 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche it should "not auto-complete values inside quotes" in { val pathWithQuotes = """a[b = "some value"]""" - replace(pathWithQuotes, pathWithQuotes.length - 4) mustBe PathToReplace(pathWithQuotes.length - 4, 0, None, insideQuotes = true) + replace(pathWithQuotes, pathWithQuotes.length - 4) mustBe PathToReplace(pathWithQuotes.length - 4, 0, None, insideQuotes = true, insideFilter = true) } it should "should not suggest anything if path in the beginning is invalid" in { From 56b1b662d8532b8e599a4c13dff6b53da27366e8 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Thu, 22 Apr 2021 17:28:13 +0200 Subject: [PATCH 43/95] Use local part as search filter extracted from URIs for auto-completion --- .../PartialSourcePathAutocompletionHelper.scala | 9 +++++++-- .../PartialSourcePathAutocompletionHelperTest.scala | 11 ++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala index ccf01f5608..283cd706c8 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala @@ -2,7 +2,7 @@ package controllers.transform.autoCompletion import org.silkframework.config.Prefixes import org.silkframework.entity.paths.{DirectionalPathOperator, PartialParseError, PartialParseResult, PathOperator, PathParser, UntypedPath} -import org.silkframework.util.StringUtils +import org.silkframework.util.{StringUtils, Uri} object PartialSourcePathAutocompletionHelper { @@ -186,7 +186,12 @@ object PartialSourcePathAutocompletionHelper { private def extractQuery(input: String): String = { var inputToProcess: String = input.trim if(input.startsWith("<") && input.endsWith(">")) { - inputToProcess = input.drop(1).dropRight(1) + // Extract local name from URI for filter query to get similarly named URIs + val uri = input.drop(1).dropRight(1) + inputToProcess = Uri(uri).localName.getOrElse(uri) + } else if((input.startsWith("http") || input.startsWith("urn")) && Uri(input).isValidUri) { + // Extract local name from URI for filter query to get similarly named URIs + inputToProcess = Uri(input).localName.getOrElse(input) } else if(!input.contains("<") && input.contains(":") && !input.contains(" ") && startsWithPrefix.findFirstMatchIn(input).isDefined && !input.startsWith("http") && !input.startsWith("urn")) { // heuristic to detect qualified names diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala index beaebe641e..c2e9705885 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala @@ -12,6 +12,8 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche PartialSourcePathAutocompletionHelper.pathToReplace(PartialSourcePathAutoCompletionRequest(inputString, cursorPosition, None), subPathOnly)(Prefixes.empty) } + def replace(inputString: String): PathToReplace = replace(inputString, inputString.length) + it should "replace simple multi word input paths" in { def replaceFull(input: String) = replace(input, input.length) replaceFull("some test") mustBe PathToReplace(0, 9, Some("some test")) @@ -57,7 +59,7 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche it should "handle URIs correctly" in { val pathWithUri = "" - val expectedQuery = Some(pathWithUri.drop(1).dropRight(1)) + val expectedQuery = Some("address") replace(pathWithUri, pathWithUri.length) mustBe PathToReplace(0, pathWithUri.length, query = expectedQuery) replace("/" + pathWithUri, pathWithUri.length - 3) mustBe PathToReplace(0, pathWithUri.length + 1, query = expectedQuery, insideUri = true) replace("/" + pathWithUri, 0, subPathOnly = true) mustBe PathToReplace(0, 0, query = Some("")) @@ -68,6 +70,13 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche replace(pathWithUri, 1) mustBe PathToReplace(0, pathWithUri.length, query = expectedQuery, insideUri = true) } + it should "extract filter queries from URIs and qnames correctly" in { + replace("").query mustBe Some("address") + replace("\\/").query mustBe Some("address") + replace("\\").query mustBe Some("prop") + replace("").query mustBe Some("prop") + } + it should "not auto-complete values inside quotes" in { val pathWithQuotes = """a[b = "some value"]""" replace(pathWithQuotes, pathWithQuotes.length - 4) mustBe PathToReplace(pathWithQuotes.length - 4, 0, None, insideQuotes = true, insideFilter = true) From df536d3b7caad4ae4cd94b711ae8884787dc020d Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 22 Apr 2021 16:38:38 +0100 Subject: [PATCH 44/95] fixed invalid value for path validation resulting in the delayed error icon --- .../components/AutoSuggestion/AutoSuggestion.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index b964a4c5df..2b62bd588c 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -90,7 +90,7 @@ const AutoSuggestion = ({ setShouldShowDropdown(true); //only change if the input has changed, regardless of the cursor change if (valueRef.current !== value) { - checkPathValidity(inputString); + checkPathValidity(value); valueRef.current = value; } onEditorParamsChange(inputString, cursorPosition); From 90d202c2d07b92a05ac5980817c9e5bc7246805a Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 22 Apr 2021 16:39:33 +0100 Subject: [PATCH 45/95] fixed editor focus on dropdown select --- .../components/AutoSuggestion/AutoSuggestion.tsx | 2 ++ .../HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx | 1 + 2 files changed, 3 insertions(+) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 2b62bd588c..8293c30e77 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -140,6 +140,7 @@ const AutoSuggestion = ({ ); setShouldShowDropdown(false); editorInstance.setCursor({ line: 0, ch: to }); + editorInstance.focus() clearMarkers(); } }; @@ -148,6 +149,7 @@ const AutoSuggestion = ({ setValue(""); }; + return (
diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx index a1226a93bd..0b7572fa8e 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx @@ -10,6 +10,7 @@ import { Spinner, Spacing, } from "@gui-elements/index"; +// import {Suggest} from '@blueprintjs/core' interface IDropdownProps { options: Array; From d52be10f5cd76b76811ea2ad770cf3478249010d Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Thu, 22 Apr 2021 17:39:36 +0200 Subject: [PATCH 46/95] Highlight matches in the completion description --- .../components/AutoSuggestion/Dropdown.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx index a1226a93bd..e6cf5a75c6 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx @@ -35,7 +35,9 @@ const Item = ({ item, query }) => { {item.description ? ( - {item.description} + ) : null} From c35532c58e8e5d536a44a3fa0fed70c467ee27b2 Mon Sep 17 00:00:00 2001 From: darausi Date: Thu, 22 Apr 2021 17:21:13 +0100 Subject: [PATCH 47/95] added valuepath label to autosuggestion --- .../components/AutoSuggestion/AutoSuggestion.tsx | 13 +++++++------ .../MappingRule/ValueRule/ValueRuleForm.tsx | 1 + 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 8293c30e77..4cc3bd1b87 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -1,6 +1,6 @@ import React from "react"; import CodeMirror from "codemirror"; -import { Icon, Spinner } from "@eccenca/gui-elements"; +import { Icon, Spinner, Label } from "@gui-elements/index"; //custom components import { CodeEditor } from "../CodeEditor"; @@ -16,6 +16,7 @@ const AutoSuggestion = ({ validationResponse, pathValidationPending, suggestionsPending, + label, }) => { const [value, setValue] = React.useState(""); const [inputString, setInputString] = React.useState(""); @@ -109,7 +110,7 @@ const AutoSuggestion = ({ const handleTextHighlighting = (focusedSuggestion: string) => { const indexes = replacementIndexesDict[focusedSuggestion]; if (indexes) { - clearMarkers() + clearMarkers(); const { from, length } = indexes; const to = from + length; const marker = editorInstance.markText( @@ -140,7 +141,7 @@ const AutoSuggestion = ({ ); setShouldShowDropdown(false); editorInstance.setCursor({ line: 0, ch: to }); - editorInstance.focus() + editorInstance.focus(); clearMarkers(); } }; @@ -149,16 +150,16 @@ const AutoSuggestion = ({ setValue(""); }; - return (
+
- {shouldShowDropdown || suggestionsPending ? ( -
-
- ) : null} +
+
); }; diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx index d4c57146ea..4e44909e90 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx @@ -1,4 +1,5 @@ import React from "react"; +import computeScrollIntoView from "compute-scroll-into-view"; import { Menu, MenuItem, @@ -10,74 +11,119 @@ import { Spinner, Spacing, } from "@gui-elements/index"; -// import {Suggest} from '@blueprintjs/core' interface IDropdownProps { options: Array; onItemSelectionChange: (item) => void; - onMouseOverItem: (value: string) => void; isOpen: boolean; query?: string; loading?: boolean; + left?: number; + currentlyFocusedIndex?: number; } -const Item = ({ item, query }) => { +const RawItem = ({ item, query }, ref) => { return ( - - - - - - - - {item.description ? ( - +
+ + + + label={item.value} + searchValue={query} + > - ) : null} - - + {item.description ? ( + + + + + + ) : null} + + +
); }; +const Item = React.forwardRef(RawItem); + const Dropdown: React.FC = ({ + isOpen, options, loading, onItemSelectionChange, - isOpen = true, query, - onMouseOverItem, + left, + currentlyFocusedIndex, }) => { + const dropdownRef = React.useRef(); + const refs = {}; + const generateRef = (index) => { + if (!refs.hasOwnProperty(`${index}`)) { + refs[`${index}`] = React.createRef(); + } + return refs[`${index}`]; + }; + + React.useEffect(() => { + const listIndexNode = refs[currentlyFocusedIndex]; + if (dropdownRef.current && listIndexNode.current) { + const actions = computeScrollIntoView(listIndexNode.current, { + boundary: dropdownRef.current, + block: "nearest", + scrollMode: "if-needed", + }); + actions.forEach(({ el, top, left }) => { + el.scrollTop = top; + el.scrollLeft = left; + }); + } + }, [currentlyFocusedIndex]); + if (!isOpen) return null; - if (loading) - return ( - - Fetching suggestions - - - - ); + + const Loader = ( + + Fetching suggestions + + + + ); + return ( - - {options.map( - (item: { value: string; description?: string }, index) => ( - onItemSelectionChange(item.value)} - text={} - onMouseEnter={() => onMouseOverItem(item.value)} - /> - ) +
+ {loading ? ( + Loader + ) : ( + + {options.map((item, index) => ( + onItemSelectionChange(item.value)} + text={ + + } + > + ))} + )} -
+
); }; -export default React.memo(Dropdown); +export default Dropdown; diff --git a/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx b/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx index 54e109b1c0..6f281a441b 100644 --- a/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx @@ -1,22 +1,21 @@ import React from "react"; import { Controlled as ControlledEditor } from "react-codemirror2"; import "codemirror/mode/sparql/sparql.js"; -import PropTypes from "prop-types"; -export function CodeEditor({ +const CodeEditor = ({ setEditorInstance, onChange, onCursorChange, mode = "sparql", value, onFocusChange, -}) { + handleSpecialKeysPress, +}) => { return (
{ editor.on("beforeChange", (_, change) => { - console.log({ change }); var newtext = change.text.join("").replace(/\n/g, ""); //failing unexpectedly during undo and redo if ( @@ -31,7 +30,7 @@ export function CodeEditor({ }} value={value} onFocus={() => onFocusChange(true)} - onBlur={() => onFocusChange(false)} + // onBlur={() => onFocusChange(false)} options={{ mode: mode, lineNumbers: false, @@ -44,15 +43,10 @@ export function CodeEditor({ const trimmedValue = value.replace(/\n/g, ""); onChange(trimmedValue); }} + onKeyDown={(_, event) => handleSpecialKeysPress(event)} />
); -} - -CodeEditor.propTypes = { - mode: PropTypes.string, - value: PropTypes.string.isRequired, - onChange: PropTypes.func.isRequired, - onCursorChange: PropTypes.func.isRequired, - onSelection: PropTypes.func, }; + +export default CodeEditor; diff --git a/silk-react-components/yarn.lock b/silk-react-components/yarn.lock index 9935d3d51a..5b6795982d 100644 --- a/silk-react-components/yarn.lock +++ b/silk-react-components/yarn.lock @@ -4389,6 +4389,11 @@ compute-scroll-into-view@^1.0.13: resolved "https://registry.yarnpkg.com/compute-scroll-into-view/-/compute-scroll-into-view-1.0.16.tgz#5b7bf4f7127ea2c19b750353d7ce6776a90ee088" integrity sha512-a85LHKY81oQnikatZYA90pufpZ6sQx++BoCxOEMsjpZx+ZnaKGQnCyCehTRr/1p9GBIAHTjcU9k71kSYWloLiQ== +compute-scroll-into-view@^1.0.17: + version "1.0.17" + resolved "https://registry.yarnpkg.com/compute-scroll-into-view/-/compute-scroll-into-view-1.0.17.tgz#6a88f18acd9d42e9cf4baa6bec7e0522607ab7ab" + integrity sha512-j4dx+Fb0URmzbwwMUrhqWM2BEWHdFGx+qZ9qqASHRPqvTYdqvWnHg0H1hIbcyLnvgnoNAVMlwkepyqM3DaIFUg== + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" From ef3ec1ab82c52e1c4d1702f89b7b6b56cb47c783 Mon Sep 17 00:00:00 2001 From: darausi Date: Mon, 26 Apr 2021 08:59:07 +0100 Subject: [PATCH 52/95] fixed valuepath highlighting --- .../AutoSuggestion/AutoSuggestion.tsx | 3 +- .../components/AutoSuggestion/Dropdown.tsx | 1 + .../components/CodeEditor.tsx | 83 +++++++++---------- 3 files changed, 42 insertions(+), 45 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 68630d4569..fc906792c8 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -51,7 +51,6 @@ const AutoSuggestion = ({ ] = React.useState(); const valueRef = React.useRef(""); - const dropdownRef = React.useRef(); const pathIsValid = validationResponse?.valid ?? true; //handle keypress @@ -215,7 +214,7 @@ const AutoSuggestion = ({ return nextIndex; }); } - const chosenSuggestion = suggestions[nextIndex]?.value + const chosenSuggestion = suggestions[nextIndex]?.value; handleTextHighlighting(chosenSuggestion); }; diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx index 4e44909e90..24db77e4e2 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/Dropdown.tsx @@ -110,6 +110,7 @@ const Dropdown: React.FC = ({ e.preventDefault()} onClick={() => onItemSelectionChange(item.value)} text={ { - return ( -
- { - editor.on("beforeChange", (_, change) => { - var newtext = change.text.join("").replace(/\n/g, ""); - //failing unexpectedly during undo and redo - if ( - change.update && - typeof change.update === "function" - ) { - change.update(change.from, change.to, [newtext]); - } - return true; - }); - setEditorInstance(editor); - }} - value={value} - onFocus={() => onFocusChange(true)} - // onBlur={() => onFocusChange(false)} - options={{ - mode: mode, - lineNumbers: false, - theme: "xq-light", - }} - onCursor={(editor, data) => { - onCursorChange(data, editor.cursorCoords(true, "div")); - }} - onBeforeChange={(editor, data, value) => { - const trimmedValue = value.replace(/\n/g, ""); - onChange(trimmedValue); - }} - onKeyDown={(_, event) => handleSpecialKeysPress(event)} - /> -
- ); + return ( +
+ { + editor.on("beforeChange", (_, change) => { + var newtext = change.text.join("").replace(/\n/g, ""); + //failing unexpectedly during undo and redo + if (change.update && typeof change.update === "function") { + change.update(change.from, change.to, [newtext]); + } + return true; + }); + setEditorInstance(editor); + }} + value={value} + onFocus={() => onFocusChange(true)} + onBlur={() => onFocusChange(false)} + options={{ + mode: mode, + lineNumbers: false, + theme: "xq-light", + }} + onCursor={(editor, data) => { + onCursorChange(data, editor.cursorCoords(true, "div")); + }} + onBeforeChange={(editor, data, value) => { + const trimmedValue = value.replace(/\n/g, ""); + onChange(trimmedValue); + }} + onKeyDown={(_, event) => handleSpecialKeysPress(event)} + /> +
+ ); }; export default CodeEditor; From 67ddd8a8e4b251d0bbaf2dc5749ce8215c3eedbe Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Mon, 26 Apr 2021 16:30:34 +0200 Subject: [PATCH 53/95] Do not rely on serialization of parsed path ops for partial auto-completion --- .../transform/AutoCompletionApi.scala | 127 +------------- ...rtialSourcePathAutoCompletionRequest.scala | 7 + ...artialSourcePathAutocompletionHelper.scala | 160 +++++++++++++++--- .../PartialAutoCompletionApiTest.scala | 11 ++ ...alSourcePathAutocompletionHelperTest.scala | 15 ++ 5 files changed, 172 insertions(+), 148 deletions(-) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala index f9e4cd23e7..113a8f224b 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/AutoCompletionApi.scala @@ -56,8 +56,6 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU sourcePath.filter(op => op.isInstanceOf[ForwardOperator] || op.isInstanceOf[BackwardOperator]) } - final val DEFAULT_AUTO_COMPLETE_RESULTS = 30 - /** A more fine-grained auto-completion of a source path that suggests auto-completion in parts of a path. */ def partialSourcePath(projectId: String, transformTaskId: String, @@ -87,13 +85,13 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU relativeForwardPaths = relativeForwardPaths.map(c => if(!c.value.startsWith("/") && !c.value.startsWith("\\")) c.copy(value = "/" + c.value) else c) } } - val dataSourceSpecialPathCompletions = specialPathCompletions(dataSourceCharacteristicsOpt, pathToReplace) + val dataSourceSpecialPathCompletions = PartialSourcePathAutocompletionHelper.specialPathCompletions(dataSourceCharacteristicsOpt, pathToReplace) // Add known paths val completions: Completions = relativeForwardPaths ++ dataSourceSpecialPathCompletions val from = pathToReplace.from val length = pathToReplace.length // Return filtered result - val filteredResults = filterResults(autoCompletionRequest, pathToReplace, completions, dataSourceSpecialPathCompletions) + val filteredResults = PartialSourcePathAutocompletionHelper.filterResults(autoCompletionRequest, pathToReplace, completions) val response = PartialSourcePathAutoCompletionResponse( autoCompletionRequest.inputString, autoCompletionRequest.cursorPosition, @@ -103,131 +101,13 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU pathToReplace.query.getOrElse(""), filteredResults.toCompletionsBase.completions ) - ) ++ operatorCompletions(dataSourceCharacteristicsOpt, pathToReplace, autoCompletionRequest) + ) ++ PartialSourcePathAutocompletionHelper.operatorCompletions(dataSourceCharacteristicsOpt, pathToReplace, autoCompletionRequest) ) Ok(Json.toJson(response)) } } } - private def specialPathCompletions(dataSourceCharacteristicsOpt: Option[DataSourceCharacteristics], - pathToReplace: PathToReplace): Seq[Completion] = { - if(pathToReplace.insideQuotes) { - Seq.empty - } else { - dataSourceCharacteristicsOpt.toSeq.flatMap { characteristics => - val pathOps = Seq("/", "\\", "[") - - def pathWithoutOperator(specialPath: String): Boolean = pathOps.forall(op => !specialPath.startsWith(op)) - - characteristics.supportedPathExpressions.specialPaths - // No backward or filter paths allowed inside filters - .filter(p => !pathToReplace.insideFilter || !pathOps.drop(1).forall(disallowedOp => p.value.startsWith(disallowedOp))) - .map { p => - val pathSegment = if (pathToReplace.from > 0 && !pathToReplace.insideFilter && pathWithoutOperator(p.value)) { - "/" + p.value - } else if ((pathToReplace.from == 0 || pathToReplace.insideFilter) && p.value.startsWith("/")) { - p.value.stripPrefix("/") - } else { - p.value - } - Completion(pathSegment, label = None, description = p.description, category = Categories.sourcePaths, isCompletion = true) - } - } - } - } - - // Filter results based on text query and limit number of results - private def filterResults(autoCompletionRequest: PartialSourcePathAutoCompletionRequest, - pathToReplace: PathToReplace, - completions: Completions, - dataSourceSpecialPathCompletions: Seq[Completion]): Completions = { - val stringToBeReplaced = autoCompletionRequest.inputString.substring(pathToReplace.from, pathToReplace.from + pathToReplace.length) - val filteredCompletions: Completions = pathToReplace.query match { - case Some(query) => - completions. - filterAndSort( - query, - autoCompletionRequest.maxSuggestions.getOrElse(DEFAULT_AUTO_COMPLETE_RESULTS), - sortEmptyTermResult = false, - multiWordFilter = true - ) - case None => - Seq.empty - } - if(filteredCompletions.values.size == 1 && filteredCompletions.values.head.value == stringToBeReplaced) { - // If only real search result has the exact same value as the string to be replaced, do not suggest anything. - Seq.empty - } else { - filteredCompletions - } - } - - private def operatorCompletions(dataSourceCharacteristicsOpt: Option[DataSourceCharacteristics], - pathToReplace: PathToReplace, - autoCompletionRequest: PartialSourcePathAutoCompletionRequest): Option[ReplacementResults] = { - def completion(predicate: Boolean, value: String, description: String): Option[CompletionBase] = { - if(predicate) Some(CompletionBase(value, description = Some(description))) else None - } - // Propose operators - if (!pathToReplace.insideQuotesOrUri && !autoCompletionRequest.charBeforeCursor.contains('/') && !autoCompletionRequest.charBeforeCursor.contains('\\')) { - val supportedPathExpressions = dataSourceCharacteristicsOpt.getOrElse(DataSourceCharacteristics()).supportedPathExpressions - val forwardOp = completion(autoCompletionRequest.cursorPosition > 0 && supportedPathExpressions.multiHopPaths && !pathToReplace.insideFilter, - "/", "Starts a forward path segment") - val backwardOp = completion(supportedPathExpressions.backwardPaths && supportedPathExpressions.multiHopPaths && !pathToReplace.insideFilter, - "\\", "Starts a backward path segment") - val langFilterOp = completion(autoCompletionRequest.cursorPosition > 0 && supportedPathExpressions.languageFilter && !pathToReplace.insideFilter, - "[@lang ", "Starts a language filter expression") - val propertyFilter = completion(autoCompletionRequest.cursorPosition > 0 && supportedPathExpressions.propertyFilter && !pathToReplace.insideFilter, - "[", "Starts a property filter expression") - val filterClose = completion(validEndOfFilter(pathToReplace, autoCompletionRequest, supportedPathExpressions), - "]", "End filter expression") - Some(ReplacementResults( - ReplacementInterval(autoCompletionRequest.cursorPosition, 0), - "", - forwardOp.toSeq ++ backwardOp ++ propertyFilter ++ langFilterOp ++ filterClose - )) - } else { - None - } - } - - private val operatorRegexReverse = ">|<|=>|=<|=|=!".r - - /** Decide if the cursor position would be a valid end of filter expression in order to propose the filter end operator. */ - private def validEndOfFilter(pathToReplace: PathToReplace, - autoCompletionRequest: PartialSourcePathAutoCompletionRequest, - supportedPathExpressions: SupportedPathExpressions): Boolean = { - if(pathToReplace.insideFilter && - (supportedPathExpressions.propertyFilter || supportedPathExpressions.languageFilter) && - !pathToReplace.insideQuotesOrUri && - !autoCompletionRequest.remainingStringInOperator.endsWith("]") && - autoCompletionRequest.remainingStringInOperator.trim == "") { - // Still need to check if there is a value and an operator to compare against - val beforeCursor = autoCompletionRequest.stringInOperatorToCursor.reverse.trim - beforeCursor.headOption match { - case Some(lastChar) => - val filterWithoutValue = if(Set('\'', '"', '>').contains(lastChar)) { - val searchChar = if(lastChar == '>') '<' else lastChar - beforeCursor.drop(1).dropWhile(_ != searchChar).drop(1) - } else { - beforeCursor.dropWhile(c => !Set(' ', '\t', '!', '=', '<', '>').contains(c)) - } - if(beforeCursor.length == filterWithoutValue.length) { - // No value found - false - } else { - // Find operator (reversed) - operatorRegexReverse.findFirstMatchIn(filterWithoutValue.trim).exists(_.start == 0) - } - case None => - false - } - } else { - false - } - } - private def dataSourceCharacteristics(task: ProjectTask[TransformSpec]) (implicit userContext: UserContext): Option[DataSourceCharacteristics] = { task.project.taskOption[GenericDatasetSpec](task.selection.inputId) @@ -476,7 +356,6 @@ class AutoCompletionApi @Inject() () extends InjectedController with ControllerU propertyCompletions.distinct } - private implicit def createCompletion(completions: Seq[Completion]): Completions = Completions(completions) } diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala index 572d15ec08..4514ffdcc5 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutoCompletionRequest.scala @@ -29,6 +29,13 @@ case class PartialSourcePathAutoCompletionRequest(inputString: String, } } + /** The full string of original representation of the path operator the cursor is currently placed in. */ + def currentOperatorString: String = { + // Zero if in first path segment with stripped forward operator + val operatorIdx = pathOperatorIdxBeforeCursor.getOrElse(0) + inputString.substring(operatorIdx, indexOfOperatorEnd) + } + /** The string in the current operator up to the cursor. */ def stringInOperatorToCursor: String = { inputString.substring(pathOperatorIdxBeforeCursor.getOrElse(0), cursorPosition) diff --git a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala index 283cd706c8..c2a3a5664b 100644 --- a/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala +++ b/silk-workbench/silk-workbench-rules/app/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelper.scala @@ -1,11 +1,18 @@ package controllers.transform.autoCompletion +import controllers.transform.AutoCompletionApi.Categories import org.silkframework.config.Prefixes +import org.silkframework.dataset.{DataSourceCharacteristics, SupportedPathExpressions} import org.silkframework.entity.paths.{DirectionalPathOperator, PartialParseError, PartialParseResult, PathOperator, PathParser, UntypedPath} import org.silkframework.util.{StringUtils, Uri} +import scala.language.implicitConversions + +/** Utility methods for the partial source path auto-completion endpoint. */ object PartialSourcePathAutocompletionHelper { + final val DEFAULT_AUTO_COMPLETE_RESULTS = 30 + /** * Returns the part of the path that should be replaced and the extracted query words that can be used for search. * @@ -49,6 +56,128 @@ object PartialSourcePathAutocompletionHelper { handleCursorStatusFlags(request, pathToReplace) } + /** Returns completion results for data source specific "special" paths, e.g. "#idx" for CSV/Excel. */ + def specialPathCompletions(dataSourceCharacteristicsOpt: Option[DataSourceCharacteristics], + pathToReplace: PathToReplace): Seq[Completion] = { + if(pathToReplace.insideQuotes) { + Seq.empty + } else { + dataSourceCharacteristicsOpt.toSeq.flatMap { characteristics => + val pathOps = Seq("/", "\\", "[") + + def pathWithoutOperator(specialPath: String): Boolean = pathOps.forall(op => !specialPath.startsWith(op)) + + characteristics.supportedPathExpressions.specialPaths + // No backward or filter paths allowed inside filters + .filter(p => !pathToReplace.insideFilter || !pathOps.drop(1).forall(disallowedOp => p.value.startsWith(disallowedOp))) + .map { p => + val pathSegment = if (pathToReplace.from > 0 && !pathToReplace.insideFilter && pathWithoutOperator(p.value)) { + "/" + p.value + } else if ((pathToReplace.from == 0 || pathToReplace.insideFilter) && p.value.startsWith("/")) { + p.value.stripPrefix("/") + } else { + p.value + } + Completion(pathSegment, label = None, description = p.description, category = Categories.sourcePaths, isCompletion = true) + } + } + } + } + + /** Returns filter results based on text query and the number of results limit. */ + def filterResults(autoCompletionRequest: PartialSourcePathAutoCompletionRequest, + pathToReplace: PathToReplace, + completions: Completions): Completions = { + val stringToBeReplaced = autoCompletionRequest.inputString.substring(pathToReplace.from, pathToReplace.from + pathToReplace.length) + val filteredCompletions: Completions = pathToReplace.query match { + case Some(query) => + completions. + filterAndSort( + query, + autoCompletionRequest.maxSuggestions.getOrElse(DEFAULT_AUTO_COMPLETE_RESULTS), + sortEmptyTermResult = false, + multiWordFilter = true + ) + case None => + Seq.empty + } + if(filteredCompletions.values.size == 1 && filteredCompletions.values.head.value == stringToBeReplaced) { + // If only real search result has the exact same value as the string to be replaced, do not suggest anything. + Seq.empty + } else { + filteredCompletions + } + } + + private implicit def createCompletion(completions: Seq[Completion]): Completions = Completions(completions) + + /** Returns completions for path operators. */ + def operatorCompletions(dataSourceCharacteristicsOpt: Option[DataSourceCharacteristics], + pathToReplace: PathToReplace, + autoCompletionRequest: PartialSourcePathAutoCompletionRequest): Option[ReplacementResults] = { + def completion(predicate: Boolean, value: String, description: String): Option[CompletionBase] = { + if(predicate) Some(CompletionBase(value, description = Some(description))) else None + } + // Propose operators + if (!pathToReplace.insideQuotesOrUri && !autoCompletionRequest.charBeforeCursor.contains('/') && !autoCompletionRequest.charBeforeCursor.contains('\\')) { + val supportedPathExpressions = dataSourceCharacteristicsOpt.getOrElse(DataSourceCharacteristics()).supportedPathExpressions + val forwardOp = completion(autoCompletionRequest.cursorPosition > 0 && supportedPathExpressions.multiHopPaths && !pathToReplace.insideFilter, + "/", "Starts a forward path segment") + val backwardOp = completion(supportedPathExpressions.backwardPaths && supportedPathExpressions.multiHopPaths && !pathToReplace.insideFilter, + "\\", "Starts a backward path segment") + val langFilterOp = completion(autoCompletionRequest.cursorPosition > 0 && supportedPathExpressions.languageFilter && !pathToReplace.insideFilter, + "[@lang ", "Starts a language filter expression") + val propertyFilter = completion(autoCompletionRequest.cursorPosition > 0 && supportedPathExpressions.propertyFilter && !pathToReplace.insideFilter, + "[", "Starts a property filter expression") + val filterClose = completion(validEndOfFilter(pathToReplace, autoCompletionRequest, supportedPathExpressions), + "]", "End filter expression") + Some(ReplacementResults( + ReplacementInterval(autoCompletionRequest.cursorPosition, 0), + "", + forwardOp.toSeq ++ backwardOp ++ propertyFilter ++ langFilterOp ++ filterClose + )) + } else { + None + } + } + + private val operatorRegexReverse = ">|<|=>|=<|=|=!".r + + /** Decide if the cursor position would be a valid end of filter expression in order to propose the filter end operator. */ + private def validEndOfFilter(pathToReplace: PathToReplace, + autoCompletionRequest: PartialSourcePathAutoCompletionRequest, + supportedPathExpressions: SupportedPathExpressions): Boolean = { + if(pathToReplace.insideFilter && + (supportedPathExpressions.propertyFilter || supportedPathExpressions.languageFilter) && + !pathToReplace.insideQuotesOrUri && + !autoCompletionRequest.remainingStringInOperator.endsWith("]") && + autoCompletionRequest.remainingStringInOperator.trim == "") { + // Still need to check if there is a value and an operator to compare against + val beforeCursor = autoCompletionRequest.stringInOperatorToCursor.reverse.trim + beforeCursor.headOption match { + case Some(lastChar) => + val filterWithoutValue = if(Set('\'', '"', '>').contains(lastChar)) { + val searchChar = if(lastChar == '>') '<' else lastChar + beforeCursor.drop(1).dropWhile(_ != searchChar).drop(1) + } else { + beforeCursor.dropWhile(c => !Set(' ', '\t', '!', '=', '<', '>').contains(c)) + } + if(beforeCursor.length == filterWithoutValue.length) { + // No value found + false + } else { + // Find operator (reversed) + operatorRegexReverse.findFirstMatchIn(filterWithoutValue.trim).exists(_.start == 0) + } + case None => + false + } + } else { + false + } + } + + private def handleCursorStatusFlags(request: PartialSourcePathAutoCompletionRequest, replacement: PathToReplace): PathToReplace = { val positionStatus = request.cursorPositionStatus @@ -145,34 +274,17 @@ object PartialSourcePathAutocompletionHelper { private def handleMiddleOfPathOp(partialResult: PartialParseResult, request: PartialSourcePathAutoCompletionRequest): PathToReplace = { - val lastPathOp = partialResult.partialPath.operators.last - val extractedTextQuery = extractTextPart(lastPathOp) - val remainingStringInOp = partialResult.error match { - case Some(error) => request.inputString.substring(error.nextParseOffset, request.indexOfOperatorEnd) - case None => request.remainingStringInOperator - } - val fullQuery = extractedTextQuery.map(q => extractQuery(q + remainingStringInOp)) + val currentPathOpString = request.currentOperatorString + val currentOpLength = currentPathOpString.length + val extractedTextQuery = currentPathOpString.stripPrefix("/").stripPrefix("\\") + val fullQuery = Some(extractQuery(extractedTextQuery)) if(extractedTextQuery.isEmpty) { // This is the end of a valid filter expression, do not replace or suggest anything besides default completions PathToReplace(request.pathUntilCursor.length, 0, None) - } else if(lastPathOp.serialize.length >= request.pathUntilCursor.length) { - // The path op is the complete input path - PathToReplace(0, request.pathUntilCursor.length + request.remainingStringInOperator.length, fullQuery) } else { - // Replace the last path operator of the input path - val lastOpLength = lastPathOp.serialize.length - val from = math.max(partialResult.error.map(_.nextParseOffset).getOrElse(request.cursorPosition) - lastOpLength, 0) - PathToReplace(from, lastOpLength + request.remainingStringInOperator.length, fullQuery) - } - } - - private def extractTextPart(pathOp: PathOperator): Option[String] = { - pathOp match { - case op: DirectionalPathOperator => - Some(op.property.uri) - case _ => - // This is the end of a complete filter expression, suggest nothing to replace it with. - None + // Replace the current path operator of the input path + val from = math.max(partialResult.error.map(_.nextParseOffset).getOrElse(request.cursorPosition) - currentOpLength, 0) + PathToReplace(request.pathOperatorIdxBeforeCursor.getOrElse(0), currentOpLength, fullQuery) } } diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala index 70065940a4..5fc61ff624 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/PartialAutoCompletionApiTest.scala @@ -151,6 +151,17 @@ class PartialAutoCompletionApiTest extends FlatSpec with MustMatchers with Singl jsonSuggestions("department/[ != \"label\"") must contain("]") } + it should "suggest the replacement of properties where the cursor is currently at" in { + val input = "/rdf:type" + val secondPathOp = "/rdf:type" + val result = partialSourcePathAutoCompleteRequest(rdfTransform, inputText = input, cursorPosition = input.length) + result.replacementResults must have size 2 + result.replacementResults.head.replacementInterval mustBe ReplacementInterval(input.length - secondPathOp.length, secondPathOp.length) + val resultFirstProp = partialSourcePathAutoCompleteRequest(rdfTransform, inputText = input, cursorPosition = 2) + resultFirstProp.replacementResults must have size 1 + resultFirstProp.replacementResults.head.replacementInterval mustBe ReplacementInterval(0, input.length - secondPathOp.length) + } + private def partialAutoCompleteResult(inputString: String = "", cursorPosition: Int = 0, replacementResult: Seq[ReplacementResults]): PartialSourcePathAutoCompletionResponse = { diff --git a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala index db8b63c40b..d91c3b5da3 100644 --- a/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala +++ b/silk-workbench/silk-workbench-rules/test/controllers/transform/autoCompletion/PartialSourcePathAutocompletionHelperTest.scala @@ -20,6 +20,12 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche replace("some test", 3) mustBe PathToReplace(0, 9, Some("some test")) } + it should "replace multi word queries in the middle of a path" in { + def replaceFull(input: String) = replace(input, input.length) + replace("a/some test/b", cursorPosition = 2) mustBe PathToReplace(1, 12, Some("some test")) + replace("a/some test/b", "a/some t".length) mustBe PathToReplace(1, 12, Some("some test")) + } + it should "correctly find out which part of a path to replace in simple forward paths for the sub path the cursor is in" in { val input = "a1/b1/c1" replace(input, 4, subPathOnly = true) mustBe PathToReplace(2, 3, Some("b1")) @@ -70,6 +76,15 @@ class PartialSourcePathAutocompletionHelperTest extends FlatSpec with MustMatche replace(pathWithUri, 1) mustBe PathToReplace(0, pathWithUri.length, query = expectedQuery, insideUri = true) } + it should "suggest replacements for inputs containing URIs correctly" in { + val input = "/rdf:type" + replace(input, input.length, subPathOnly = true) mustBe PathToReplace( + "".length, + "/rdf:type".length, + query = Some("type") + ) + } + it should "extract filter queries from URIs and qnames correctly" in { replace("").query mustBe Some("address") replace("\\/").query mustBe Some("address") From c751c32d3b5a804335c2a348171aaec190d9da34 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Mon, 26 Apr 2021 16:55:18 +0200 Subject: [PATCH 54/95] Fix auto-complete search highlighting in value suggest component --- .../AutoSuggestion/AutoSuggestion.tsx | 25 +++++++++++-------- .../components/AutoSuggestion/Dropdown.tsx | 13 ++++------ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index fc906792c8..f4af32ae78 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -4,7 +4,7 @@ import { Icon, Spinner, Label } from "@gui-elements/index"; //custom components import CodeEditor from "../CodeEditor"; -import Dropdown from "./Dropdown"; +import {Dropdown} from "./Dropdown"; //styles require("./AutoSuggestion.scss"); @@ -16,6 +16,13 @@ export enum OVERWRITTEN_KEYS { Tab = "Tab", } +export interface ISuggestion { + value: string + label?: string + description?: string + query: string +} + const AutoSuggestion = ({ onEditorParamsChange, data, @@ -33,9 +40,7 @@ const AutoSuggestion = ({ const [replacementIndexesDict, setReplacementIndexesDict] = React.useState( {} ); - const [suggestions, setSuggestions] = React.useState< - Array<{ value: string; description?: string; label?: string }> - >([]); + const [suggestions, setSuggestions] = React.useState([]); const [markers, setMarkers] = React.useState([]); const [ editorInstance, @@ -76,7 +81,7 @@ const AutoSuggestion = ({ /** generate suggestions and also populate the replacement indexes dict */ React.useEffect(() => { - let newSuggestions = []; + let newSuggestions: ISuggestion[] = []; let newReplacementIndexesDict = {}; if ( data?.replacementResults?.length === 1 && @@ -86,8 +91,9 @@ const AutoSuggestion = ({ } if (data?.replacementResults?.length) { data.replacementResults.forEach( - ({ replacements, replacementInterval: { from, length } }) => { - newSuggestions = [...newSuggestions, ...replacements]; + ({ replacements, replacementInterval: { from, length }, extractedQuery }) => { + const replacementsWithMetaData = replacements.map(r => ({...r, query: extractedQuery})) + newSuggestions = [...newSuggestions, ...replacementsWithMetaData]; replacements.forEach((replacement) => { newReplacementIndexesDict = { ...newReplacementIndexesDict, @@ -99,8 +105,8 @@ const AutoSuggestion = ({ }); } ); - setSuggestions(() => newSuggestions); - setReplacementIndexesDict(() => newReplacementIndexesDict); + setSuggestions(newSuggestions); + setReplacementIndexesDict(newReplacementIndexesDict) } }, [data]); @@ -285,7 +291,6 @@ const AutoSuggestion = ({ ; + options: Array; onItemSelectionChange: (item) => void; isOpen: boolean; - query?: string; loading?: boolean; left?: number; currentlyFocusedIndex?: number; @@ -53,15 +53,14 @@ const RawItem = ({ item, query }, ref) => { const Item = React.forwardRef(RawItem); -const Dropdown: React.FC = ({ +export const Dropdown = ({ isOpen, options, loading, onItemSelectionChange, - query, left, currentlyFocusedIndex, -}) => { +}: IDropdownProps) => { const dropdownRef = React.useRef(); const refs = {}; const generateRef = (index) => { @@ -116,7 +115,7 @@ const Dropdown: React.FC = ({ } > @@ -126,5 +125,3 @@ const Dropdown: React.FC = ({
); }; - -export default Dropdown; From c84f00c11dea8f250a1bf9917c9cee4f15fbf8c7 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Mon, 26 Apr 2021 17:13:13 +0200 Subject: [PATCH 55/95] Underline full error when inputLeadingToError > 1 --- .../scala/org/silkframework/entity/paths/PathParser.scala | 2 +- .../components/AutoSuggestion/AutoSuggestion.tsx | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala b/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala index b686ccef77..198dc3b590 100644 --- a/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala +++ b/silk-core/src/main/scala/org/silkframework/entity/paths/PathParser.scala @@ -136,7 +136,7 @@ private[entity] class PathParser(prefixes: Prefixes) extends RegexParsers { private def languageTag = """[a-zA-Z]+('-'[a-zA-Z0-9]+)*""".r // A value that is either an identifier or a literal value enclosed in quotes (e.g., "literal"). - private def value = identifier | "\"[^\"]+\"".r + private def value = identifier | "\"[^\"]*\"".r // A comparison operator private def compOperator = ">" | "<" | ">=" | "<=" | "=" | "!=" diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index f4af32ae78..701154581c 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -68,8 +68,9 @@ const AutoSuggestion = ({ const parseError = validationResponse?.parseError; if (parseError) { clearMarkers(); - const { offset: start, inputLeadingToError } = parseError; - const end = start + inputLeadingToError?.length; + const { offset, inputLeadingToError, message } = parseError; + const start = inputLeadingToError.length > 1 ? offset - inputLeadingToError.length + 1 : offset + const end = offset + 2; const marker = editorInstance.markText( { line: 0, ch: start }, { line: 0, ch: end }, From e2ce1774da39ec5c0058cf30f3757daa7b0d40c9 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Mon, 26 Apr 2021 17:20:30 +0200 Subject: [PATCH 56/95] Add tooltip to clear icon of auto suggest component --- .../components/AutoSuggestion/AutoSuggestion.tsx | 3 +++ .../containers/MappingRule/ValueRule/ValueRuleForm.tsx | 1 + 2 files changed, 4 insertions(+) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 701154581c..c1828a9f55 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -31,6 +31,7 @@ const AutoSuggestion = ({ pathValidationPending, suggestionsPending, label, + clearIconText, }) => { const [value, setValue] = React.useState(""); const [inputString, setInputString] = React.useState(""); @@ -286,6 +287,8 @@ const AutoSuggestion = ({ small className="editor__icon clear" name="operation-clear" + tooltipText={clearIconText} + tooltipProperties={{usePortal: false}} />
diff --git a/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx b/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx index 4a82b66472..03bee57632 100644 --- a/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx +++ b/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx @@ -317,6 +317,7 @@ export function ValueRuleForm(props: IProps) { sourcePropertyInput = ( Date: Mon, 26 Apr 2021 17:42:10 +0200 Subject: [PATCH 57/95] Add tooltip to value path error icon --- .../components/AutoSuggestion/AutoSuggestion.tsx | 3 +++ .../containers/MappingRule/ValueRule/ValueRuleForm.tsx | 1 + 2 files changed, 4 insertions(+) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index c1828a9f55..9e1681f30b 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -32,6 +32,7 @@ const AutoSuggestion = ({ suggestionsPending, label, clearIconText, + validationErrorText, }) => { const [value, setValue] = React.useState(""); const [inputString, setInputString] = React.useState(""); @@ -270,6 +271,8 @@ const AutoSuggestion = ({ small className="editor__icon error" name="operation-clear" + tooltipText={validationErrorText} + tooltipProperties={{usePortal: false}} /> ) : null}
diff --git a/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx b/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx index 03bee57632..c2890ed70b 100644 --- a/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx +++ b/silk-react-components/src/HierarchicalMapping/containers/MappingRule/ValueRule/ValueRuleForm.tsx @@ -318,6 +318,7 @@ export function ValueRuleForm(props: IProps) { Date: Tue, 27 Apr 2021 09:12:46 +0200 Subject: [PATCH 58/95] Fix error underline is shown for valid paths --- .../AutoSuggestion/AutoSuggestion.tsx | 25 ++++++++++++++----- ...odeEditor.tsx => SingleLineCodeEditor.tsx} | 13 +++++----- 2 files changed, 26 insertions(+), 12 deletions(-) rename silk-react-components/src/HierarchicalMapping/components/{CodeEditor.tsx => SingleLineCodeEditor.tsx} (77%) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index 9e1681f30b..d09e8daa22 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -3,7 +3,7 @@ import CodeMirror from "codemirror"; import { Icon, Spinner, Label } from "@gui-elements/index"; //custom components -import CodeEditor from "../CodeEditor"; +import SingleLineCodeEditor from "../SingleLineCodeEditor"; import {Dropdown} from "./Dropdown"; //styles @@ -23,6 +23,8 @@ export interface ISuggestion { query: string } +/** Input component that allows partial, fine-grained auto-completion, i.e. of sub-strings of the input string. + * This is comparable to a one line code editor. */ const AutoSuggestion = ({ onEditorParamsChange, data, @@ -43,7 +45,8 @@ const AutoSuggestion = ({ {} ); const [suggestions, setSuggestions] = React.useState([]); - const [markers, setMarkers] = React.useState([]); + const [markers, setMarkers] = React.useState([]); + const [, setErrorMarkers] = React.useState([]); const [ editorInstance, setEditorInstance, @@ -73,14 +76,24 @@ const AutoSuggestion = ({ const { offset, inputLeadingToError, message } = parseError; const start = inputLeadingToError.length > 1 ? offset - inputLeadingToError.length + 1 : offset const end = offset + 2; + editorInstance.getDoc().getEditor() const marker = editorInstance.markText( { line: 0, ch: start }, { line: 0, ch: end }, { className: "ecc-text-error-highlighting" } ); - setMarkers((previousMarkers) => [...previousMarkers, marker]); + setErrorMarkers((previousMarkers) => { + previousMarkers.forEach(marker => marker.clear()) + return [marker] + }); + } else { + // Valid, clear all error markers + setErrorMarkers((previous) => { + previous.forEach(marker => marker.clear()) + return [] + }) } - }, [validationResponse?.parseError]); + }, [validationResponse?.valid, validationResponse?.parseError]); /** generate suggestions and also populate the replacement indexes dict */ React.useEffect(() => { @@ -276,12 +289,12 @@ const AutoSuggestion = ({ /> ) : null}
- diff --git a/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx b/silk-react-components/src/HierarchicalMapping/components/SingleLineCodeEditor.tsx similarity index 77% rename from silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx rename to silk-react-components/src/HierarchicalMapping/components/SingleLineCodeEditor.tsx index f568314596..bd9a123322 100644 --- a/silk-react-components/src/HierarchicalMapping/components/CodeEditor.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/SingleLineCodeEditor.tsx @@ -2,12 +2,12 @@ import React from "react"; import { Controlled as ControlledEditor } from "react-codemirror2"; import "codemirror/mode/sparql/sparql.js"; -const CodeEditor = ({ +const SingleLineCodeEditor = ({ setEditorInstance, onChange, onCursorChange, mode = "sparql", - value, + initialValue, onFocusChange, handleSpecialKeysPress, }) => { @@ -16,16 +16,17 @@ const CodeEditor = ({ { editor.on("beforeChange", (_, change) => { - var newtext = change.text.join("").replace(/\n/g, ""); + // Prevent the user from entering new-line characters, since this is supposed to be a one-line editor. + const newText = change.text.join("").replace(/\n/g, ""); //failing unexpectedly during undo and redo if (change.update && typeof change.update === "function") { - change.update(change.from, change.to, [newtext]); + change.update(change.from, change.to, [newText]); } return true; }); setEditorInstance(editor); }} - value={value} + value={initialValue} onFocus={() => onFocusChange(true)} onBlur={() => onFocusChange(false)} options={{ @@ -46,4 +47,4 @@ const CodeEditor = ({ ); }; -export default CodeEditor; +export default SingleLineCodeEditor; From ab97f2ea5ca8f5150609453af8dc9e3792b18799 Mon Sep 17 00:00:00 2001 From: Andreas Schultz Date: Tue, 27 Apr 2021 09:54:02 +0200 Subject: [PATCH 59/95] AutoSuggest component: Handle actual change and initial setting of the value for the outside --- .../AutoSuggestion/AutoSuggestion.tsx | 42 ++++++++++++------- .../MappingRule/ValueRule/ValueRuleForm.tsx | 6 +++ ...alSourcePathAutocompletionHelperTest.scala | 4 +- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx index d09e8daa22..2c0031937a 100644 --- a/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx +++ b/silk-react-components/src/HierarchicalMapping/components/AutoSuggestion/AutoSuggestion.tsx @@ -23,11 +23,27 @@ export interface ISuggestion { query: string } +interface IProps { + // Text that should be shown if the input validation failed. + validationErrorText: string + // Text that should be shown when hovering over the clear icon + clearIconText: string + // Optional label to be shown for the input (above) + label?: string + // The value the component is initialized with, do not use this to control value changes. + initialValue: string + // Callback on value change + onChange: (currentValue: string) => any + // TODO: Add remaining parameters + [key: string]: any +} + /** Input component that allows partial, fine-grained auto-completion, i.e. of sub-strings of the input string. * This is comparable to a one line code editor. */ const AutoSuggestion = ({ onEditorParamsChange, data, + onChange, checkPathValidity, validationResponse, pathValidationPending, @@ -35,9 +51,10 @@ const AutoSuggestion = ({ label, clearIconText, validationErrorText, -}) => { - const [value, setValue] = React.useState(""); - const [inputString, setInputString] = React.useState(""); + initialValue, +}: IProps) => { + const [value, setValue] = React.useState(initialValue); + const [inputString, setInputString] = React.useState(initialValue); const [cursorPosition, setCursorPosition] = React.useState(0); const [coords, setCoords] = React.useState({ left: 0 }); const [shouldShowDropdown, setShouldShowDropdown] = React.useState(false); @@ -128,7 +145,7 @@ const AutoSuggestion = ({ React.useEffect(() => { if (isFocused) { - setInputString(() => value); + setInputString(value); setShouldShowDropdown(true); //only change if the input has changed, regardless of the cursor change if (valueRef.current !== value) { @@ -139,7 +156,8 @@ const AutoSuggestion = ({ } }, [cursorPosition, value, inputString, isFocused]); - const handleChange = (val) => { + const handleChange = (val: string) => { + onChange(val) setValue(val); }; @@ -182,22 +200,16 @@ const AutoSuggestion = ({ if (indexes) { const { from, length } = indexes; const to = from + length; - setValue( - (value) => - `${value.substring( - 0, - from - )}${selectedSuggestion}${value.substring(to)}` - ); + editorInstance.replaceRange(selectedSuggestion, {line: 0, ch: from}, {line: 0, ch: to}) setShouldShowDropdown(false); - editorInstance.setCursor({ line: 0, ch: to }); + editorInstance.setCursor({ line: 0, ch: from + selectedSuggestion.length }); editorInstance.focus(); clearMarkers(); } }; const handleInputEditorClear = () => { - setValue(""); + editorInstance.getDoc().setValue("") }; const handleInputFocus = (focusState: boolean) => { @@ -273,7 +285,7 @@ const AutoSuggestion = ({ return (
-