From 01cc3277b3cbf21e90a489549c163173bd494b98 Mon Sep 17 00:00:00 2001 From: Marvin Hofer Date: Mon, 25 Sep 2023 11:21:59 +0200 Subject: [PATCH] docu --- .../server/core/execution/Singleton.scala | 11 --- .../{DatabusFile.scala => DatasetPart.scala} | 36 ++++++-- ...Repository.scala => DatasetPartRepo.scala} | 6 +- .../{ModRepository.scala => ModRepo.scala} | 0 .../{TaskRepository.scala => TaskRepo.scala} | 0 ...orkerRepository.scala => WorkerRepo.scala} | 2 +- .../core/persistence/WorkerStatus.scala | 6 -- .../server/core/utils/TestUtil.scala | 44 --------- .../server/core/utils/VOSUtil.scala | 31 ------- doc/arch.md | 19 ---- doc/worker-api.md | 89 +++++++++++++++++++ 11 files changed, 120 insertions(+), 124 deletions(-) delete mode 100644 databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/execution/Singleton.scala rename databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/{DatabusFile.scala => DatasetPart.scala} (78%) rename databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/{DatabusFileRepository.scala => DatasetPartRepo.scala} (75%) rename databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/{ModRepository.scala => ModRepo.scala} (100%) rename databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/{TaskRepository.scala => TaskRepo.scala} (100%) rename databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/{WorkerRepository.scala => WorkerRepo.scala} (80%) delete mode 100644 databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/WorkerStatus.scala delete mode 100644 databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/utils/TestUtil.scala delete mode 100644 databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/utils/VOSUtil.scala delete mode 100644 doc/arch.md diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/execution/Singleton.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/execution/Singleton.scala deleted file mode 100644 index 4962110..0000000 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/execution/Singleton.scala +++ /dev/null @@ -1,11 +0,0 @@ -package org.dbpedia.databus_mods.server.core.execution - -import org.dbpedia.databus_mods.server.core.service.{MetadataService, TaskService} - -object Singleton { - - var taskService: TaskService = _ - - var metadataService: MetadataService = _ - -} diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatabusFile.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatasetPart.scala similarity index 78% rename from databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatabusFile.scala rename to databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatasetPart.scala index 7be6181..129a13d 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatabusFile.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatasetPart.scala @@ -9,6 +9,18 @@ import org.dbpedia.databus_mods.server.core.views.Views import scala.annotation.meta.field import scala.beans.BeanProperty +/** + * Java representation of dataid core metadata for a file registered on the DBpedia databus + * @param dataIdSingleFile + * @param publisher + * @param dataIdGroup + * @param dataIdArtifact + * @param dataIdVersion + * @param downloadUrl + * @param name + * @param checksum + * @param issued + */ @Entity @Table( name = "databus_file", @@ -16,29 +28,37 @@ import scala.beans.BeanProperty // // TODO // new UniqueConstraint(columnNames = Array("dataIdSingleFile", "checkSum"))) ) -class DatabusFile +class DataIdPart ( // TODO rename to databusFileID or fileID @BeanProperty @(JsonView@field)(value = Array(classOf[Views.Default])) @(Column@field)(length = 4000) var dataIdSingleFile: String, + @BeanProperty var publisher: String, + @BeanProperty var dataIdGroup: String, + @BeanProperty var dataIdArtifact: String, + @BeanProperty var dataIdVersion: String, + @BeanProperty @(Column@field)(length = 4000) var downloadUrl: String, + @BeanProperty var name: String, + @BeanProperty @(JsonView@field)(value = Array(classOf[Views.Default])) var checksum: String, + @BeanProperty @Temporal(TemporalType.TIMESTAMP) var issued: java.sql.Timestamp @@ -50,10 +70,10 @@ class DatabusFile @(JsonView@field)(value = Array(classOf[Views.Default])) var id: Long = _ - @(JsonView@field)(value = Array(classOf[Views.DatabusFileView])) - @(OneToMany@field)(mappedBy = "databusFile", cascade = Array(CascadeType.ALL), fetch = FetchType.EAGER, orphanRemoval = true) - @BeanProperty - var tasks: java.util.List[Task] = new util.ArrayList[Task]() +// @(JsonView@field)(value = Array(classOf[Views.DatabusFileView])) +// @(OneToMany@field)(mappedBy = "databusFile", cascade = Array(CascadeType.ALL), fetch = FetchType.EAGER, orphanRemoval = true) +// @BeanProperty +// var tasks: java.util.List[Task] = new util.ArrayList[Task]() def this() { this(null, null, null, null, null, null, null, null, null) @@ -81,15 +101,15 @@ class DatabusFile |+ dataIdVersion : $dataIdVersion |+ downloadUrl : $downloadUrl |+ checksum : $checksum - |+ taskIds : ${tasks.map(_.getId).mkString(", ")} |""".stripMargin + // + taskIds : ${tasks.map(_.getId).mkString(", ")} } def getDatabusPath: String = { List(publisher,dataIdGroup,dataIdArtifact,dataIdVersion,name).mkString("/") } - def copyOf(df: DatabusFile): Unit = { + def copyOf(df: DataIdPart): Unit = { setId(df.getId) setIssued(df.getIssued) setPublisher(df.getPublisher) @@ -98,6 +118,6 @@ class DatabusFile setDataIdVersion(df.getDataIdVersion) setDownloadUrl(df.getDownloadUrl) setChecksum(df.getChecksum) - setTasks(df.getTasks) +// setTasks(df.getTasks) } } diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatabusFileRepository.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatasetPartRepo.scala similarity index 75% rename from databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatabusFileRepository.scala rename to databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatasetPartRepo.scala index 67c6d94..bae8374 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatabusFileRepository.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatasetPartRepo.scala @@ -8,9 +8,7 @@ import org.springframework.transaction.annotation.Transactional @Repository @Transactional -trait DatabusFileRepository extends CrudRepository[DatabusFile,Long] { - - def findByDataIdSingleFileAndChecksum(dataIdSingleFile: String, checksum: String): Optional[DatabusFile] - +trait DataIdPartRepo extends CrudRepository[DataIdPart,Long] { + def findByDataIdSingleFileAndChecksum(dataIdSingleFile: String, checksum: String): Optional[DataIdPart] } diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/ModRepository.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/ModRepo.scala similarity index 100% rename from databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/ModRepository.scala rename to databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/ModRepo.scala diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/TaskRepository.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/TaskRepo.scala similarity index 100% rename from databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/TaskRepository.scala rename to databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/TaskRepo.scala diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/WorkerRepository.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/WorkerRepo.scala similarity index 80% rename from databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/WorkerRepository.scala rename to databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/WorkerRepo.scala index 6fc930a..5e586e6 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/WorkerRepository.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/WorkerRepo.scala @@ -6,7 +6,7 @@ import org.springframework.data.repository.CrudRepository import org.springframework.stereotype.Repository @Repository -trait WorkerRepository extends CrudRepository[Worker,Long] { +trait WorkerRepo extends CrudRepository[Worker,Long] { def findByUrl(url: String): Optional[Worker] } diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/WorkerStatus.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/WorkerStatus.scala deleted file mode 100644 index 537e8ea..0000000 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/WorkerStatus.scala +++ /dev/null @@ -1,6 +0,0 @@ -package org.dbpedia.databus_mods.server.core.persistence - -object WorkerStatus extends Enumeration { - val UP,DOWN = Value -} - diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/utils/TestUtil.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/utils/TestUtil.scala deleted file mode 100644 index 5e81ccf..0000000 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/utils/TestUtil.scala +++ /dev/null @@ -1,44 +0,0 @@ -package org.dbpedia.databus_mods.server.core.utils - -import java.util.Calendar - -object TestUtil extends App { - - object Util { - - def doSomething() { - synchronized { - Thread.sleep(3000) - println(s"${Thread.currentThread().getName} ${Calendar.getInstance().getTime}") - } - } - - def doSomethingDifferent(): Unit = { - synchronized { -// Thread.sleep(1000) - println(s"${Thread.currentThread().getName} ${Calendar.getInstance().getTime}") - } - } - } - - val t1: Thread = new Thread(new Runnable { - override def run(): Unit = { - while (true) { - Util.doSomething() - Thread.sleep(0,1) -// Thread.sleep(100) - } - } - }) - t1.setName("A") - t1.start() - - new Thread(new Runnable { - override def run(): Unit = { - while (true) { - Util.doSomethingDifferent() -// Thread.sleep(100) - } - } - }).start() -} diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/utils/VOSUtil.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/utils/VOSUtil.scala deleted file mode 100644 index 0dc5233..0000000 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/utils/VOSUtil.scala +++ /dev/null @@ -1,31 +0,0 @@ -package org.dbpedia.databus_mods.server.core.utils - -import java.io.ByteArrayInputStream -import java.nio.charset.StandardCharsets - -import org.apache.jena.rdf.model.ModelFactory -import org.apache.jena.riot.{Lang, RDFDataMgr, RDFLanguages} -import virtuoso.jena.driver.VirtDataset - -object VOSUtil extends App { - - val ttl = - """ - | "foobar2" .""".stripMargin - - val model = ModelFactory.createDefaultModel() - - RDFDataMgr.read(model, new ByteArrayInputStream(ttl.getBytes(StandardCharsets.UTF_8)),RDFLanguages.NTRIPLES) - - model.write(System.out,Lang.NTRIPLES.getName) - - val virtDataset = new VirtDataset("jdbc:virtuoso://localhost:32972/charset=UTF-8/","dba","myDbaPassword") - - println(virtDataset.containsNamedModel("http://localhost:8890/test")) - - virtDataset.removeNamedModel("http://localhost:8890/test") - - virtDataset.addNamedModel("http://localhost:8890/test",model,false) - virtDataset.commit() - virtDataset.close() -} diff --git a/doc/arch.md b/doc/arch.md deleted file mode 100644 index 373e739..0000000 --- a/doc/arch.md +++ /dev/null @@ -1,19 +0,0 @@ -# Databus Mods Architecture - -## Components - -* 1 Mod Server -* N Mod Workers -* Linked Data Services - * SPARQL Endpoint - * HTTP File Server - -## Mod Server - -## Mod Worker - -## Linked Data Services - -### SPARQL Endpoint - -### HTTP File Server diff --git a/doc/worker-api.md b/doc/worker-api.md index a56d128..ddf560a 100644 --- a/doc/worker-api.md +++ b/doc/worker-api.md @@ -26,3 +26,92 @@ ## Pollingbased API Profile +## Backlog +- configuration of boot starter +- documentation: server setup, boot starter, re-implementation of APIs +- push metadata over master +- refactor ActivityController to be a blank trait +- create VocabFactory to help extend the existing Mod Vocab + +### Components + +- core: core components for master and workers + - model + - rdf generation + - validation + - worker client + - + +## Springboot Starter + +### Http Api Documentation + +--- + +#### Execute Mod Activity at Worker + +
+ POST /activity (execute activity at worker) + +##### Parameters + +> | name | type | data type | description | +> |--------| -------|-----------|-----------| +> | dataId | required | string | IRI of dataid entity | +> | accessIRI | optional | string | access IRI of data | + +##### Responses + +> | http code | content-type | response | +> |--------------------|-------------------------------------------|-----------------------------------| +> | `200` | `text/turtle` | `Descriptive Mod Activity Metadata` | +> | `202` | `text/plain` | `Descriptive Mod Activity Metadata` | +> | `400` | `application/json` | `{"code":"400","message":"Bad Request"}` | +> | `500` | `application/json` | `{"code":"500","message":"${Exception}"}` | + +##### Example cURL + +> ```javascript +> curl -X POST --data-urlencode "dataId=TODO" http://localhost:8080/ +> ``` + +
+ +--- + +#### Polling Activity Status + +
+ GET /${jobid}/activity (check activity at worker) + +##### Responses + +> | http code | content-type | response | +> |---------------------------|-----------------------------------------------------------------|-----------------------------------| +> | `202` | `text/turtle` | `Configuration created successfully` | +> | `202` | `text/plain` | `Configuration created successfully` | +> | `400` | `application/json` | `{"code":"400","message":"Bad Request"}` | +> | `500` |`application/json` | `{"code":"500","message":"${Exception}"}` | + +##### Example cURL + +> ```javascript +> curl -X POST -H "Content-Type: application/json" --data @post.json http://localhost:8889/ +> ``` + +
+ +--- + +## HTTP API Re-Implementation + +[//]: # (The master will follow redirects (includes 202) and obeys retry after headers) + +Requirements: +- Post request that ends with /activity + +Pitfalls/Features: +- Activity Deduplication +- Resource Management: Delete local files after they have been accessed once +- +