Merge pull request #1448 from adpi2/speed-up-republish

Speed up republishing all artifacts

modules/core/shared/src/main/scala/scaladex/core/service/SonatypeClient.scala → modules/core/shared/src/main/scala/scaladex/core/service/MavenCentralClient.scala

@@ -8,7 +8,7 @@ import scaladex.core.model.Artifact
 import scaladex.core.model.Artifact.MavenReference
 import scaladex.core.model.SemanticVersion
 
-trait SonatypeClient {
+trait MavenCentralClient {
   def getAllArtifactIds(groupId: Artifact.GroupId): Future[Seq[Artifact.ArtifactId]]
   def getAllVersions(groupId: Artifact.GroupId, artifactId: Artifact.ArtifactId): Future[Seq[SemanticVersion]]
   def getPomFile(mavenReference: MavenReference): Future[Option[(String, Instant)]]

modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala

@@ -29,8 +29,9 @@ trait SchedulerDatabase extends WebDatabase {
   def updateArtifacts(artifacts: Seq[Artifact], newRef: Project.Reference): Future[Int]
   def updateArtifactReleaseDate(reference: MavenReference, releaseDate: Instant): Future[Int]
   def getAllGroupIds(): Future[Seq[Artifact.GroupId]]
-  def getAllArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]]
+  def getArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]]
   def getAllMavenReferences(): Future[Seq[Artifact.MavenReference]]
+  def getMavenReferences(ref: Project.Reference): Future[Seq[Artifact.MavenReference]]
   def getDependencies(projectRef: Project.Reference): Future[Seq[ArtifactDependency]]
   def updateLatestVersion(ref: MavenReference): Future[Unit]
 }

modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala

@@ -139,6 +139,8 @@ class InMemoryDatabase extends SchedulerDatabase {
   override def updateArtifacts(allArtifacts: Seq[Artifact], newRef: Project.Reference): Future[Int] = ???
   override def getAllGroupIds(): Future[Seq[Artifact.GroupId]] = ???
   override def getAllMavenReferences(): Future[Seq[Artifact.MavenReference]] = ???
+
+  override def getMavenReferences(ref: Project.Reference): Future[Seq[Artifact.MavenReference]] = ???
   override def insertUser(userId: UUID, userInfo: UserInfo): Future[Unit] = ???
   override def updateUser(userId: UUID, userInfo: UserState): Future[Unit] = ???
   override def getUser(userId: UUID): Future[Option[UserState]] = ???
@@ -208,7 +210,7 @@ class InMemoryDatabase extends SchedulerDatabase {
     Future.successful(res)
   }
 
-  override def getAllArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]] =
+  override def getArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]] =
     Future.successful(allArtifacts(ref).map(a => (a.groupId, a.artifactId)).toSeq)
 
   override def updateLatestVersion(ref: Artifact.MavenReference): Future[Unit] =

modules/data/src/main/scala/scaladex/data/maven/PomsReader.scala

@@ -77,16 +77,12 @@ class PomsReader(resolver: PomResolver) extends LazyLogging {
   private val jdk = new Properties
   jdk.setProperty("java.version", "1.8") // << ???
 
-  def loadOne(path: Path): Try[(ArtifactModel, String)] = {
-    val sha1 = path.getFileName.toString.stripSuffix(".pom")
-
-    Try {
-      val request = (new DefaultModelBuildingRequest)
-        .setModelResolver(modelResolver)
-        .setSystemProperties(jdk)
-        .setPomFile(path.toFile)
-
-      builder.build(request).getEffectiveModel
-    }.map(pom => (PomConvert(pom), sha1))
-  }
+  def loadOne(path: Path): Try[ArtifactModel] = Try {
+    val request = (new DefaultModelBuildingRequest)
+      .setModelResolver(modelResolver)
+      .setSystemProperties(jdk)
+      .setPomFile(path.toFile)
+
+    builder.build(request).getEffectiveModel
+  }.map(pom => PomConvert(pom))
 }

modules/infra/src/main/scala/scaladex/infra/SonatypeClientImpl.scala → modules/infra/src/main/scala/scaladex/infra/MavenCentralClientImpl.scala

@@ -25,15 +25,15 @@ import scaladex.core.model.Artifact
 import scaladex.core.model.Artifact.MavenReference
 import scaladex.core.model.SbtPlugin
 import scaladex.core.model.SemanticVersion
-import scaladex.core.service.SonatypeClient
+import scaladex.core.service.MavenCentralClient
 import scaladex.core.util.JsoupUtils
 
-class SonatypeClientImpl()(implicit val system: ActorSystem)
+class MavenCentralClientImpl()(implicit val system: ActorSystem)
     extends CommonAkkaHttpClient
-    with SonatypeClient
+    with MavenCentralClient
     with LazyLogging {
   private implicit val ec: ExecutionContextExecutor = system.dispatcher
-  private val sonatypeUri = "https://repo1.maven.org/maven2"
+  private val baseUri = "https://repo1.maven.org/maven2"
   lazy val poolClientFlow: Flow[
     (HttpRequest, Promise[HttpResponse]),
     (Try[HttpResponse], Promise[HttpResponse]),
@@ -46,7 +46,7 @@ class SonatypeClientImpl()(implicit val system: ActorSystem)
       )
 
   def getAllArtifactIds(groupId: Artifact.GroupId): Future[Seq[Artifact.ArtifactId]] = {
-    val uri = s"$sonatypeUri/${groupId.mavenUrl}/"
+    val uri = s"$baseUri/${groupId.mavenUrl}/"
     val request =
       HttpRequest(uri = uri)
 
@@ -60,7 +60,7 @@ class SonatypeClientImpl()(implicit val system: ActorSystem)
   }
 
   def getAllVersions(groupId: Artifact.GroupId, artifactId: Artifact.ArtifactId): Future[Seq[SemanticVersion]] = {
-    val uri = s"$sonatypeUri/${groupId.mavenUrl}/${artifactId.value}/"
+    val uri = s"$baseUri/${groupId.mavenUrl}/${artifactId.value}/"
     val request = HttpRequest(uri = uri)
 
     val future = for {
@@ -104,7 +104,7 @@ class SonatypeClientImpl()(implicit val system: ActorSystem)
       .parse(mavenReference.artifactId)
       .map { artifactId =>
         val pomUrl = getPomUrl(artifactId, mavenReference.version)
-        val uri = s"$sonatypeUri/${groupIdUrl}/${mavenReference.artifactId}/${mavenReference.version}/$pomUrl"
+        val uri = s"$baseUri/${groupIdUrl}/${mavenReference.artifactId}/${mavenReference.version}/$pomUrl"
         val request = HttpRequest(uri = uri)
         queueRequest(request).map(Option.apply)
       }

modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala

@@ -180,12 +180,14 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
   override def getAllGroupIds(): Future[Seq[Artifact.GroupId]] =
     run(ArtifactTable.selectGroupIds.to[Seq])
 
-  override def getAllArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]] =
+  override def getArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]] =
     run(ArtifactTable.selectArtifactIds.to[Seq](ref))
 
   override def getAllMavenReferences(): Future[Seq[Artifact.MavenReference]] =
-    run(ArtifactTable.selectMavenReference.to[Seq])
+    run(ArtifactTable.selectAllMavenReferences.to[Seq])
 
+  override def getMavenReferences(ref: Project.Reference): Future[Seq[Artifact.MavenReference]] =
+    run(ArtifactTable.selectMavenReferences.to[Seq](ref))
   override def insertUser(userId: UUID, userInfo: UserInfo): Future[Unit] =
     run(UserSessionsTable.insert.run((userId, userInfo)).map(_ => ()))
 

modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala

@@ -107,9 +107,12 @@ object ArtifactTable {
   val selectArtifactIds: Query[Project.Reference, (Artifact.GroupId, String)] =
     selectRequest(table, Seq("DISTINCT group_id", "artifact_id"), keys = projectReferenceFields)
 
-  val selectMavenReference: Query0[Artifact.MavenReference] =
+  val selectAllMavenReferences: Query0[Artifact.MavenReference] =
     selectRequest(table, Seq("DISTINCT group_id", "artifact_id", "\"version\""))
 
+  val selectMavenReferences: Query[Project.Reference, Artifact.MavenReference] =
+    selectRequest(table, Seq("DISTINCT group_id", "artifact_id", "\"version\""), keys = projectReferenceFields)
+
   val selectMavenReferenceWithNoReleaseDate: Query0[Artifact.MavenReference] =
     selectRequest(table, Seq("group_id", "artifact_id", "\"version\""), where = Seq("release_date is NULL"))
 

modules/infra/src/test/scala/scaladex/infra/SonatypeClientImplTests.scala → modules/infra/src/test/scala/scaladex/infra/MavenCentralClientImplTests.scala

@@ -9,28 +9,28 @@ import scaladex.core.model.Artifact
 import scaladex.core.model.Artifact._
 import scaladex.core.model.SemanticVersion
 
-class SonatypeClientImplTests extends AsyncFunSpec with Matchers {
-  implicit val system: ActorSystem = ActorSystem("sonatype-client-tests")
-  val sonatypeClient = new SonatypeClientImpl()
+class MavenCentralClientImplTests extends AsyncFunSpec with Matchers {
+  implicit val system: ActorSystem = ActorSystem("maven-central-client-tests")
+  val client = new MavenCentralClientImpl()
   val groupId: GroupId = GroupId("ch.epfl.scala")
   val artifactId: ArtifactId = ArtifactId.parse("sbt-scalafix_2.12_1.0").get
   val version: SemanticVersion = SemanticVersion.parse("0.9.23").get
 
   it(s"retrieve artifactIds for ${groupId.value}") {
     for {
-      res <- sonatypeClient.getAllArtifactIds(groupId)
+      res <- client.getAllArtifactIds(groupId)
     } yield res should contain(artifactId)
   }
 
   it(s"retrieve versions for groupId: ${groupId.value}, artifactId: ${artifactId.value}") {
     for {
-      res <- sonatypeClient.getAllVersions(groupId, artifactId)
+      res <- client.getAllVersions(groupId, artifactId)
     } yield res should contain(version)
   }
 
   it(s"retrieve versions for ru.tinkoff:typed-schema-swagger-typesafe_2.12") {
     for {
-      res <- sonatypeClient.getAllVersions(
+      res <- client.getAllVersions(
         GroupId("ru.tinkoff"),
         ArtifactId.parse("typed-schema-swagger-typesafe_2.12").get
       )
@@ -39,31 +39,31 @@ class SonatypeClientImplTests extends AsyncFunSpec with Matchers {
 
   it(s"retrieve pomfile for maven reference of sbt plugin") {
     for {
-      res <- sonatypeClient.getPomFile(Artifact.MavenReference("ch.epfl.scala", "sbt-scalafix_2.12_1.0", "0.9.23"))
+      res <- client.getPomFile(Artifact.MavenReference("ch.epfl.scala", "sbt-scalafix_2.12_1.0", "0.9.23"))
     } yield res.get._1.startsWith("<?xml") shouldBe true
   }
 
   it(s"retrieve pomfile for maven reference of jvm") {
     for {
-      res <- sonatypeClient.getPomFile(Artifact.MavenReference("ch.epfl.scala", "scalafix-core_2.13", "0.9.23"))
+      res <- client.getPomFile(Artifact.MavenReference("ch.epfl.scala", "scalafix-core_2.13", "0.9.23"))
     } yield res.get._1.startsWith("<?xml") shouldBe true
   }
 
   it(s"retrieve pomfile for maven reference of ScalaJs") {
     for {
-      res <- sonatypeClient.getPomFile(Artifact.MavenReference("ch.epfl.scala", "bloop-config_sjs1_2.13", "1.4.11"))
+      res <- client.getPomFile(Artifact.MavenReference("ch.epfl.scala", "bloop-config_sjs1_2.13", "1.4.11"))
     } yield res.get._1.startsWith("<?xml") shouldBe true
   }
 
   it(s"retrieve pomfile for maven reference of Scala Native") {
     for {
-      res <- sonatypeClient.getPomFile(
+      res <- client.getPomFile(
         Artifact.MavenReference("ch.epfl.scala", "bloop-native-bridge-0-4_2.12", "1.3.4")
       )
     } yield res.get._1.startsWith("<?xml") shouldBe true
   }
 
   it(s"parse date time") {
-    sonatypeClient.parseDate("Wed, 23 Sep 2020 11:40:44 GMT") shouldBe Instant.ofEpochSecond(1600861244L)
+    client.parseDate("Wed, 23 Sep 2020 11:40:44 GMT") shouldBe Instant.ofEpochSecond(1600861244L)
   }
 }

modules/infra/src/test/scala/scaladex/infra/sql/ArtifactTableTests.scala

@@ -22,7 +22,8 @@ class ArtifactTableTests extends AnyFunSpec with BaseDatabaseSuite with Matchers
   it("check selectOldestByProject")(check(selectOldestByProject))
   it("check updateProjectRef")(check(updateProjectRef))
   it("check selectGroupIds")(check(selectGroupIds))
-  it("check selectMavenReference")(check(selectMavenReference))
+  it("check selectAllMavenReferences")(check(selectAllMavenReferences))
+  it("check selectMavenReferences")(check(selectMavenReferences))
   it("check updateReleaseDate")(check(updateReleaseDate))
   it("check selectByMavenReference")(check(selectByMavenReference))
   it("check countVersionsByProject")(check(countVersionsByProject))

modules/server/src/main/scala/scaladex/server/Server.scala

@@ -19,16 +19,16 @@ import scaladex.infra.DataPaths
 import scaladex.infra.ElasticsearchEngine
 import scaladex.infra.FilesystemStorage
 import scaladex.infra.GithubClientImpl
-import scaladex.infra.SonatypeClientImpl
+import scaladex.infra.MavenCentralClientImpl
 import scaladex.infra.SqlDatabase
 import scaladex.infra.sql.DoobieUtils
 import scaladex.server.config.ServerConfig
 import scaladex.server.route.AuthenticationApi
 import scaladex.server.route._
 import scaladex.server.route.api._
 import scaladex.server.service.AdminService
+import scaladex.server.service.MavenCentralService
 import scaladex.server.service.PublishProcess
-import scaladex.server.service.SonatypeService
 import scaladex.view.html.notfound
 object Server extends LazyLogging {
 
@@ -67,10 +67,11 @@ object Server extends LazyLogging {
             val paths = DataPaths.from(config.filesystem)
             val filesystem = FilesystemStorage(config.filesystem)
             val publishProcess = PublishProcess(paths, filesystem, webDatabase, config.env)(publishPool, system)
-            val sonatypeClient = new SonatypeClientImpl()
-            val sonatypeSynchronizer = new SonatypeService(paths, schedulerDatabase, sonatypeClient, publishProcess)
+            val mavenCentralClient = new MavenCentralClientImpl()
+            val mavenCentralService =
+              new MavenCentralService(paths, schedulerDatabase, mavenCentralClient, publishProcess)
             val adminService =
-              new AdminService(config.env, schedulerDatabase, searchEngine, githubClient, sonatypeSynchronizer)
+              new AdminService(config.env, schedulerDatabase, searchEngine, githubClient, mavenCentralService)
 
             for {
               _ <- init(webDatabase, adminService, searchEngine, config.elasticsearch.reset)

modules/server/src/main/scala/scaladex/server/route/AdminPage.scala

@@ -67,8 +67,8 @@ class AdminPage(env: Env, adminService: AdminService) {
               }
             } ~
             post {
-              path("tasks" / Task.updateMavenArtifacts.name) {
-                adminService.updateMavenArtifacts(user)
+              path("tasks" / Task.republishArtifacts.name) {
+                adminService.republishArtifacts(user)
                 redirect(Uri("/admin"), StatusCodes.SeeOther)
               }
             }

modules/server/src/main/scala/scaladex/server/service/AdminService.scala

@@ -23,7 +23,7 @@ class AdminService(
     database: SchedulerDatabase,
     searchEngine: SearchEngine,
     githubClientOpt: Option[GithubClient],
-    sonatypeSynchronizer: SonatypeService
+    mavenCentralService: MavenCentralService
 )(implicit actorSystem: ActorSystem)
     extends LazyLogging {
   import actorSystem.dispatcher
@@ -50,8 +50,8 @@ class AdminService(
       } ++ (
         if (!env.isLocal) {
           Seq(
-            new JobScheduler(Job.missingMavenArtifacts, sonatypeSynchronizer.findMissing),
-            new JobScheduler(Job.nonStandardArtifacts, sonatypeSynchronizer.findNonStandard)
+            new JobScheduler(Job.missingMavenArtifacts, mavenCentralService.findMissing),
+            new JobScheduler(Job.nonStandardArtifacts, mavenCentralService.findNonStandard)
           )
         } else Seq.empty
       )
@@ -80,7 +80,7 @@ class AdminService(
     val input = Seq("Group ID" -> groupId.value) ++
       artifactNameOpt.map(name => "Artifact Name" -> name.value)
     val task = TaskRunner.run(Task.findMissingArtifacts, user.info.login, input) { () =>
-      sonatypeSynchronizer.syncOne(groupId, artifactNameOpt)
+      mavenCentralService.syncOne(groupId, artifactNameOpt)
     }
     tasks = tasks :+ task
   }
@@ -137,9 +137,9 @@ class AdminService(
     tasks = tasks :+ task
   }
 
-  def updateMavenArtifacts(user: UserState): Unit = {
-    val task = TaskRunner.run(Task.updateMavenArtifacts, user.info.login, input = Seq.empty) { () =>
-      sonatypeSynchronizer.updateAllArtifacts()
+  def republishArtifacts(user: UserState): Unit = {
+    val task = TaskRunner.run(Task.republishArtifacts, user.info.login, input = Seq.empty) { () =>
+      mavenCentralService.republishArtifacts()
     }
     tasks = tasks :+ task
   }

modules/server/src/main/scala/scaladex/server/service/ArtifactService.scala

@@ -52,7 +52,7 @@ class ArtifactService(database: SchedulerDatabase)(implicit ec: ExecutionContext
 
   def updateLatestVersions(ref: Project.Reference, preferStableVersion: Boolean): Future[Int] =
     for {
-      artifactIds <- database.getAllArtifactIds(ref)
+      artifactIds <- database.getArtifactIds(ref)
       _ <- artifactIds.mapSync {
         case (groupId, artifactId) => updateLatestVersion(groupId, artifactId, preferStableVersion)
       }

modules/server/src/main/scala/scaladex/server/service/GithubUpdater.scala

@@ -52,7 +52,7 @@ class GithubUpdater(database: WebDatabase, github: GithubClient)(implicit ec: Ex
 
       case GithubResponse.MovedPermanently((destination, info)) =>
         val status = GithubStatus.Moved(now, destination)
-        logger.info(s"$repo moved to $status")
+        logger.info(s"$repo moved to $destination")
         database.moveProject(repo, info, status).map(_ => status)
 
       case GithubResponse.Failed(code, reason) =>