From 7c8f0dd1f91c522700d2a7db6c58f14e880e0043 Mon Sep 17 00:00:00 2001 From: Siddharth Ingle <29544714+skingle@users.noreply.github.com> Date: Fri, 16 Aug 2024 16:50:35 +0530 Subject: [PATCH 1/3] feat: Added an admin task to sync all artifacts for new metadata fields --- .../scala/scaladex/core/test/Values.scala | 6 +- .../scaladex/infra/sql/ArtifactTable.scala | 8 +-- .../scaladex/server/route/AdminPage.scala | 7 +++ .../server/service/AdminService.scala | 7 +++ .../server/service/SonatypeService.scala | 62 +++++++++++-------- .../src/main/scala/scaladex/view/Task.scala | 5 ++ .../scaladex/view/admin/admin.scala.html | 12 +++- 7 files changed, 74 insertions(+), 33 deletions(-) diff --git a/modules/core/shared/src/test/scala/scaladex/core/test/Values.scala b/modules/core/shared/src/test/scala/scaladex/core/test/Values.scala index dd5b2744a..6e1fd2eca 100644 --- a/modules/core/shared/src/test/scala/scaladex/core/test/Values.scala +++ b/modules/core/shared/src/test/scala/scaladex/core/test/Values.scala @@ -1,5 +1,8 @@ package scaladex.core.test +import java.time.Instant +import java.time.temporal.ChronoUnit + import scaladex.core.model.Artifact import scaladex.core.model.Artifact._ import scaladex.core.model.ArtifactDependency @@ -26,9 +29,6 @@ import scaladex.core.model.SemanticVersion import scaladex.core.model.Url import scaladex.core.model.search.ProjectDocument -import java.time.Instant -import java.time.temporal.ChronoUnit - object Values { val now: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS) val ok: GithubStatus = GithubStatus.Ok(now) diff --git a/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala b/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala index 051527545..9fcc60d7a 100644 --- a/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala +++ b/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala @@ -42,13 +42,13 @@ object ArtifactTable { val versionFields: Seq[String] = Seq("is_semantic", "is_prerelease") def insertIfNotExist(artifact: Artifact): ConnectionIO[Int] = - insertIfNotExist.run((artifact, artifact.version.isSemantic, artifact.version.isPreRelease)) + insertIfNotExist.run((artifact, artifact.version.isSemantic, artifact.version.isPreRelease, artifact)) def insertIfNotExist(artifacts: Seq[Artifact]): ConnectionIO[Int] = - insertIfNotExist.updateMany(artifacts.map(a => (a, a.version.isSemantic, a.version.isPreRelease))) + insertIfNotExist.updateMany(artifacts.map(a => (a, a.version.isSemantic, a.version.isPreRelease, a))) - private[sql] val insertIfNotExist: Update[(Artifact, Boolean, Boolean)] = - insertOrUpdateRequest(table, fields ++ versionFields, mavenReferenceFields) + private[sql] val insertIfNotExist: Update[(Artifact, Boolean, Boolean, Artifact)] = + insertOrUpdateRequest(table, fields ++ versionFields, mavenReferenceFields, fields) val count: Query0[Long] = selectRequest(table, Seq("COUNT(*)")) diff --git a/modules/server/src/main/scala/scaladex/server/route/AdminPage.scala b/modules/server/src/main/scala/scaladex/server/route/AdminPage.scala index eefbc786e..f2bf88041 100644 --- a/modules/server/src/main/scala/scaladex/server/route/AdminPage.scala +++ b/modules/server/src/main/scala/scaladex/server/route/AdminPage.scala @@ -65,7 +65,14 @@ class AdminPage(env: Env, adminService: AdminService) { redirect(Uri("/admin"), StatusCodes.SeeOther) } } + } ~ + post { + path("tasks" / Task.updateMavenArtifacts.name) { + adminService.updateMavenArtifacts(user) + redirect(Uri("/admin"), StatusCodes.SeeOther) + } } + case _ => complete(StatusCodes.Forbidden, view.html.forbidden(env, user)) } diff --git a/modules/server/src/main/scala/scaladex/server/service/AdminService.scala b/modules/server/src/main/scala/scaladex/server/service/AdminService.scala index 32789c475..f3496e5f7 100644 --- a/modules/server/src/main/scala/scaladex/server/service/AdminService.scala +++ b/modules/server/src/main/scala/scaladex/server/service/AdminService.scala @@ -136,6 +136,13 @@ class AdminService( tasks = tasks :+ task } + def updateMavenArtifacts(user: UserState): Unit = { + val task = TaskRunner.run(Task.updateMavenArtifacts, user.info.login, input = Seq.empty) { () => + sonatypeSynchronizer.updateAllArtifacts() + } + tasks = tasks :+ task + } + private def updateProjectCreationDate(): Future[String] = for { creationDates <- database.computeAllProjectsCreationDates() diff --git a/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala b/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala index 1173f7215..d8e731aa1 100644 --- a/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala +++ b/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala @@ -33,6 +33,29 @@ class SonatypeService( } yield s"Inserted ${result.sum} missing poms" } + private def findAndIndexMissingArtifacts( + groupId: GroupId, + artifactId: ArtifactId, + knownRefs: Set[MavenReference] + ): Future[Int] = + for { + versions <- sonatypeService.getAllVersions(groupId, artifactId) + mavenReferences = versions.map(v => + MavenReference(groupId = groupId.value, artifactId = artifactId.value, version = v.toString) + ) + missingVersions = mavenReferences.filterNot(knownRefs) + _ = if (missingVersions.nonEmpty) + logger.warn(s"${missingVersions.size} artifacts are missing for ${groupId.value}:${artifactId.value}") + missingPomFiles <- missingVersions.map(ref => sonatypeService.getPomFile(ref).map(_.map(ref -> _))).sequence + publishResult <- missingPomFiles.flatten.mapSync { + case (mavenRef, (pomFile, creationDate)) => + publishProcess.publishPom(mavenRef.toString(), pomFile, creationDate, None) + } + } yield publishResult.count { + case PublishResult.Success => true + case _ => false + } + def findMissing(): Future[String] = for { mavenReferenceFromDatabase <- database.getAllMavenReferences().map(_.toSet) @@ -41,12 +64,6 @@ class SonatypeService( result <- groupIds.mapSync(g => findAndIndexMissingArtifacts(g, None, mavenReferenceFromDatabase)) } yield s"Inserted ${result.sum} missing poms" - def syncOne(groupId: GroupId, artifactNameOpt: Option[Artifact.Name]): Future[String] = - for { - mavenReferenceFromDatabase <- database.getAllMavenReferences() - result <- findAndIndexMissingArtifacts(groupId, artifactNameOpt, mavenReferenceFromDatabase.toSet) - } yield s"Inserted ${result} poms" - private def findAndIndexMissingArtifacts( groupId: GroupId, artifactNameOpt: Option[Artifact.Name], @@ -61,26 +78,21 @@ class SonatypeService( .mapSync(id => findAndIndexMissingArtifacts(groupId, id, knownRefs)) } yield result.sum - private def findAndIndexMissingArtifacts( - groupId: GroupId, - artifactId: ArtifactId, - knownRefs: Set[MavenReference] - ): Future[Int] = + def syncOne(groupId: GroupId, artifactNameOpt: Option[Artifact.Name]): Future[String] = for { - versions <- sonatypeService.getAllVersions(groupId, artifactId) - mavenReferences = versions.map(v => - MavenReference(groupId = groupId.value, artifactId = artifactId.value, version = v.toString) - ) - missingVersions = mavenReferences.filterNot(knownRefs) - _ = if (missingVersions.nonEmpty) - logger.warn(s"${missingVersions.size} artifacts are missing for ${groupId.value}:${artifactId.value}") - missingPomFiles <- missingVersions.map(ref => sonatypeService.getPomFile(ref).map(_.map(ref -> _))).sequence - publishResult <- missingPomFiles.flatten.mapSync { + mavenReferenceFromDatabase <- database.getAllMavenReferences() + result <- findAndIndexMissingArtifacts(groupId, artifactNameOpt, mavenReferenceFromDatabase.toSet) + } yield s"Inserted $result poms" + + def updateAllArtifacts(): Future[String] = + for { + mavenReferences <- database.getAllMavenReferences() + _ = logger.info(s"${mavenReferences.size} artifacts will be synced for new metadata.") + poms <- mavenReferences.map(ref => sonatypeService.getPomFile(ref).map(_.map(ref -> _))).sequence + _ = logger.info(s"publishing poms now") + publishResult <- poms.flatten.mapSync { case (mavenRef, (pomFile, creationDate)) => - publishProcess.publishPom(mavenRef.toString, pomFile, creationDate, None) + publishProcess.publishPom(mavenRef.toString(), pomFile, creationDate, None) } - } yield publishResult.count { - case PublishResult.Success => true - case _ => false - } + } yield s"Updated ${publishResult.size} poms" } diff --git a/modules/template/src/main/scala/scaladex/view/Task.scala b/modules/template/src/main/scala/scaladex/view/Task.scala index 78d7be6b4..76761c44f 100644 --- a/modules/template/src/main/scala/scaladex/view/Task.scala +++ b/modules/template/src/main/scala/scaladex/view/Task.scala @@ -24,6 +24,11 @@ object Task { "Update the Github info of an existing project" ) + val updateMavenArtifacts: Task = Task( + "update-maven-artifact", + "Update existing artifacts with new fields" + ) + case class Status(name: String, user: String, start: Instant, input: Seq[(String, String)], state: State) { def fromNow: FiniteDuration = TimeUtils.toFiniteDuration(start, Instant.now()) } diff --git a/modules/template/src/main/twirl/scaladex/view/admin/admin.scala.html b/modules/template/src/main/twirl/scaladex/view/admin/admin.scala.html index f790bc265..6b1a0db37 100644 --- a/modules/template/src/main/twirl/scaladex/view/admin/admin.scala.html +++ b/modules/template/src/main/twirl/scaladex/view/admin/admin.scala.html @@ -22,7 +22,7 @@

Background jobs

@jobs.map { case (job, status) => - +
@job.name every @job.frequency.shortPrint

@job.description

@@ -120,6 +120,16 @@
@Task.updateGithubInfo.name
+ + +
@Task.updateMavenArtifacts.name
+

@Task.updateMavenArtifacts.description

+ + +
+ +
+

Task History

From 4fdb0bbfb08a90a6df768765259d91c95fcab4a3 Mon Sep 17 00:00:00 2001 From: Siddharth Ingle <29544714+skingle@users.noreply.github.com> Date: Mon, 19 Aug 2024 19:41:24 +0530 Subject: [PATCH 2/3] feat: synced the POM meta-data update task --- .../server/service/SonatypeService.scala | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala b/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala index d8e731aa1..39b687d7d 100644 --- a/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala +++ b/modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala @@ -88,11 +88,15 @@ class SonatypeService( for { mavenReferences <- database.getAllMavenReferences() _ = logger.info(s"${mavenReferences.size} artifacts will be synced for new metadata.") - poms <- mavenReferences.map(ref => sonatypeService.getPomFile(ref).map(_.map(ref -> _))).sequence - _ = logger.info(s"publishing poms now") - publishResult <- poms.flatten.mapSync { - case (mavenRef, (pomFile, creationDate)) => - publishProcess.publishPom(mavenRef.toString(), pomFile, creationDate, None) - } - } yield s"Updated ${publishResult.size} poms" + publishResult <- mavenReferences.mapSync(updateArtifact) + successCount = publishResult.count(_ == PublishResult.Success) + failedCount = publishResult.size - successCount + } yield s"Synced $successCount poms, while $failedCount poms failed to update." + + private def updateArtifact(ref: MavenReference): Future[PublishResult] = + sonatypeService.getPomFile(ref).map(ref -> _).flatMap { + case (mavenRef, Some((pomFile, creationDate))) => + publishProcess.publishPom(mavenRef.toString(), pomFile, creationDate, None) + case _ => Future.successful(PublishResult.InvalidPom) + } } From cbc7c53c9ca66f6eda4075250e7694328b375ed1 Mon Sep 17 00:00:00 2001 From: Siddharth Ingle <29544714+skingle@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:44:04 +0530 Subject: [PATCH 3/3] Update modules/template/src/main/scala/scaladex/view/Task.scala Co-authored-by: adpi2 --- modules/template/src/main/scala/scaladex/view/Task.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/template/src/main/scala/scaladex/view/Task.scala b/modules/template/src/main/scala/scaladex/view/Task.scala index 76761c44f..f3ca5ebcc 100644 --- a/modules/template/src/main/scala/scaladex/view/Task.scala +++ b/modules/template/src/main/scala/scaladex/view/Task.scala @@ -26,7 +26,7 @@ object Task { val updateMavenArtifacts: Task = Task( "update-maven-artifact", - "Update existing artifacts with new fields" + "Download all pom files to update existing artifacts with new fields" ) case class Status(name: String, user: String, start: Instant, input: Seq[(String, String)], state: State) {