Merge pull request #1446 from skingle/main

1407 :  Background task to reindex all artifacts &  979 : UI Changes

build.sbt

@@ -83,7 +83,9 @@ lazy val infra = project
       "io.circe" %% "circe-core",
       "io.circe" %% "circe-generic",
       "io.circe" %% "circe-parser"
-    ).map(_ % V.circe),
+    ).map(_ % V.circe) ++ Seq(
+      "io.circe" %% "circe-generic-extras"
+    ).map(_ % V.circeGenericExtra),
     Elasticsearch.settings(defaultPort = 9200),
     Postgres.settings(Compile, defaultPort = 5432, database = "scaladex"),
     javaOptions ++= {
@@ -243,6 +245,7 @@ lazy val V = new {
   val nscalaTime = "2.32.0"
   val scalatest = "3.2.19"
   val circe = "0.14.9"
+  val circeGenericExtra = "0.14.4"
   val json4s = "4.0.7"
   val coursier = "2.1.6"
 }

modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala

@@ -32,7 +32,7 @@ case class Artifact(
     fullScalaVersion: Option[SemanticVersion],
     scaladocUrl: Option[Url],
     versionScheme: Option[String],
-    developers: Seq[Contributor]
+    developers: Seq[Contributor] = Seq.empty
 ) {
   val binaryVersion: BinaryVersion = BinaryVersion(platform, language)
   val mavenReference: Artifact.MavenReference = Artifact.MavenReference(groupId.value, artifactId, version.encode)

modules/core/shared/src/main/scala/scaladex/core/model/Project.scala

@@ -44,10 +44,13 @@ case class Project(
     image = githubInfo.flatMap(_.logo).orElse(Some(Url("https://index.scala-lang.org/assets/img/scaladex-brand.svg")))
   )
 
-  def scaladoc(artifact: Artifact): Option[LabeledLink] =
-    settings.customScalaDoc
-      .map(DocumentationPattern("Scaladoc", _).eval(artifact))
-      .orElse(artifact.defaultScaladoc.map(LabeledLink("Scaladoc", _)))
+  def scaladoc(artifact: Artifact): Option[LabeledLink] = artifact.scaladocUrl
+    .map(_.labeled("Scaladoc"))
+    .orElse(
+      settings.customScalaDoc
+        .map(DocumentationPattern("Scaladoc", _).eval(artifact))
+        .orElse(artifact.defaultScaladoc.map(LabeledLink("Scaladoc", _)))
+    )
 
   private def globalDocumentation: Seq[LabeledLink] =
     settings.customScalaDoc.flatMap(DocumentationPattern("Scaladoc", _).asGlobal).toSeq ++

modules/infra/src/main/scala/scaladex/infra/Codecs.scala

@@ -3,6 +3,8 @@ package scaladex.infra
 import java.time.Instant
 
 import io.circe._
+import io.circe.generic.extras.Configuration
+import io.circe.generic.extras.semiauto.deriveConfiguredCodec
 import io.circe.generic.semiauto._
 import scaladex.core.model._
 import scaladex.core.model.search.GithubInfoDocument
@@ -44,7 +46,8 @@ object Codecs {
   implicit val languageCodec: Codec[Language] = fromString(_.label, Language.fromLabel(_).get)
   implicit val resolverCodec: Codec[Resolver] = deriveCodec
   implicit val licenseCodec: Codec[License] = fromString(_.shortName, License.allByShortName.apply)
-  implicit val artifactCodec: Codec[Artifact] = deriveCodec
+  implicit val customConfig: Configuration = Configuration.default.withDefaults
+  implicit val artifactCodec: Codec[Artifact] = deriveConfiguredCodec[Artifact]
   implicit val scopeCodec: Codec[ArtifactDependency.Scope] = fromString(_.value, ArtifactDependency.Scope.apply)
 
   implicit val mavenRefCodec: Codec[Artifact.MavenReference] = deriveCodec
@@ -62,4 +65,5 @@ object Codecs {
 
   private def fromString[A](encode: A => String, decode: String => A): Codec[A] =
     Codec.from(Decoder[String].map(decode), Encoder[String].contramap(encode))
+
 }

modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala

@@ -41,13 +41,13 @@ object ArtifactTable {
   val isLatestVersion: String = "is_latest_version"
 
   def insertIfNotExist(artifact: Artifact): ConnectionIO[Int] =
-    insertIfNotExist.run((artifact, artifact.version.isSemantic, artifact.version.isPreRelease))
+    insertIfNotExist.run((artifact, artifact.version.isSemantic, artifact.version.isPreRelease, artifact))
 
   def insertIfNotExist(artifacts: Seq[Artifact]): ConnectionIO[Int] =
-    insertIfNotExist.updateMany(artifacts.map(a => (a, a.version.isSemantic, a.version.isPreRelease)))
+    insertIfNotExist.updateMany(artifacts.map(a => (a, a.version.isSemantic, a.version.isPreRelease, a)))
 
-  private[sql] val insertIfNotExist: Update[(Artifact, Boolean, Boolean)] =
-    insertOrUpdateRequest(table, fields ++ versionFields, mavenReferenceFields)
+  private[sql] val insertIfNotExist: Update[(Artifact, Boolean, Boolean, Artifact)] =
+    insertOrUpdateRequest(table, fields ++ versionFields, mavenReferenceFields, fields)
 
   val count: Query0[Long] =
     selectRequest(table, Seq("COUNT(*)"))

modules/server/src/main/assets/css/partials/_project.scss

@@ -431,4 +431,15 @@
       display: block;
     }
   }
+
+  .developers {
+    a {
+      color: $gray;
+      &:hover,
+      &:focus {
+        color: $brand-primary;
+        text-decoration: none;
+      }
+    }
+  }
 }

modules/server/src/main/scala/scaladex/server/route/AdminPage.scala

@@ -65,7 +65,14 @@ class AdminPage(env: Env, adminService: AdminService) {
                   redirect(Uri("/admin"), StatusCodes.SeeOther)
                 }
               }
+            } ~
+            post {
+              path("tasks" / Task.updateMavenArtifacts.name) {
+                adminService.updateMavenArtifacts(user)
+                redirect(Uri("/admin"), StatusCodes.SeeOther)
+              }
             }
+
         case _ =>
           complete(StatusCodes.Forbidden, view.html.forbidden(env, user))
       }

modules/server/src/main/scala/scaladex/server/service/AdminService.scala

@@ -137,6 +137,13 @@ class AdminService(
     tasks = tasks :+ task
   }
 
+  def updateMavenArtifacts(user: UserState): Unit = {
+    val task = TaskRunner.run(Task.updateMavenArtifacts, user.info.login, input = Seq.empty) { () =>
+      sonatypeSynchronizer.updateAllArtifacts()
+    }
+    tasks = tasks :+ task
+  }
+
   private def updateProjectCreationDate(): Future[String] =
     for {
       creationDates <- database.computeAllProjectsCreationDates()

modules/server/src/main/scala/scaladex/server/service/SonatypeService.scala

@@ -33,6 +33,29 @@ class SonatypeService(
     } yield s"Inserted ${result.sum} missing poms"
   }
 
+  private def findAndIndexMissingArtifacts(
+      groupId: GroupId,
+      artifactId: ArtifactId,
+      knownRefs: Set[MavenReference]
+  ): Future[Int] =
+    for {
+      versions <- sonatypeService.getAllVersions(groupId, artifactId)
+      mavenReferences = versions.map(v =>
+        MavenReference(groupId = groupId.value, artifactId = artifactId.value, version = v.toString)
+      )
+      missingVersions = mavenReferences.filterNot(knownRefs)
+      _ = if (missingVersions.nonEmpty)
+        logger.warn(s"${missingVersions.size} artifacts are missing for ${groupId.value}:${artifactId.value}")
+      missingPomFiles <- missingVersions.map(ref => sonatypeService.getPomFile(ref).map(_.map(ref -> _))).sequence
+      publishResult <- missingPomFiles.flatten.mapSync {
+        case (mavenRef, (pomFile, creationDate)) =>
+          publishProcess.publishPom(mavenRef.toString(), pomFile, creationDate, None)
+      }
+    } yield publishResult.count {
+      case PublishResult.Success => true
+      case _                     => false
+    }
+
   def findMissing(): Future[String] =
     for {
       mavenReferenceFromDatabase <- database.getAllMavenReferences().map(_.toSet)
@@ -41,12 +64,6 @@ class SonatypeService(
       result <- groupIds.mapSync(g => findAndIndexMissingArtifacts(g, None, mavenReferenceFromDatabase))
     } yield s"Inserted ${result.sum} missing poms"
 
-  def syncOne(groupId: GroupId, artifactNameOpt: Option[Artifact.Name]): Future[String] =
-    for {
-      mavenReferenceFromDatabase <- database.getAllMavenReferences()
-      result <- findAndIndexMissingArtifacts(groupId, artifactNameOpt, mavenReferenceFromDatabase.toSet)
-    } yield s"Inserted ${result} poms"
-
   private def findAndIndexMissingArtifacts(
       groupId: GroupId,
       artifactNameOpt: Option[Artifact.Name],
@@ -61,26 +78,25 @@ class SonatypeService(
         .mapSync(id => findAndIndexMissingArtifacts(groupId, id, knownRefs))
     } yield result.sum
 
-  private def findAndIndexMissingArtifacts(
-      groupId: GroupId,
-      artifactId: ArtifactId,
-      knownRefs: Set[MavenReference]
-  ): Future[Int] =
+  def syncOne(groupId: GroupId, artifactNameOpt: Option[Artifact.Name]): Future[String] =
     for {
-      versions <- sonatypeService.getAllVersions(groupId, artifactId)
-      mavenReferences = versions.map(v =>
-        MavenReference(groupId = groupId.value, artifactId = artifactId.value, version = v.toString)
-      )
-      missingVersions = mavenReferences.filterNot(knownRefs)
-      _ = if (missingVersions.nonEmpty)
-        logger.warn(s"${missingVersions.size} artifacts are missing for ${groupId.value}:${artifactId.value}")
-      missingPomFiles <- missingVersions.map(ref => sonatypeService.getPomFile(ref).map(_.map(ref -> _))).sequence
-      publishResult <- missingPomFiles.flatten.mapSync {
-        case (mavenRef, (pomFile, creationDate)) =>
-          publishProcess.publishPom(mavenRef.toString, pomFile, creationDate, None)
-      }
-    } yield publishResult.count {
-      case PublishResult.Success => true
-      case _                     => false
+      mavenReferenceFromDatabase <- database.getAllMavenReferences()
+      result <- findAndIndexMissingArtifacts(groupId, artifactNameOpt, mavenReferenceFromDatabase.toSet)
+    } yield s"Inserted $result poms"
+
+  def updateAllArtifacts(): Future[String] =
+    for {
+      mavenReferences <- database.getAllMavenReferences()
+      _ = logger.info(s"${mavenReferences.size} artifacts will be synced for new metadata.")
+      publishResult <- mavenReferences.mapSync(updateArtifact)
+      successCount = publishResult.count(_ == PublishResult.Success)
+      failedCount = publishResult.size - successCount
+    } yield s"Synced $successCount poms, while $failedCount poms failed to update."
+
+  private def updateArtifact(ref: MavenReference): Future[PublishResult] =
+    sonatypeService.getPomFile(ref).map(ref -> _).flatMap {
+      case (mavenRef, Some((pomFile, creationDate))) =>
+        publishProcess.publishPom(mavenRef.toString(), pomFile, creationDate, None)
+      case _ => Future.successful(PublishResult.InvalidPom)
     }
 }

modules/template/src/main/scala/scaladex/view/Task.scala

@@ -24,6 +24,11 @@ object Task {
     "Update the Github info of an existing project"
   )
 
+  val updateMavenArtifacts: Task = Task(
+    "update-maven-artifact",
+    "Download all pom files to update existing artifacts with new fields"
+  )
+
   case class Status(name: String, user: String, start: Instant, input: Seq[(String, String)], state: State) {
     def fromNow: FiniteDuration = TimeUtils.toFiniteDuration(start, Instant.now())
   }

modules/template/src/main/twirl/scaladex/view/admin/admin.scala.html

@@ -22,7 +22,7 @@ <h2>Background jobs</h2>
       <tbody>
         @jobs.map { case (job, status) =>
           <tr>
-            <td>
+            <th>
               <h5>@job.name <small>every @job.frequency.shortPrint</small></h5>
               <p>@job.description</p>
             </th>
@@ -120,6 +120,16 @@ <h5>@Task.updateGithubInfo.name</h5>
             <td><button type="submit" class="btn btn-primary">Submit</button></td>
           </form>
         </tr>
+        <tr>
+          <td>
+            <h5>@Task.updateMavenArtifacts.name</h5>
+            <p>@Task.updateMavenArtifacts.description</p>
+          </td>
+          <td></td>
+          <form action="/admin/tasks/@Task.updateMavenArtifacts.name" method="POST">
+            <td><button type="submit" class="btn btn-primary">Submit</button></td>
+          </form>
+        </tr>
       </tbody>
     </table>
     <h2>Task History</h2>

modules/template/src/main/twirl/scaladex/view/project/artifact.scala.html

@@ -52,6 +52,13 @@ <h3>
               @info("Group ID") { @artifact.groupId }
               @info("Artifact ID") { @artifact.artifactId }
               @info("Version") { @artifact.version }
+              @artifact.versionScheme.map { scheme =>
+                @info("Version Scheme"){
+                  <a href="https://www.scala-sbt.org/1.x/docs/Publishing.html#Version+scheme" rel="nofollow">
+                    @scheme
+                  </a>
+                }
+              }
               @info("Release Date") { @artifact.releaseDateFormat }
               @info("Licenses") {
                 @for(license <- artifact.licenses) {
@@ -62,6 +69,7 @@ <h3>
                 @info("Files") { <a href="@artifact.mavenReference.repoUrl">View all</a> }
               }
               @artifact.fullScalaVersion.map{ version => @info("Full Scala Version") { @version.toString }}
+              @developers
             </div>
             @installBox(InstallTab.allOf(artifact, project.settings.cliArtifacts))
           </div>
@@ -97,7 +105,21 @@ <h4>Documentation</h4>
     </div>
   }
 }
-
+@developers = {
+  @if(artifact.developers.nonEmpty){
+    @info("Developers"){
+      <div class="developers">
+        @for(developer <- artifact.developers) {
+          <span>
+            <a href="@developer.url">
+              @developer.name
+            </a> |
+          </span>
+        }
+      </div>
+    }
+  }
+}
 @scastieBox = {
   @if(artifact.scastieURL.nonEmpty) {
     <div class="box">