Merge remote-tracking branch 'origin/main' into 19-background-task-fo…

…r-reindexing

build.sbt

@@ -23,7 +23,7 @@ lazy val loggingSettings = Seq(
     "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5"
   ),
   // Drop and replace commons-logging with slf4j
-  libraryDependencies += "org.slf4j" % "jcl-over-slf4j" % "2.0.13",
+  libraryDependencies += "org.slf4j" % "jcl-over-slf4j" % "2.0.16",
   excludeDependencies += ExclusionRule("commons-logging", "commons-logging")
 )
 

modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala

@@ -48,9 +48,6 @@ case class Artifact(
     s"$groupId$sep$artifactName"
   }
 
-  def release: Release =
-    Release(projectRef.organization, projectRef.repository, platform, language, version, releaseDate)
-
   def releaseDateFormat: String = Artifact.dateFormatter.format(releaseDate)
 
   def httpUrl: String = {
@@ -302,4 +299,9 @@ object Artifact {
     def repoUrl: String =
       s"https://repo1.maven.org/maven2/${groupId.replace('.', '/')}/$artifactId/$version/"
   }
+
+  object MavenReference {
+    def apply(groupId: GroupId, artifactId: String, version: SemanticVersion): MavenReference =
+      MavenReference(groupId.value, artifactId, version.encode)
+  }
 }

modules/core/shared/src/main/scala/scaladex/core/model/Project.scala

@@ -93,6 +93,9 @@ object Project {
       settings = settings.getOrElse(Settings.empty)
     )
 
+  def default(ref: Project.Reference, githubStatus: GithubStatus): Project =
+    Project(ref.organization, ref.repository, None, githubStatus, None, Settings.empty)
+
   case class Settings(
       preferStableVersion: Boolean,
       defaultArtifact: Option[Artifact.Name],

modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala

@@ -9,7 +9,7 @@ class ProjectService(database: WebDatabase)(implicit context: ExecutionContext)
   def getProjectHeader(project: Project): Future[Option[ProjectHeader]] = {
     val ref = project.reference
     for {
-      latestArtifacts <- database.getLatestArtifacts(ref, project.settings.preferStableVersion)
+      latestArtifacts <- database.getLatestArtifacts(ref)
       versionCount <- database.countVersions(ref)
     } yield ProjectHeader(
       project.reference,

modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala

@@ -9,20 +9,18 @@ import scaladex.core.model.Artifact.MavenReference
 import scaladex.core.model.ArtifactDependency
 import scaladex.core.model.Project
 import scaladex.core.model.ProjectDependency
-import scaladex.core.model.ReleaseDependency
 import scaladex.core.model.SemanticVersion
 
 trait SchedulerDatabase extends WebDatabase {
   // project and github
+  def getAllProjects(): Future[Seq[Project]]
   def insertProject(project: Project): Future[Unit]
   def updateProjectCreationDate(ref: Project.Reference, creationDate: Instant): Future[Unit]
   def computeAllProjectsCreationDates(): Future[Seq[(Instant, Project.Reference)]]
 
-  // project dependencies (and release dependencies)
+  // project dependencies
   def computeProjectDependencies(reference: Project.Reference, version: SemanticVersion): Future[Seq[ProjectDependency]]
-  def computeReleaseDependencies(): Future[Seq[ReleaseDependency]]
   def insertProjectDependencies(projectDependencies: Seq[ProjectDependency]): Future[Int]
-  def insertReleaseDependencies(projectDependencies: Seq[ReleaseDependency]): Future[Int]
   def deleteProjectDependencies(ref: Project.Reference): Future[Int]
 
   // artifacts and its dependencies
@@ -31,6 +29,8 @@ trait SchedulerDatabase extends WebDatabase {
   def updateArtifacts(artifacts: Seq[Artifact], newRef: Project.Reference): Future[Int]
   def updateArtifactReleaseDate(reference: MavenReference, releaseDate: Instant): Future[Int]
   def getAllGroupIds(): Future[Seq[Artifact.GroupId]]
+  def getAllArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]]
   def getAllMavenReferences(): Future[Seq[Artifact.MavenReference]]
   def getDependencies(projectRef: Project.Reference): Future[Seq[ArtifactDependency]]
+  def updateLatestVersion(ref: MavenReference): Future[Unit]
 }

modules/core/shared/src/main/scala/scaladex/core/service/WebDatabase.scala

@@ -1,27 +1,20 @@
 package scaladex.core.service
 
-import java.time.Instant
 import java.util.UUID
 
 import scala.concurrent.Future
 
 import scaladex.core.model._
-import scaladex.core.web.ArtifactsPageParams
 
 trait WebDatabase {
   // artifacts
-  // insertArtifact return a boolean. It's true if a new project is inserted, false otherwise
-  def insertArtifact(artifact: Artifact, dependencies: Seq[ArtifactDependency], time: Instant): Future[Boolean]
-  def getArtifacts(groupId: Artifact.GroupId, artifactId: Artifact.ArtifactId): Future[Seq[Artifact]]
+  def insertArtifact(artifact: Artifact): Future[Boolean]
+  def getArtifacts(groupId: Artifact.GroupId, artifactId: String): Future[Seq[Artifact]]
   def getArtifacts(projectRef: Project.Reference): Future[Seq[Artifact]]
-  def getArtifacts(
-      ref: Project.Reference,
-      artifactName: Artifact.Name,
-      params: ArtifactsPageParams
-  ): Future[Seq[Artifact]]
+  def getArtifacts(ref: Project.Reference, artifactName: Artifact.Name, preReleases: Boolean): Future[Seq[Artifact]]
   def getArtifacts(ref: Project.Reference, artifactName: Artifact.Name, version: SemanticVersion): Future[Seq[Artifact]]
   def getArtifactsByName(projectRef: Project.Reference, artifactName: Artifact.Name): Future[Seq[Artifact]]
-  def getLatestArtifacts(ref: Project.Reference, preferStableVersions: Boolean): Future[Seq[Artifact]]
+  def getLatestArtifacts(ref: Project.Reference): Future[Seq[Artifact]]
   def getArtifactByMavenReference(mavenRef: Artifact.MavenReference): Future[Option[Artifact]]
   def getAllArtifacts(language: Option[Language], platform: Option[Platform]): Future[Seq[Artifact]]
   def countArtifacts(): Future[Long]
@@ -31,8 +24,8 @@ trait WebDatabase {
   def getReverseDependencies(artifact: Artifact): Future[Seq[ArtifactDependency.Reverse]]
 
   // projects
+  def insertProjectRef(ref: Project.Reference, status: GithubStatus): Future[Boolean]
   def updateProjectSettings(ref: Project.Reference, settings: Project.Settings): Future[Unit]
-  def getAllProjects(): Future[Seq[Project]]
   def getAllProjectsStatuses(): Future[Map[Project.Reference, GithubStatus]]
   def getProject(projectRef: Project.Reference): Future[Option[Project]]
   def getFormerReferences(projectRef: Project.Reference): Future[Seq[Project.Reference]]

modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala

@@ -14,12 +14,10 @@ import scaladex.core.model.Language
 import scaladex.core.model.Platform
 import scaladex.core.model.Project
 import scaladex.core.model.ProjectDependency
-import scaladex.core.model.ReleaseDependency
 import scaladex.core.model.SemanticVersion
 import scaladex.core.model.UserInfo
 import scaladex.core.model.UserState
 import scaladex.core.service.SchedulerDatabase
-import scaladex.core.web.ArtifactsPageParams
 
 class InMemoryDatabase extends SchedulerDatabase {
   private val allProjects = mutable.Map[Project.Reference, Project]()
@@ -32,24 +30,26 @@ class InMemoryDatabase extends SchedulerDatabase {
     allDependencies.clear()
   }
 
-  override def insertArtifact(
-      artifact: Artifact,
-      dependencies: Seq[ArtifactDependency],
-      now: Instant
-  ): Future[Boolean] = {
-    val ref = artifact.projectRef
+  override def insertArtifact(artifact: Artifact): Future[Boolean] = {
+    val isNewArtifact = !allArtifacts.values.flatten.exists(a => a.mavenReference == artifact.mavenReference)
+    allArtifacts.addOne(artifact.projectRef -> (allArtifacts.getOrElse(artifact.projectRef, Seq.empty) :+ artifact))
+    Future.successful(isNewArtifact)
+  }
+
+  override def insertProjectRef(ref: Project.Reference, status: GithubStatus): Future[Boolean] = {
     val isNewProject = !allProjects.contains(ref)
-    if (isNewProject) allProjects.addOne(ref -> Project.default(ref, now = now))
-    allArtifacts.addOne(ref -> (allArtifacts.getOrElse(ref, Seq.empty) :+ artifact))
-    dependencies.appendedAll(dependencies)
+    if (isNewProject) allProjects.addOne(ref -> Project.default(ref, status))
     Future.successful(isNewProject)
   }
 
   override def insertProject(project: Project): Future[Unit] = ???
 
   override def insertArtifacts(allArtifacts: Seq[Artifact]): Future[Unit] = ???
 
-  override def insertDependencies(dependencies: Seq[ArtifactDependency]): Future[Unit] = ???
+  override def insertDependencies(dependencies: Seq[ArtifactDependency]): Future[Unit] = {
+    allDependencies ++= dependencies
+    Future.successful(())
+  }
 
   override def deleteProjectDependencies(ref: Project.Reference): Future[Int] = ???
 
@@ -61,10 +61,10 @@ class InMemoryDatabase extends SchedulerDatabase {
   override def getProject(projectRef: Project.Reference): Future[Option[Project]] =
     Future.successful(allProjects.get(projectRef))
 
-  override def getArtifacts(groupId: Artifact.GroupId, artifactId: Artifact.ArtifactId): Future[Seq[Artifact]] =
+  override def getArtifacts(groupId: Artifact.GroupId, artifactId: String): Future[Seq[Artifact]] =
     Future.successful {
       allArtifacts.values.flatten.filter { artifact: Artifact =>
-        artifact.groupId == groupId && artifact.artifactId == artifactId.value
+        artifact.groupId == groupId && artifact.artifactId == artifactId
       }.toSeq
     }
 
@@ -125,7 +125,6 @@ class InMemoryDatabase extends SchedulerDatabase {
       ref: Project.Reference,
       version: SemanticVersion
   ): Future[Seq[ProjectDependency]] = ???
-  override def computeReleaseDependencies(): Future[Seq[ReleaseDependency]] = ???
 
   override def computeAllProjectsCreationDates(): Future[Seq[(Instant, Project.Reference)]] = ???
 
@@ -134,8 +133,6 @@ class InMemoryDatabase extends SchedulerDatabase {
 
   override def insertProjectDependencies(projectDependencies: Seq[ProjectDependency]): Future[Int] = ???
 
-  override def insertReleaseDependencies(releaseDependency: Seq[ReleaseDependency]): Future[Int] = ???
-
   override def countProjectDependents(ref: Project.Reference): Future[Long] =
     Future.successful(0)
 
@@ -161,9 +158,9 @@ class InMemoryDatabase extends SchedulerDatabase {
   override def getArtifacts(
       ref: Project.Reference,
       artifactName: Artifact.Name,
-      params: ArtifactsPageParams
+      preReleases: Boolean
   ): Future[Seq[Artifact]] =
-    // does not filter with params
+    // TODO: use preReleases to filter
     Future.successful(allArtifacts.getOrElse(ref, Seq.empty).filter(_.artifactName == artifactName))
   override def getProjectDependencies(
       ref: Project.Reference,
@@ -202,12 +199,18 @@ class InMemoryDatabase extends SchedulerDatabase {
     Future.successful(())
   }
 
-  override def getLatestArtifacts(ref: Project.Reference, preferStableVersion: Boolean): Future[Seq[Artifact]] = {
+  override def getLatestArtifacts(ref: Project.Reference): Future[Seq[Artifact]] = {
     val res = allArtifacts(ref)
       .groupBy(a => (a.groupId, a.artifactId))
       .values
       .map(artifacts => artifacts.maxBy(_.releaseDate))
       .toSeq
     Future.successful(res)
   }
+
+  override def getAllArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]] =
+    Future.successful(allArtifacts(ref).map(a => (a.groupId, a.artifactId)).toSeq)
+
+  override def updateLatestVersion(ref: Artifact.MavenReference): Future[Unit] =
+    Future.successful(())
 }

modules/core/shared/src/test/scala/scaladex/core/test/Values.scala

@@ -32,6 +32,7 @@ import scaladex.core.model.search.ProjectDocument
 object Values {
   val now: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS)
   val ok: GithubStatus = GithubStatus.Ok(now)
+  val unknown: GithubStatus = GithubStatus.Unknown(now)
 
   val `2.6.1` = PatchVersion(2, 6, 1)
   val `4`: SemanticVersion = MajorVersion(4)

modules/data/src/main/resources/subindex.txt

@@ -128,7 +128,7 @@ foundweekends/giter8
 cb372/scalacache
 blemale/scaffeine
 scala/scala
-lampepfl/dotty
+scala/scala3
 sbt/zinc
 scala-js/scala-js
 scala-native/scala-native

modules/infra/src/main/resources/migrations/V21__drop_release_and_release_dependencies.sql

@@ -0,0 +1,2 @@
+DROP TABLE releases, release_dependencies;
+

modules/infra/src/main/resources/migrations/V22__new_artifact_fields.sql

@@ -0,0 +1,4 @@
+ALTER TABLE artifacts
+ADD COLUMN IF NOT EXISTS scaladoc_url VARCHAR,
+ADD COLUMN IF NOT EXISTS version_scheme VARCHAR,
+ADD COLUMN IF NOT EXISTS developers VARCHAR NOT NULL DEFAULT '[]';

modules/infra/src/main/resources/migrations/V23__add_is_latest_version.sql

@@ -0,0 +1,2 @@
+ALTER TABLE artifacts
+ADD COLUMN IF NOT EXISTS is_latest_version BOOLEAN NOT NULL default 'false';

modules/infra/src/main/scala/scaladex/infra/GithubClientImpl.scala

@@ -143,14 +143,15 @@ class GithubClientImpl(token: Secret)(implicit val system: ActorSystem)
       .flatMap {
         case (headers, entity) =>
           val lastPage = headers.find(_.is("link")).map(_.value()).flatMap(extractLastPage)
+          val contributors = Unmarshal(entity).to[List[GithubModel.Contributor]]
           lastPage match {
             case Some(lastPage) if lastPage > 1 =>
               for {
-                page1 <- Unmarshal(entity).to[List[GithubModel.Contributor]]
+                page1 <- contributors
                 nextPages <- (2 to lastPage).mapSync(getContributionPage).map(_.flatten)
               } yield page1 ++ nextPages
 
-            case _ => Unmarshal(entity).to[List[GithubModel.Contributor]]
+            case _ => contributors
           }
       }
       .fallbackTo(Future.successful(List.empty))
@@ -180,14 +181,15 @@ class GithubClientImpl(token: Secret)(implicit val system: ActorSystem)
       .flatMap {
         case (headers, entity) =>
           val lastPage = headers.find(_.is("link")).map(_.value()).flatMap(extractLastPage)
+          val issues = Unmarshal(entity).to[Seq[Option[GithubModel.OpenIssue]]]
           lastPage match {
             case Some(lastPage) if lastPage > 1 =>
               for {
-                page1 <- Unmarshal(entity).to[Seq[Option[GithubModel.OpenIssue]]]
+                page1 <- issues
                 nextPages <- (2 to lastPage).mapSync(getOpenIssuePage).map(_.flatten)
               } yield page1.flatten ++ nextPages
 
-            case _ => Unmarshal(entity).to[Seq[Option[GithubModel.OpenIssue]]].map(_.flatten)
+            case _ => issues.map(_.flatten)
           }
       }
       .fallbackTo(Future.successful(Seq.empty))