diff --git a/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala b/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala index ccfc9018f..a8bbdde1c 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala @@ -298,4 +298,9 @@ object Artifact { def repoUrl: String = s"https://repo1.maven.org/maven2/${groupId.replace('.', '/')}/$artifactId/$version/" } + + object MavenReference { + def apply(groupId: GroupId, artifactId: String, version: SemanticVersion): MavenReference = + MavenReference(groupId.value, artifactId, version.encode) + } } diff --git a/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala b/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala index d38f0dd9d..fa77e018c 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala @@ -93,6 +93,9 @@ object Project { settings = settings.getOrElse(Settings.empty) ) + def default(ref: Project.Reference, githubStatus: GithubStatus): Project = + Project(ref.organization, ref.repository, None, githubStatus, None, Settings.empty) + case class Settings( preferStableVersion: Boolean, defaultArtifact: Option[Artifact.Name], diff --git a/modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala b/modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala index 7001e8f3f..f9e0976ec 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala @@ -9,7 +9,7 @@ class ProjectService(database: WebDatabase)(implicit context: ExecutionContext) def getProjectHeader(project: Project): Future[Option[ProjectHeader]] = { val ref = project.reference for { - latestArtifacts <- database.getLatestArtifacts(ref, project.settings.preferStableVersion) + latestArtifacts <- database.getLatestArtifacts(ref) versionCount <- database.countVersions(ref) } yield ProjectHeader( project.reference, diff --git a/modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala b/modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala index 2dee10e16..58c8c5a20 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala @@ -13,6 +13,7 @@ import scaladex.core.model.SemanticVersion trait SchedulerDatabase extends WebDatabase { // project and github + def getAllProjects(): Future[Seq[Project]] def insertProject(project: Project): Future[Unit] def updateProjectCreationDate(ref: Project.Reference, creationDate: Instant): Future[Unit] def computeAllProjectsCreationDates(): Future[Seq[(Instant, Project.Reference)]] @@ -28,6 +29,8 @@ trait SchedulerDatabase extends WebDatabase { def updateArtifacts(artifacts: Seq[Artifact], newRef: Project.Reference): Future[Int] def updateArtifactReleaseDate(reference: MavenReference, releaseDate: Instant): Future[Int] def getAllGroupIds(): Future[Seq[Artifact.GroupId]] + def getAllArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]] def getAllMavenReferences(): Future[Seq[Artifact.MavenReference]] def getDependencies(projectRef: Project.Reference): Future[Seq[ArtifactDependency]] + def updateLatestVersion(ref: MavenReference): Future[Unit] } diff --git a/modules/core/shared/src/main/scala/scaladex/core/service/WebDatabase.scala b/modules/core/shared/src/main/scala/scaladex/core/service/WebDatabase.scala index 2abaac473..55aa3d35b 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/service/WebDatabase.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/service/WebDatabase.scala @@ -1,6 +1,5 @@ package scaladex.core.service -import java.time.Instant import java.util.UUID import scala.concurrent.Future @@ -9,14 +8,13 @@ import scaladex.core.model._ trait WebDatabase { // artifacts - // insertArtifact return a boolean. It's true if a new project is inserted, false otherwise - def insertArtifact(artifact: Artifact, dependencies: Seq[ArtifactDependency], time: Instant): Future[Boolean] - def getArtifacts(groupId: Artifact.GroupId, artifactId: Artifact.ArtifactId): Future[Seq[Artifact]] + def insertArtifact(artifact: Artifact): Future[Boolean] + def getArtifacts(groupId: Artifact.GroupId, artifactId: String): Future[Seq[Artifact]] def getArtifacts(projectRef: Project.Reference): Future[Seq[Artifact]] def getArtifacts(ref: Project.Reference, artifactName: Artifact.Name, preReleases: Boolean): Future[Seq[Artifact]] def getArtifacts(ref: Project.Reference, artifactName: Artifact.Name, version: SemanticVersion): Future[Seq[Artifact]] def getArtifactsByName(projectRef: Project.Reference, artifactName: Artifact.Name): Future[Seq[Artifact]] - def getLatestArtifacts(ref: Project.Reference, preferStableVersions: Boolean): Future[Seq[Artifact]] + def getLatestArtifacts(ref: Project.Reference): Future[Seq[Artifact]] def getArtifactByMavenReference(mavenRef: Artifact.MavenReference): Future[Option[Artifact]] def getAllArtifacts(language: Option[Language], platform: Option[Platform]): Future[Seq[Artifact]] def countArtifacts(): Future[Long] @@ -26,8 +24,8 @@ trait WebDatabase { def getReverseDependencies(artifact: Artifact): Future[Seq[ArtifactDependency.Reverse]] // projects + def insertProjectRef(ref: Project.Reference, status: GithubStatus): Future[Boolean] def updateProjectSettings(ref: Project.Reference, settings: Project.Settings): Future[Unit] - def getAllProjects(): Future[Seq[Project]] def getAllProjectsStatuses(): Future[Map[Project.Reference, GithubStatus]] def getProject(projectRef: Project.Reference): Future[Option[Project]] def getFormerReferences(projectRef: Project.Reference): Future[Seq[Project.Reference]] diff --git a/modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala b/modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala index b0a4d4327..716dd8b69 100644 --- a/modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala +++ b/modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala @@ -30,16 +30,15 @@ class InMemoryDatabase extends SchedulerDatabase { allDependencies.clear() } - override def insertArtifact( - artifact: Artifact, - dependencies: Seq[ArtifactDependency], - now: Instant - ): Future[Boolean] = { - val ref = artifact.projectRef + override def insertArtifact(artifact: Artifact): Future[Boolean] = { + val isNewArtifact = !allArtifacts.values.flatten.exists(a => a.mavenReference == artifact.mavenReference) + allArtifacts.addOne(artifact.projectRef -> (allArtifacts.getOrElse(artifact.projectRef, Seq.empty) :+ artifact)) + Future.successful(isNewArtifact) + } + + override def insertProjectRef(ref: Project.Reference, status: GithubStatus): Future[Boolean] = { val isNewProject = !allProjects.contains(ref) - if (isNewProject) allProjects.addOne(ref -> Project.default(ref, now = now)) - allArtifacts.addOne(ref -> (allArtifacts.getOrElse(ref, Seq.empty) :+ artifact)) - dependencies.appendedAll(dependencies) + if (isNewProject) allProjects.addOne(ref -> Project.default(ref, status)) Future.successful(isNewProject) } @@ -47,7 +46,10 @@ class InMemoryDatabase extends SchedulerDatabase { override def insertArtifacts(allArtifacts: Seq[Artifact]): Future[Unit] = ??? - override def insertDependencies(dependencies: Seq[ArtifactDependency]): Future[Unit] = ??? + override def insertDependencies(dependencies: Seq[ArtifactDependency]): Future[Unit] = { + allDependencies ++= dependencies + Future.successful(()) + } override def deleteProjectDependencies(ref: Project.Reference): Future[Int] = ??? @@ -59,10 +61,10 @@ class InMemoryDatabase extends SchedulerDatabase { override def getProject(projectRef: Project.Reference): Future[Option[Project]] = Future.successful(allProjects.get(projectRef)) - override def getArtifacts(groupId: Artifact.GroupId, artifactId: Artifact.ArtifactId): Future[Seq[Artifact]] = + override def getArtifacts(groupId: Artifact.GroupId, artifactId: String): Future[Seq[Artifact]] = Future.successful { allArtifacts.values.flatten.filter { artifact: Artifact => - artifact.groupId == groupId && artifact.artifactId == artifactId.value + artifact.groupId == groupId && artifact.artifactId == artifactId }.toSeq } @@ -197,7 +199,7 @@ class InMemoryDatabase extends SchedulerDatabase { Future.successful(()) } - override def getLatestArtifacts(ref: Project.Reference, preferStableVersion: Boolean): Future[Seq[Artifact]] = { + override def getLatestArtifacts(ref: Project.Reference): Future[Seq[Artifact]] = { val res = allArtifacts(ref) .groupBy(a => (a.groupId, a.artifactId)) .values @@ -205,4 +207,10 @@ class InMemoryDatabase extends SchedulerDatabase { .toSeq Future.successful(res) } + + override def getAllArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]] = + Future.successful(allArtifacts(ref).map(a => (a.groupId, a.artifactId)).toSeq) + + override def updateLatestVersion(ref: Artifact.MavenReference): Future[Unit] = + Future.successful(()) } diff --git a/modules/core/shared/src/test/scala/scaladex/core/test/Values.scala b/modules/core/shared/src/test/scala/scaladex/core/test/Values.scala index 521e2397d..004fe53ac 100644 --- a/modules/core/shared/src/test/scala/scaladex/core/test/Values.scala +++ b/modules/core/shared/src/test/scala/scaladex/core/test/Values.scala @@ -31,6 +31,7 @@ import scaladex.core.model.search.ProjectDocument object Values { val now: Instant = Instant.now().truncatedTo(ChronoUnit.MILLIS) val ok: GithubStatus = GithubStatus.Ok(now) + val unknown: GithubStatus = GithubStatus.Unknown(now) val `2.6.1` = PatchVersion(2, 6, 1) val `4`: SemanticVersion = MajorVersion(4) diff --git a/modules/infra/src/main/resources/migrations/V22__add_is_latest_version.sql b/modules/infra/src/main/resources/migrations/V22__add_is_latest_version.sql new file mode 100644 index 000000000..c2116fc30 --- /dev/null +++ b/modules/infra/src/main/resources/migrations/V22__add_is_latest_version.sql @@ -0,0 +1,2 @@ +ALTER TABLE artifacts +ADD COLUMN is_latest_version BOOLEAN NOT NULL default 'false'; diff --git a/modules/infra/src/main/scala/scaladex/infra/GithubClientImpl.scala b/modules/infra/src/main/scala/scaladex/infra/GithubClientImpl.scala index 05d1adb77..bc317b0ed 100644 --- a/modules/infra/src/main/scala/scaladex/infra/GithubClientImpl.scala +++ b/modules/infra/src/main/scala/scaladex/infra/GithubClientImpl.scala @@ -143,14 +143,15 @@ class GithubClientImpl(token: Secret)(implicit val system: ActorSystem) .flatMap { case (headers, entity) => val lastPage = headers.find(_.is("link")).map(_.value()).flatMap(extractLastPage) + val contributors = Unmarshal(entity).to[List[GithubModel.Contributor]] lastPage match { case Some(lastPage) if lastPage > 1 => for { - page1 <- Unmarshal(entity).to[List[GithubModel.Contributor]] + page1 <- contributors nextPages <- (2 to lastPage).mapSync(getContributionPage).map(_.flatten) } yield page1 ++ nextPages - case _ => Unmarshal(entity).to[List[GithubModel.Contributor]] + case _ => contributors } } .fallbackTo(Future.successful(List.empty)) @@ -180,14 +181,15 @@ class GithubClientImpl(token: Secret)(implicit val system: ActorSystem) .flatMap { case (headers, entity) => val lastPage = headers.find(_.is("link")).map(_.value()).flatMap(extractLastPage) + val issues = Unmarshal(entity).to[Seq[Option[GithubModel.OpenIssue]]] lastPage match { case Some(lastPage) if lastPage > 1 => for { - page1 <- Unmarshal(entity).to[Seq[Option[GithubModel.OpenIssue]]] + page1 <- issues nextPages <- (2 to lastPage).mapSync(getOpenIssuePage).map(_.flatten) } yield page1.flatten ++ nextPages - case _ => Unmarshal(entity).to[Seq[Option[GithubModel.OpenIssue]]].map(_.flatten) + case _ => issues.map(_.flatten) } } .fallbackTo(Future.successful(Seq.empty)) diff --git a/modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala b/modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala index 412625137..f25a570be 100644 --- a/modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala +++ b/modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala @@ -33,24 +33,11 @@ import scaladex.infra.sql.UserSessionsTable class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) extends SchedulerDatabase with LazyLogging { private val flyway = DoobieUtils.flyway(datasource) - def migrate: IO[Unit] = IO { - flyway.repair() - flyway.migrate() - } + def migrate: IO[Unit] = IO(flyway.migrate()) def dropTables: IO[Unit] = IO(flyway.clean()) - override def insertArtifact( - artifact: Artifact, - dependencies: Seq[ArtifactDependency], - time: Instant - ): Future[Boolean] = { - val unknownStatus = GithubStatus.Unknown(time) - for { - isNewProject <- insertProjectRef(artifact.projectRef, unknownStatus) - _ <- run(ArtifactTable.insertIfNotExist(artifact)) - _ <- insertDependencies(dependencies) - } yield isNewProject - } + override def insertArtifact(artifact: Artifact): Future[Boolean] = + run(ArtifactTable.insertIfNotExist(artifact)).map(_ >= 1) override def insertArtifacts(artifacts: Seq[Artifact]): Future[Unit] = run(ArtifactTable.insertIfNotExist(artifacts)).map(_ => ()) @@ -63,7 +50,7 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten override def updateArtifactReleaseDate(reference: Artifact.MavenReference, releaseDate: Instant): Future[Int] = run(ArtifactTable.updateReleaseDate.run((releaseDate, reference))) - override def getArtifacts(groupId: Artifact.GroupId, artifactId: Artifact.ArtifactId): Future[Seq[Artifact]] = + override def getArtifacts(groupId: Artifact.GroupId, artifactId: String): Future[Seq[Artifact]] = run(ArtifactTable.selectArtifactByGroupIdAndArtifactId.to[Seq](groupId, artifactId)) override def getArtifacts(projectRef: Project.Reference): Future[Seq[Artifact]] = @@ -79,21 +66,8 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten override def getArtifactsByName(ref: Project.Reference, artifactName: Artifact.Name): Future[Seq[Artifact]] = run(ArtifactTable.selectArtifactByProjectAndName.to[Seq]((ref, artifactName))) - override def getLatestArtifacts(ref: Project.Reference, preferStableVersions: Boolean): Future[Seq[Artifact]] = { - val latestArtifactsF = run(ArtifactTable.selectLatestArtifacts(stableOnly = false).to[Seq](ref)) - if (preferStableVersions) { - for { - latestStableArtifacts <- run(ArtifactTable.selectLatestArtifacts(stableOnly = true).to[Seq](ref)) - latestArtifacts <- latestArtifactsF - } yield - // override non-stable version with the latest stable version - (latestStableArtifacts ++ latestArtifacts) - .groupBy(a => (a.groupId, a.artifactId)) - .valuesIterator - .map(_.head) - .toSeq - } else latestArtifactsF - } + override def getLatestArtifacts(ref: Project.Reference): Future[Seq[Artifact]] = + run(ArtifactTable.selectLatestArtifacts.to[Seq](ref)) override def getArtifactByMavenReference(mavenRef: Artifact.MavenReference): Future[Option[Artifact]] = run(ArtifactTable.selectByMavenReference.option(mavenRef)) @@ -120,7 +94,7 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten run(ArtifactDependencyTable.insertIfNotExist.updateMany(dependencies)).map(_ => ()) // return true if inserted, false if it already existed - private def insertProjectRef(ref: Project.Reference, status: GithubStatus): Future[Boolean] = + override def insertProjectRef(ref: Project.Reference, status: GithubStatus): Future[Boolean] = run(ProjectTable.insertIfNotExists.run((ref, status))).map(x => x >= 1) override def getAllProjectsStatuses(): Future[Map[Project.Reference, GithubStatus]] = @@ -206,6 +180,9 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten override def getAllGroupIds(): Future[Seq[Artifact.GroupId]] = run(ArtifactTable.selectGroupIds.to[Seq]) + override def getAllArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, String)]] = + run(ArtifactTable.selectArtifactIds.to[Seq](ref)) + override def getAllMavenReferences(): Future[Seq[Artifact.MavenReference]] = run(ArtifactTable.selectMavenReference.to[Seq]) @@ -231,6 +208,14 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten ): Future[Seq[Artifact]] = run(ArtifactTable.selectArtifactByParams(preReleases).to[Seq](ref, artifactName)) + override def updateLatestVersion(ref: Artifact.MavenReference): Future[Unit] = { + val transaction = for { + _ <- ArtifactTable.setLatestVersion.run(ref) + _ <- ArtifactTable.unsetOthersLatestVersion.run(ref) + } yield () + run(transaction) + } + override def countVersions(ref: Project.Reference): Future[Long] = run(ArtifactTable.countVersionsByProject.unique(ref)) diff --git a/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala b/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala index 1363d66db..0da4ef272 100644 --- a/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala +++ b/modules/infra/src/main/scala/scaladex/infra/sql/ArtifactTable.scala @@ -35,6 +35,7 @@ object ArtifactTable { ) // these field are usually excluded when we read artifacts from the artifacts table. val versionFields: Seq[String] = Seq("is_semantic", "is_prerelease") + val isLatestVersion: String = "is_latest_version" def insertIfNotExist(artifact: Artifact): ConnectionIO[Int] = insertIfNotExist.run((artifact, artifact.version.isSemantic, artifact.version.isPreRelease)) @@ -62,7 +63,7 @@ object ArtifactTable { selectRequest(table, fields, where = where) } - val selectArtifactByGroupIdAndArtifactId: Query[(Artifact.GroupId, Artifact.ArtifactId), Artifact] = + val selectArtifactByGroupIdAndArtifactId: Query[(Artifact.GroupId, String), Artifact] = selectRequest(table, fields, Seq("group_id", "artifact_id")) val selectArtifactByProject: Query[Project.Reference, Artifact] = @@ -100,6 +101,9 @@ object ArtifactTable { val selectGroupIds: Query0[Artifact.GroupId] = selectRequest(table, Seq("DISTINCT group_id")) + val selectArtifactIds: Query[Project.Reference, (Artifact.GroupId, String)] = + selectRequest(table, Seq("DISTINCT group_id", "artifact_id"), keys = projectReferenceFields) + val selectMavenReference: Query0[Artifact.MavenReference] = selectRequest(table, Seq("DISTINCT group_id", "artifact_id", "\"version\"")) @@ -114,26 +118,16 @@ object ArtifactTable { groupBy = projectReferenceFields ) - def selectLatestArtifacts(stableOnly: Boolean): Query[Project.Reference, Artifact] = - selectRequest1(latestDateTable(stableOnly), fields.map(c => s"a.$c")) - - // the latest release date of all artifact IDs - private def latestDateTable(stableOnly: Boolean): String = - s"($table a " + - s"INNER JOIN (${selectLatestDate(stableOnly).sql}) d " + - s"ON a.group_id=d.group_id " + - s"AND a.artifact_id=d.artifact_id " + - s"AND a.release_date=d.release_date)" - - private def selectLatestDate( - stableOnly: Boolean - ): Query[Project.Reference, (Artifact.GroupId, String, Instant)] = { - val isReleaseFilters = if (stableOnly) Seq("is_semantic='true'", "is_prerelease='false'") else Seq.empty - selectRequest1( + def selectLatestArtifacts: Query[Project.Reference, Artifact] = + selectRequest1(table, fields, where = Seq("organization=?", "repository=?", "is_latest_version=true")) + + def setLatestVersion: Update[Artifact.MavenReference] = + updateRequest0(table, set = Seq("is_latest_version=true"), where = Seq("group_id=?", "artifact_id=?", "version=?")) + + def unsetOthersLatestVersion: Update[Artifact.MavenReference] = + updateRequest0( table, - Seq("group_id", "artifact_id", "MAX(release_date) as release_date"), - where = Seq("release_date IS NOT NULL") ++ isReleaseFilters ++ Seq("organization=?", "repository=?"), - groupBy = Seq("group_id", "artifact_id") + set = Seq("is_latest_version=false"), + where = Seq("group_id=?", "artifact_id=?", "version<>?") ) - } } diff --git a/modules/infra/src/main/scala/scaladex/infra/sql/DoobieUtils.scala b/modules/infra/src/main/scala/scaladex/infra/sql/DoobieUtils.scala index f24779628..b973b79fc 100644 --- a/modules/infra/src/main/scala/scaladex/infra/sql/DoobieUtils.scala +++ b/modules/infra/src/main/scala/scaladex/infra/sql/DoobieUtils.scala @@ -93,6 +93,12 @@ object DoobieUtils { Update(s"UPDATE $table SET $fieldsStr WHERE $keysStr") } + def updateRequest0[T: Write](table: String, set: Seq[String], where: Seq[String]): Update[T] = { + val setStr = set.mkString(", ") + val whereStr = where.mkString(" AND ") + Update(s"UPDATE $table SET $setStr WHERE $whereStr") + } + def selectRequest[A: Read](table: String, fields: Seq[String]): Query0[A] = { val fieldsStr = fields.mkString(", ") Query0(s"SELECT $fieldsStr FROM $table") diff --git a/modules/infra/src/test/scala/scaladex/infra/SqlDatabaseTests.scala b/modules/infra/src/test/scala/scaladex/infra/SqlDatabaseTests.scala index 202429170..2ca2ffabd 100644 --- a/modules/infra/src/test/scala/scaladex/infra/SqlDatabaseTests.scala +++ b/modules/infra/src/test/scala/scaladex/infra/SqlDatabaseTests.scala @@ -34,26 +34,22 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers it("insert artifact and its dependencies") { for { - _ <- database.insertArtifact(Cats.`core_3:2.6.1`, Cats.dependencies, now) - project <- database.getProject(Cats.reference) + _ <- database.insertArtifact(Cats.`core_3:2.6.1`) artifacts <- database.getArtifacts(Cats.reference) - } yield { - project should not be empty - artifacts should contain theSameElementsAs Seq(Cats.`core_3:2.6.1`) - } + } yield artifacts should contain theSameElementsAs Seq(Cats.`core_3:2.6.1`) } it("should get all project statuses") { for { - _ <- database.insertArtifact(Cats.`core_3:2.6.1`, Cats.dependencies, now) - _ <- database.insertArtifact(PlayJsonExtra.artifact, Seq.empty, now) + _ <- database.insertProjectRef(Cats.reference, unknown) + _ <- database.insertProjectRef(PlayJsonExtra.reference, unknown) projectStatuses <- database.getAllProjectsStatuses() } yield projectStatuses.keys should contain theSameElementsAs Seq(PlayJsonExtra.reference, Cats.reference) } it("should update project settings") { for { - _ <- database.insertArtifact(Scalafix.artifact, Seq.empty, now) + _ <- database.insertProjectRef(Scalafix.reference, unknown) _ <- database.updateGithubInfoAndStatus(Scalafix.reference, Scalafix.githubInfo, ok) _ <- database.updateProjectSettings(Scalafix.reference, Scalafix.settings) scalafix <- database.getProject(Scalafix.reference) @@ -63,8 +59,8 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers it("should update artifacts") { val newRef = Project.Reference.from("kindlevel", "dogs") for { - _ <- database.insertArtifact(Cats.`core_3:2.6.1`, Cats.dependencies, now) - _ <- database.insertArtifact(Cats.`core_sjs1_3:2.6.1`, Seq.empty, now) + _ <- database.insertArtifact(Cats.`core_3:2.6.1`) + _ <- database.insertArtifact(Cats.`core_sjs1_3:2.6.1`) _ <- database.updateArtifacts(Seq(Cats.`core_3:2.6.1`, Cats.`core_sjs1_3:2.6.1`), newRef) oldArtifacts <- database.getArtifacts(Cats.reference) newArtifacts <- database.getArtifacts(newRef) @@ -80,7 +76,7 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers it("should update github status") { val failed = GithubStatus.Failed(now, 405, "Unauthorized") for { - _ <- database.insertArtifact(Scalafix.artifact, Seq.empty, now) + _ <- database.insertProjectRef(Scalafix.reference, unknown) _ <- database.updateGithubInfoAndStatus(Scalafix.reference, Scalafix.githubInfo, ok) _ <- database.updateGithubStatus(Scalafix.reference, failed) scalafix <- database.getProject(Scalafix.reference) @@ -89,18 +85,22 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers it("should find artifacts by name") { for { - _ <- database.insertArtifact(Cats.`core_3:2.6.1`, Cats.dependencies, now) - _ <- database.insertArtifact(Cats.`core_sjs1_3:2.6.1`, Seq.empty, now) + _ <- database.insertArtifact(Cats.`core_3:2.6.1`) + _ <- database.insertArtifact(Cats.`core_sjs1_3:2.6.1`) artifacts <- database.getArtifactsByName(Cats.reference, Cats.`core_3:2.6.1`.artifactName) } yield artifacts should contain theSameElementsAs Seq(Cats.`core_3:2.6.1`, Cats.`core_sjs1_3:2.6.1`) } it("should count projects, artifacts, dependencies, github infos and data forms") { for { - _ <- database.insertArtifact(Cats.`core_3:2.6.1`, Cats.dependencies, now) - _ <- database.insertArtifact(Cats.`core_sjs1_3:2.6.1`, Seq.empty, now) - _ <- database.insertArtifact(Scalafix.artifact, Seq.empty, now) - _ <- database.insertArtifact(PlayJsonExtra.artifact, Seq.empty, now) + _ <- database.insertProjectRef(Cats.reference, unknown) + _ <- database.insertArtifact(Cats.`core_3:2.6.1`) + _ <- database.insertDependencies(Cats.dependencies) + _ <- database.insertArtifact(Cats.`core_sjs1_3:2.6.1`) + _ <- database.insertProjectRef(Scalafix.reference, unknown) + _ <- database.insertArtifact(Scalafix.artifact) + _ <- database.insertProjectRef(PlayJsonExtra.reference, unknown) + _ <- database.insertArtifact(PlayJsonExtra.artifact) _ <- database.updateGithubInfoAndStatus(Scalafix.reference, Scalafix.githubInfo, GithubStatus.Ok(now)) _ <- database.updateProjectSettings(Scalafix.reference, Scalafix.settings) projects <- database.countProjects() @@ -119,8 +119,9 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers it("should find directDependencies") { for { - _ <- database.insertArtifact(Cats.`core_3:2.6.1`, Cats.dependencies, now) - _ <- database.insertArtifact(Cats.`kernel_3:2.6.1`, Seq.empty, now) + _ <- database.insertArtifact(Cats.`core_3:2.6.1`) + _ <- database.insertDependencies(Cats.dependencies) + _ <- database.insertArtifact(Cats.`kernel_3:2.6.1`) directDependencies <- database.getDirectDependencies(Cats.`core_3:2.6.1`) } yield { val targets = directDependencies.map(_.target) @@ -130,28 +131,29 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers it("should find reverseDependencies") { for { - _ <- database.insertArtifact(Cats.`core_3:2.6.1`, Cats.dependencies, now) - _ <- database.insertArtifact(Cats.`kernel_3:2.6.1`, Seq.empty, now) + _ <- database.insertArtifact(Cats.`core_3:2.6.1`) + _ <- database.insertDependencies(Cats.dependencies) + _ <- database.insertArtifact(Cats.`kernel_3:2.6.1`) reverseDependencies <- database.getReverseDependencies(Cats.`kernel_3:2.6.1`) } yield reverseDependencies.map(_.source) should contain theSameElementsAs List(Cats.`core_3:2.6.1`) } it("should compute project dependencies") { for { - _ <- database.insertArtifact(Cats.`core_2.13:2.6.1`, Seq.empty, now) - _ <- database.insertArtifact( - Cats.`kernel_2.13`, + _ <- database.insertProjectRef(Cats.reference, unknown) + _ <- database.insertArtifact(Cats.`core_2.13:2.6.1`) + _ <- database.insertArtifact(Cats.`kernel_2.13`) + _ <- database.insertDependencies( Seq( ArtifactDependency(Cats.`kernel_2.13`.mavenReference, Cats.`core_2.13:2.6.1`.mavenReference, Scope("compile")) - ), - now + ) ) - _ <- database.insertArtifact( - Scalafix.artifact, + _ <- database.insertProjectRef(Scalafix.reference, unknown) + _ <- database.insertArtifact(Scalafix.artifact) + _ <- database.insertDependencies( Seq( ArtifactDependency(Scalafix.artifact.mavenReference, Cats.`core_2.13:2.6.1`.mavenReference, Scope("compile")) - ), - now + ) ) scalafixDependencies <- database.computeProjectDependencies(Scalafix.reference, Scalafix.artifact.version) catsDependencies <- database.computeProjectDependencies(Cats.reference, Cats.`core_2.13:2.6.1`.version) @@ -174,9 +176,12 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers } it("should update creation date") { for { - _ <- database.insertArtifact(Scalafix.artifact, Seq.empty, now) - _ <- database.insertArtifact(Cats.`core_3:2.6.1`, Seq.empty, now) - _ <- database.insertArtifact(PlayJsonExtra.artifact, Seq.empty, now) + _ <- database.insertProjectRef(Scalafix.reference, unknown) + _ <- database.insertArtifact(Scalafix.artifact) + _ <- database.insertProjectRef(Cats.reference, unknown) + _ <- database.insertArtifact(Cats.`core_3:2.6.1`) + _ <- database.insertProjectRef(PlayJsonExtra.reference, unknown) + _ <- database.insertArtifact(PlayJsonExtra.artifact) creationDates <- database.computeAllProjectsCreationDates() _ <- creationDates.mapSync { case (creationDate, ref) => database.updateProjectCreationDate(ref, creationDate) } projects <- database.getAllProjects() @@ -190,11 +195,11 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers creationDates should contain theSameElementsAs expected } } - it("should createMovedProject") { + it("should create moved project") { val destination = Project.Reference.from("scala", "fix") val moved = GithubStatus.Moved(now, destination) for { - _ <- database.insertArtifact(Scalafix.artifact, Seq.empty, now) + _ <- database.insertProjectRef(Scalafix.reference, unknown) _ <- database.updateGithubInfoAndStatus(Scalafix.reference, Scalafix.githubInfo, GithubStatus.Ok(now)) _ <- database.moveProject(Scalafix.reference, Scalafix.githubInfo, moved) newProject <- database.getProject(destination) @@ -217,7 +222,7 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers ) ) for { - _ <- database.insertArtifact(Scalafix.artifact, Seq.empty, now) + _ <- database.insertProjectRef(Scalafix.reference, unknown) _ <- database.insertProjectDependencies(dependencies) _ <- database.deleteProjectDependencies(Scalafix.reference) catsDependents <- database.countProjectDependents(Cats.reference) @@ -259,21 +264,21 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers } yield obtained shouldBe None } - it("should return artifact from maven reference") { + it("get artifact from maven reference") { for { - _ <- database.insertArtifact(Cats.`core_3:2.6.1`, Seq.empty, now) + _ <- database.insertArtifact(Cats.`core_3:2.6.1`) catsCore <- database.getArtifactByMavenReference(Cats.`core_3:2.6.1`.mavenReference) } yield catsCore should contain(Cats.`core_3:2.6.1`) } - it("should return artifact from maven reference if version is only major") { + it("get artifact from maven reference if version is only major") { for { - _ <- database.insertArtifact(Cats.`core_3:4`, Seq.empty, now) + _ <- database.insertArtifact(Cats.`core_3:4`) catsCore <- database.getArtifactByMavenReference(Cats.`core_3:4`.mavenReference) } yield catsCore should contain(Cats.`core_3:4`) } - it("should return all artifacts given no language or platform") { + it("get all artifacts given no language or platform") { val testArtifacts = Seq(Scalafix.artifact, Cats.`core_3:4`, PlayJsonExtra.artifact) for { _ <- database.insertArtifacts(testArtifacts) @@ -283,14 +288,14 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers it("should return no artifacts given a language that has no artifacts stored") { for { - _ <- database.insertArtifact(Scalafix.artifact, Seq.empty, now) // Scalafix has Scala version 2.13 + _ <- database.insertArtifact(Scalafix.artifact) // Scalafix has Scala version 2.13 storedArtifacts <- database.getAllArtifacts(Some(Scala.`3`), None) } yield storedArtifacts.size shouldBe 0 } it("should return no artifacts given a platform that has no artifacts stored") { for { - _ <- database.insertArtifact(Scalafix.artifact, Seq.empty, now) // Scalafix is a JVM-platform artifact + _ <- database.insertArtifact(Scalafix.artifact) // Scalafix is a JVM-platform artifact storedArtifacts <- database.getAllArtifacts(None, Some(ScalaJs.`1.x`)) } yield storedArtifacts.size shouldBe 0 } @@ -320,39 +325,27 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers } it("should not return any artifacts when the database is empty, given a group id and artifact id") { - val testArtifact = Cats.`core_3:4` - val testArtifactId = Artifact.ArtifactId - .parse(testArtifact.artifactId) - .getOrElse(fail("Parsing an artifact id should not have failed")) for { - retrievedArtifacts <- database.getArtifacts(testArtifact.groupId, testArtifactId) + retrievedArtifacts <- database.getArtifacts(Cats.`core_3:4`.groupId, Cats.`core_3:4`.artifactId) } yield retrievedArtifacts.size shouldBe 0 } - it("should return an artifact, given a group id an artifact id of a stored artifact") { - val testArtifact = Cats.`core_3:4` - val testArtifactId = Artifact.ArtifactId - .parse(testArtifact.artifactId) - .getOrElse(fail("Parsing an artifact id should not have failed")) + it("get an artifact from its group id and artifact id") { for { - isStoredSuccessfully <- database.insertArtifact(testArtifact, dependencies = Cats.dependencies, now) - retrievedArtifacts <- database.getArtifacts(testArtifact.groupId, testArtifactId) + isStoredSuccessfully <- database.insertArtifact(Cats.`core_3:4`) + retrievedArtifacts <- database.getArtifacts(Cats.`core_3:4`.groupId, Cats.`core_3:4`.artifactId) } yield { isStoredSuccessfully shouldBe true retrievedArtifacts.size shouldBe 1 - retrievedArtifacts.headOption shouldBe Some(testArtifact) + retrievedArtifacts.headOption shouldBe Some(Cats.`core_3:4`) } } - it("should return all versions of an artifact given a group id and an artifact id") { + it("get all artifacts from group id and artifact id") { val testArtifacts = Seq(Cats.`core_3:4`, Cats.`core_3:2.7.0`) - val groupId = Artifact.GroupId("org.typelevel") - val artifactId = Artifact.ArtifactId - .parse("cats-core_3") - .getOrElse(fail("Parsing an artifact id should not have failed")) for { _ <- database.insertArtifacts(testArtifacts) - retrievedArtifacts <- database.getArtifacts(groupId, artifactId) + retrievedArtifacts <- database.getArtifacts(Artifact.GroupId("org.typelevel"), "cats-core_3") } yield { retrievedArtifacts.size shouldBe 2 retrievedArtifacts should contain theSameElementsAs testArtifacts diff --git a/modules/infra/src/test/scala/scaladex/infra/sql/ArtifactTableTests.scala b/modules/infra/src/test/scala/scaladex/infra/sql/ArtifactTableTests.scala index 440553416..d50acfb41 100644 --- a/modules/infra/src/test/scala/scaladex/infra/sql/ArtifactTableTests.scala +++ b/modules/infra/src/test/scala/scaladex/infra/sql/ArtifactTableTests.scala @@ -31,8 +31,7 @@ class ArtifactTableTests extends AnyFunSpec with BaseDatabaseSuite with Matchers check(selectArtifactByParams(true)) } it("check selectMavenReferenceWithNoReleaseDate")(check(selectMavenReferenceWithNoReleaseDate)) - it("check selectLatestArtifacts") { - check(selectLatestArtifacts(true)) - check(selectLatestArtifacts(false)) - } + it("check selectLatestArtifacts")(check(selectLatestArtifacts)) + it("check setLatestVersion")(check(setLatestVersion)) + it("check unsetOthersLatestVersion")(check(unsetOthersLatestVersion)) } diff --git a/modules/server/src/it/scala/scaladex/RelevanceTest.scala b/modules/server/src/it/scala/scaladex/RelevanceTest.scala index c7d1fa1b9..c8ad420e2 100644 --- a/modules/server/src/it/scala/scaladex/RelevanceTest.scala +++ b/modules/server/src/it/scala/scaladex/RelevanceTest.scala @@ -39,7 +39,7 @@ class RelevanceTest extends TestKit(ActorSystem("SbtActorTest")) with AsyncFunSu val projectService = new ProjectService(database) val searchSync = new SearchSynchronizer(database, projectService, searchEngine) - val projectDependenciesUpdater = new DependencyUpdater(database) + val projectDependenciesUpdater = new DependencyUpdater(database, projectService) IO.fromFuture(IO { for { diff --git a/modules/server/src/main/scala/scaladex/server/Server.scala b/modules/server/src/main/scala/scaladex/server/Server.scala index e6fcbddde..58d274a35 100644 --- a/modules/server/src/main/scala/scaladex/server/Server.scala +++ b/modules/server/src/main/scala/scaladex/server/Server.scala @@ -14,7 +14,6 @@ import org.apache.pekko.http.scaladsl._ import org.apache.pekko.http.scaladsl.model.StatusCodes import org.apache.pekko.http.scaladsl.server.Directives._ import org.apache.pekko.http.scaladsl.server._ -import scaladex.core.service.WebDatabase import scaladex.data.util.PidLock import scaladex.infra.DataPaths import scaladex.infra.ElasticsearchEngine @@ -128,7 +127,7 @@ object Server extends LazyLogging { private def configureRoutes( config: ServerConfig, searchEngine: ElasticsearchEngine, - webDatabase: WebDatabase, + webDatabase: SqlDatabase, adminService: AdminService, publishProcess: PublishProcess )( diff --git a/modules/server/src/main/scala/scaladex/server/route/ProjectPages.scala b/modules/server/src/main/scala/scaladex/server/route/ProjectPages.scala index 35ee95295..fd2402c79 100644 --- a/modules/server/src/main/scala/scaladex/server/route/ProjectPages.scala +++ b/modules/server/src/main/scala/scaladex/server/route/ProjectPages.scala @@ -16,21 +16,23 @@ import org.apache.pekko.http.scaladsl.server.Directives._ import org.apache.pekko.http.scaladsl.server._ import scaladex.core.model._ import scaladex.core.service.ProjectService +import scaladex.core.service.SchedulerDatabase import scaladex.core.service.SearchEngine -import scaladex.core.service.WebDatabase import scaladex.core.web.ArtifactPageParams import scaladex.core.web.ArtifactsPageParams import scaladex.server.TwirlSupport._ +import scaladex.server.service.ArtifactsService import scaladex.server.service.SearchSynchronizer import scaladex.view.html.forbidden import scaladex.view.html.notfound import scaladex.view.project.html -class ProjectPages(env: Env, database: WebDatabase, searchEngine: SearchEngine)( +class ProjectPages(env: Env, database: SchedulerDatabase, searchEngine: SearchEngine)( implicit executionContext: ExecutionContext ) extends LazyLogging { - private val service = new ProjectService(database) - private val searchSynchronizer = new SearchSynchronizer(database, service, searchEngine) + private val projectService = new ProjectService(database) + private val artifactsService = new ArtifactsService(database) + private val searchSynchronizer = new SearchSynchronizer(database, projectService, searchEngine) def route(user: Option[UserState]): Route = concat( @@ -42,7 +44,7 @@ class ProjectPages(env: Env, database: WebDatabase, searchEngine: SearchEngine)( artifactsParams { params => getProjectOrRedirect(ref, user) { project => val artifactsF = database.getArtifacts(ref, artifactName, params.preReleases) - val headerF = service.getProjectHeader(project).map(_.get) + val headerF = projectService.getProjectHeader(project).map(_.get) for (artifacts <- artifactsF; header <- headerF) yield { val binaryVersions = artifacts .map(_.binaryVersion) @@ -81,7 +83,7 @@ class ProjectPages(env: Env, database: WebDatabase, searchEngine: SearchEngine)( path(projectM / "artifacts" / artifactNameM / versionM) { (ref, artifactName, artifactVersion) => artifactParams { params => getProjectOrRedirect(ref, user) { project => - val headerF = service.getProjectHeader(project) + val headerF = projectService.getProjectHeader(project) val artifactsF = database.getArtifacts(ref, artifactName, artifactVersion) for { artifacts <- artifactsF @@ -116,7 +118,7 @@ class ProjectPages(env: Env, database: WebDatabase, searchEngine: SearchEngine)( getProjectOrRedirect(ref, user) { project => for { artifacts <- database.getArtifacts(project.reference) - header <- service.getProjectHeader(project) + header <- projectService.getProjectHeader(project) } yield { val binaryVersionByPlatforms = artifacts .map(_.binaryVersion) @@ -156,6 +158,7 @@ class ProjectPages(env: Env, database: WebDatabase, searchEngine: SearchEngine)( editForm { form => val updateF = for { _ <- database.updateProjectSettings(projectRef, form) + _ <- artifactsService.updateLatestVersions(projectRef, form.preferStableVersion) _ <- searchSynchronizer.syncProject(projectRef) } yield () onComplete(updateF) { @@ -234,7 +237,7 @@ class ProjectPages(env: Env, database: WebDatabase, searchEngine: SearchEngine)( getProjectOrRedirect(ref, Some(user)) { project => for { artifacts <- database.getArtifacts(ref) - header <- service.getProjectHeader(project) + header <- projectService.getProjectHeader(project) } yield { val page = html.editproject(env, user, project, header, artifacts) complete(page) @@ -244,7 +247,7 @@ class ProjectPages(env: Env, database: WebDatabase, searchEngine: SearchEngine)( private def getProjectPage(ref: Project.Reference, user: Option[UserState]): Route = getProjectOrRedirect(ref, user) { project => for { - header <- service.getProjectHeader(project) + header <- projectService.getProjectHeader(project) directDependencies <- header .map(h => database.getProjectDependencies(ref, h.latestVersion)) @@ -275,7 +278,7 @@ class ProjectPages(env: Env, database: WebDatabase, searchEngine: SearchEngine)( private def getBadges(ref: Project.Reference, user: Option[UserState]): Route = getProjectOrRedirect(ref, user) { project => - for (header <- service.getProjectHeader(project).map(_.get)) yield { + for (header <- projectService.getProjectHeader(project).map(_.get)) yield { val artifact = header.getDefaultArtifact(None, None) val page = html.badges(env, user, project, header, artifact) complete(StatusCodes.OK, page) diff --git a/modules/server/src/main/scala/scaladex/server/service/AdminService.scala b/modules/server/src/main/scala/scaladex/server/service/AdminService.scala index 32789c475..af361f848 100644 --- a/modules/server/src/main/scala/scaladex/server/service/AdminService.scala +++ b/modules/server/src/main/scala/scaladex/server/service/AdminService.scala @@ -30,7 +30,7 @@ class AdminService( val projectService = new ProjectService(database) val searchSynchronizer = new SearchSynchronizer(database, projectService, searchEngine) - val projectDependenciesUpdater = new DependencyUpdater(database) + val projectDependenciesUpdater = new DependencyUpdater(database, projectService) val userSessionService = new UserSessionService(database) val artifactsService = new ArtifactsService(database) val githubUpdaterOpt: Option[GithubUpdater] = githubClientOpt.map(client => new GithubUpdater(database, client)) @@ -41,7 +41,8 @@ class AdminService( new JobScheduler(Job.projectDependencies, projectDependenciesUpdater.updateAll), new JobScheduler(Job.projectCreationDates, updateProjectCreationDate), new JobScheduler(Job.moveArtifacts, artifactsService.moveAll), - new JobScheduler(Job.userSessions, userSessionService.updateAll) + new JobScheduler(Job.userSessions, userSessionService.updateAll), + new JobScheduler(Job.latestArtifacts, artifactsService.updateAllLatestVersions) ) ++ githubClientOpt.map { client => val githubUpdater = new GithubUpdater(database, client) diff --git a/modules/server/src/main/scala/scaladex/server/service/ArtifactsService.scala b/modules/server/src/main/scala/scaladex/server/service/ArtifactsService.scala index ca3b4a768..0f9a5a090 100644 --- a/modules/server/src/main/scala/scaladex/server/service/ArtifactsService.scala +++ b/modules/server/src/main/scala/scaladex/server/service/ArtifactsService.scala @@ -1,14 +1,28 @@ package scaladex.server.service +import java.time.Instant + import scala.concurrent.ExecutionContext import scala.concurrent.Future import com.typesafe.scalalogging.LazyLogging import scaladex.core.model.GithubStatus +import scaladex.core.model._ import scaladex.core.service.SchedulerDatabase import scaladex.core.util.ScalaExtensions._ class ArtifactsService(database: SchedulerDatabase)(implicit ec: ExecutionContext) extends LazyLogging { + def insertArtifact(artifact: Artifact, dependencies: Seq[ArtifactDependency]): Future[Boolean] = { + val unknownStatus = GithubStatus.Unknown(Instant.now) + for { + isNewProject <- database.insertProjectRef(artifact.projectRef, unknownStatus) + project <- database.getProject(artifact.projectRef).map(_.get) + _ <- database.insertArtifact(artifact) + _ <- database.insertDependencies(dependencies) + _ <- updateLatestVersion(artifact.groupId, artifact.artifactId, project.settings.preferStableVersion) + } yield isNewProject + } + def moveAll(): Future[String] = for { projectStatuses <- database.getAllProjectsStatuses() @@ -21,4 +35,38 @@ class ArtifactsService(database: SchedulerDatabase)(implicit ec: ExecutionContex .sequence .map(_.sum) } yield s"Moved $total artifacts" + + def updateAllLatestVersions(): Future[String] = + for { + projectStatuses <- database.getAllProjectsStatuses() + refs = projectStatuses.collect { case (ref, status) if status.isOk || status.isUnknown || status.isFailed => ref } + total <- refs.mapSync(updateLatestVersions).map(_.sum) + } yield s"Updated $total artifacts in ${refs.size} projects" + + def updateLatestVersions(ref: Project.Reference): Future[Int] = + for { + project <- database.getProject(ref).map(_.get) + total <- updateLatestVersions(ref, project.settings.preferStableVersion) + } yield total + + def updateLatestVersions(ref: Project.Reference, preferStableVersion: Boolean): Future[Int] = + for { + artifactIds <- database.getAllArtifactIds(ref) + _ <- artifactIds.mapSync { + case (groupId, artifactId) => updateLatestVersion(groupId, artifactId, preferStableVersion) + } + } yield artifactIds.size + + def updateLatestVersion(groupId: Artifact.GroupId, artifactId: String, preferStableVersion: Boolean): Future[Unit] = + for { + artifacts <- database.getArtifacts(groupId, artifactId) + latestVersion = computeLatestVersion(artifacts.map(_.version), preferStableVersion) + _ <- database.updateLatestVersion(Artifact.MavenReference(groupId, artifactId, latestVersion)) + } yield () + + def computeLatestVersion(versions: Seq[SemanticVersion], preferStableVersion: Boolean): SemanticVersion = { + def maxStable = versions.filter(_.isStable).maxOption + def max = versions.max + if (preferStableVersion) maxStable.getOrElse(max) else max + } } diff --git a/modules/server/src/main/scala/scaladex/server/service/DependencyUpdater.scala b/modules/server/src/main/scala/scaladex/server/service/DependencyUpdater.scala index 9f2a17ac2..bcfe42f1c 100644 --- a/modules/server/src/main/scala/scaladex/server/service/DependencyUpdater.scala +++ b/modules/server/src/main/scala/scaladex/server/service/DependencyUpdater.scala @@ -6,11 +6,12 @@ import scala.util.control.NonFatal import com.typesafe.scalalogging.LazyLogging import scaladex.core.model.Project -import scaladex.core.model.ProjectHeader +import scaladex.core.service.ProjectService import scaladex.core.service.SchedulerDatabase import scaladex.core.util.ScalaExtensions._ -class DependencyUpdater(database: SchedulerDatabase)(implicit ec: ExecutionContext) extends LazyLogging { +class DependencyUpdater(database: SchedulerDatabase, projectService: ProjectService)(implicit ec: ExecutionContext) + extends LazyLogging { def updateAll(): Future[String] = for { @@ -30,14 +31,7 @@ class DependencyUpdater(database: SchedulerDatabase)(implicit ec: ExecutionConte database.deleteProjectDependencies(project.reference).map(_ => ()) else for { - latestArtifacts <- database.getLatestArtifacts(project.reference, project.settings.preferStableVersion) - header = ProjectHeader( - project.reference, - latestArtifacts, - 0, - project.settings.defaultArtifact, - project.settings.preferStableVersion - ) + header <- projectService.getProjectHeader(project) dependencies <- header .map(h => database.computeProjectDependencies(project.reference, h.latestVersion)) .getOrElse(Future.successful(Seq.empty)) diff --git a/modules/server/src/main/scala/scaladex/server/service/PublishProcess.scala b/modules/server/src/main/scala/scaladex/server/service/PublishProcess.scala index 34779f3fa..5854545d8 100644 --- a/modules/server/src/main/scala/scaladex/server/service/PublishProcess.scala +++ b/modules/server/src/main/scala/scaladex/server/service/PublishProcess.scala @@ -11,6 +11,7 @@ import scaladex.core.model.Env import scaladex.core.model.Project import scaladex.core.model.Sha1 import scaladex.core.model.UserState +import scaladex.core.service.SchedulerDatabase import scaladex.core.service.Storage import scaladex.core.service.WebDatabase import scaladex.data.cleanup.GithubRepoExtractor @@ -34,6 +35,7 @@ class PublishProcess( githubExtractor: GithubRepoExtractor, converter: ArtifactConverter, database: WebDatabase, + artifactsService: ArtifactsService, pomsReader: PomsReader, env: Env )(implicit system: ActorSystem) @@ -76,7 +78,7 @@ class PublishProcess( converter.convert(pom, repo, creationDate) match { case Some((artifact, deps)) => for { - isNewProject <- database.insertArtifact(artifact, deps, Instant.now) + isNewProject <- artifactsService.insertArtifact(artifact, deps) _ <- if (isNewProject && userState.nonEmpty) { val githubUpdater = new GithubUpdater(database, new GithubClientImpl(userState.get.info.token)) @@ -99,13 +101,14 @@ class PublishProcess( } object PublishProcess { - def apply(paths: DataPaths, filesystem: Storage, database: WebDatabase, env: Env)( + def apply(paths: DataPaths, filesystem: Storage, database: SchedulerDatabase, env: Env)( implicit ec: ExecutionContext, actorSystem: ActorSystem ): PublishProcess = { val githubExtractor = new GithubRepoExtractor(paths) val converter = new ArtifactConverter(paths) val pomsReader = new PomsReader(new CoursierResolver) - new PublishProcess(filesystem, githubExtractor, converter, database, pomsReader, env) + val artifactsService = new ArtifactsService(database) + new PublishProcess(filesystem, githubExtractor, converter, database, artifactsService, pomsReader, env) } } diff --git a/modules/server/src/main/scala/scaladex/server/service/SearchSynchronizer.scala b/modules/server/src/main/scala/scaladex/server/service/SearchSynchronizer.scala index 76d814bec..9de44fb94 100644 --- a/modules/server/src/main/scala/scaladex/server/service/SearchSynchronizer.scala +++ b/modules/server/src/main/scala/scaladex/server/service/SearchSynchronizer.scala @@ -8,11 +8,11 @@ import scaladex.core.model.GithubStatus import scaladex.core.model.Project import scaladex.core.model.search.ProjectDocument import scaladex.core.service.ProjectService +import scaladex.core.service.SchedulerDatabase import scaladex.core.service.SearchEngine -import scaladex.core.service.WebDatabase import scaladex.core.util.ScalaExtensions._ -class SearchSynchronizer(database: WebDatabase, service: ProjectService, searchEngine: SearchEngine)( +class SearchSynchronizer(database: SchedulerDatabase, service: ProjectService, searchEngine: SearchEngine)( implicit ec: ExecutionContext ) extends LazyLogging { def syncAll(): Future[String] = diff --git a/modules/server/src/test/scala/scaladex/server/route/BadgesTests.scala b/modules/server/src/test/scala/scaladex/server/route/BadgesTests.scala index cad9f8c51..d02f3519b 100644 --- a/modules/server/src/test/scala/scaladex/server/route/BadgesTests.scala +++ b/modules/server/src/test/scala/scaladex/server/route/BadgesTests.scala @@ -13,17 +13,19 @@ import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers import scaladex.core.model.Scala._ import scaladex.core.test.Values._ -import scaladex.core.util.ScalaExtensions._ import scaladex.server.route.Badges.summaryOfLatestVersions class BadgesTests extends ControllerBaseSuite with BeforeAndAfterAll { val badgesRoute: Route = new Badges(database).route - override protected def beforeAll(): Unit = Await.result(insertCats(), Duration.Inf) + override protected def beforeAll(): Unit = Await.result(insertCatsArtifacts(), Duration.Inf) - def insertCats(): Future[Unit] = - Cats.allArtifacts.map(database.insertArtifact(_, Seq.empty, now)).sequence.map(_ => ()) + def insertCatsArtifacts(): Future[Unit] = + for { + _ <- database.insertProjectRef(Cats.reference, unknown) + _ <- Future.traverse(Cats.allArtifacts)(database.insertArtifact(_)) + } yield () it("should fallback to JVM artifacts") { Get(s"/${Cats.reference}/cats-core/latest-by-scala-version.svg") ~> badgesRoute ~> check { diff --git a/modules/server/src/test/scala/scaladex/server/route/ProjectPagesTests.scala b/modules/server/src/test/scala/scaladex/server/route/ProjectPagesTests.scala index 775d097c5..af76e6db4 100644 --- a/modules/server/src/test/scala/scaladex/server/route/ProjectPagesTests.scala +++ b/modules/server/src/test/scala/scaladex/server/route/ProjectPagesTests.scala @@ -25,7 +25,8 @@ class ProjectPagesTests extends ControllerBaseSuite with BeforeAndAfterEach { private def insertPlayJsonExtra(): Future[Unit] = for { - _ <- database.insertArtifact(PlayJsonExtra.artifact, Seq.empty, Values.now) + _ <- database.insertProjectRef(PlayJsonExtra.reference, unknown) + _ <- database.insertArtifact(PlayJsonExtra.artifact) _ <- database.updateProjectCreationDate(PlayJsonExtra.reference, PlayJsonExtra.creationDate) _ <- database.updateGithubInfoAndStatus(PlayJsonExtra.reference, PlayJsonExtra.githubInfo, ok) } yield () diff --git a/modules/server/src/test/scala/scaladex/server/route/api/ApiEndpointsImplTests.scala b/modules/server/src/test/scala/scaladex/server/route/api/ApiEndpointsImplTests.scala index 954d11052..a841f6ccd 100644 --- a/modules/server/src/test/scala/scaladex/server/route/api/ApiEndpointsImplTests.scala +++ b/modules/server/src/test/scala/scaladex/server/route/api/ApiEndpointsImplTests.scala @@ -20,8 +20,11 @@ class ApiEndpointsImplTests extends ControllerBaseSuite with BeforeAndAfterEach import endpoints._ override protected def beforeAll(): Unit = { - val insertCatsArtifacts = Future.traverse(Cats.allArtifacts)(database.insertArtifact(_, Seq.empty, now)) - Await.result(insertCatsArtifacts, Duration.Inf) + val insertions = for { + _ <- database.insertProjectRef(Cats.reference, unknown) + _ <- Future.traverse(Cats.allArtifacts)(database.insertArtifact(_)) + } yield () + Await.result(insertions, Duration.Inf) } implicit def jsonCodecToUnmarshaller[T: JsonCodec]: FromEntityUnmarshaller[T] = diff --git a/modules/server/src/test/scala/scaladex/server/route/api/OldSearchApiTests.scala b/modules/server/src/test/scala/scaladex/server/route/api/OldSearchApiTests.scala index 45b69de78..fc409a7a9 100644 --- a/modules/server/src/test/scala/scaladex/server/route/api/OldSearchApiTests.scala +++ b/modules/server/src/test/scala/scaladex/server/route/api/OldSearchApiTests.scala @@ -8,7 +8,6 @@ import scaladex.core.model.BinaryVersion import scaladex.core.model.Jvm import scaladex.core.model.Scala import scaladex.core.test.Values -import scaladex.core.util.ScalaExtensions._ import scaladex.server.route.ControllerBaseSuite class OldSearchApiTests extends ControllerBaseSuite with PlayJsonSupport { @@ -33,7 +32,10 @@ class OldSearchApiTests extends ControllerBaseSuite with PlayJsonSupport { } def insertAllCatsArtifacts(): Future[Unit] = - Cats.allArtifacts.map(database.insertArtifact(_, Seq.empty, now)).sequence.map(_ => ()) + for { + _ <- database.insertProjectRef(Cats.reference, unknown) + _ <- Future.traverse(Cats.allArtifacts)(database.insertArtifact(_)) + } yield () describe("route") { it("should find project") { diff --git a/modules/template/src/main/scala/scaladex/view/Job.scala b/modules/template/src/main/scala/scaladex/view/Job.scala index d035d33c3..f6984045f 100644 --- a/modules/template/src/main/scala/scaladex/view/Job.scala +++ b/modules/template/src/main/scala/scaladex/view/Job.scala @@ -49,6 +49,11 @@ object Job { "Find missing non-standard artifacts from Maven Central", 2.hours ) + val latestArtifacts: Job = Job( + "latest-artifacts", + "Update latest version of artifacts", + 24.hours + ) case class Status(state: State, results: Seq[Result], progress: Option[Progress]) { def isStarted: Boolean = state.isInstanceOf[Started]