Merge pull request #1501 from adpi2/add-cache

Add cache to reduce pressure on SQL database on high load

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
           submodules: true
       - uses: coursier/setup-action@v1
         with:
-          jvm: "adopt:1.8"
+          jvm: "adoptium:1.17"
       - uses: coursier/cache-action@v6
       - name: Unit tests
         run: sbt test
@@ -40,7 +40,7 @@ jobs:
           submodules: true
       - uses: coursier/setup-action@v1
         with:
-          jvm: "adopt:1.8"
+          jvm: "adoptium:1.17"
       - name: Integration tests
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CONTRIBUTING.md

@@ -25,7 +25,7 @@ For development you only need to clone [scaladex]:
 You will need the following tools installed:
 
 * git
-* Java 1.8
+* Java 1.17
 * the [sbt](https://www.scala-sbt.org/) build tool
 * a Scala code editor:
   * [VSCode](https://code.visualstudio.com/) with [Metals](https://scalameta.org/metals/)

build.sbt

@@ -72,6 +72,7 @@ lazy val infra = project
       "com.github.pjfanning" %% "pekko-http-circe" % "2.8.0",
       "io.get-coursier" %% "coursier" % V.coursier,
       "io.get-coursier" %% "coursier-sbt-maven-repository" % V.coursier,
+      "com.github.blemale" %% "scaffeine" % "5.3.0",
       "org.tpolecat" %% "doobie-scalatest" % V.doobie % Test,
       "org.scalatest" %% "scalatest" % V.scalatest % "test,it"
     ) ++ Seq(

modules/core/shared/src/main/scala/scaladex/core/model/ProjectHeader.scala

@@ -6,19 +6,17 @@ object ProjectHeader {
   def apply(
       ref: Project.Reference,
       artifacts: Seq[Artifact],
-      versionCount: Long,
       defaultArtifactName: Option[Artifact.Name],
       preferStableVersion: Boolean
   ): Option[ProjectHeader] =
     Option.when(artifacts.nonEmpty) {
-      new ProjectHeader(ref, artifacts, versionCount, defaultArtifactName, preferStableVersion)
+      new ProjectHeader(ref, artifacts, defaultArtifactName, preferStableVersion)
     }
 }
 
 final case class ProjectHeader(
     ref: Project.Reference,
     artifacts: Seq[Artifact],
-    versionCount: Long,
     defaultArtifactName: Option[Artifact.Name],
     preferStableVersion: Boolean
 ) {

modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala

@@ -67,15 +67,12 @@ class ProjectService(database: WebDatabase, searchEngine: SearchEngine)(implicit
 
   def getHeader(project: Project): Future[Option[ProjectHeader]] = {
     val ref = project.reference
-    for {
-      latestArtifacts <- database.getProjectLatestArtifacts(ref)
-      versionCount <- database.countVersions(ref)
-    } yield ProjectHeader(
-      ref,
-      latestArtifacts,
-      versionCount,
-      project.settings.defaultArtifact,
-      project.settings.preferStableVersion
-    )
+    for (latestArtifacts <- database.getProjectLatestArtifacts(ref))
+      yield ProjectHeader(
+        ref,
+        latestArtifacts,
+        project.settings.defaultArtifact,
+        project.settings.preferStableVersion
+      )
   }
 }

modules/core/shared/src/main/scala/scaladex/core/service/SchedulerDatabase.scala

@@ -28,9 +28,9 @@ trait SchedulerDatabase extends WebDatabase {
   def insertArtifacts(artifacts: Seq[Artifact]): Future[Unit] // for init process
   def insertDependencies(dependencies: Seq[ArtifactDependency]): Future[Unit]
   def updateArtifacts(artifacts: Seq[Artifact.Reference], newRef: Project.Reference): Future[Int]
-  def updateArtifactReleaseDate(reference: Artifact.Reference, releaseDate: Instant): Future[Int]
+  def updateArtifactReleaseDate(ref: Artifact.Reference, releaseDate: Instant): Future[Int]
   def getGroupIds(): Future[Seq[Artifact.GroupId]]
   def getArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, Artifact.ArtifactId)]]
   def getArtifactRefs(): Future[Seq[Artifact.Reference]]
-  def updateLatestVersion(ref: Artifact.Reference): Future[Unit]
+  def updateLatestVersion(ref: Project.Reference, artifact: Artifact.Reference): Future[Unit]
 }

modules/core/shared/src/test/scala/scaladex/core/model/ProjectHeaderTests.scala

@@ -21,7 +21,7 @@ class ProjectHeaderTests extends AnyFunSpec with Matchers {
   }
 
   private def getDefaultArtifact(artifacts: Artifact*): Artifact = {
-    val header = ProjectHeader(reference, artifacts, 10, None, false).get
+    val header = ProjectHeader(reference, artifacts, None, false).get
     header.getDefaultArtifact(None, None)
   }
 }

modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala

@@ -216,8 +216,8 @@ class InMemoryDatabase extends SchedulerDatabase {
   override def getArtifactIds(ref: Project.Reference): Future[Seq[(Artifact.GroupId, Artifact.ArtifactId)]] =
     Future.successful(getProjectArtifactsSync(ref).map(a => (a.groupId, a.artifactId)).distinct.toSeq)
 
-  override def updateLatestVersion(ref: Artifact.Reference): Future[Unit] = {
-    latestArtifacts += (ref.groupId, ref.artifactId) -> ref
+  override def updateLatestVersion(ref: Project.Reference, artifact: Artifact.Reference): Future[Unit] = {
+    latestArtifacts += (artifact.groupId, artifact.artifactId) -> artifact
     Future.unit
   }
 

modules/core/shared/src/test/scala/scaladex/core/test/Values.scala

@@ -80,7 +80,6 @@ object Values {
     val projectHeader: ProjectHeader = ProjectHeader(
       reference,
       Seq(artifact),
-      1,
       settings.defaultArtifact,
       settings.preferStableVersion
     ).get
@@ -210,8 +209,7 @@ object Values {
         Scope("compile")
       )
     )
-    val versionCount: Int = allArtifacts.map(_.version).distinct.size
-    val projectHeader: ProjectHeader = ProjectHeader(reference, allArtifacts, versionCount, None, true).get
+    val projectHeader: ProjectHeader = ProjectHeader(reference, allArtifacts, None, true).get
     val projectDocument: ProjectDocument =
       ProjectDocument(
         project.copy(creationDate = Some(now.minus(10, ChronoUnit.MINUTES))),

modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala

@@ -5,8 +5,11 @@ import java.util.UUID
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
+import scala.concurrent.duration._
 
 import cats.effect.IO
+import com.github.blemale.scaffeine.AsyncLoadingCache
+import com.github.blemale.scaffeine.Scaffeine
 import com.typesafe.scalalogging.LazyLogging
 import com.zaxxer.hikari.HikariDataSource
 import doobie.implicits._
@@ -125,8 +128,10 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
   ): Future[Seq[Artifact]] =
     run(ArtifactTable.selectArtifactByProjectAndNameAndVersion.to[Seq](ref, name, version))
 
+  private val projectLatestArtifactsCache: AsyncLoadingCache[Project.Reference, Seq[Artifact]] =
+    Scaffeine().buildAsyncFuture(ref => run(ArtifactTable.selectLatestArtifacts.to[Seq](ref)))
   override def getProjectLatestArtifacts(ref: Project.Reference): Future[Seq[Artifact]] =
-    run(ArtifactTable.selectLatestArtifacts.to[Seq](ref))
+    projectLatestArtifactsCache.get(ref)
 
   override def getProjectDependencies(projectRef: Project.Reference): Future[Seq[ArtifactDependency]] =
     run(ArtifactDependencyTable.selectDependencyFromProject.to[Seq](projectRef))
@@ -143,8 +148,9 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
   def countProjects(): Future[Long] =
     run(ProjectTable.countProjects.unique)
 
-  override def countArtifacts(): Future[Long] =
-    run(ArtifactTable.count.unique)
+  private val countArtifactsCache: AsyncLoadingCache[Unit, Long] =
+    Scaffeine().refreshAfterWrite(5.minutes).buildAsyncFuture[Unit, Long](_ => run(ArtifactTable.count.unique))
+  override def countArtifacts(): Future[Long] = countArtifactsCache.get(())
 
   def countDependencies(): Future[Long] =
     run(ArtifactDependencyTable.count.unique)
@@ -210,12 +216,12 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
   override def deleteUser(userId: UUID): Future[Unit] =
     run(UserSessionsTable.deleteById.run(userId).map(_ => ()))
 
-  override def updateLatestVersion(ref: Artifact.Reference): Future[Unit] = {
+  override def updateLatestVersion(ref: Project.Reference, artifact: Artifact.Reference): Future[Unit] = {
     val transaction = for {
-      _ <- ArtifactTable.setLatestVersion.run(ref)
-      _ <- ArtifactTable.unsetOthersLatestVersion.run(ref)
+      _ <- ArtifactTable.setLatestVersion.run(artifact)
+      _ <- ArtifactTable.unsetOthersLatestVersion.run(artifact)
     } yield ()
-    run(transaction)
+    run(transaction).map(_ => projectLatestArtifactsCache.underlying.synchronous().invalidate(ref))
   }
 
   override def countVersions(ref: Project.Reference): Future[Long] =

modules/server/src/main/scala/scaladex/server/service/ArtifactService.scala

@@ -32,7 +32,12 @@ class ArtifactService(database: SchedulerDatabase)(implicit ec: ExecutionContext
       project <- database.getProject(artifact.projectRef).map(_.get)
       _ <- database.insertArtifact(artifact)
       _ <- database.insertDependencies(dependencies)
-      _ <- updateLatestVersion(artifact.groupId, artifact.artifactId, project.settings.preferStableVersion)
+      _ <- updateLatestVersion(
+        project.reference,
+        artifact.groupId,
+        artifact.artifactId,
+        project.settings.preferStableVersion
+      )
     } yield isNewProject
   }
 
@@ -69,19 +74,20 @@ class ArtifactService(database: SchedulerDatabase)(implicit ec: ExecutionContext
     for {
       artifactIds <- database.getArtifactIds(ref)
       _ <- artifactIds.mapSync {
-        case (groupId, artifactId) => updateLatestVersion(groupId, artifactId, preferStableVersion)
+        case (groupId, artifactId) => updateLatestVersion(ref, groupId, artifactId, preferStableVersion)
       }
     } yield artifactIds.size
 
   def updateLatestVersion(
+      ref: Project.Reference,
       groupId: Artifact.GroupId,
       artifactId: Artifact.ArtifactId,
       preferStableVersion: Boolean
   ): Future[Unit] =
     for {
       artifacts <- database.getArtifacts(groupId, artifactId)
       latestVersion = computeLatestVersion(artifacts.map(_.version), preferStableVersion)
-      _ <- database.updateLatestVersion(Artifact.Reference(groupId, artifactId, latestVersion))
+      _ <- database.updateLatestVersion(ref, Artifact.Reference(groupId, artifactId, latestVersion))
     } yield ()
 
   def computeLatestVersion(versions: Seq[Version], preferStableVersion: Boolean): Version = {

modules/template/src/main/twirl/scaladex/view/project/headproject.scala.html

@@ -113,7 +113,7 @@ <h1>
             <li role="project" class="@isActive(ProjectTab.Main)"><a href="/@project.reference">Project</a></li>
             @for(header <- header) {
               <li role="artifacts" class="@isActive(ProjectTab.Artifacts)"><a href="/@project.reference/artifacts">Artifacts</a></li>
-              <li role="versions" class="@isActive(ProjectTab.Versions)"><a href="@header.versionsUrl">@plural(header.versionCount, "Version")</a></li>
+              <li role="versions" class="@isActive(ProjectTab.Versions)"><a href="@header.versionsUrl">Versions</a></li>
               <!-- <li role="version-matrix" class="@isActive(ProjectTab.VersionMatrix)"><a href="/@project.reference/version-matrix">Version Matrix</a></li> -->
               <li role="badges" class="@isActive(ProjectTab.Badges)"><a href="/@project.reference/badges">Badges</a></li>
             }