Index artifacts with no associated projects

modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala

@@ -23,7 +23,7 @@ case class Artifact(
     artifactId: String,
     version: SemanticVersion,
     artifactName: Artifact.Name,
-    projectRef: Project.Reference,
+    projectRef: Option[Project.Reference],
     description: Option[String],
     releaseDate: Instant,
     resolver: Option[Resolver],
@@ -45,12 +45,15 @@ case class Artifact(
     s"$groupId$sep$artifactName"
   }
 
-  private def artifactHttpPath: String = s"/${projectRef.organization}/${projectRef.repository}/$artifactName"
+  private def artifactHttpPath: Option[String] = projectRef.map { ref =>
+    s"/${ref.organization}/${ref.repository}/$artifactName"
+  }
 
   val mavenReference: Artifact.MavenReference = Artifact.MavenReference(groupId.value, artifactId, version.encode)
 
-  def release: Release =
-    Release(projectRef.organization, projectRef.repository, platform, language, version, releaseDate)
+  def release: Option[Release] = projectRef.map { ref =>
+    Release(ref.organization, ref.repository, platform, language, version, releaseDate)
+  }
 
   def releaseDateFormat: String = Artifact.dateFormatter.format(releaseDate)
 

modules/core/shared/src/test/scala/scaladex/core/model/ArtifactSelectionTests.scala

@@ -20,7 +20,7 @@ class ArtifactSelectionTests extends AsyncFunSpec with Matchers {
       artifactId.value,
       version,
       artifactId.name,
-      reference,
+      Some(reference),
       None,
       Instant.ofEpochMilli(1475505237265L),
       None,

modules/core/shared/src/test/scala/scaladex/core/model/ArtifactTests.scala

@@ -161,7 +161,7 @@ class ArtifactTests extends AnyFunSpec with Matchers {
       artifactName = artifactIdResult.name,
       platform = artifactIdResult.binaryVersion.platform,
       language = artifactIdResult.binaryVersion.language,
-      projectRef = Project.Reference.from("", ""),
+      projectRef = Some(Project.Reference.from("", "")),
       description = None,
       releaseDate = Instant.now(),
       resolver = resolver,

modules/core/shared/src/test/scala/scaladex/core/test/InMemoryDatabase.scala

@@ -38,10 +38,13 @@ class InMemoryDatabase extends SchedulerDatabase {
       dependencies: Seq[ArtifactDependency],
       now: Instant
   ): Future[Boolean] = {
-    val ref = artifact.projectRef
-    val isNewProject = !projects.contains(ref)
-    if (isNewProject) projects.addOne(ref -> Project.default(ref, now = now))
-    allArtifacts.addOne(ref -> (allArtifacts.getOrElse(ref, Seq.empty) :+ artifact))
+    val projectRef = artifact.projectRef
+    val isNewProject = projectRef.fold(false)(projectRef => !projects.contains(projectRef))
+    if (isNewProject) projectRef.foreach(ref => projects.addOne(ref -> Project.default(ref, now = now)))
+    projectRef.foreach { ref =>
+      val prevArtifactsForProject = allArtifacts.get(ref).fold(Seq[Artifact]())(identity)
+      allArtifacts.addOne(ref -> (prevArtifactsForProject :+ artifact))
+    }
     dependencies.appendedAll(dependencies)
     Future.successful(isNewProject)
   }

modules/core/shared/src/test/scala/scaladex/core/test/Values.scala

@@ -55,7 +55,7 @@ object Values {
       artifactName = artifactId.name,
       platform = artifactId.binaryVersion.platform,
       language = artifactId.binaryVersion.language,
-      projectRef = reference,
+      projectRef = Some(reference),
       description = None,
       releaseDate = creationDate,
       resolver = None,
@@ -101,7 +101,7 @@ object Values {
       artifactName = artifactId.name,
       platform = artifactId.binaryVersion.platform,
       language = artifactId.binaryVersion.language,
-      projectRef = reference,
+      projectRef = Some(reference),
       description = None,
       releaseDate = creationDate,
       resolver = None,
@@ -159,7 +159,7 @@ object Values {
         artifactName = artifactId.name,
         platform = binaryVersion.platform,
         language = binaryVersion.language,
-        projectRef = reference,
+        projectRef = Some(reference),
         description = description,
         releaseDate = Instant.ofEpochMilli(1620911032000L),
         resolver = None,

modules/data/src/main/scala/scaladex/data/meta/ArtifactConverter.scala

@@ -17,7 +17,7 @@ class ArtifactConverter(paths: DataPaths) {
 
   def convert(
       pom: ArtifactModel,
-      projectRef: Project.Reference,
+      projectRef: Option[Project.Reference],
       creationDate: Instant
   ): Option[(Artifact, Seq[ArtifactDependency])] =
     for {

modules/infra/src/main/resources/migrations/V14__make_artifact_project_nullable.sql

@@ -0,0 +1 @@
+ALTER TABLE artifacts ALTER COLUMN repository DROP NOT NULL;

modules/infra/src/main/scala/scaladex/infra/SqlDatabase.scala

@@ -7,6 +7,7 @@ import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
 
 import cats.effect.IO
+import cats.implicits._
 import com.typesafe.scalalogging.LazyLogging
 import com.zaxxer.hikari.HikariDataSource
 import doobie.implicits._
@@ -48,11 +49,11 @@ class SqlDatabase(datasource: HikariDataSource, xa: doobie.Transactor[IO]) exten
   ): Future[Boolean] = {
     val unknownStatus = GithubStatus.Unknown(time)
     for {
-      isNewProject <- insertProjectRef(artifact.projectRef, unknownStatus)
+      maybeIsNewProject <- artifact.projectRef.traverse(ref => insertProjectRef(ref, unknownStatus))
       _ <- run(ArtifactTable.insertIfNotExist(artifact))
-      _ <- run(ReleaseTable.insertIfNotExists.run(artifact.release))
+      _ <- artifact.release.traverse(release => run(ReleaseTable.insertIfNotExists.run(release)))
       _ <- insertDependencies(dependencies)
-    } yield isNewProject
+    } yield maybeIsNewProject.fold(true)(identity)
   }
 
   override def insertProject(project: Project): Future[Unit] =

modules/infra/src/test/scala/scaladex/infra/SqlDatabaseTests.scala

@@ -71,8 +71,8 @@ class SqlDatabaseTests extends AsyncFunSpec with BaseDatabaseSuite with Matchers
     } yield {
       oldArtifacts shouldBe empty
       newArtifacts should contain theSameElementsAs Seq(
-        Cats.`core_3:2.6.1`.copy(projectRef = newRef),
-        Cats.core_sjs1_3.copy(projectRef = newRef)
+        Cats.`core_3:2.6.1`.copy(projectRef = Some(newRef)),
+        Cats.core_sjs1_3.copy(projectRef = Some(newRef))
       )
     }
   }

modules/server/src/main/scala/scaladex/server/route/ArtifactPages.scala

@@ -7,6 +7,7 @@ import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.model._
 import akka.http.scaladsl.server.Directives._
 import akka.http.scaladsl.server.Route
+import cats.implicits._
 import com.typesafe.scalalogging.LazyLogging
 import scaladex.core.model.Env
 import scaladex.core.model.UserState
@@ -21,7 +22,7 @@ class ArtifactPages(env: Env, database: WebDatabase)(implicit executionContext:
         path("artifacts" / mavenReferenceM / "scaladoc" ~ RemainingPath) { (mavenRef, dri) =>
           val scaladocUriF = for {
             artifact <- database.getArtifactByMavenReference(mavenRef).map(_.get)
-            project <- database.getProject(artifact.projectRef)
+            project <- artifact.projectRef.flatTraverse(database.getProject)
           } yield project.flatMap(_.scaladoc(artifact).map(doc => Uri(doc.link)))
 
           onComplete(scaladocUriF) {

modules/server/src/main/scala/scaladex/server/service/PublishProcess.scala

@@ -6,7 +6,10 @@ import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
 
 import akka.actor.ActorSystem
+import cats.implicits._
 import com.typesafe.scalalogging.LazyLogging
+import scaladex.core.model.Artifact
+import scaladex.core.model.ArtifactDependency
 import scaladex.core.model.Env
 import scaladex.core.model.Project
 import scaladex.core.model.Sha1
@@ -27,7 +30,7 @@ object PublishResult {
   object InvalidPom extends PublishResult
   object NoGithubRepo extends PublishResult
   object Success extends PublishResult
-  case class Forbidden(login: String, repo: Project.Reference) extends PublishResult
+  case class Forbidden(login: String, repo: Option[Project.Reference]) extends PublishResult
 }
 
 class PublishProcess(
@@ -65,35 +68,50 @@ class PublishProcess(
       userState: Option[UserState]
   ): Future[PublishResult] = {
     val pomRef = s"${pom.groupId}:${pom.artifactId}:${pom.version}"
-    githubExtractor.extract(pom) match {
-      case None =>
-        // TODO: save artifact with no github information
-        Future.successful(PublishResult.NoGithubRepo)
-      case Some(repo) =>
-        // userState can be empty when the request of publish is done through the scheduler
-        if (userState.isEmpty || userState.get.hasPublishingAuthority(env) || userState.get.repos.contains(repo)) {
-          converter.convert(pom, repo, creationDate) match {
-            case Some((artifact, deps)) =>
-              for {
-                isNewProject <- database.insertArtifact(artifact, deps, Instant.now)
-                _ <-
-                  if (isNewProject && userState.nonEmpty)
-                    updateGithubInfo(new GithubClient(userState.get.info.token), artifact.projectRef, Instant.now())
-                  else Future.successful(())
-              } yield {
-                logger.info(s"Published $pomRef")
-                PublishResult.Success
-              }
-            case None =>
-              logger.warn(s"Cannot convert $pomRef to valid Scala artifact.")
-              Future.successful(PublishResult.InvalidPom)
+    val maybeProjectRef = githubExtractor.extract(pom)
+    if (isPermittedToPublish(userState, maybeProjectRef)) {
+      converter.convert(pom, maybeProjectRef, creationDate) match {
+        case Some((artifact, deps)) =>
+          insertArtifactWithDependencies(artifact, deps, maybeProjectRef, userState).map { _ =>
+            logger.info(s"Published $pomRef")
+            PublishResult.Success
           }
-        } else {
-          logger.warn(s"User ${userState.get.info.login} attempted to publish to $repo")
-          Future.successful(PublishResult.Forbidden(userState.get.info.login, repo))
-        }
+        case None =>
+          logger.warn(s"Cannot convert $pomRef to a valid Scala artifact")
+          Future.successful(PublishResult.InvalidPom)
+      }
+    } else {
+      logger.warn(s"User ${userState.get.info.login} attempted to publish to $maybeProjectRef")
+      Future.successful(PublishResult.Forbidden(userState.get.info.login, maybeProjectRef))
+    }
+  }
+
+  private def insertArtifactWithDependencies(
+      artifact: Artifact,
+      dependencies: Seq[ArtifactDependency],
+      maybeProjectRef: Option[Project.Reference],
+      maybeUserState: Option[UserState]
+  ): Future[Unit] =
+    database.insertArtifact(artifact, dependencies, Instant.now).flatMap { isNewProject =>
+      if (isNewProject && maybeProjectRef.isDefined && maybeUserState.isDefined)
+        (maybeUserState, maybeProjectRef)
+          .traverseN((userState, ref) => updateGithubInfo(new GithubClient(userState.info.token), ref, Instant.now))
+          .map(_ => ())
+      else Future.successful(())
     }
+
+  private def isPermittedToPublish(
+      maybeUserState: Option[UserState],
+      maybeProjectRef: Option[Project.Reference]
+  ): Boolean = {
+    val userCanPublish = maybeUserState.map(_.hasPublishingAuthority(env)).fold(false)(identity)
+    val userHasRepo = (maybeUserState, maybeProjectRef)
+      .mapN((userState, projectRef) => userState.repos.contains(projectRef))
+      .fold(false)(identity)
+    // UserState may be empty if a publish is performed via a scheduler.
+    maybeUserState.isEmpty || userCanPublish || userHasRepo
   }
+
   private def updateGithubInfo(githubClient: GithubClient, ref: Project.Reference, now: Instant): Future[Unit] =
     for {
       githubResponse <- githubClient.getProjectInfo(ref)