From fa496b298ab445156f89f20ee5e4183c45cb1020 Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Wed, 23 Oct 2024 15:27:06 +0200 Subject: [PATCH 1/2] Fix perf of /api/projects --- .../scala/scaladex/core/model/Project.scala | 9 ++++-- .../core/service/ProjectService.scala | 8 +---- .../scaladex/core/service/SearchEngine.scala | 19 +++++++----- .../core/test/InMemorySearchEngine.scala | 27 ++++++++++------ .../scaladex/infra/ElasticsearchEngine.scala | 31 ++++++++++++++++++- 5 files changed, 66 insertions(+), 28 deletions(-) diff --git a/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala b/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala index d2d0b63e6..b296cef68 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/model/Project.scala @@ -72,11 +72,14 @@ object Project { def from(org: String, repo: String): Reference = Reference(Organization(org), Repository(repo)) - def unsafe(string: String): Reference = - string.split('/') match { - case Array(org, repo) => from(org, repo) + def parse(value: String): Option[Reference] = + value.split('/') match { + case Array(org, repo) => Some(from(org, repo)) + case _ => None } + def unsafe(value: String): Reference = parse(value).get + implicit val ordering: Ordering[Reference] = Ordering.by(ref => (ref.organization.value, ref.repository.value)) } diff --git a/modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala b/modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala index 5633af1bc..abf622780 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/service/ProjectService.scala @@ -4,18 +4,12 @@ import scala.concurrent.ExecutionContext import scala.concurrent.Future import scaladex.core.model._ -import scaladex.core.model.search.PageParams import scaladex.core.model.search.SearchParams -import scaladex.core.util.ScalaExtensions._ class ProjectService(database: WebDatabase, searchEngine: SearchEngine)(implicit context: ExecutionContext) { def getProjects(languages: Seq[Language], platforms: Seq[Platform]): Future[Seq[Project.Reference]] = { val searchParams = SearchParams(languages = languages, platforms = platforms) - for { - firstPage <- searchEngine.find(searchParams, PageParams(0, 10000)) - p = firstPage.pagination - otherPages <- 1.until(p.pageCount).map(PageParams(_, 10000)).mapSync(p => searchEngine.find(searchParams, p)) - } yield (firstPage +: otherPages).flatMap(_.items).map(_.document.reference) + searchEngine.findRefs(searchParams) } def getProject(ref: Project.Reference): Future[Option[Project]] = database.getProject(ref) diff --git a/modules/core/shared/src/main/scala/scaladex/core/service/SearchEngine.scala b/modules/core/shared/src/main/scala/scaladex/core/service/SearchEngine.scala index 957ffc53d..c13f46991 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/service/SearchEngine.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/service/SearchEngine.scala @@ -28,14 +28,6 @@ trait SearchEngine { def getMostDependedUpon(limit: Int): Future[Seq[ProjectDocument]] def getLatest(limit: Int): Future[Seq[ProjectDocument]] - // Old Search API - def find( - query: String, - binaryVersion: Option[BinaryVersion], - cli: Boolean, - page: PageParams - ): Future[Page[ProjectDocument]] - // Search Page def find(params: SearchParams, page: PageParams): Future[Page[ProjectHit]] def autocomplete(params: SearchParams, limit: Int): Future[Seq[ProjectDocument]] @@ -47,4 +39,15 @@ trait SearchEngine { def find(category: Category, params: AwesomeParams, page: PageParams): Future[Page[ProjectDocument]] def countByLanguages(category: Category, params: AwesomeParams): Future[Seq[(Language, Int)]] def countByPlatforms(category: Category, params: AwesomeParams): Future[Seq[(Platform, Int)]] + + // Old Search API + def find( + query: String, + binaryVersion: Option[BinaryVersion], + cli: Boolean, + page: PageParams + ): Future[Page[ProjectDocument]] + + // API + def findRefs(params: SearchParams): Future[Seq[Project.Reference]] } diff --git a/modules/core/shared/src/test/scala/scaladex/core/test/InMemorySearchEngine.scala b/modules/core/shared/src/test/scala/scaladex/core/test/InMemorySearchEngine.scala index 18f9ea7ae..d2b214e1b 100644 --- a/modules/core/shared/src/test/scala/scaladex/core/test/InMemorySearchEngine.scala +++ b/modules/core/shared/src/test/scala/scaladex/core/test/InMemorySearchEngine.scala @@ -43,15 +43,24 @@ class InMemorySearchEngine extends SearchEngine { page: PageParams ): Future[Page[ProjectDocument]] = ??? - override def find(params: SearchParams, page: PageParams): Future[Page[ProjectHit]] = - Future.successful { - val hits = allDocuments.values - .filter(doc => (params.languages.toSet -- doc.languages).isEmpty) - .filter(doc => (params.platforms.toSet -- doc.platforms).isEmpty) - .toSeq - .map(ProjectHit(_, Seq.empty)) - Page(Pagination(1, 1, hits.size), hits) - } + override def find(params: SearchParams, page: PageParams): Future[Page[ProjectHit]] = { + val hits = allDocuments.values + .filter(doc => (params.languages.toSet -- doc.languages).isEmpty) + .filter(doc => (params.platforms.toSet -- doc.platforms).isEmpty) + .toSeq + .map(ProjectHit(_, Seq.empty)) + val res = Page(Pagination(1, 1, hits.size), hits) + Future.successful(res) + } + + override def findRefs(params: SearchParams): Future[Seq[Project.Reference]] = { + val res = allDocuments.values + .filter(doc => (params.languages.toSet -- doc.languages).isEmpty) + .filter(doc => (params.platforms.toSet -- doc.platforms).isEmpty) + .toSeq + .map(_.reference) + Future.successful(res) + } override def autocomplete(params: SearchParams, limit: Int): Future[Seq[ProjectDocument]] = ??? diff --git a/modules/infra/src/main/scala/scaladex/infra/ElasticsearchEngine.scala b/modules/infra/src/main/scala/scaladex/infra/ElasticsearchEngine.scala index a3e834e49..f31800ac2 100644 --- a/modules/infra/src/main/scala/scaladex/infra/ElasticsearchEngine.scala +++ b/modules/infra/src/main/scala/scaladex/infra/ElasticsearchEngine.scala @@ -5,10 +5,12 @@ import java.io.Closeable import scala.annotation.nowarn import scala.concurrent.ExecutionContext import scala.concurrent.Future +import scala.concurrent.duration._ import com.sksamuel.elastic4s.ElasticClient import com.sksamuel.elastic4s.ElasticDsl._ import com.sksamuel.elastic4s.ElasticProperties +import com.sksamuel.elastic4s.Hit import com.sksamuel.elastic4s.Response import com.sksamuel.elastic4s.analysis.Analysis import com.sksamuel.elastic4s.http.JavaClient @@ -173,6 +175,30 @@ class ElasticsearchEngine(esClient: ElasticClient, index: String)(implicit ec: E findPage(request, page).map(_.flatMap(toProjectHit)) } + override def findRefs(params: SearchParams): Future[Seq[Project.Reference]] = { + val request = searchRequest(filteredSearchQuery(params), params.sorting) + .sourceInclude("organization", "repository") + .limit(10000) + scroll(request, 30.seconds).map(_.flatMap(hit => Project.Reference.parse(hit.id))) + } + + private def scroll(request: SearchRequest, timeout: FiniteDuration): Future[Seq[Hit]] = { + val r0 = request.keepAlive(timeout) + val keepAlive = r0.keepAlive.get + def recur(resp: Response[SearchResponse]): Future[Seq[Hit]] = { + val hits = resp.result.hits.hits.toSeq + resp.result.scrollId match { + case None => Future.successful(hits) + case Some(id) => + for { + r <- esClient.execute(searchScroll(id, keepAlive)) + nextHits <- recur(r) + } yield hits ++ nextHits + } + } + esClient.execute(request).flatMap(recur) + } + private def findPage(request: SearchRequest, page: PageParams): Future[Page[SearchHit]] = { val clamp = if (page.page <= 0) 1 else page.page val pagedRequest = request.from(page.size * (clamp - 1)).size(page.size) @@ -212,7 +238,10 @@ class ElasticsearchEngine(esClient: ElasticClient, index: String)(implicit ec: E case _ => scoreSort().order(SortOrder.Desc) } - search(index).query(scoringQuery).sortBy(sortQuery) + search(index) + .sourceExclude("githubInfo.readme") + .query(scoringQuery) + .sortBy(sortQuery) } private def extractDocuments(response: Response[SearchResponse]): Seq[ProjectDocument] = From b6f017c6804a6dde394817deca6352af9739f9f2 Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Wed, 23 Oct 2024 15:27:31 +0200 Subject: [PATCH 2/2] Add API Doc --- build.sbt | 3 +- .../scala/scaladex/core/model/Artifact.scala | 9 +-- .../main/scala/scaladex/core/model/Env.scala | 4 + .../src/main/assets/css/partials/_header.scss | 16 ++-- .../main/resources/lib/swagger-initializer.js | 18 +++++ .../scala/scaladex/server/route/Assets.scala | 11 ++- .../route/api/DocumentationRoutes.scala | 39 ++++++---- .../main/twirl/scaladex/view/main.scala.html | 75 +++++++++---------- 8 files changed, 103 insertions(+), 72 deletions(-) create mode 100644 modules/server/src/main/resources/lib/swagger-initializer.js diff --git a/build.sbt b/build.sbt index ef50759d3..d0b063c84 100644 --- a/build.sbt +++ b/build.sbt @@ -177,7 +177,8 @@ lazy val server = project "org.webjars.npm" % "chartjs-adapter-date-fns" % "3.0.0", "org.webjars" % "font-awesome" % "6.5.2", "org.webjars" % "jquery" % "3.7.1", - "org.webjars.bower" % "select2" % "4.0.13" + "org.webjars.bower" % "select2" % "4.0.13", + "org.webjars" % "swagger-ui" % "5.17.14" ), Compile / unmanagedResourceDirectories += (Assets / WebKeys.public).value, Compile / resourceGenerators += (Assets / WebKeys.assets).map(Seq(_)), diff --git a/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala b/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala index bf28efcfd..cb66c8d0e 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/model/Artifact.scala @@ -52,14 +52,7 @@ case class Artifact( s"${fullHttpUrl(env)}/latest-by-scala-version.svg?platform=${platform.map(_.value).getOrElse(this.platform.value)}" // TODO move this out - def fullHttpUrl(env: Env): String = - env match { - case Env.Prod => s"https://index.scala-lang.org$artifactHttpPath" - case Env.Dev => - s"https://index-dev.scala-lang.org$artifactHttpPath" // todo: fix locally - case Env.Local => - s"http://localhost:8080$artifactHttpPath" // todo: fix locally - } + def fullHttpUrl(env: Env): String = env.rootUrl + artifactHttpPath private def artifactHttpPath: String = s"/${projectRef.organization}/${projectRef.repository}/$name" diff --git a/modules/core/shared/src/main/scala/scaladex/core/model/Env.scala b/modules/core/shared/src/main/scala/scaladex/core/model/Env.scala index 5aefd50e6..f27ab5fa0 100644 --- a/modules/core/shared/src/main/scala/scaladex/core/model/Env.scala +++ b/modules/core/shared/src/main/scala/scaladex/core/model/Env.scala @@ -5,16 +5,20 @@ sealed trait Env { def isDev: Boolean = false def isLocal: Boolean = false def isDevOrProd: Boolean = isDev || isProd + def rootUrl: String } object Env { case object Local extends Env { override def isLocal: Boolean = true + override def rootUrl: String = "http://localhost:8080" } case object Dev extends Env { override def isDev: Boolean = true + override def rootUrl: String = "https://index-dev.scala-lang.org" } case object Prod extends Env { override def isProd: Boolean = true + override def rootUrl: String = "https://index.scala-lang.org" } def from(s: String): Env = diff --git a/modules/server/src/main/assets/css/partials/_header.scss b/modules/server/src/main/assets/css/partials/_header.scss index ef9a32e42..785587d6a 100644 --- a/modules/server/src/main/assets/css/partials/_header.scss +++ b/modules/server/src/main/assets/css/partials/_header.scss @@ -54,19 +54,15 @@ .logo { margin-bottom: 15px; + margin-left: 10px; @media screen and (min-width: $screen-md-min) { margin-bottom: 0; } } - .awesome { - font-family: Caveat; - font-size: 21px; - padding: 3px 10px; - } - .btn-default { + padding: 7px 8px; background-color: #224951; color: white; &:hover, &:focus { @@ -75,8 +71,14 @@ } } + .awesome { + padding: 3px 8px; + font-family: Caveat; + font-size: 21px; + } + .btn { - margin-left: 16px; + margin-right: 8px; min-height: 39px; i { margin-right: 8px; diff --git a/modules/server/src/main/resources/lib/swagger-initializer.js b/modules/server/src/main/resources/lib/swagger-initializer.js new file mode 100644 index 000000000..906440cdd --- /dev/null +++ b/modules/server/src/main/resources/lib/swagger-initializer.js @@ -0,0 +1,18 @@ +window.onload = function() { + window.ui = SwaggerUIBundle({ + urls: [ + { name: "Scaladex API v1", url: "/api/v1/open-api.json" }, + { name: "Scaladex API v0", url: "/api/open-api.json" } + ], + dom_id: '#swagger-ui', + deepLinking: true, + presets: [ + SwaggerUIBundle.presets.apis, + SwaggerUIStandalonePreset + ], + plugins: [ + SwaggerUIBundle.plugins.DownloadUrl + ], + layout: "StandaloneLayout" + }); +}; diff --git a/modules/server/src/main/scala/scaladex/server/route/Assets.scala b/modules/server/src/main/scala/scaladex/server/route/Assets.scala index 91789021e..50d337b2a 100644 --- a/modules/server/src/main/scala/scaladex/server/route/Assets.scala +++ b/modules/server/src/main/scala/scaladex/server/route/Assets.scala @@ -1,5 +1,6 @@ package scaladex.server.route +import org.apache.pekko.http.scaladsl.model.StatusCodes import org.apache.pekko.http.scaladsl.server.Directives._ import org.apache.pekko.http.scaladsl.server.Route @@ -8,10 +9,12 @@ object Assets { pathPrefix("assets") { get( concat( - path("lib" / Remaining)(path => getFromResource("lib/" + path)), - path("img" / Remaining)(path => getFromResource("img/" + path)), - path("css" / Remaining)(path => getFromResource("css/" + path)), - path("js" / Remaining)(path => getFromResource("js/" + path)), + pathPrefix("lib" / "swagger-ui")(redirect("/api/doc", StatusCodes.PermanentRedirect)), + // be explicit on what we can get to avoid security leak + pathPrefix("lib")(getFromResourceDirectory("lib")), + pathPrefix("img")(getFromResourceDirectory("img")), + pathPrefix("css")(getFromResourceDirectory("css")), + pathPrefix("js")(getFromResourceDirectory("js")), path("webclient-opt.js")( getFromResource("webclient-opt.js") ), diff --git a/modules/server/src/main/scala/scaladex/server/route/api/DocumentationRoutes.scala b/modules/server/src/main/scala/scaladex/server/route/api/DocumentationRoutes.scala index a9dd2afaa..526856ead 100644 --- a/modules/server/src/main/scala/scaladex/server/route/api/DocumentationRoutes.scala +++ b/modules/server/src/main/scala/scaladex/server/route/api/DocumentationRoutes.scala @@ -1,25 +1,36 @@ package scaladex.server.route.api -import endpoints4s.openapi.model.OpenApi -import endpoints4s.pekkohttp.server +import endpoints4s.Encoder import org.apache.pekko.http.cors.scaladsl.CorsDirectives.cors -import org.apache.pekko.http.scaladsl.server.Directives.concat +import org.apache.pekko.http.scaladsl.marshalling.Marshaller +import org.apache.pekko.http.scaladsl.marshalling.ToEntityMarshaller +import org.apache.pekko.http.scaladsl.model.MediaTypes +import org.apache.pekko.http.scaladsl.model.StatusCodes +import org.apache.pekko.http.scaladsl.server.Directives._ import org.apache.pekko.http.scaladsl.server.Route /** * Akka-Http routes serving the documentation of the public HTTP API of Scaladex */ -object DocumentationRoute extends server.Endpoints with server.JsonEntitiesFromEncodersAndDecoders { +object DocumentationRoute { + implicit def marshallerFromEncoder[T](implicit encoder: Encoder[T, String]): ToEntityMarshaller[T] = + Marshaller + .stringMarshaller(MediaTypes.`application/json`) + .compose(c => encoder.encode(c)) + val route: Route = cors() { - concat( - endpoint( - get(path / "api" / "open-api.json"), - ok(jsonResponse[OpenApi]) - ).implementedBy(_ => ApiDocumentation.apiV0), - endpoint( - get(path / "api" / "v1" / "open-api.json"), - ok(jsonResponse[OpenApi]) - ).implementedBy(_ => ApiDocumentation.apiV1) - ) + get { + concat( + pathPrefix("api" / "doc")( + pathEnd(redirect("/api/doc/", StatusCodes.PermanentRedirect)) ~ + pathSingleSlash(getFromResource("lib/swagger-ui/index.html")) ~ + // override default swagger-initializer + path("swagger-initializer.js")(getFromResource("lib/swagger-initializer.js")) ~ + getFromResourceDirectory("lib/swagger-ui") + ), + path("api" / "open-api.json")(complete(StatusCodes.OK, ApiDocumentation.apiV0)), + path("api" / "v1" / "open-api.json")(complete(StatusCodes.OK, ApiDocumentation.apiV1)) + ) + } } } diff --git a/modules/template/src/main/twirl/scaladex/view/main.scala.html b/modules/template/src/main/twirl/scaladex/view/main.scala.html index 3036675ec..e5a1dae3d 100644 --- a/modules/template/src/main/twirl/scaladex/view/main.scala.html +++ b/modules/template/src/main/twirl/scaladex/view/main.scala.html @@ -62,46 +62,45 @@