diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala index 3930efc7..6825659f 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala @@ -4,14 +4,16 @@ import org.scalawiki.dto.Page object PageFilter { - def titles(titles: Set[String]) = (p: Page) => titles.contains(p.title) + type PageFilter = Page => Boolean - def ids(ids: Set[Long]) = (p: Page) => p.id.exists(ids.contains) + def titles(titles: Set[String]): PageFilter = (p: Page) => titles.contains(p.title) - def ns(namespaces: Set[Int]) = (p: Page) => p.ns.exists(namespaces.contains) + def ids(ids: Set[Long]): PageFilter = (p: Page) => p.id.exists(ids.contains) - val all = (p: Page) => true + def ns(namespaces: Set[Int]): PageFilter = (p: Page) => p.ns.exists(namespaces.contains) - val none = (p: Page) => false + val all: PageFilter = _ => true + + val none: PageFilter = _ => false } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala index bad165af..2ad9edd5 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala @@ -7,7 +7,7 @@ import org.scalawiki.dto.filter.RevisionFilter class History(val revisions: Seq[Revision]) { - def hasPageCreation = revisions.headOption.exists(_.isNewPage) + def hasPageCreation: Boolean = revisions.headOption.exists(_.isNewPage) def users(revisionFilter: RevisionFilter): Set[String] = { val filtered = revisionFilter.apply(revisions) @@ -45,10 +45,10 @@ class History(val revisions: Seq[Revision]) { def updated: Option[ZonedDateTime] = revisions.headOption.flatMap(_.timestamp) - def createdAfter(from: Option[ZonedDateTime]) = + def createdAfter(from: Option[ZonedDateTime]): Boolean = created.exists(rev => from.forall(rev.isAfter)) - def editedIn(revisionFilter: RevisionFilter) = + def editedIn(revisionFilter: RevisionFilter): Boolean = revisionFilter.apply(revisions).nonEmpty } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala index 4802e862..b219b6e5 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala @@ -12,7 +12,7 @@ case class SwTemplate(wtNode: WtTemplate) extends SwNode { wtNode.getArgs.asScala.collect { case arg: WtTemplateArgument => arg } val template = getTemplate - def getTemplate = { + def getTemplate: Template = { val argsMap = args.zipWithIndex.map { case (arg, index) => val name = if (arg.hasName) @@ -25,13 +25,13 @@ case class SwTemplate(wtNode: WtTemplate) extends SwNode { name -> value }.toMap - new Template(name, argsMap) + Template(name, argsMap) } def getArg(name: String): Option[WtTemplateArgument] = args.find(p => p.getName.isResolved && p.getName.getAsString.trim == name) - def setTemplateParam(name: String, value: String) = { + def setTemplateParam(name: String, value: String): Unit = { getArg(name).foreach { arg => val orig = getText(arg.getValue) diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala index 99392855..cb18265b 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala @@ -7,7 +7,7 @@ case class Table( cssClass: String = "wikitable sortable" ) { - def asWiki = { + def asWiki: String = { "{|" + (if (cssClass.nonEmpty) s" class='$cssClass'" else "") + (if (title.nonEmpty) "\n|+ " + title else "") + @@ -22,7 +22,7 @@ case class Table( "\n|}" } - def asHtml = { + def asHtml: String = { "" + (if (cssClass.nonEmpty) s" class='$cssClass'" else "") + (if (title.nonEmpty) s"\n" else "") + diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala index 7249a262..cdbb001b 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala @@ -1,9 +1,11 @@ package org.scalawiki.wlx import org.scalawiki.dto.Image +import org.scalawiki.wlx.ImageDB.allowList import org.scalawiki.wlx.dto.{Contest, Monument} import org.scalawiki.wlx.query.ImageQuery +import java.time.ZonedDateTime import scala.concurrent.Future case class ImageDB( @@ -21,29 +23,38 @@ case class ImageDB( lazy val _byMegaPixels: Grouping[Int, Image] = new Grouping("mpx", ImageGrouping.byMpx, images) - lazy val withCorrectIds: Iterable[Image] = monumentDb.fold(images) { db => - images.filter(_.monumentId.exists(db.ids.contains)) - } + private lazy val withCorrectIds: Seq[Image] = monumentDb + .fold(images) { db => + images.filter(_.monumentId.exists(db.ids.contains)) + } + .toSeq - lazy val sansIneligible: Iterable[Image] = withCorrectIds - .filterNot( - _.categories - .contains(s"Ineligible submissions for WLM ${contest.year} in Ukraine") - ) - .filter(_.atLeastMpx(minMpx)) + lazy val sansIneligible: Seq[Image] = + withCorrectIds.filterNot(_.pageId.exists(ineligibleIds.contains)) - lazy val ineligible: Iterable[Image] = withCorrectIds - .filter( - _.categories - .contains(s"Ineligible submissions for WLM ${contest.year} in Ukraine") - ) - .filterNot(_.atLeastMpx(minMpx)) + private val jun30 = ZonedDateTime.parse(s"2023-06-30T23:59:59Z") + + lazy val ineligible: Seq[Image] = withCorrectIds.filter { i => + val after30 = + i.metadata.exists(_.date.exists(_.isAfter(jun30))) && + !i.specialNominations.contains(s"WLM${contest.year}-UA-interior") && + !i.pageId.exists(allowList.contains) + + val category = i.categories + .contains(s"Ineligible submissions for WLM ${contest.year} in Ukraine") + + val mpx = !i.atLeastMpx(minMpx) + + after30 || category || mpx + } + + lazy val ineligibleIds: Set[Long] = ineligible.flatMap(_.pageId).toSet lazy val _byId: Grouping[String, Image] = - new Grouping("monument", ImageGrouping.byMonument, withCorrectIds) + new Grouping("monument", ImageGrouping.byMonument, sansIneligible) lazy val _byRegion: Grouping[String, Image] = - new Grouping("monument", ImageGrouping.byRegion, withCorrectIds) + new Grouping("monument", ImageGrouping.byRegion, sansIneligible) lazy val _byAuthor: Grouping[String, Image] = new Grouping("author", ImageGrouping.byAuthor, sansIneligible) @@ -216,4 +227,27 @@ object ImageDB { new ImageDB(contest, images, monumentDb, minMpx) } } + + private val allowList = Set[Long](139033190, 139033189, 138896313, 138896491, 138679587, + 139500804, 139899874, 139899873, 139643873, 139473382, 139843450, 138561292, 138543453, + 139756664, 138561778, 138561777, 138543414, 138543415, 139842986, 139431707, 138397222, + 139431706, 139431716, 139431703, 138393699, 139676985, 139676986, 139641232, 139643516, + 139685413, 139590379, 138393495, 139590381, 139590378, 138909394, 138478400, 139033188, + 138397226, 138943312, 139913130, 139640691, 138942396, 138896118, 139783542, 139783543, + 138896202, 138944112, 138896244, 139783534, 139430660, 138376808, 138942397, 139363471, + 139363470, 139363466, 139237579, 139313124, 138393537, 139684886, 138394378, 138397346, + 139723578, 139723568, 139723569, 138394430, 138393538, 139721126, 138397221, 139295608, + 138394203, 139363709, 138397220, 138397227, 139363710, 139363713, 139363711, 138376523, + 138397390, 138942398, 138950033, 138376807, 138950032, 138397349, 139276786, 139684887, + 138942408, 139431710, 139363714, 139431708, 138943859, 138394379, 138943647, 138909161, + 139646725, 138393972, 138909396, 139723573, 139431705, 139313128, 139723576, 139783536, + 139723572, 139723570, 139313132, 139173215, 139723577, 139723571, 139660924, 139660925, + 139913381, 138942403, 138943310, 139778492, 138376590, 139751221, 139721718, 139721714, + 139783537, 139751228, 138942404, 138393704, 138393697, 138393702, 138888154, 139025954, + 139590377, 139913380, 139643684, 139643683, 139643872, 139751220, 139751222, 139363468, + 139363467, 139643871, 139237811, 139785053, 139091352, 138943860, 138397224, 139401104, + 139400816, 139313126, 138909243, 138376524, 139313129, 139641093, 139640690, 139313125, + 139721715, 139721713, 138397215, 138393861, 138393700, 139313130, 138943316, 138397350, + 138949957, 139723584, 139751226, 138478405, 138943315, 138943311, 138943648, 138376522, + 138376615, 138393776, 138397345, 139237058, 138943314, 138943309, 138943308) } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala index 1d3af163..350b1f9a 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala @@ -75,7 +75,7 @@ class Statistics( contest: Contest, startYear: Option[Int], monumentQuery: MonumentQuery, - imageQuery: ImageQuery, + imageQuery: Option[ImageQuery], imageQueryWiki: Option[ImageQuery], bot: MwBot, config: StatConfig @@ -85,7 +85,7 @@ class Statistics( contest: Contest, startYear: Option[Int] = None, monumentQuery: MonumentQuery, - imageQuery: ImageQuery = ImageQuery.create, + imageQuery: Option[ImageQuery] = Some(ImageQuery.create), imageQueryWiki: Option[ImageQuery] = None, bot: MwBot = MwBot.fromHost(MwBot.commons), config: Option[StatConfig] = None @@ -100,6 +100,11 @@ class Statistics( config.getOrElse(StatConfig(contest.campaign)) ) + def getImageQuery(year: Option[Int]): ImageQuery = { + val cacheName = s"${contest.campaign}-${year.getOrElse("all")}" + ImageQuery.create(new CachedBot(Site.commons, cacheName, true)) + } + private val currentYear = contest.year private val contests = @@ -121,7 +126,7 @@ class Statistics( for ( byYear <- Future.sequence(contests.map(contestImages(monumentDb))); totalImages <- - if (total) imagesByTemplate(monumentDb) + if (total) imagesByTemplate(monumentDb, imageQuery.getOrElse(getImageQuery(None))) else Future.successful( Some( @@ -150,11 +155,16 @@ class Statistics( } private def contestImages(monumentDb: Some[MonumentDB])(contest: Contest) = - ImageDB.create(contest, imageQuery, monumentDb, config.minMpx) + ImageDB.create( + contest, + imageQuery.getOrElse(getImageQuery(Some(contest.year))), + monumentDb, + config.minMpx + ) private def imagesByTemplate( monumentDb: Some[MonumentDB], - imageQuery: ImageQuery = imageQuery + imageQuery: ImageQuery ) = for ( commons <- imageQuery.imagesWithTemplateAsync( @@ -224,7 +234,7 @@ object Statistics { startYear = Some(cfg.years.head), monumentQuery = MonumentQuery.create(contest, reportDifferentRegionIds = true), config = Some(cfg), - imageQuery = imageQuery, + imageQuery = Option.empty[ImageQuery], imageQueryWiki = Some(imageQueryWiki) ) diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala index 3e1f0577..e917b782 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala @@ -34,7 +34,7 @@ class StatisticsSpec(implicit ee: ExecutionEnv) val cfg = StatConfig(campaign = contest.campaign) - new Statistics(contest, None, monumentQuery, imageQuery, None, bot, cfg) + new Statistics(contest, None, monumentQuery, Some(imageQuery), None, bot, cfg) } "give current year stat empty" in { @@ -81,7 +81,7 @@ class StatisticsSpec(implicit ee: ExecutionEnv) monumentQuery.byMonumentTemplate(date = None) returns monuments val stat = - new Statistics(contest, None, monumentQuery, imageQuery, None, bot) + new Statistics(contest, None, monumentQuery, Some(imageQuery), None, bot) stat.gatherData(false) must throwA[RuntimeException].await }
$title