diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala
index 3930efc7..6825659f 100644
--- a/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala
+++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala
@@ -4,14 +4,16 @@ import org.scalawiki.dto.Page
object PageFilter {
- def titles(titles: Set[String]) = (p: Page) => titles.contains(p.title)
+ type PageFilter = Page => Boolean
- def ids(ids: Set[Long]) = (p: Page) => p.id.exists(ids.contains)
+ def titles(titles: Set[String]): PageFilter = (p: Page) => titles.contains(p.title)
- def ns(namespaces: Set[Int]) = (p: Page) => p.ns.exists(namespaces.contains)
+ def ids(ids: Set[Long]): PageFilter = (p: Page) => p.id.exists(ids.contains)
- val all = (p: Page) => true
+ def ns(namespaces: Set[Int]): PageFilter = (p: Page) => p.ns.exists(namespaces.contains)
- val none = (p: Page) => false
+ val all: PageFilter = _ => true
+
+ val none: PageFilter = _ => false
}
diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala
index bad165af..2ad9edd5 100644
--- a/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala
+++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala
@@ -7,7 +7,7 @@ import org.scalawiki.dto.filter.RevisionFilter
class History(val revisions: Seq[Revision]) {
- def hasPageCreation = revisions.headOption.exists(_.isNewPage)
+ def hasPageCreation: Boolean = revisions.headOption.exists(_.isNewPage)
def users(revisionFilter: RevisionFilter): Set[String] = {
val filtered = revisionFilter.apply(revisions)
@@ -45,10 +45,10 @@ class History(val revisions: Seq[Revision]) {
def updated: Option[ZonedDateTime] = revisions.headOption.flatMap(_.timestamp)
- def createdAfter(from: Option[ZonedDateTime]) =
+ def createdAfter(from: Option[ZonedDateTime]): Boolean =
created.exists(rev => from.forall(rev.isAfter))
- def editedIn(revisionFilter: RevisionFilter) =
+ def editedIn(revisionFilter: RevisionFilter): Boolean =
revisionFilter.apply(revisions).nonEmpty
}
diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala
index 4802e862..b219b6e5 100644
--- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala
+++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala
@@ -12,7 +12,7 @@ case class SwTemplate(wtNode: WtTemplate) extends SwNode {
wtNode.getArgs.asScala.collect { case arg: WtTemplateArgument => arg }
val template = getTemplate
- def getTemplate = {
+ def getTemplate: Template = {
val argsMap = args.zipWithIndex.map { case (arg, index) =>
val name =
if (arg.hasName)
@@ -25,13 +25,13 @@ case class SwTemplate(wtNode: WtTemplate) extends SwNode {
name -> value
}.toMap
- new Template(name, argsMap)
+ Template(name, argsMap)
}
def getArg(name: String): Option[WtTemplateArgument] =
args.find(p => p.getName.isResolved && p.getName.getAsString.trim == name)
- def setTemplateParam(name: String, value: String) = {
+ def setTemplateParam(name: String, value: String): Unit = {
getArg(name).foreach { arg =>
val orig = getText(arg.getValue)
diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala
index 99392855..cb18265b 100644
--- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala
+++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala
@@ -7,7 +7,7 @@ case class Table(
cssClass: String = "wikitable sortable"
) {
- def asWiki = {
+ def asWiki: String = {
"{|" +
(if (cssClass.nonEmpty) s" class='$cssClass'" else "") +
(if (title.nonEmpty) "\n|+ " + title else "") +
@@ -22,7 +22,7 @@ case class Table(
"\n|}"
}
- def asHtml = {
+ def asHtml: String = {
"
" +
(if (cssClass.nonEmpty) s" class='$cssClass'" else "") +
(if (title.nonEmpty) s"\n $title " else "") +
diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala
index 7249a262..cdbb001b 100644
--- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala
+++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala
@@ -1,9 +1,11 @@
package org.scalawiki.wlx
import org.scalawiki.dto.Image
+import org.scalawiki.wlx.ImageDB.allowList
import org.scalawiki.wlx.dto.{Contest, Monument}
import org.scalawiki.wlx.query.ImageQuery
+import java.time.ZonedDateTime
import scala.concurrent.Future
case class ImageDB(
@@ -21,29 +23,38 @@ case class ImageDB(
lazy val _byMegaPixels: Grouping[Int, Image] =
new Grouping("mpx", ImageGrouping.byMpx, images)
- lazy val withCorrectIds: Iterable[Image] = monumentDb.fold(images) { db =>
- images.filter(_.monumentId.exists(db.ids.contains))
- }
+ private lazy val withCorrectIds: Seq[Image] = monumentDb
+ .fold(images) { db =>
+ images.filter(_.monumentId.exists(db.ids.contains))
+ }
+ .toSeq
- lazy val sansIneligible: Iterable[Image] = withCorrectIds
- .filterNot(
- _.categories
- .contains(s"Ineligible submissions for WLM ${contest.year} in Ukraine")
- )
- .filter(_.atLeastMpx(minMpx))
+ lazy val sansIneligible: Seq[Image] =
+ withCorrectIds.filterNot(_.pageId.exists(ineligibleIds.contains))
- lazy val ineligible: Iterable[Image] = withCorrectIds
- .filter(
- _.categories
- .contains(s"Ineligible submissions for WLM ${contest.year} in Ukraine")
- )
- .filterNot(_.atLeastMpx(minMpx))
+ private val jun30 = ZonedDateTime.parse(s"2023-06-30T23:59:59Z")
+
+ lazy val ineligible: Seq[Image] = withCorrectIds.filter { i =>
+ val after30 =
+ i.metadata.exists(_.date.exists(_.isAfter(jun30))) &&
+ !i.specialNominations.contains(s"WLM${contest.year}-UA-interior") &&
+ !i.pageId.exists(allowList.contains)
+
+ val category = i.categories
+ .contains(s"Ineligible submissions for WLM ${contest.year} in Ukraine")
+
+ val mpx = !i.atLeastMpx(minMpx)
+
+ after30 || category || mpx
+ }
+
+ lazy val ineligibleIds: Set[Long] = ineligible.flatMap(_.pageId).toSet
lazy val _byId: Grouping[String, Image] =
- new Grouping("monument", ImageGrouping.byMonument, withCorrectIds)
+ new Grouping("monument", ImageGrouping.byMonument, sansIneligible)
lazy val _byRegion: Grouping[String, Image] =
- new Grouping("monument", ImageGrouping.byRegion, withCorrectIds)
+ new Grouping("monument", ImageGrouping.byRegion, sansIneligible)
lazy val _byAuthor: Grouping[String, Image] =
new Grouping("author", ImageGrouping.byAuthor, sansIneligible)
@@ -216,4 +227,27 @@ object ImageDB {
new ImageDB(contest, images, monumentDb, minMpx)
}
}
+
+ private val allowList = Set[Long](139033190, 139033189, 138896313, 138896491, 138679587,
+ 139500804, 139899874, 139899873, 139643873, 139473382, 139843450, 138561292, 138543453,
+ 139756664, 138561778, 138561777, 138543414, 138543415, 139842986, 139431707, 138397222,
+ 139431706, 139431716, 139431703, 138393699, 139676985, 139676986, 139641232, 139643516,
+ 139685413, 139590379, 138393495, 139590381, 139590378, 138909394, 138478400, 139033188,
+ 138397226, 138943312, 139913130, 139640691, 138942396, 138896118, 139783542, 139783543,
+ 138896202, 138944112, 138896244, 139783534, 139430660, 138376808, 138942397, 139363471,
+ 139363470, 139363466, 139237579, 139313124, 138393537, 139684886, 138394378, 138397346,
+ 139723578, 139723568, 139723569, 138394430, 138393538, 139721126, 138397221, 139295608,
+ 138394203, 139363709, 138397220, 138397227, 139363710, 139363713, 139363711, 138376523,
+ 138397390, 138942398, 138950033, 138376807, 138950032, 138397349, 139276786, 139684887,
+ 138942408, 139431710, 139363714, 139431708, 138943859, 138394379, 138943647, 138909161,
+ 139646725, 138393972, 138909396, 139723573, 139431705, 139313128, 139723576, 139783536,
+ 139723572, 139723570, 139313132, 139173215, 139723577, 139723571, 139660924, 139660925,
+ 139913381, 138942403, 138943310, 139778492, 138376590, 139751221, 139721718, 139721714,
+ 139783537, 139751228, 138942404, 138393704, 138393697, 138393702, 138888154, 139025954,
+ 139590377, 139913380, 139643684, 139643683, 139643872, 139751220, 139751222, 139363468,
+ 139363467, 139643871, 139237811, 139785053, 139091352, 138943860, 138397224, 139401104,
+ 139400816, 139313126, 138909243, 138376524, 139313129, 139641093, 139640690, 139313125,
+ 139721715, 139721713, 138397215, 138393861, 138393700, 139313130, 138943316, 138397350,
+ 138949957, 139723584, 139751226, 138478405, 138943315, 138943311, 138943648, 138376522,
+ 138376615, 138393776, 138397345, 139237058, 138943314, 138943309, 138943308)
}
diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala
index 1d3af163..350b1f9a 100644
--- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala
+++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala
@@ -75,7 +75,7 @@ class Statistics(
contest: Contest,
startYear: Option[Int],
monumentQuery: MonumentQuery,
- imageQuery: ImageQuery,
+ imageQuery: Option[ImageQuery],
imageQueryWiki: Option[ImageQuery],
bot: MwBot,
config: StatConfig
@@ -85,7 +85,7 @@ class Statistics(
contest: Contest,
startYear: Option[Int] = None,
monumentQuery: MonumentQuery,
- imageQuery: ImageQuery = ImageQuery.create,
+ imageQuery: Option[ImageQuery] = Some(ImageQuery.create),
imageQueryWiki: Option[ImageQuery] = None,
bot: MwBot = MwBot.fromHost(MwBot.commons),
config: Option[StatConfig] = None
@@ -100,6 +100,11 @@ class Statistics(
config.getOrElse(StatConfig(contest.campaign))
)
+ def getImageQuery(year: Option[Int]): ImageQuery = {
+ val cacheName = s"${contest.campaign}-${year.getOrElse("all")}"
+ ImageQuery.create(new CachedBot(Site.commons, cacheName, true))
+ }
+
private val currentYear = contest.year
private val contests =
@@ -121,7 +126,7 @@ class Statistics(
for (
byYear <- Future.sequence(contests.map(contestImages(monumentDb)));
totalImages <-
- if (total) imagesByTemplate(monumentDb)
+ if (total) imagesByTemplate(monumentDb, imageQuery.getOrElse(getImageQuery(None)))
else
Future.successful(
Some(
@@ -150,11 +155,16 @@ class Statistics(
}
private def contestImages(monumentDb: Some[MonumentDB])(contest: Contest) =
- ImageDB.create(contest, imageQuery, monumentDb, config.minMpx)
+ ImageDB.create(
+ contest,
+ imageQuery.getOrElse(getImageQuery(Some(contest.year))),
+ monumentDb,
+ config.minMpx
+ )
private def imagesByTemplate(
monumentDb: Some[MonumentDB],
- imageQuery: ImageQuery = imageQuery
+ imageQuery: ImageQuery
) =
for (
commons <- imageQuery.imagesWithTemplateAsync(
@@ -224,7 +234,7 @@ object Statistics {
startYear = Some(cfg.years.head),
monumentQuery = MonumentQuery.create(contest, reportDifferentRegionIds = true),
config = Some(cfg),
- imageQuery = imageQuery,
+ imageQuery = Option.empty[ImageQuery],
imageQueryWiki = Some(imageQueryWiki)
)
diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala
index 3e1f0577..e917b782 100644
--- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala
+++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala
@@ -34,7 +34,7 @@ class StatisticsSpec(implicit ee: ExecutionEnv)
val cfg = StatConfig(campaign = contest.campaign)
- new Statistics(contest, None, monumentQuery, imageQuery, None, bot, cfg)
+ new Statistics(contest, None, monumentQuery, Some(imageQuery), None, bot, cfg)
}
"give current year stat empty" in {
@@ -81,7 +81,7 @@ class StatisticsSpec(implicit ee: ExecutionEnv)
monumentQuery.byMonumentTemplate(date = None) returns monuments
val stat =
- new Statistics(contest, None, monumentQuery, imageQuery, None, bot)
+ new Statistics(contest, None, monumentQuery, Some(imageQuery), None, bot)
stat.gatherData(false) must throwA[RuntimeException].await
}