From 8c26d75756f00ab33e6dd8c35922591b87d47d27 Mon Sep 17 00:00:00 2001
From: ilya <intracer@gmail.com>
Date: Wed, 3 Apr 2024 21:12:21 +0300
Subject: [PATCH 1/2] fixes

---
 .../main/scala/org/scalawiki/dto/Image.scala  |  4 +-
 .../org/scalawiki/dto/filter/PageFilter.scala | 12 ++--
 .../org/scalawiki/dto/history/History.scala   |  6 +-
 .../org/scalawiki/dto/markup/SwTemplate.scala |  6 +-
 .../org/scalawiki/dto/markup/Table.scala      |  4 +-
 .../scala/org/scalawiki/wlx/ImageDB.scala     | 68 ++++++++++++++-----
 .../org/scalawiki/wlx/stat/Statistics.scala   | 22 ++++--
 .../scalawiki/wlx/stat/StatisticsSpec.scala   |  4 +-
 8 files changed, 87 insertions(+), 39 deletions(-)

diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/Image.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/Image.scala
index 1386c303..9a817363 100644
--- a/scalawiki-core/src/main/scala/org/scalawiki/dto/Image.scala
+++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/Image.scala
@@ -8,6 +8,8 @@ import org.scalawiki.dto.markup.Gallery
 import org.scalawiki.wikitext.TemplateParser
 import org.sweble.wikitext.engine.nodes.EngPage
 
+import scala.util.Try
+
 case class ImageMetadata(data: Map[String, String]) {
 
   def camera: Option[String] = data.get("Model")
@@ -15,7 +17,7 @@ case class ImageMetadata(data: Map[String, String]) {
   def date: Option[ZonedDateTime] =
     data
       .get("DateTime")
-      .map(s => LocalDateTime.parse(s, ImageMetadata.df).atZone(ZoneOffset.UTC))
+      .flatMap(s => Try(LocalDateTime.parse(s, ImageMetadata.df).atZone(ZoneOffset.UTC)).toOption)
 }
 
 object ImageMetadata {
diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala
index 3930efc7..6825659f 100644
--- a/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala
+++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala
@@ -4,14 +4,16 @@ import org.scalawiki.dto.Page
 
 object PageFilter {
 
-  def titles(titles: Set[String]) = (p: Page) => titles.contains(p.title)
+  type PageFilter = Page => Boolean
 
-  def ids(ids: Set[Long]) = (p: Page) => p.id.exists(ids.contains)
+  def titles(titles: Set[String]): PageFilter = (p: Page) => titles.contains(p.title)
 
-  def ns(namespaces: Set[Int]) = (p: Page) => p.ns.exists(namespaces.contains)
+  def ids(ids: Set[Long]): PageFilter = (p: Page) => p.id.exists(ids.contains)
 
-  val all = (p: Page) => true
+  def ns(namespaces: Set[Int]): PageFilter = (p: Page) => p.ns.exists(namespaces.contains)
 
-  val none = (p: Page) => false
+  val all: PageFilter = _ => true
+
+  val none: PageFilter = _ => false
 
 }
diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala
index bad165af..2ad9edd5 100644
--- a/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala
+++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala
@@ -7,7 +7,7 @@ import org.scalawiki.dto.filter.RevisionFilter
 
 class History(val revisions: Seq[Revision]) {
 
-  def hasPageCreation = revisions.headOption.exists(_.isNewPage)
+  def hasPageCreation: Boolean = revisions.headOption.exists(_.isNewPage)
 
   def users(revisionFilter: RevisionFilter): Set[String] = {
     val filtered = revisionFilter.apply(revisions)
@@ -45,10 +45,10 @@ class History(val revisions: Seq[Revision]) {
 
   def updated: Option[ZonedDateTime] = revisions.headOption.flatMap(_.timestamp)
 
-  def createdAfter(from: Option[ZonedDateTime]) =
+  def createdAfter(from: Option[ZonedDateTime]): Boolean =
     created.exists(rev => from.forall(rev.isAfter))
 
-  def editedIn(revisionFilter: RevisionFilter) =
+  def editedIn(revisionFilter: RevisionFilter): Boolean =
     revisionFilter.apply(revisions).nonEmpty
 
 }
diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala
index 4802e862..b219b6e5 100644
--- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala
+++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala
@@ -12,7 +12,7 @@ case class SwTemplate(wtNode: WtTemplate) extends SwNode {
     wtNode.getArgs.asScala.collect { case arg: WtTemplateArgument => arg }
   val template = getTemplate
 
-  def getTemplate = {
+  def getTemplate: Template = {
     val argsMap = args.zipWithIndex.map { case (arg, index) =>
       val name =
         if (arg.hasName)
@@ -25,13 +25,13 @@ case class SwTemplate(wtNode: WtTemplate) extends SwNode {
       name -> value
     }.toMap
 
-    new Template(name, argsMap)
+    Template(name, argsMap)
   }
 
   def getArg(name: String): Option[WtTemplateArgument] =
     args.find(p => p.getName.isResolved && p.getName.getAsString.trim == name)
 
-  def setTemplateParam(name: String, value: String) = {
+  def setTemplateParam(name: String, value: String): Unit = {
     getArg(name).foreach { arg =>
       val orig = getText(arg.getValue)
 
diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala
index 99392855..cb18265b 100644
--- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala
+++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala
@@ -7,7 +7,7 @@ case class Table(
     cssClass: String = "wikitable sortable"
 ) {
 
-  def asWiki = {
+  def asWiki: String = {
     "{|" +
       (if (cssClass.nonEmpty) s" class='$cssClass'" else "") +
       (if (title.nonEmpty) "\n|+ " + title else "") +
@@ -22,7 +22,7 @@ case class Table(
       "\n|}"
   }
 
-  def asHtml = {
+  def asHtml: String = {
     "<table>" +
       (if (cssClass.nonEmpty) s" class='$cssClass'" else "") +
       (if (title.nonEmpty) s"\n<caption> $title </caption>" else "") +
diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala
index 7249a262..77d1d8f1 100644
--- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala
+++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala
@@ -1,9 +1,11 @@
 package org.scalawiki.wlx
 
 import org.scalawiki.dto.Image
+import org.scalawiki.wlx.ImageDB.allowList
 import org.scalawiki.wlx.dto.{Contest, Monument}
 import org.scalawiki.wlx.query.ImageQuery
 
+import java.time.ZonedDateTime
 import scala.concurrent.Future
 
 case class ImageDB(
@@ -21,29 +23,38 @@ case class ImageDB(
   lazy val _byMegaPixels: Grouping[Int, Image] =
     new Grouping("mpx", ImageGrouping.byMpx, images)
 
-  lazy val withCorrectIds: Iterable[Image] = monumentDb.fold(images) { db =>
-    images.filter(_.monumentId.exists(db.ids.contains))
-  }
+  private lazy val withCorrectIds: Seq[Image] = monumentDb
+    .fold(images) { db =>
+      images.filter(_.monumentId.exists(db.ids.contains))
+    }
+    .toSeq
 
-  lazy val sansIneligible: Iterable[Image] = withCorrectIds
-    .filterNot(
-      _.categories
-        .contains(s"Ineligible submissions for WLM ${contest.year} in Ukraine")
-    )
-    .filter(_.atLeastMpx(minMpx))
+  lazy val sansIneligible: Seq[Image] =
+    withCorrectIds.filterNot(_.pageId.exists(ineligibleIds.contains))
 
-  lazy val ineligible: Iterable[Image] = withCorrectIds
-    .filter(
-      _.categories
-        .contains(s"Ineligible submissions for WLM ${contest.year} in Ukraine")
-    )
-    .filterNot(_.atLeastMpx(minMpx))
+  private val jun30 = ZonedDateTime.parse(s"${contest.year}-06-30T23:59:59Z")
+
+  lazy val ineligible: Seq[Image] = withCorrectIds.filter { i =>
+    val after30 =
+      i.metadata.exists(_.date.exists(_.isAfter(jun30))) &&
+        !i.specialNominations.contains(s"WLM${contest.year}-UA-interior") &&
+        !i.pageId.exists(allowList.contains)
+
+    val category = i.categories
+      .contains(s"Ineligible submissions for WLM ${contest.year} in Ukraine")
+
+    val mpx = !i.atLeastMpx(minMpx)
+
+    after30 || category || mpx
+  }
+
+  lazy val ineligibleIds: Set[Long] = ineligible.flatMap(_.pageId).toSet
 
   lazy val _byId: Grouping[String, Image] =
-    new Grouping("monument", ImageGrouping.byMonument, withCorrectIds)
+    new Grouping("monument", ImageGrouping.byMonument, sansIneligible)
 
   lazy val _byRegion: Grouping[String, Image] =
-    new Grouping("monument", ImageGrouping.byRegion, withCorrectIds)
+    new Grouping("monument", ImageGrouping.byRegion, sansIneligible)
 
   lazy val _byAuthor: Grouping[String, Image] =
     new Grouping("author", ImageGrouping.byAuthor, sansIneligible)
@@ -216,4 +227,27 @@ object ImageDB {
       new ImageDB(contest, images, monumentDb, minMpx)
     }
   }
+
+  private val allowList = Set[Long](139033190, 139033189, 138896313, 138896491, 138679587,
+    139500804, 139899874, 139899873, 139643873, 139473382, 139843450, 138561292, 138543453,
+    139756664, 138561778, 138561777, 138543414, 138543415, 139842986, 139431707, 138397222,
+    139431706, 139431716, 139431703, 138393699, 139676985, 139676986, 139641232, 139643516,
+    139685413, 139590379, 138393495, 139590381, 139590378, 138909394, 138478400, 139033188,
+    138397226, 138943312, 139913130, 139640691, 138942396, 138896118, 139783542, 139783543,
+    138896202, 138944112, 138896244, 139783534, 139430660, 138376808, 138942397, 139363471,
+    139363470, 139363466, 139237579, 139313124, 138393537, 139684886, 138394378, 138397346,
+    139723578, 139723568, 139723569, 138394430, 138393538, 139721126, 138397221, 139295608,
+    138394203, 139363709, 138397220, 138397227, 139363710, 139363713, 139363711, 138376523,
+    138397390, 138942398, 138950033, 138376807, 138950032, 138397349, 139276786, 139684887,
+    138942408, 139431710, 139363714, 139431708, 138943859, 138394379, 138943647, 138909161,
+    139646725, 138393972, 138909396, 139723573, 139431705, 139313128, 139723576, 139783536,
+    139723572, 139723570, 139313132, 139173215, 139723577, 139723571, 139660924, 139660925,
+    139913381, 138942403, 138943310, 139778492, 138376590, 139751221, 139721718, 139721714,
+    139783537, 139751228, 138942404, 138393704, 138393697, 138393702, 138888154, 139025954,
+    139590377, 139913380, 139643684, 139643683, 139643872, 139751220, 139751222, 139363468,
+    139363467, 139643871, 139237811, 139785053, 139091352, 138943860, 138397224, 139401104,
+    139400816, 139313126, 138909243, 138376524, 139313129, 139641093, 139640690, 139313125,
+    139721715, 139721713, 138397215, 138393861, 138393700, 139313130, 138943316, 138397350,
+    138949957, 139723584, 139751226, 138478405, 138943315, 138943311, 138943648, 138376522,
+    138376615, 138393776, 138397345, 139237058, 138943314, 138943309, 138943308)
 }
diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala
index 1d3af163..350b1f9a 100644
--- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala
+++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala
@@ -75,7 +75,7 @@ class Statistics(
     contest: Contest,
     startYear: Option[Int],
     monumentQuery: MonumentQuery,
-    imageQuery: ImageQuery,
+    imageQuery: Option[ImageQuery],
     imageQueryWiki: Option[ImageQuery],
     bot: MwBot,
     config: StatConfig
@@ -85,7 +85,7 @@ class Statistics(
       contest: Contest,
       startYear: Option[Int] = None,
       monumentQuery: MonumentQuery,
-      imageQuery: ImageQuery = ImageQuery.create,
+      imageQuery: Option[ImageQuery] = Some(ImageQuery.create),
       imageQueryWiki: Option[ImageQuery] = None,
       bot: MwBot = MwBot.fromHost(MwBot.commons),
       config: Option[StatConfig] = None
@@ -100,6 +100,11 @@ class Statistics(
       config.getOrElse(StatConfig(contest.campaign))
     )
 
+  def getImageQuery(year: Option[Int]): ImageQuery = {
+    val cacheName = s"${contest.campaign}-${year.getOrElse("all")}"
+    ImageQuery.create(new CachedBot(Site.commons, cacheName, true))
+  }
+
   private val currentYear = contest.year
 
   private val contests =
@@ -121,7 +126,7 @@ class Statistics(
     for (
       byYear <- Future.sequence(contests.map(contestImages(monumentDb)));
       totalImages <-
-        if (total) imagesByTemplate(monumentDb)
+        if (total) imagesByTemplate(monumentDb, imageQuery.getOrElse(getImageQuery(None)))
         else
           Future.successful(
             Some(
@@ -150,11 +155,16 @@ class Statistics(
   }
 
   private def contestImages(monumentDb: Some[MonumentDB])(contest: Contest) =
-    ImageDB.create(contest, imageQuery, monumentDb, config.minMpx)
+    ImageDB.create(
+      contest,
+      imageQuery.getOrElse(getImageQuery(Some(contest.year))),
+      monumentDb,
+      config.minMpx
+    )
 
   private def imagesByTemplate(
       monumentDb: Some[MonumentDB],
-      imageQuery: ImageQuery = imageQuery
+      imageQuery: ImageQuery
   ) =
     for (
       commons <- imageQuery.imagesWithTemplateAsync(
@@ -224,7 +234,7 @@ object Statistics {
       startYear = Some(cfg.years.head),
       monumentQuery = MonumentQuery.create(contest, reportDifferentRegionIds = true),
       config = Some(cfg),
-      imageQuery = imageQuery,
+      imageQuery = Option.empty[ImageQuery],
       imageQueryWiki = Some(imageQueryWiki)
     )
 
diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala
index 3e1f0577..e917b782 100644
--- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala
+++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala
@@ -34,7 +34,7 @@ class StatisticsSpec(implicit ee: ExecutionEnv)
 
     val cfg = StatConfig(campaign = contest.campaign)
 
-    new Statistics(contest, None, monumentQuery, imageQuery, None, bot, cfg)
+    new Statistics(contest, None, monumentQuery, Some(imageQuery), None, bot, cfg)
   }
 
   "give current year stat empty" in {
@@ -81,7 +81,7 @@ class StatisticsSpec(implicit ee: ExecutionEnv)
     monumentQuery.byMonumentTemplate(date = None) returns monuments
 
     val stat =
-      new Statistics(contest, None, monumentQuery, imageQuery, None, bot)
+      new Statistics(contest, None, monumentQuery, Some(imageQuery), None, bot)
 
     stat.gatherData(false) must throwA[RuntimeException].await
   }

From c99edc55165306761d77e277166c752d91f09512 Mon Sep 17 00:00:00 2001
From: ilya <intracer@gmail.com>
Date: Fri, 12 Apr 2024 17:36:46 +0300
Subject: [PATCH 2/2] fix

---
 scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala
index 77d1d8f1..cdbb001b 100644
--- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala
+++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala
@@ -32,7 +32,7 @@ case class ImageDB(
   lazy val sansIneligible: Seq[Image] =
     withCorrectIds.filterNot(_.pageId.exists(ineligibleIds.contains))
 
-  private val jun30 = ZonedDateTime.parse(s"${contest.year}-06-30T23:59:59Z")
+  private val jun30 = ZonedDateTime.parse(s"2023-06-30T23:59:59Z")
 
   lazy val ineligible: Seq[Image] = withCorrectIds.filter { i =>
     val after30 =