From 9d082502e09a2fce857a27cd951c9fbbdbd5a132 Mon Sep 17 00:00:00 2001 From: ilya Date: Sat, 16 Mar 2024 19:07:52 +0200 Subject: [PATCH] cleanup --- .travis.yml | 24 ---------- .../org/scalawiki/wlx/CountryParser.scala | 12 ++--- .../scala/org/scalawiki/wlx/ImageDB.scala | 10 ++-- .../org/scalawiki/wlx/ListFileNSRemover.scala | 16 +++---- .../scala/org/scalawiki/wlx/MonumentDB.scala | 46 ++++++------------- .../scala/org/scalawiki/wlx/RegionFixer.scala | 19 ++++---- .../org/scalawiki/wlx/SubsetCreator.scala | 8 ++-- .../org/scalawiki/wlx/WlxTableParser.scala | 2 +- .../org/scalawiki/wlx/WlxTemplateParser.scala | 36 +++++++-------- .../org/scalawiki/wlx/dto/Coordinate.scala | 2 +- 10 files changed, 65 insertions(+), 110 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 18394ceb..00000000 --- a/.travis.yml +++ /dev/null @@ -1,24 +0,0 @@ -sudo: required - -services: - - docker - -cache: - directories: - - $HOME/.m2/repository - - $HOME/.sbt - - $HOME/.ivy2 - -language: scala - -scala: - - 2.13.1 - - 2.12.10 - -jdk: - - openjdk8 - - openjdk11 - -script: "sbt ++$TRAVIS_SCALA_VERSION clean coverage 'testOnly -- xonly timefactor 5' coverageReport" -after_success: - - bash <(curl -s https://codecov.io/bash) diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CountryParser.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CountryParser.scala index f573438d..7565a2bc 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CountryParser.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CountryParser.scala @@ -12,18 +12,18 @@ import scala.util.matching.Regex.Match */ object CountryParser { - val countries = Country.fromJavaLocales + val countries: Seq[Country] = Country.fromJavaLocales - val contestCategory = "Category\\:Images from ([a-zA-Z ]+) (\\d+)" + val contestCategory: String = "Category\\:Images from ([a-zA-Z ]+) (\\d+)" - val contestRegex = (contestCategory + "$").r - val contestWithCountryRegex = + val contestRegex: Regex = (contestCategory + "$").r + val contestWithCountryRegex: Regex = (contestCategory + " in ([a-zA-Z\\&\\- ]+)($|\\|)").r - val contestLinkRegex = + val contestLinkRegex: Regex = "\\[\\[Commons\\:([a-zA-Z ]+) (\\d+) in ([a-zA-Z\\& ]+)\\|".r - def isContestCategory(s: String) = contestRegex.pattern.matcher(s).matches() + def isContestCategory(s: String): Boolean = contestRegex.pattern.matcher(s).matches() val parser = new RegexCampaignParser(contestWithCountryRegex) diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala index 59e4f70d..7249a262 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala @@ -156,7 +156,7 @@ class Grouping[T, F](name: String, val f: F => T, data: Iterable[F]) { val grouped: Map[T, Seq[F]] = data.groupBy(f).mapValues(_.toSeq).toMap - def by(key: T): Seq[F] = grouped.getOrElse(key, Seq.empty) + def by(key: T): Seq[F] = grouped.getOrElse(key, Nil) def contains(id: T): Boolean = grouped.contains(id) @@ -196,8 +196,7 @@ object ImageGrouping { def byMonument: Image => String = (i: Image) => i.monumentId.getOrElse("") - def byRegion: Image => String = (i: Image) => - Monument.getRegionId(i.monumentId) + def byRegion: Image => String = (i: Image) => Monument.getRegionId(i.monumentId) def byAuthor: Image => String = (i: Image) => i.author.getOrElse("") @@ -213,9 +212,8 @@ object ImageDB { monumentDb: Option[MonumentDB], minMpx: Option[Float] = None ): Future[ImageDB] = { - imageQuery.imagesFromCategoryAsync(contest.imagesCategory, contest).map { - images => - new ImageDB(contest, images, monumentDb, minMpx) + imageQuery.imagesFromCategoryAsync(contest.imagesCategory, contest).map { images => + new ImageDB(contest, images, monumentDb, minMpx) } } } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListFileNSRemover.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListFileNSRemover.scala index 813a940a..a1321fca 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListFileNSRemover.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListFileNSRemover.scala @@ -4,14 +4,14 @@ import org.scalawiki.MwBot import org.scalawiki.dto.markup.SwTemplate import org.scalawiki.edit.{PageUpdateTask, PageUpdater} import org.scalawiki.wikitext.SwebleParser -import org.scalawiki.wlx.dto.Monument +import org.scalawiki.wlx.dto.{Monument, UploadConfig} import org.sweble.wikitext.engine.config.WikiConfig import org.sweble.wikitext.engine.utils.DefaultConfigEnWp import org.sweble.wikitext.parser.nodes.WtTemplate object ListFileNSRemover { - def updateLists(monumentDb: MonumentDB) { + def updateLists(monumentDb: MonumentDB): Unit = { val task = new ListFileNSRemoverTask(MwBot.ukWiki, monumentDb) val updater = new PageUpdater(task) updater.update() @@ -24,17 +24,17 @@ class ListFileNSRemoverTask(val host: String, monumentDb: MonumentDB) val config: WikiConfig = DefaultConfigEnWp.generate - val titles = pagesToFix(monumentDb) + val titles: Set[String] = pagesToFix(monumentDb) - val uploadConfig = monumentDb.contest.uploadConfigs.head + val uploadConfig: UploadConfig = monumentDb.contest.uploadConfigs.head override def updatePage(title: String, pageText: String): (String, String) = { val template = uploadConfig.listTemplate val wlxParser = new WlxTemplateParser(uploadConfig.listConfig, title) var added: Int = 0 - def mapper(wtTemplate: WtTemplate) = { - val swTemplate = new SwTemplate(wtTemplate) + def mapper(wtTemplate: WtTemplate): Unit = { + val swTemplate = SwTemplate(wtTemplate) val monument = wlxParser.templateToMonument(swTemplate.template) if (needsUpdate(monument)) { @@ -56,9 +56,7 @@ class ListFileNSRemoverTask(val host: String, monumentDb: MonumentDB) } def needsUpdate(m: Monument): Boolean = - m.photo.exists(photo => - photo.trim.startsWith("File:") || photo.trim.startsWith("Файл:") - ) + m.photo.exists(photo => photo.trim.startsWith("File:") || photo.trim.startsWith("Файл:")) def pagesToFix(monumentDb: MonumentDB): Set[String] = { diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/MonumentDB.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/MonumentDB.scala index c7dbfb02..7c1ee512 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/MonumentDB.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/MonumentDB.scala @@ -1,9 +1,8 @@ package org.scalawiki.wlx import java.time.{ZoneOffset, ZonedDateTime} - import org.scalawiki.dto.markup.Table -import org.scalawiki.wlx.dto.{AdmDivision, Contest, Monument} +import org.scalawiki.wlx.dto.{AdmDivision, Contest, Country, Monument} import org.scalawiki.wlx.query.MonumentQuery import org.scalawiki.wlx.stat.rating.PerPlaceStat @@ -13,12 +12,12 @@ class MonumentDB( withFalseIds: Boolean = true ) { - val monuments = allMonuments.filter(m => withFalseIds || isIdCorrect(m.id)) - val wrongIdMonuments = allMonuments.filterNot(m => isIdCorrect(m.id)) - val withArticles = allMonuments + val monuments: Seq[Monument] = allMonuments.filter(m => withFalseIds || isIdCorrect(m.id)) + val wrongIdMonuments: Seq[Monument] = allMonuments.filterNot(m => isIdCorrect(m.id)) + val withArticles: Map[String, Seq[Monument]] = allMonuments .filter(m => m.name.contains("[[")) .groupBy(m => Monument.getRegionId(m.id)) - val country = contest.country + val country: Country = contest.country val _byId: Map[String, Iterable[Monument]] = monuments.groupBy(_.id) val _byRegion: Map[String, Iterable[Monument]] = @@ -37,9 +36,9 @@ class MonumentDB( def ids: Set[String] = _byId.keySet - def byId(id: String) = _byId.getOrElse(id, Seq.empty[Monument]).headOption + def byId(id: String): Option[Monument] = _byId.getOrElse(id, Seq.empty[Monument]).headOption - def byRegion(regId: String) = { + def byRegion(regId: String): Iterable[Monument] = { if (regId.length == 2) { _byRegion.getOrElse(regId, Seq.empty[Monument]) } else { @@ -49,21 +48,21 @@ class MonumentDB( } } - def regionIds = + def regionIds: Seq[String] = _byRegion.keySet.toSeq.filter(contest.country.regionIds.contains).sorted - def isIdCorrect(id: String) = { + def isIdCorrect(id: String): Boolean = { val idRegex = """(\d\d)-(\d\d\d)-(\d\d\d\d)""" id.matches(idRegex) && contest.country.regionIds.contains( Monument.getRegionId(id) ) } - def withImages = monuments.filter(_.photo.isDefined) + def withImages: Seq[Monument] = monuments.filter(_.photo.isDefined) - def picturedIds = withImages.map(_.id).toSet + def picturedIds: Set[String] = withImages.map(_.id).toSet - def picturedInRegion(regionId: String) = + def picturedInRegion(regionId: String): Set[String] = byRegion(regionId).map(_.id).toSet intersect picturedIds def getAdmDivision(monumentId: String): Option[AdmDivision] = { @@ -155,7 +154,7 @@ case class UnknownPlace( def parents: Set[String] = candidates.map(_.parent().map(_.name).getOrElse("")).toSet - override def toString = { + override def toString: String = { val candidatesStr = candidates .map { c => c.parent() @@ -163,6 +162,7 @@ case class UnknownPlace( .getOrElse("") + s"${c.name}(${c.code})" } .mkString(", ") + s"* [[$page]]/$regionId/$name. \n** ${monuments.size} monument(s): ${monuments .map(_.id) .mkString(", ")}" + (if (candidates.nonEmpty) @@ -181,8 +181,7 @@ object MonumentDB { var allMonuments = monumentQuery.byMonumentTemplate(date = date) if (contest.country.code == "ru") { - allMonuments = - allMonuments.filter(_.page.contains("Природные памятники России")) + allMonuments = allMonuments.filter(_.page.contains("Природные памятники России")) } new MonumentDB(contest, allMonuments.toSeq) @@ -191,19 +190,4 @@ object MonumentDB { def getMonumentDb(contest: Contest, date: Option[ZonedDateTime]): MonumentDB = getMonumentDb(contest, MonumentQuery.create(contest), date) - def getMonumentDbRange( - contest: Contest - ): (Option[MonumentDB], Option[MonumentDB]) = { - if (contest.uploadConfigs.nonEmpty) { - val date = - ZonedDateTime.of(contest.year, 9, 1, 0, 0, 0, 0, ZoneOffset.UTC) - ( - Some(getMonumentDb(contest, None)), - Some(getMonumentDb(contest, Some(date))) - ) - } else { - (None, None) - } - } - } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/RegionFixer.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/RegionFixer.scala index 81a87212..ee3c82c1 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/RegionFixer.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/RegionFixer.scala @@ -1,25 +1,24 @@ package org.scalawiki.wlx -import org.scalawiki.wlx.dto.Monument +import org.scalawiki.wlx.dto.{AdmDivision, Contest, Country, Monument} object RegionFixer { - def fixLists(monumentDb: MonumentDB) { + def fixLists(monumentDb: MonumentDB): Unit = { ListUpdater.updateLists(monumentDb, new RegionFixerUpdater(monumentDb)) } } class RegionFixerUpdater(monumentDb: MonumentDB) extends MonumentUpdater { - val contest = monumentDb.contest - val country = contest.country - val oblasts = country.regions.filter(adm => - !Set("Київ", "Севастополь").contains(adm.name) - ) - val raions = oblasts.flatMap(_.regions).filter(_.name.endsWith("район")) - val raionNames = raions.map(_.name).toSet + val contest: Contest = monumentDb.contest + val country: Country = contest.country + val oblasts: Seq[AdmDivision] = + country.regions.filter(adm => !Set("Київ", "Севастополь").contains(adm.name)) + val raions: Seq[AdmDivision] = oblasts.flatMap(_.regions).filter(_.name.endsWith("район")) + val raionNames: Set[String] = raions.map(_.name).toSet - val nameParam = contest.uploadConfigs.head.listConfig.namesMap("city") + val nameParam: String = contest.uploadConfigs.head.listConfig.namesMap("city") val maxIndex = 2 def updatedParams(m: Monument): Map[String, String] = { diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/SubsetCreator.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/SubsetCreator.scala index 445e88aa..02342d0a 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/SubsetCreator.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/SubsetCreator.scala @@ -10,7 +10,7 @@ object SubsetCreator { import scala.concurrent.ExecutionContext.Implicits.global - def main(args: Array[String]) { + def main(args: Array[String]): Unit = { val specialNomination = "конкурс статей" @@ -34,7 +34,7 @@ object SubsetCreator { contest: Contest, specialNomination: String, monumentFilter: Monument => Boolean - ) = { + ): Unit = { val ukWiki = MwBot.fromHost("uk.wikipedia.org") val subset = monuments.filter(monumentFilter) @@ -62,7 +62,7 @@ object SubsetCreator { contest: Contest, byRegion: Map[String, Seq[Monument]], regionIds: SortedSet[String] - ) { + ): Unit = { val buf = new StringBuffer buf.append("__TOC__\n") @@ -112,7 +112,7 @@ object SubsetCreator { contest: Contest, byRegion: Map[String, Iterable[Monument]], regionIds: SortedSet[String] - ) { + ): Unit = { for (regionId <- regionIds) { val regionTitle = contest.country.regionName(regionId) diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/WlxTableParser.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/WlxTableParser.scala index da6f96e7..1d0b7e04 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/WlxTableParser.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/WlxTableParser.scala @@ -15,7 +15,7 @@ class WlxTableParser(val config: ListConfig) { val headers = table.headers.toIndexedSeq - def getIndex(name: String) = { + def getIndex(name: String): Option[Int] = { config.namesMap.get(name).flatMap { mappedName => val index = headers.indexOf(mappedName) if (index >= 0) diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/WlxTemplateParser.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/WlxTemplateParser.scala index c7fbe952..5b5d6ef3 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/WlxTemplateParser.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/WlxTemplateParser.scala @@ -11,25 +11,25 @@ import org.scalawiki.wlx.dto.lists.ListConfig */ class WlxTemplateParser(val config: ListConfig, val page: String) { - def getMappedName(name: String) = config.namesMap.get(name) + private def getMappedName(name: String): Option[String] = config.namesMap.get(name) - val id = getMappedName("ID") - val name = getMappedName("name") - val year = getMappedName("year") - val description = getMappedName("description") - val city = getMappedName("city") - val cityType = getMappedName("cityType") - val place = getMappedName("place") - val user = getMappedName("user") - val area = getMappedName("area") - val lat = getMappedName("lat") - val lon = getMappedName("lon") - val image = getMappedName("photo") - val gallery = getMappedName("gallery") - val stateId = getMappedName("stateId") - val typ = getMappedName("type") - val subType = getMappedName("subType") - val resolution = getMappedName("resolution") + private val id = getMappedName("ID") + private val name = getMappedName("name") + private val year = getMappedName("year") + private val description = getMappedName("description") + private val city = getMappedName("city") + private val cityType = getMappedName("cityType") + private val place = getMappedName("place") + private val user = getMappedName("user") + private val area = getMappedName("area") + private val lat = getMappedName("lat") + private val lon = getMappedName("lon") + private val image = getMappedName("photo") + private val gallery = getMappedName("gallery") + private val stateId = getMappedName("stateId") + private val typ = getMappedName("type") + private val subType = getMappedName("subType") + private val resolution = getMappedName("resolution") def parse(wiki: String): Iterable[Monument] = { val templates = TemplateParser.parse(wiki, config.templateName) diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Coordinate.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Coordinate.scala index 5a5a82bf..33f08660 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Coordinate.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Coordinate.scala @@ -6,7 +6,7 @@ case class Coordinate(lat: String, lon: String) { } object Coordinate { - def apply(latOpt: Option[String], lonOpt: Option[String]) = for ( + def apply(latOpt: Option[String], lonOpt: Option[String]): Option[Coordinate] = for ( lat <- latOpt; lon <- lonOpt ) yield new Coordinate(lat, lon)