From 2b7c8bc5f30a6c124b70c00bc4362dda112c7c50 Mon Sep 17 00:00:00 2001 From: Scala Steward Date: Mon, 19 Feb 2024 21:13:31 +0000 Subject: [PATCH] Reformat with scalafmt 3.8.0 Executed command: scalafmt --non-interactive --- build.sbt | 83 +- .../net/spraycookies/CookieHandling.scala | 43 +- .../scala/net/spraycookies/CookieJar.scala | 72 +- .../tldlist/EffectiveTldList.scala | 11 +- .../net/spraycookies/tldlist/TldTrie.scala | 38 +- .../spraycookies/CookieJarSpecification.scala | 105 +- .../EffectiveTldListSpecification.scala | 3 +- project/CompatibleJavaVersion.scala | 5 +- project/plugins.sbt | 2 +- .../org/scalawiki/bots/CsvComparator.scala | 42 +- .../scala/org/scalawiki/bots/FileUtils.scala | 68 +- .../org/scalawiki/bots/GlobalContrib.scala | 56 +- .../scala/org/scalawiki/bots/MessageBot.scala | 40 +- .../org/scalawiki/bots/PageFromFileBot.scala | 59 +- .../org/scalawiki/bots/PageGenerators.scala | 103 +- .../org/scalawiki/bots/PartnersBot.scala | 24 +- .../org/scalawiki/bots/RedLinksBot.scala | 19 +- .../scala/org/scalawiki/bots/Replace.scala | 105 +- .../org/scalawiki/bots/ShortLinksBot.scala | 54 +- .../main/scala/org/scalawiki/bots/Spain.scala | 86 +- .../scalawiki/bots/edu/InstitutionList.scala | 31 +- .../scalawiki/bots/finance/Resolution.scala | 69 +- .../scalawiki/bots/finance/SecretaryBot.scala | 11 +- .../scalawiki/bots/finance/UserGroup.scala | 8 +- .../org/scalawiki/bots/finance/Votes.scala | 12 +- .../org/scalawiki/bots/museum/Entry.scala | 175 ++-- .../scalawiki/bots/museum/HtmlOutput.scala | 15 +- .../scalawiki/bots/museum/HtmlParser.scala | 28 +- .../bots/museum/ImageListParser.scala | 43 +- .../scalawiki/bots/museum/ImageTemplate.scala | 19 +- .../scalawiki/bots/museum/Pereiaslav.scala | 75 +- .../scala/org/scalawiki/bots/np/TTN.scala | 26 +- .../org/scalawiki/bots/np/TTNReader.scala | 16 +- .../org/scalawiki/bots/stat/Annotation.scala | 39 +- .../org/scalawiki/bots/stat/ArticleStat.scala | 9 +- .../scalawiki/bots/stat/ArticleStatBot.scala | 49 +- .../org/scalawiki/bots/stat/CatScan.scala | 113 +- .../scala/org/scalawiki/bots/stat/Event.scala | 16 +- .../bots/stat/NumericArrayStat.scala | 5 +- .../bots/stat/RevisionAnnotation.scala | 19 +- .../scalawiki/bots/stat/RevisionStat.scala | 42 +- .../scalawiki/bots/stat/UserContribList.scala | 46 +- .../org/scalawiki/bots/stat/UserStat.scala | 27 +- .../bots/stat/zachte/UkWikiStat.scala | 15 +- .../org/scalawiki/bots/vote/VoteList.scala | 11 +- .../scala/org/scalawiki/copyvio/CopyVio.scala | 39 +- .../org/scalawiki/copyvio/CopyVioSource.scala | 10 +- .../scala/util/matching/RegexFactory.scala | 2 +- .../org/scalawiki/bots/FileUtilsSpec.scala | 2 +- .../org/scalawiki/bots/MessageBotSpec.scala | 5 +- .../scalawiki/bots/PageFromFileBotSpec.scala | 32 +- .../scalawiki/bots/PageGeneratorsSpec.scala | 2 +- .../org/scalawiki/bots/ReplaceSpec.scala | 25 +- .../org/scalawiki/bots/museum/EntrySpec.scala | 191 +++- .../bots/museum/HtmlParserSpec.scala | 13 +- .../bots/museum/ImageTemplateSpec.scala | 4 +- .../bots/museum/PereiaslavSpec.scala | 48 +- .../scalawiki/bots/np/WlmContactsSpec.scala | 36 +- .../scalawiki/bots/stat/AnnotationSpec.scala | 1 - .../bots/stat/ArticleStatBotSpec.scala | 49 +- .../bots/stat/RevisionStatSpec.scala | 74 +- .../org/scalawiki/BaseIntegrationSpec.scala | 9 +- .../scalawiki/EditTokensIntegrationTest.scala | 3 +- .../scala/org/scalawiki/IntegrationSpec.scala | 18 +- .../scala/org/scalawiki/ActionLibrary.scala | 21 +- .../main/scala/org/scalawiki/LoginInfo.scala | 14 +- .../src/main/scala/org/scalawiki/MwBot.scala | 176 ++-- .../main/scala/org/scalawiki/MwUtils.scala | 16 +- .../main/scala/org/scalawiki/WithBot.scala | 1 - .../scala/org/scalawiki/cache/CachedBot.scala | 46 +- .../scala/org/scalawiki/dto/Contributor.scala | 11 +- .../org/scalawiki/dto/GlobalUserInfo.scala | 28 +- .../main/scala/org/scalawiki/dto/Image.scala | 97 +- .../org/scalawiki/dto/IpContributor.scala | 8 +- .../org/scalawiki/dto/LoginResponse.scala | 16 +- .../scala/org/scalawiki/dto/MwException.scala | 11 +- .../main/scala/org/scalawiki/dto/Page.scala | 92 +- .../scala/org/scalawiki/dto/PageList.scala | 13 +- .../scala/org/scalawiki/dto/Revision.scala | 58 +- .../main/scala/org/scalawiki/dto/Site.scala | 41 +- .../main/scala/org/scalawiki/dto/User.scala | 21 +- .../scala/org/scalawiki/dto/UserContrib.scala | 44 +- .../scala/org/scalawiki/dto/cmd/Cmd.scala | 56 +- .../org/scalawiki/dto/cmd/WatchParam.scala | 56 +- .../org/scalawiki/dto/cmd/edit/Edit.scala | 210 ++-- .../scalawiki/dto/cmd/email/EmailUser.scala | 30 +- .../scalawiki/dto/cmd/query/Generator.scala | 16 +- .../org/scalawiki/dto/cmd/query/Module.scala | 7 +- .../org/scalawiki/dto/cmd/query/Query.scala | 43 +- .../dto/cmd/query/list/AllUsers.scala | 96 +- .../dto/cmd/query/list/CategoryMembers.scala | 142 ++- .../dto/cmd/query/list/EmbeddedIn.scala | 72 +- .../dto/cmd/query/list/ListArgs.scala | 9 +- .../dto/cmd/query/list/ListParam.scala | 20 +- .../dto/cmd/query/list/UserContribs.scala | 82 +- .../scalawiki/dto/cmd/query/list/Users.scala | 108 +- .../dto/cmd/query/meta/GlobalUserInfo.scala | 84 +- .../dto/cmd/query/meta/MetaParam.scala | 4 +- .../dto/cmd/query/prop/Categories.scala | 25 +- .../dto/cmd/query/prop/CategoryInfo.scala | 16 +- .../dto/cmd/query/prop/ImageInfo.scala | 348 +++++-- .../scalawiki/dto/cmd/query/prop/Images.scala | 28 +- .../scalawiki/dto/cmd/query/prop/Info.scala | 114 +- .../dto/cmd/query/prop/LangLinks.scala | 37 +- .../scalawiki/dto/cmd/query/prop/Links.scala | 48 +- .../scalawiki/dto/cmd/query/prop/Prop.scala | 15 +- .../dto/cmd/query/prop/Revisions.scala | 310 ++++-- .../dto/cmd/query/prop/RvPropArgs.scala | 18 +- .../org/scalawiki/dto/cmd/upload/Upload.scala | 152 ++- .../org/scalawiki/dto/filter/PageFilter.scala | 10 +- .../scalawiki/dto/filter/RevisionFilter.scala | 30 +- .../org/scalawiki/dto/history/History.scala | 24 +- .../org/scalawiki/dto/markup/Gallery.scala | 71 +- .../scalawiki/dto/markup/NodeFactory.scala | 3 +- .../org/scalawiki/dto/markup/SwNode.scala | 1 - .../org/scalawiki/dto/markup/SwTemplate.scala | 47 +- .../org/scalawiki/dto/markup/Table.scala | 43 +- .../org/scalawiki/dto/markup/Template.scala | 10 +- .../org/scalawiki/edit/PageUpdater.scala | 7 +- .../scala/org/scalawiki/http/HttpClient.scala | 21 +- .../org/scalawiki/http/HttpClientAkka.scala | 51 +- .../scala/org/scalawiki/json/MwReads.scala | 18 +- .../scala/org/scalawiki/json/Parser.scala | 137 ++- .../scala/org/scalawiki/json/WikiReads.scala | 167 ++- .../scala/org/scalawiki/query/DslQuery.scala | 87 +- .../scala/org/scalawiki/query/PageQuery.scala | 3 +- .../scalawiki/query/PageQueryImplDsl.scala | 103 +- .../org/scalawiki/query/QueryLibrary.scala | 217 ++-- .../org/scalawiki/query/SinglePageQuery.scala | 41 +- .../scala/org/scalawiki/time/TimeRange.scala | 2 +- .../org/scalawiki/wikitext/SwebleParser.scala | 26 +- .../org/scalawiki/wikitext/TableParser.scala | 24 +- .../scalawiki/wikitext/TemplateParser.scala | 35 +- .../src/main/scala/spray/util.scala | 3 +- .../scala/spray/util/pimps/PimpedFuture.scala | 2 +- .../test/scala/org/scalawiki/DockerSpec.scala | 13 +- .../test/scala/org/scalawiki/EditSpec.scala | 86 +- .../test/scala/org/scalawiki/LoginSpec.scala | 34 +- .../test/scala/org/scalawiki/MwBotSpec.scala | 22 +- .../scala/org/scalawiki/dto/ImageSpec.scala | 48 +- .../scala/org/scalawiki/dto/SiteSpec.scala | 26 +- .../scala/org/scalawiki/dto/cmd/DslSpec.scala | 25 +- .../scalawiki/dto/cmd/query/QuerySpec.scala | 6 +- .../cmd/query/list/CategoryMembersSpec.scala | 17 +- .../dto/history/RevisionFilterSpec.scala | 13 +- .../dto/markup/GalleryHtmlSpec.scala | 16 +- .../dto/markup/GalleryThumbSpec.scala | 112 +- .../dto/markup/GalleryWikiSpec.scala | 9 +- .../org/scalawiki/dto/markup/LineUtil.scala | 4 +- .../scalawiki/dto/markup/TableHtmlSpec.scala | 42 +- .../scalawiki/dto/markup/TableWikiSpec.scala | 6 +- .../scala/org/scalawiki/json/ParserSpec.scala | 47 +- .../org/scalawiki/json/WikiReadsSpec.scala | 32 +- .../mockserver/BaseMockServerSpec.scala | 65 +- .../ImagesEmbeddedInMockServerSpec.scala | 30 +- .../mockserver/ImagesMockServerSpec.scala | 33 +- .../mockserver/LoginMockServerSpec.scala | 7 +- .../LongRevisionsMockServerSpec.scala | 3 +- .../org/scalawiki/mockserver/StubServer.scala | 4 +- .../scalawiki/parser/TableParserSpec.scala | 42 +- .../scalawiki/parser/TemplateParserSpec.scala | 22 +- .../scalawiki/query/ListAllUsersSpec.scala | 57 +- .../query/ListCategoryMembersSpec.scala | 58 +- .../query/ListUserContribsSpec.scala | 26 +- .../org/scalawiki/query/ListUsersSpec.scala | 67 +- .../query/MetaGlobalUserInfoSpec.scala | 22 +- .../org/scalawiki/query/PageQuerySpec.scala | 91 +- .../scalawiki/query/PropEmbeddedInSpec.scala | 52 +- .../scalawiki/query/PropImageInfoSpec.scala | 167 ++- .../org/scalawiki/query/PropImagesSpec.scala | 63 +- .../scalawiki/query/PropLangLinksSpec.scala | 47 +- .../org/scalawiki/query/PropLinksSpec.scala | 54 +- .../scalawiki/query/PropRevisionsSpec.scala | 78 +- .../scalawiki/time/DateTimeReadersSpec.scala | 14 +- .../scala/org/scalawiki/util/HttpStub.scala | 10 +- .../org/scalawiki/util/TestHttpClient.scala | 53 +- .../scala/org/scalawiki/util/TestUtils.scala | 8 +- .../org/scalawiki/compress/Compression.scala | 23 +- .../main/scala/org/scalawiki/xml/Block.scala | 1 - .../scala/org/scalawiki/xml/DumpFile.scala | 29 +- .../scala/org/scalawiki/xml/PageIndex.scala | 3 +- .../scalawiki/xml/SkippingInputStream.scala | 18 +- .../scala/org/scalawiki/xml/XmlIndex.scala | 6 +- .../scala/org/scalawiki/xml/XmlParser.scala | 128 ++- .../scala/org/scalawiki/xml/XmlWriter.scala | 31 +- .../org/scalawiki/xml/DataDumpSpec.scala | 15 +- .../xml/SkippingInputStreamSpec.scala | 84 +- .../scala/org/scalawiki/xml/XmlHelper.scala | 21 +- .../org/scalawiki/xml/XmlIndexSpec.scala | 3 +- .../org/scalawiki/xml/XmlParserSpec.scala | 364 ++++++- .../org/scalawiki/xml/XmlWriterSpec.scala | 49 +- .../scala/org/scalawiki/sql/Categories.scala | 69 +- .../org/scalawiki/sql/DslQueryDbCache.scala | 105 +- .../main/scala/org/scalawiki/sql/Images.scala | 204 ++-- .../scala/org/scalawiki/sql/MwDatabase.scala | 57 +- .../main/scala/org/scalawiki/sql/Pages.scala | 144 +-- .../scala/org/scalawiki/sql/Revisions.scala | 171 ++- .../main/scala/org/scalawiki/sql/Texts.scala | 78 +- .../main/scala/org/scalawiki/sql/Users.scala | 168 ++- .../org/scalawiki/sql/dao/ImageDao.scala | 7 +- .../scala/org/scalawiki/sql/dao/PageDao.scala | 129 ++- .../org/scalawiki/sql/dao/RevisionDao.scala | 28 +- .../scala/org/scalawiki/sql/dao/TextDao.scala | 6 +- .../scala/org/scalawiki/sql/dao/UserDao.scala | 9 +- .../sql/DslQueryDbCacheBlackBoxSpec.scala | 633 +++++++++--- .../sql/DslQueryDbCacheModularSpec.scala | 257 +++-- .../org/scalawiki/sql/ImageDaoSpec.scala | 26 +- .../org/scalawiki/sql/MwDatabaseSpec.scala | 13 +- .../org/scalawiki/sql/PageDaoBulkSpec.scala | 63 +- .../scala/org/scalawiki/sql/PageDaoSpec.scala | 75 +- .../scala/org/scalawiki/sql/UserDaoSpec.scala | 14 +- .../org/scalawiki/wlx/CampaignList.scala | 40 +- .../scala/org/scalawiki/wlx/Checker.scala | 10 +- .../org/scalawiki/wlx/CountryParser.scala | 62 +- .../scala/org/scalawiki/wlx/ImageDB.scala | 57 +- .../wlx/KatotthMonumentListCreator.scala | 117 ++- .../org/scalawiki/wlx/ListFileNSRemover.scala | 17 +- .../scala/org/scalawiki/wlx/ListUpdater.scala | 14 +- .../scala/org/scalawiki/wlx/MonumentDB.scala | 143 ++- .../scala/org/scalawiki/wlx/RegionFixer.scala | 22 +- .../org/scalawiki/wlx/SubsetCreator.scala | 80 +- .../org/scalawiki/wlx/WlxTableParser.scala | 56 +- .../org/scalawiki/wlx/WlxTemplateParser.scala | 19 +- .../scalawiki/wlx/dto/AdmDivisionFlat.scala | 62 +- .../org/scalawiki/wlx/dto/Campaign.scala | 20 +- .../scala/org/scalawiki/wlx/dto/Contest.scala | 96 +- .../org/scalawiki/wlx/dto/ContestType.scala | 3 +- .../org/scalawiki/wlx/dto/Coordinate.scala | 4 +- .../scala/org/scalawiki/wlx/dto/Country.scala | 181 ++-- .../scala/org/scalawiki/wlx/dto/Katotth.scala | 36 +- .../scala/org/scalawiki/wlx/dto/Koatuu.scala | 68 +- .../org/scalawiki/wlx/dto/KoatuuNew.scala | 9 +- .../org/scalawiki/wlx/dto/Monument.scala | 108 +- .../scalawiki/wlx/dto/SpecialNomination.scala | 128 ++- .../org/scalawiki/wlx/dto/UploadConfig.scala | 27 +- .../scalawiki/wlx/dto/lists/ListConfig.scala | 21 +- .../org/scalawiki/wlx/query/ImageQuery.scala | 56 +- .../scalawiki/wlx/query/MonumentQuery.scala | 173 +++- .../org/scalawiki/wlx/stat/RateRanges.scala | 55 +- .../scala/org/scalawiki/wlx/stat/Rater.scala | 337 ++++-- .../org/scalawiki/wlx/stat/Statistics.scala | 245 +++-- .../scala/org/scalawiki/wlx/stat/Stats.scala | 17 +- .../scalawiki/wlx/stat/generic/Records.scala | 46 +- .../wlx/stat/reports/AuthorMonuments.scala | 98 +- .../scalawiki/wlx/stat/reports/Charts.scala | 127 ++- .../DesnaRegionSpecialNomination.scala | 80 +- .../scalawiki/wlx/stat/reports/Gallery.scala | 51 +- .../wlx/stat/reports/MonumentDbStat.scala | 3 +- .../reports/MonumentsPicturedByRegion.scala | 130 ++- .../stat/reports/MostPopularMonuments.scala | 103 +- .../wlx/stat/reports/NewMonuments.scala | 7 +- .../scalawiki/wlx/stat/reports/Output.scala | 423 +++++--- .../stat/reports/RateInputDistribution.scala | 21 +- .../wlx/stat/reports/ReporterRegistry.scala | 54 +- .../wlx/stat/reports/SpecialNominations.scala | 86 +- .../org/scalawiki/wlx/CampaignListSpec.scala | 82 +- .../scalawiki/wlx/CountryListParserSpec.scala | 50 +- .../org/scalawiki/wlx/DesnaRegionSpec.scala | 15 +- .../scala/org/scalawiki/wlx/ImageDbSpec.scala | 180 +++- .../org/scalawiki/wlx/ImageFillerSpec.scala | 168 ++- .../scalawiki/wlx/ListFileNSRemoverSpec.scala | 61 +- .../org/scalawiki/wlx/MonumentDbSpec.scala | 28 +- .../org/scalawiki/wlx/MonumentSpec.scala | 49 +- .../org/scalawiki/wlx/RegionFixerSpec.scala | 16 +- .../org/scalawiki/wlx/TestListConfigs.scala | 1 - .../org/scalawiki/wlx/UnknownPlacesSpec.scala | 41 +- .../org/scalawiki/wlx/WlmUaListsSpec.scala | 11 +- .../scalawiki/wlx/WlxTableParserSpec.scala | 5 +- .../scalawiki/wlx/WlxTemplateParserSpec.scala | 9 +- .../org/scalawiki/wlx/dto/CampaignSpec.scala | 10 +- .../org/scalawiki/wlx/dto/CountrySpec.scala | 3 +- .../org/scalawiki/wlx/dto/KatotthSpec.scala | 103 +- .../scalawiki/wlx/dto/KoatuuComparator.scala | 8 +- .../wlx/dto/KoatuuFlatParserSpec.scala | 37 +- .../org/scalawiki/wlx/dto/KoatuuNewSpec.scala | 190 +++- .../org/scalawiki/wlx/dto/KoatuuSpec.scala | 200 +++- .../wlx/dto/KoatuuToKatotthSpec.scala | 6 +- .../wlx/stat/AuthorsMonumentsSpec.scala | 973 ++++++++++-------- .../org/scalawiki/wlx/stat/ChartsSpec.scala | 36 +- .../org/scalawiki/wlx/stat/GroupingSpec.scala | 20 +- .../stat/MonumentsPicturedByRegionSpec.scala | 213 +++- .../wlx/stat/MostPopularMonumentsSpec.scala | 192 ++-- .../wlx/stat/NumberOfAuthorsBonusSpec.scala | 44 +- .../scalawiki/wlx/stat/RateRangesSpec.scala | 22 +- .../org/scalawiki/wlx/stat/RaterSpec.scala | 24 +- .../scalawiki/wlx/stat/StatParamsSpec.scala | 119 ++- .../scalawiki/wlx/stat/StatisticsSpec.scala | 80 +- .../wlx/stat/generic/MonumentStatSpec.scala | 40 +- .../wlx/stat/generic/RecordsSpec.scala | 2 - 289 files changed, 11878 insertions(+), 5877 deletions(-) diff --git a/build.sbt b/build.sbt index 1f21ee55..6ff0a94a 100644 --- a/build.sbt +++ b/build.sbt @@ -15,9 +15,7 @@ lazy val commonSettings = Seq( scalacOptions := Seq("-target:jvm-1.8"), // conflictManager := ConflictManager.strict, licenses += ("Apache-2.0", url("http://opensource.org/licenses/Apache-2.0")), - Keys.resolvers ++= Dependencies.resolvers, - libraryDependencies ++= Seq( Library.Specs2.core % Test, Library.Specs2.matcherExtra % Test, @@ -25,22 +23,22 @@ lazy val commonSettings = Seq( "com.google.jimfs" % "jimfs" % JimFsV % Test, "org.mock-server" % "mockserver-netty" % MockServerV % Test ), - dependencyOverrides ++= Dependencies.overrides, - initialize := { val _ = initialize.value val required = VersionNumber("1.8") val curr = VersionNumber(sys.props("java.specification.version")) - assert(CompatibleJavaVersion(curr, required), s"Java $required or above required") + assert( + CompatibleJavaVersion(curr, required), + s"Java $required or above required" + ) }, - assembly / assemblyJarName := { s"${name.value}-${version.value}.jar" }, assembly / test := {}, assembly / assemblyMergeStrategy := { - case PathList("org", "xmlpull", "v1", xs@_*) => MergeStrategy.first + case PathList("org", "xmlpull", "v1", xs @ _*) => MergeStrategy.first case x => val oldStrategy = (assembly / assemblyMergeStrategy).value oldStrategy(x) @@ -65,7 +63,7 @@ lazy val core = Project("scalawiki-core", file("scalawiki-core")) "com.iheart" %% "ficus" % FicusV, "jp.ne.opt" %% "chronoscala" % ChronoScalaV, "ch.qos.logback" % "logback-classic" % LogbackClassicV, - "org.sweble.wikitext" % "swc-engine" % SwcEngineV exclude("org.jsoup", "jsoup"), + "org.sweble.wikitext" % "swc-engine" % SwcEngineV exclude ("org.jsoup", "jsoup"), "javax.xml.bind" % "jaxb-api" % "2.3.1", "de.fau.cs.osr.ptk" % "ptk-common" % "3.0.8", Library.Commons.codec, @@ -74,34 +72,39 @@ lazy val core = Project("scalawiki-core", file("scalawiki-core")) "net.openhft" % "chronicle-map" % ChronicleMapV, "org.rogach" %% "scallop" % ScallopV ) - }).dependsOn(`http-extensions`) + }) + .dependsOn(`http-extensions`) lazy val bots = Project("scalawiki-bots", file("scalawiki-bots")) .dependsOn(core % "compile->compile;test->test", wlx) .settings(commonSettings: _*) - .settings(libraryDependencies ++= Seq( - "com.github.pathikrit" %% "better-files" % BetterFilesV, - "org.rogach" %% "scallop" % ScallopV, - "org.xwiki.commons" % "xwiki-commons-blame-api" % BlameApiV, - Library.Commons.io, - Library.Poi.scratchpad, - Library.Poi.ooxml, - Library.Poi.ooxmlFull, - Library.Poi.poi, - Library.Poi.converter, - Library.Play.twirlApi(isScala213.value), - "com.github.tototoshi" %% "scala-csv" % ScalaCsvV - )) + .settings( + libraryDependencies ++= Seq( + "com.github.pathikrit" %% "better-files" % BetterFilesV, + "org.rogach" %% "scallop" % ScallopV, + "org.xwiki.commons" % "xwiki-commons-blame-api" % BlameApiV, + Library.Commons.io, + Library.Poi.scratchpad, + Library.Poi.ooxml, + Library.Poi.ooxmlFull, + Library.Poi.poi, + Library.Poi.converter, + Library.Play.twirlApi(isScala213.value), + "com.github.tototoshi" %% "scala-csv" % ScalaCsvV + ) + ) .enablePlugins(SbtTwirl) lazy val dumps = Project("scalawiki-dumps", file("scalawiki-dumps")) .dependsOn(core % "compile->compile;test->test") .settings(commonSettings: _*) - .settings(libraryDependencies ++= - Seq("com.fasterxml" % "aalto-xml" % AaltoXmlV, - Library.Commons.compress, - "org.glassfish.jaxb" % "txw2" % "3.0.2" // scala-steward:off - ) + .settings( + libraryDependencies ++= + Seq( + "com.fasterxml" % "aalto-xml" % AaltoXmlV, + Library.Commons.compress, + "org.glassfish.jaxb" % "txw2" % "3.0.2" // scala-steward:off + ) ) lazy val wlx = Project("scalawiki-wlx", file("scalawiki-wlx")) @@ -118,17 +121,21 @@ lazy val wlx = Project("scalawiki-wlx", file("scalawiki-wlx")) lazy val sql = Project("scalawiki-sql", file("scalawiki-sql")) .dependsOn(core % "compile->compile;test->test") .settings(commonSettings: _*) - .settings(libraryDependencies ++= Seq( - Library.Slick.slick, - Library.Slick.hikaricp, - "com.h2database" % "h2" % H2V - )) + .settings( + libraryDependencies ++= Seq( + Library.Slick.slick, + Library.Slick.hikaricp, + "com.h2database" % "h2" % H2V + ) + ) lazy val `http-extensions` = (project in file("http-extensions")) .settings(commonSettings: _*) - .settings(libraryDependencies ++= Seq( - Library.Akka.actor, - Library.Akka.stream, - Library.Akka.http, - "org.scalacheck" %% "scalacheck" % ScalaCheckV % Test - )) \ No newline at end of file + .settings( + libraryDependencies ++= Seq( + Library.Akka.actor, + Library.Akka.stream, + Library.Akka.http, + "org.scalacheck" %% "scalacheck" % ScalaCheckV % Test + ) + ) diff --git a/http-extensions/src/main/scala/net/spraycookies/CookieHandling.scala b/http-extensions/src/main/scala/net/spraycookies/CookieHandling.scala index e4a15326..ae3e81d9 100644 --- a/http-extensions/src/main/scala/net/spraycookies/CookieHandling.scala +++ b/http-extensions/src/main/scala/net/spraycookies/CookieHandling.scala @@ -8,17 +8,20 @@ import scala.concurrent.{ExecutionContext, Future} object CookieHandling { - def withCookies(cookieSource: Option[CookieJar], cookieTarget: Option[CookieJar]) - (innerPipeline: HttpRequest ⇒ Future[HttpResponse]) - (implicit context: ExecutionContext) = { - req: HttpRequest ⇒ - { - val cookiedReq = cookieSource.foldLeft(req)((_, jar) ⇒ addCookies(jar)(req)) - val fResp = innerPipeline(cookiedReq) - fResp.map(res ⇒ { - cookieTarget.foldLeft(res)((_, jar) ⇒ storeCookies(jar, req.uri)(res)) - }) - } + def withCookies( + cookieSource: Option[CookieJar], + cookieTarget: Option[CookieJar] + )( + innerPipeline: HttpRequest ⇒ Future[HttpResponse] + )(implicit context: ExecutionContext) = { req: HttpRequest ⇒ + { + val cookiedReq = + cookieSource.foldLeft(req)((_, jar) ⇒ addCookies(jar)(req)) + val fResp = innerPipeline(cookiedReq) + fResp.map(res ⇒ { + cookieTarget.foldLeft(res)((_, jar) ⇒ storeCookies(jar, req.uri)(res)) + }) + } } def addCookies(cookieJar: CookieJar): HttpRequest ⇒ HttpRequest = { @@ -33,14 +36,16 @@ object CookieHandling { } } - def storeCookies(cookieJar: CookieJar, uri: ⇒ Uri): HttpResponse ⇒ HttpResponse = { - res: HttpResponse ⇒ - { - val cookieHeaders = res.headers collect { case c: `Set-Cookie` ⇒ c } - for (c ← cookieHeaders.map(ch ⇒ ch.cookie)) { - cookieJar.setCookie(c, uri) - } - res + def storeCookies( + cookieJar: CookieJar, + uri: ⇒ Uri + ): HttpResponse ⇒ HttpResponse = { res: HttpResponse ⇒ + { + val cookieHeaders = res.headers collect { case c: `Set-Cookie` ⇒ c } + for (c ← cookieHeaders.map(ch ⇒ ch.cookie)) { + cookieJar.setCookie(c, uri) } + res + } } } diff --git a/http-extensions/src/main/scala/net/spraycookies/CookieJar.scala b/http-extensions/src/main/scala/net/spraycookies/CookieJar.scala index 96adaaf9..4af5b390 100644 --- a/http-extensions/src/main/scala/net/spraycookies/CookieJar.scala +++ b/http-extensions/src/main/scala/net/spraycookies/CookieJar.scala @@ -9,29 +9,56 @@ import scala.language.implicitConversions class CookieJar(blacklist: EffectiveTldList) { - private case class StoredCookie(name: String, content: String, expires: Option[DateTime], - domain: String, path: String, httpOnly: Boolean, secure: Boolean) + private case class StoredCookie( + name: String, + content: String, + expires: Option[DateTime], + domain: String, + path: String, + httpOnly: Boolean, + secure: Boolean + ) private object StoredCookie { implicit def toHttpCookie(src: StoredCookie): HttpCookie = { - HttpCookie(src.name, src.content, src.expires, - None, Some(src.domain), Some(src.path), src.secure, src.httpOnly, None) + HttpCookie( + src.name, + src.content, + src.expires, + None, + Some(src.domain), + Some(src.path), + src.secure, + src.httpOnly, + None + ) } - implicit def toStoredCookie(src: HttpCookie)(implicit uri: Uri): StoredCookie = { + implicit def toStoredCookie( + src: HttpCookie + )(implicit uri: Uri): StoredCookie = { val domain = src.domain.getOrElse(uri.authority.host.address) val path = src.path.getOrElse(uri.path.toString) val expiration = src.expires match { case x: Some[DateTime] ⇒ x case None ⇒ src.maxAge.map(age ⇒ DateTime.now + age) } - StoredCookie(src.name, src.value, expiration, domain, path, src.httpOnly, src.secure) + StoredCookie( + src.name, + src.value, + expiration, + domain, + path, + src.httpOnly, + src.secure + ) } } private var jar: CookieJar_ = CookieJar_("", Map.empty, Map.empty) - def cookiesFor(uri: Uri) = jar.cookiesFor(uri).map(c ⇒ StoredCookie.toHttpCookie(c)) + def cookiesFor(uri: Uri) = + jar.cookiesFor(uri).map(c ⇒ StoredCookie.toHttpCookie(c)) def setCookie(cookie: HttpCookie, source: Uri) = { val storedCookie = StoredCookie.toStoredCookie(cookie)(source) @@ -43,8 +70,8 @@ class CookieJar(blacklist: EffectiveTldList) { private def isAllowedFor(cookie: StoredCookie, source: Uri): Boolean = { !blacklist.contains(cookie.domain) && - isDomainPostfix(cookie.domain, source.authority.host.address) - //status of cookie paths is not clear to me, not implemented + isDomainPostfix(cookie.domain, source.authority.host.address) + // status of cookie paths is not clear to me, not implemented } private def isDomainPostfix(needle: String, haystack: String) = { @@ -53,7 +80,11 @@ class CookieJar(blacklist: EffectiveTldList) { haystackElements.startsWith(needleElements) } - private case class CookieJar_(domainElement: String, subdomains: Map[String, CookieJar_], cookies: Map[String, StoredCookie]) { + private case class CookieJar_( + domainElement: String, + subdomains: Map[String, CookieJar_], + cookies: Map[String, StoredCookie] + ) { def cookiesFor(uri: Uri) = { val domain = uri.authority.host.address val domainElements = domain.split('.').toList.reverse @@ -61,7 +92,11 @@ class CookieJar(blacklist: EffectiveTldList) { } @tailrec - private def _getCookies(domain: List[String], uri: Uri, accum: Map[String, StoredCookie]): Map[String, StoredCookie] = { + private def _getCookies( + domain: List[String], + uri: Uri, + accum: Map[String, StoredCookie] + ): Map[String, StoredCookie] = { val now = DateTime.now val newCookies = removeStale(cookies, now) .filter(c ⇒ uri.scheme == "https" || !c._2.secure) @@ -78,12 +113,17 @@ class CookieJar(blacklist: EffectiveTldList) { } def setCookie(cookie: StoredCookie) = { - val trimmed = if (cookie.domain.indexOf('.') == 0) cookie.domain.substring(1) else cookie.domain + val trimmed = + if (cookie.domain.indexOf('.') == 0) cookie.domain.substring(1) + else cookie.domain val domainElements = trimmed.split('.').toList.reverse _setCookie(domainElements, cookie) } - private def _setCookie(domain: List[String], cookie: StoredCookie): CookieJar_ = { + private def _setCookie( + domain: List[String], + cookie: StoredCookie + ): CookieJar_ = { val now = DateTime.now domain match { case Nil ⇒ @@ -92,7 +132,9 @@ class CookieJar(blacklist: EffectiveTldList) { case head :: tail ⇒ lazy val newSubJar = CookieJar_(head, Map.empty, Map.empty) val subJar = subdomains.getOrElse(head, newSubJar) - this.copy(subdomains = subdomains + (head -> subJar._setCookie(tail, cookie))) + this.copy(subdomains = + subdomains + (head -> subJar._setCookie(tail, cookie)) + ) } } @@ -100,4 +142,4 @@ class CookieJar(blacklist: EffectiveTldList) { cookies.filter(c ⇒ c._2.expires.forall(_ > cutoff)) } -} \ No newline at end of file +} diff --git a/http-extensions/src/main/scala/net/spraycookies/tldlist/EffectiveTldList.scala b/http-extensions/src/main/scala/net/spraycookies/tldlist/EffectiveTldList.scala index 6c3fdc3c..ae579359 100644 --- a/http-extensions/src/main/scala/net/spraycookies/tldlist/EffectiveTldList.scala +++ b/http-extensions/src/main/scala/net/spraycookies/tldlist/EffectiveTldList.scala @@ -10,16 +10,21 @@ trait TrieTldList extends EffectiveTldList { def domainTrie: TldTrie private def contains(domain: List[String]) = domainTrie.contains(domain) - def contains(domain: String) = domainTrie.contains(domain.split('.').reverse.toList) + def contains(domain: String) = + domainTrie.contains(domain.split('.').reverse.toList) } object DefaultEffectiveTldList extends TrieTldList { private val lines = { val inputStream = getClass.getResourceAsStream("/effectivetlds.lst") - Source.fromInputStream(inputStream, "UTF-8").getLines.filterNot(_.startsWith("//")).filterNot(_.isEmpty) + Source + .fromInputStream(inputStream, "UTF-8") + .getLines + .filterNot(_.startsWith("//")) + .filterNot(_.isEmpty) } val domainTrie = TldTrie(lines) -} \ No newline at end of file +} diff --git a/http-extensions/src/main/scala/net/spraycookies/tldlist/TldTrie.scala b/http-extensions/src/main/scala/net/spraycookies/tldlist/TldTrie.scala index 6ec2f747..85d0ba08 100644 --- a/http-extensions/src/main/scala/net/spraycookies/tldlist/TldTrie.scala +++ b/http-extensions/src/main/scala/net/spraycookies/tldlist/TldTrie.scala @@ -4,14 +4,16 @@ private[tldlist] sealed trait TldTrie { def contains(list: List[String]): Boolean = { list match { case Nil ⇒ true - case head :: tail ⇒ this match { - case Leaf ⇒ false - case Wildcard(negations) ⇒ tail == Nil && !negations.contains(head) - case Node(m) ⇒ m.get(head) match { - case None ⇒ false - case Some(trieTail) ⇒ trieTail.contains(tail) + case head :: tail ⇒ + this match { + case Leaf ⇒ false + case Wildcard(negations) ⇒ tail == Nil && !negations.contains(head) + case Node(m) ⇒ + m.get(head) match { + case None ⇒ false + case Some(trieTail) ⇒ trieTail.contains(tail) + } } - } } } def merge(that: TldTrie): TldTrie @@ -20,9 +22,11 @@ private[tldlist] sealed trait TldTrie { private case class Node(map: Map[String, TldTrie]) extends TldTrie { def merge(that: TldTrie): TldTrie = { that match { - case Node(thatMap) ⇒ Node(TldTrie.mapMerge(map, thatMap, (_: TldTrie).merge(_: TldTrie))) - case Leaf ⇒ this - case x: Wildcard ⇒ throw new Exception(s"tries $x and $this not mergeable") + case Node(thatMap) ⇒ + Node(TldTrie.mapMerge(map, thatMap, (_: TldTrie).merge(_: TldTrie))) + case Leaf ⇒ this + case x: Wildcard ⇒ + throw new Exception(s"tries $x and $this not mergeable") } } } @@ -33,18 +37,22 @@ private case object Leaf extends TldTrie { private case class Wildcard(whitelist: Set[String]) extends TldTrie { def merge(that: TldTrie) = that match { - case Leaf ⇒ this + case Leaf ⇒ this case Wildcard(thatWhitelist) ⇒ Wildcard(whitelist ++ thatWhitelist) - case x: Node ⇒ throw new Exception(s"tries $x and $this not mergeable") + case x: Node ⇒ throw new Exception(s"tries $x and $this not mergeable") } } private[tldlist] object TldTrie { - def mapMerge[T, U](left: Map[T, U], right: Map[T, U], merge: (U, U) ⇒ U): Map[T, U] = { + def mapMerge[T, U]( + left: Map[T, U], + right: Map[T, U], + merge: (U, U) ⇒ U + ): Map[T, U] = { left.foldLeft(right)((nm, kvp) ⇒ { nm.get(kvp._1) match { - case None ⇒ nm + kvp + case None ⇒ nm + kvp case Some(val2) ⇒ nm + (kvp._1 -> merge(val2, kvp._2)) } }) @@ -66,4 +74,4 @@ private[tldlist] object TldTrie { r.merge(toTrie(elems)) }) } -} \ No newline at end of file +} diff --git a/http-extensions/src/test/scala/net/spraycookies/CookieJarSpecification.scala b/http-extensions/src/test/scala/net/spraycookies/CookieJarSpecification.scala index f02bcc4a..5f7e7f36 100644 --- a/http-extensions/src/test/scala/net/spraycookies/CookieJarSpecification.scala +++ b/http-extensions/src/test/scala/net/spraycookies/CookieJarSpecification.scala @@ -22,7 +22,8 @@ object CookieJarSpecification extends Properties("CookieHandling") { val genToken = { val separators = List( - '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=', '{', '}', ' ', '\t' + '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', + '=', '{', '}', ' ', '\t' ) val allowedChars = Range(32, 126).map(_.toChar).toSet -- separators val genTokenChar = Gen.oneOf(allowedChars.toSeq) @@ -30,14 +31,19 @@ object CookieJarSpecification extends Properties("CookieHandling") { } val genCookieValue = { - val genCookieValueChar = Gen.oneOf( - Gen.choose(0x21, 0x21), - Gen.choose(0x23, 0x2B), - Gen.choose(0x2D, 0x3A), - Gen.choose(0x3C, 0x5B), - Gen.choose(0x5D, 0x7E) - ).map(i ⇒ i.toChar) - Gen.containerOf[List, Char](genCookieValueChar).suchThat(_ != Nil).map(_.mkString) + val genCookieValueChar = Gen + .oneOf( + Gen.choose(0x21, 0x21), + Gen.choose(0x23, 0x2b), + Gen.choose(0x2d, 0x3a), + Gen.choose(0x3c, 0x5b), + Gen.choose(0x5d, 0x7e) + ) + .map(i ⇒ i.toChar) + Gen + .containerOf[List, Char](genCookieValueChar) + .suchThat(_ != Nil) + .map(_.mkString) } val genBareCookie = for { @@ -47,43 +53,60 @@ object CookieJarSpecification extends Properties("CookieHandling") { val genBareCookieList = Gen.containerOf[List, HttpCookie](genBareCookie) - property("withCookies") = forAll(genBareCookieList) { cookies ⇒ { - val jar = new CookieJar(emptyTldlist) + property("withCookies") = forAll(genBareCookieList) { cookies ⇒ + { + val jar = new CookieJar(emptyTldlist) - val addingPipeline = (req: HttpRequest) ⇒ { - val resp = HttpResponse() - Future { - val setCookieHeaders = cookies.map(`Set-Cookie`(_)) - resp.withHeaders(setCookieHeaders) + val addingPipeline = (req: HttpRequest) ⇒ { + val resp = HttpResponse() + Future { + val setCookieHeaders = cookies.map(`Set-Cookie`(_)) + resp.withHeaders(setCookieHeaders) + } } - } - val testingPipeline = (req: HttpRequest) ⇒ { - Future { - val httpCookies = req.headers.collect({ case Cookie(httpCookies) ⇒ httpCookies }).flatten - if (httpCookies.length > cookies.length) throw new Exception("received more cookies than expected") - else if (!cookies.forall(expected ⇒ httpCookies.exists(received ⇒ received.name == expected.name))) - throw new Exception("reponse didn't contain cookies for all names") - else if (!httpCookies.forall(received ⇒ cookies.exists(testcookie ⇒ testcookie.name == received.name))) - throw new Exception("reponse contained a cookie with a name that is not expected") - else HttpResponse() + val testingPipeline = (req: HttpRequest) ⇒ { + Future { + val httpCookies = req.headers + .collect({ case Cookie(httpCookies) ⇒ httpCookies }) + .flatten + if (httpCookies.length > cookies.length) + throw new Exception("received more cookies than expected") + else if ( + !cookies.forall(expected ⇒ + httpCookies.exists(received ⇒ received.name == expected.name) + ) + ) + throw new Exception("reponse didn't contain cookies for all names") + else if ( + !httpCookies.forall(received ⇒ + cookies.exists(testcookie ⇒ testcookie.name == received.name) + ) + ) + throw new Exception( + "reponse contained a cookie with a name that is not expected" + ) + else HttpResponse() + } } - } - val cookiedPipeline = CookieHandling.withCookies(None, Some(jar))(addingPipeline) - Await.result(cookiedPipeline(emptyRequest), 10.seconds) - val cookiedTestPipeline = CookieHandling.withCookies(Some(jar), None)(testingPipeline) - val fTestResult = cookiedTestPipeline(HttpRequest()) - val testResult = try { - Await.result(fTestResult, 10.seconds) - true - } catch { - case t: Exception ⇒ - println(s"test failed $t") - false - } + val cookiedPipeline = + CookieHandling.withCookies(None, Some(jar))(addingPipeline) + Await.result(cookiedPipeline(emptyRequest), 10.seconds) + val cookiedTestPipeline = + CookieHandling.withCookies(Some(jar), None)(testingPipeline) + val fTestResult = cookiedTestPipeline(HttpRequest()) + val testResult = + try { + Await.result(fTestResult, 10.seconds) + true + } catch { + case t: Exception ⇒ + println(s"test failed $t") + false + } - testResult - } + testResult + } } } diff --git a/http-extensions/src/test/scala/net/spraycookies/tldlist/EffectiveTldListSpecification.scala b/http-extensions/src/test/scala/net/spraycookies/tldlist/EffectiveTldListSpecification.scala index 487d54e2..f0f37f04 100644 --- a/http-extensions/src/test/scala/net/spraycookies/tldlist/EffectiveTldListSpecification.scala +++ b/http-extensions/src/test/scala/net/spraycookies/tldlist/EffectiveTldListSpecification.scala @@ -3,7 +3,8 @@ package net.spraycookies.tldlist import org.scalacheck.Prop._ import org.scalacheck.Properties -object EffectiveTldListSpecification extends Properties("EffectiveTldListSpecification") { +object EffectiveTldListSpecification + extends Properties("EffectiveTldListSpecification") { property("contains") = forAll { (a: String, b: String) ⇒ val tldList = new TrieTldList { diff --git a/project/CompatibleJavaVersion.scala b/project/CompatibleJavaVersion.scala index c61b9069..bcf3c786 100644 --- a/project/CompatibleJavaVersion.scala +++ b/project/CompatibleJavaVersion.scala @@ -7,6 +7,7 @@ object CompatibleJavaVersion extends VersionNumberCompatibility { def isCompatible(current: VersionNumber, required: VersionNumber) = current.numbers.zip(required.numbers).forall(n => n._1 >= n._2) - def apply(current: VersionNumber, required: VersionNumber) = isCompatible(current, required) + def apply(current: VersionNumber, required: VersionNumber) = + isCompatible(current, required) -} \ No newline at end of file +} diff --git a/project/plugins.sbt b/project/plugins.sbt index 4fbad48e..951fadf8 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -8,4 +8,4 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-twirl" % "1.5.1") addSbtPlugin("org.foundweekends" % "sbt-bintray" % "0.6.1") -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5") \ No newline at end of file +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.5") diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/CsvComparator.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/CsvComparator.scala index 2ba604f4..16efd1a7 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/CsvComparator.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/CsvComparator.scala @@ -24,7 +24,7 @@ abstract class OpTable() { def toDouble(s: String) = s match { case x if x.nonEmpty => x.toDouble - case _ => 0.0 + case _ => 0.0 } lazy val byDate = ops.groupBy(opDate) @@ -37,29 +37,38 @@ abstract class OpTable() { def sumPlus = ops.map(opAmount).filter(_ > 0).sum - def sumByDate: Map[String, Double] = byDate.mapValues(_.map(opAmount).filter(_ > 0).sum).toMap + def sumByDate: Map[String, Double] = + byDate.mapValues(_.map(opAmount).filter(_ > 0).sum).toMap def diffByDate(that: OpTable) = { val thisMap = this.sumByDate val thatMap = that.sumByDate val allKeys = (thisMap.keySet ++ thatMap.keySet).toSeq.sorted - allKeys.map { - date => + allKeys + .map { date => val thisV = thisMap.getOrElse(date, 0.0) val thatV = thatMap.getOrElse(date, 0.0) if (thisV - thatV <= 0.01) - "" //s"the same: $thisV" + "" // s"the same: $thisV" else { val tab = List.fill(10)(" ").mkString("") s"$date Diff: ${thisV - thatV}; $name :$thisV; ${that.name}: $thatV \n" + this.name + ":\n" + - this.byDate.getOrElse(date, Nil).map(op => tab + opDescr(op)).mkString("\n") + "\n" + + this.byDate + .getOrElse(date, Nil) + .map(op => tab + opDescr(op)) + .mkString("\n") + "\n" + that.name + ":\n" + - that.byDate.getOrElse(date, Nil).map(op => tab + that.opDescr(op)).mkString("\n") + that.byDate + .getOrElse(date, Nil) + .map(op => tab + that.opDescr(op)) + .mkString("\n") } - }.filter(_.nonEmpty).mkString("\n") + } + .filter(_.nonEmpty) + .mkString("\n") } def diff[T](f: OpTable => Double, that: OpTable) = { @@ -84,7 +93,7 @@ class AppOps(val rows: Seq[Seq[String]]) extends OpTable() { override val amountCol = indexOf("Amount") override val descrCol = indexOf("Description") - override val ops = rows.tail.sortBy(_ (dateCol)) + override val ops = rows.tail.sortBy(_(dateCol)) } @@ -105,7 +114,7 @@ class BankOps(val rows: Seq[Seq[String]]) extends OpTable() { override val ops = mapDates(rows.tail) - val byDate2 = ops.groupBy(_ (dateCol2)) + val byDate2 = ops.groupBy(_(dateCol2)) private def mapDates(rows: Seq[Seq[String]]) = { def dateConv(s: String) = @@ -123,8 +132,11 @@ class BankOps(val rows: Seq[Seq[String]]) extends OpTable() { def rate(op: Seq[String]): Option[Exchange] = { val uah = credit(op) val descr = op(descrCol) - for (usd <- "(\\d+\\.\\d+) USD".r.findFirstMatchIn(descr).map(_.group(1).toDouble)) yield - Exchange(op(dateCol), uah, usd, uah/usd) + for ( + usd <- "(\\d+\\.\\d+) USD".r + .findFirstMatchIn(descr) + .map(_.group(1).toDouble) + ) yield Exchange(op(dateCol), uah, usd, uah / usd) } def rates = ops.flatMap(rate) @@ -135,7 +147,9 @@ object CsvComparator { override val delimiter = ';' } - val bankRaw = CSVReader.open("/mnt/mint/home/ilya/wmua-finance/2016-2202.csv", "cp1251")(UA_CSV).all() + val bankRaw = CSVReader + .open("/mnt/mint/home/ilya/wmua-finance/2016-2202.csv", "cp1251")(UA_CSV) + .all() // val appRaw = CSVReader.open("/home/ilya/Downloads/2016-2202-app.csv").all() val appRaw = CSVReader.open("/home/ilya/Downloads/wmua_fin (5).csv").all() @@ -152,4 +166,4 @@ object CsvComparator { println(appOps.diffByDate(bankOps)) bankOps.rates.foreach(println) } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/FileUtils.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/FileUtils.scala index 6e121ccf..8eb46908 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/FileUtils.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/FileUtils.scala @@ -10,8 +10,7 @@ import org.scalawiki.AlphaNumOrdering import scala.collection.mutable import scala.io.{Codec, Source} -/** - * Save and load lines to/from file +/** Save and load lines to/from file */ object FileUtils { @@ -19,22 +18,28 @@ object FileUtils { val byAlphaNumName = Ordering.by[SFile, String](_.name)(AlphaNumOrdering) - /** - * Read lines from file + /** Read lines from file * - * @param filename file to read - * @param codec character encoding/decoding preferences, default is [[scala.io.Codec.defaultCharsetCodec()]] - * @return lines from the file + * @param filename + * file to read + * @param codec + * character encoding/decoding preferences, default is + * [[scala.io.Codec.defaultCharsetCodec()]] + * @return + * lines from the file */ def read(filename: String)(implicit codec: Codec): mutable.Buffer[String] = Source.fromFile(filename).getLines.toBuffer - /** - * Save lines to file, overwriting it + /** Save lines to file, overwriting it * - * @param filename file to write to - * @param lines lines to save - * @param codec character encoding/decoding preferences, default is [[scala.io.Codec.defaultCharsetCodec()]] + * @param filename + * file to write to + * @param lines + * lines to save + * @param codec + * character encoding/decoding preferences, default is + * [[scala.io.Codec.defaultCharsetCodec()]] * @return */ def write(filename: String, lines: Seq[String])(implicit codec: Codec): Path = @@ -55,37 +60,44 @@ object FileUtils { def backupName(file: SFile): String = { val pattern = Pattern.quote(file.pathAsString) + "\\.(\\d+)" - val maybeLast = file.parent.glob(pattern)(PathMatcherSyntax.regex) - .toSeq.sortBy(_.name)(AlphaNumOrdering) + val maybeLast = file.parent + .glob(pattern)(PathMatcherSyntax.regex) + .toSeq + .sortBy(_.name)(AlphaNumOrdering) .lastOption - val number = maybeLast.fold(1) { - last => - pattern.r.findFirstMatchIn(last.pathAsString) - .fold(1)(_.group(1).toInt + 1) + val number = maybeLast.fold(1) { last => + pattern.r + .findFirstMatchIn(last.pathAsString) + .fold(1)(_.group(1).toInt + 1) } file.name + "." + number } - /** - * @param dir directory - * @return subdirectories, sorted by name + /** @param dir + * directory + * @return + * subdirectories, sorted by name */ def subDirs(dir: SFile): Seq[SFile] = list(dir, _.isDirectory) - /** - * @param dir directory - * @return regular files, sorted by name + /** @param dir + * directory + * @return + * regular files, sorted by name */ def getFiles(dir: SFile): Seq[SFile] = list(dir, _.isRegularFile) - /** - * @param dir directory + /** @param dir + * directory * @param predicate - * @return directory members filtered by predicate + * @return + * directory members filtered by predicate */ - def list(dir: SFile, predicate: SFile => Boolean)(implicit order: Order = byAlphaNumName): Seq[SFile] = + def list(dir: SFile, predicate: SFile => Boolean)(implicit + order: Order = byAlphaNumName + ): Seq[SFile] = dir.list.filter(predicate).toSeq.sorted(order) def isImage(f: SFile): Boolean = hasExt(f, Set(".jpg", ".tif")) diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/GlobalContrib.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/GlobalContrib.scala index c275ffd7..d2ca1f55 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/GlobalContrib.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/GlobalContrib.scala @@ -14,11 +14,20 @@ class GlobalContrib extends QueryLibrary { val bot = MwBot.fromHost(MwBot.commons) - def editsBefore(user: User, acc: SulAccount, start: ZonedDateTime): Future[Long] = { + def editsBefore( + user: User, + acc: SulAccount, + start: ZonedDateTime + ): Future[Long] = { val host = acc.url.replace("https://", "") val siteBot = MwBot.fromHost(host) - val action = userContribs(user.name.get, TimeRange(None, Some(start)), limit = "500", dir = "newer") + val action = userContribs( + user.name.get, + TimeRange(None, Some(start)), + limit = "500", + dir = "newer" + ) siteBot.run(action).map { edits => println(s"$host - ${edits.size}") edits.size.toLong @@ -26,15 +35,19 @@ class GlobalContrib extends QueryLibrary { } def checkContribs(username: String, start: ZonedDateTime): Future[Long] = { - bot.run(globalUserInfo(username)).flatMap { - pages => - pagesToUsers(pages).collect { case u: User => u }.headOption.fold(Future(0L)) { - user => - val activeAccounts = user.sulAccounts.filter(_.editCount > 0) - - println(s"${user.name.get} - accounts ${activeAccounts.size}: ${activeAccounts.map(_.wiki)}") - - Future.traverse(activeAccounts)(acc => editsBefore(user, acc, start)).map(_.sum) + bot.run(globalUserInfo(username)).flatMap { pages => + pagesToUsers(pages) + .collect { case u: User => u } + .headOption + .fold(Future(0L)) { user => + val activeAccounts = user.sulAccounts.filter(_.editCount > 0) + + println(s"${user.name.get} - accounts ${activeAccounts.size}: ${activeAccounts + .map(_.wiki)}") + + Future + .traverse(activeAccounts)(acc => editsBefore(user, acc, start)) + .map(_.sum) } } } @@ -44,19 +57,24 @@ class GlobalContrib extends QueryLibrary { object GlobalContrib { def main(args: Array[String]) { - new GlobalContrib().checkContribs("Ilya", ZonedDateTime.of(2016, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC)).map { - size => + new GlobalContrib() + .checkContribs( + "Ilya", + ZonedDateTime.of(2016, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC) + ) + .map { size => println(size) - } + } } def countNewComers(authors: Set[String], date: ZonedDateTime): Unit = { val gc = new GlobalContrib() - Future.traverse(authors) { author => - gc.checkContribs(author, date).map(author -> _) - }.map { - set => + Future + .traverse(authors) { author => + gc.checkContribs(author, date).map(author -> _) + } + .map { set => val seq = set.toSeq val sorted = seq.sortBy(-_._2) sorted.foreach(println) @@ -68,6 +86,6 @@ object GlobalContrib { val newComers = seq.filter(_._2 == 0) println(s"newComers ${newComers.size}") - } + } } } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/MessageBot.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/MessageBot.scala index ab10c309..09144572 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/MessageBot.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/MessageBot.scala @@ -15,39 +15,36 @@ import scala.concurrent.Future case class Message(subject: String, body: String) -/** - * Send messages to users either via talk page or email +/** Send messages to users either via talk page or email * - * @param conf configuration + * @param conf + * configuration */ class MessageBot(val conf: Config) extends ActionLibrary with QueryLibrary { - /** - * Mediawiki host, e.g. en.wikipedia.org + /** Mediawiki host, e.g. en.wikipedia.org */ val host = conf.getString("host") - /** - * Page that contains links to user pages of users we are going to notify + /** Page that contains links to user pages of users we are going to notify */ val userListPage = conf.getString("users.list") - /** - * optional start and end of time range that user contributions are queried + /** optional start and end of time range that user contributions are queried */ val (start: Option[ZonedDateTime], end: Option[ZonedDateTime]) = ( - conf.as[Option[LocalDate]]("users.start").map(_.atStartOfDay(ZoneOffset.UTC)), + conf + .as[Option[LocalDate]]("users.start") + .map(_.atStartOfDay(ZoneOffset.UTC)), conf.as[Option[LocalDate]]("users.end").map(_.atStartOfDay(ZoneOffset.UTC)) ) val range = TimeRange(start, end) - /** - * Email message + /** Email message */ val mail = conf.as[Message]("email") - /** - * Talk page message + /** Talk page message */ val talkPageMessage = conf.as[Message]("talk-page") @@ -68,11 +65,14 @@ class MessageBot(val conf: Config) extends ActionLibrary with QueryLibrary { def processUsers(users: Iterable[User], conf: Config) = { val pages = users.map(u => userCreatedPages(u.name.get, range)) - val folded = Future.fold(pages)(Seq.empty[(String, Set[String])])(_ :+ _).map(_.toMap) + val folded = + Future.fold(pages)(Seq.empty[(String, Set[String])])(_ :+ _).map(_.toMap) for (createdPagesByUser <- folded) { - val withContribution = users.filter(u => createdPagesByUser(u.name.get).nonEmpty) - val (withEmail, withoutEmail) = withContribution.partition(_.emailable.getOrElse(false)) + val withContribution = + users.filter(u => createdPagesByUser(u.name.get).nonEmpty) + val (withEmail, withoutEmail) = + withContribution.partition(_.emailable.getOrElse(false)) logUsers(users, withEmail, withoutEmail) @@ -85,7 +85,11 @@ class MessageBot(val conf: Config) extends ActionLibrary with QueryLibrary { } } - def logUsers(users: Iterable[User], withEmail: Iterable[User], withoutEmail: Iterable[User]): Unit = { + def logUsers( + users: Iterable[User], + withEmail: Iterable[User], + withoutEmail: Iterable[User] + ): Unit = { println("AllUsers: " + users.size) println("WithEmail: " + withEmail.size) println("WithoutEmail: " + withoutEmail.size) diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/PageFromFileBot.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/PageFromFileBot.scala index f919cb9f..40de7bbf 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/PageFromFileBot.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/PageFromFileBot.scala @@ -4,17 +4,22 @@ import java.util.regex.Pattern.quote import org.scalawiki.dto.Page import scala.collection.TraversableOnce -case class PageFromFileFormat(start: String = "{{-start-}}", - end: String = "{{-end-}}", - titleStart: String = "'''", - titleEnd: String = "'''") +case class PageFromFileFormat( + start: String = "{{-start-}}", + end: String = "{{-end-}}", + titleStart: String = "'''", + titleEnd: String = "'''" +) object PageFromFileBot { - def pages(content: CharSequence, - fmt: PageFromFileFormat = PageFromFileFormat(), - noTitle: Boolean = false): TraversableOnce[Page] = { - val pageRegex = ("(?s)" + quote(fmt.start) + "\r?\n(.*?)\r?\n" + quote(fmt.end)).r + def pages( + content: CharSequence, + fmt: PageFromFileFormat = PageFromFileFormat(), + noTitle: Boolean = false + ): TraversableOnce[Page] = { + val pageRegex = + ("(?s)" + quote(fmt.start) + "\r?\n(.*?)\r?\n" + quote(fmt.end)).r val titleRegex = (quote(fmt.titleStart) + "(.*?)" + quote(fmt.titleEnd)).r pageRegex.findAllMatchIn(content).flatMap { m => @@ -22,27 +27,33 @@ object PageFromFileBot { val titleOption = titleRegex.findFirstMatchIn(text).map(_.group(1)) titleOption.map { title => - val pageText = if (noTitle) - titleRegex.replaceFirstIn(text, "") - else text + val pageText = + if (noTitle) + titleRegex.replaceFirstIn(text, "") + else text Page(title).withText(pageText) } } } - def join(pages: Seq[Page], - fmt: PageFromFileFormat = PageFromFileFormat(), - includeTitle: Boolean = true): String = { - pages.map { page => - val title = if (includeTitle) - fmt.titleStart + page.title + fmt.titleEnd - else "" - - fmt.start + "\n" + - title + - page.text.getOrElse("") + - "\n" + fmt.end - }.mkString("\n") + def join( + pages: Seq[Page], + fmt: PageFromFileFormat = PageFromFileFormat(), + includeTitle: Boolean = true + ): String = { + pages + .map { page => + val title = + if (includeTitle) + fmt.titleStart + page.title + fmt.titleEnd + else "" + + fmt.start + "\n" + + title + + page.text.getOrElse("") + + "\n" + fmt.end + } + .mkString("\n") } } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/PageGenerators.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/PageGenerators.scala index 56ff32f6..c91b7929 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/PageGenerators.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/PageGenerators.scala @@ -2,53 +2,62 @@ package org.scalawiki.bots import org.rogach.scallop.ScallopConf -case class PageGenConfig(cat: Seq[String] = Seq.empty, //, - // catR: String, - // subCats: String, - // subCatsR: String, - // unCat: String, - // unCatCat: String, - // unCatFiles: String, - // file: String, - // fileLinks: String, - // search: String, - // logEvents: String, - namespaces: Seq[String] = Seq.empty - // interwiki: String, - // limit: Int, - // links: String, - // liveRecentChanges: Option[Int], - // imagesUsed: String, - // newImages: Option[Int], - // newPages: Option[Int], - // recentChanges: String, - // unconnectedPages: Option[Int], - // ref: String, - // start: String, - // prefixIndex: String, - // subPage: Int, - // titleRegex: String, - // transcludes: String, - // unusedFiles: Option[Int], - // lonelyPages: Option[Int], - // unwatched: Option[Int], - // userContribs: String, - // weblink: String, - // withoutInterwiki: Option[Int], - // sqlQuery: String, - // wikidataQuery: String, - // sparqlQuery: String, - // random: Option[Int], - // randomRedirect: Option[Int], - // page: Seq[String], - // pageId : Seq[Long], - // grep: String, - // quality: Int, - // onlyIf: String, - // onlyIfNot: String - ) +case class PageGenConfig( + cat: Seq[String] = Seq.empty, // , + // catR: String, + // subCats: String, + // subCatsR: String, + // unCat: String, + // unCatCat: String, + // unCatFiles: String, + // file: String, + // fileLinks: String, + // search: String, + // logEvents: String, + namespaces: Seq[String] = Seq.empty + // interwiki: String, + // limit: Int, + // links: String, + // liveRecentChanges: Option[Int], + // imagesUsed: String, + // newImages: Option[Int], + // newPages: Option[Int], + // recentChanges: String, + // unconnectedPages: Option[Int], + // ref: String, + // start: String, + // prefixIndex: String, + // subPage: Int, + // titleRegex: String, + // transcludes: String, + // unusedFiles: Option[Int], + // lonelyPages: Option[Int], + // unwatched: Option[Int], + // userContribs: String, + // weblink: String, + // withoutInterwiki: Option[Int], + // sqlQuery: String, + // wikidataQuery: String, + // sparqlQuery: String, + // random: Option[Int], + // randomRedirect: Option[Int], + // page: Seq[String], + // pageId : Seq[Long], + // grep: String, + // quality: Int, + // onlyIf: String, + // onlyIfNot: String +) class PageGenerators(arguments: Seq[String]) extends ScallopConf(arguments) { - val category = opt[String]("cat", descr = "Work on all pages which are in a specific category.", required = false) - val namespace = opt[String]("namespace", descr = "Work on all pages in given namespaces.", required = false) + val category = opt[String]( + "cat", + descr = "Work on all pages which are in a specific category.", + required = false + ) + val namespace = opt[String]( + "namespace", + descr = "Work on all pages in given namespaces.", + required = false + ) } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/PartnersBot.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/PartnersBot.scala index fc5b604a..b1c4f458 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/PartnersBot.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/PartnersBot.scala @@ -15,15 +15,18 @@ object PartnersBot { val http = HttpClient.get() def main(args: Array[String]): Unit = { - http.get(partnersLinkPage).foreach { - html => - val doc = Jsoup.parse(html) - val links = doc.select("a[href]").asScala - val hrefs = links.map(_.attr("abs:href")) - - val external = hrefs.map(_.trim).filterNot(href => href.isEmpty || href.contains(ourDomain)).toSet - - Future.traverse(external) { link => + http.get(partnersLinkPage).foreach { html => + val doc = Jsoup.parse(html) + val links = doc.select("a[href]").asScala + val hrefs = links.map(_.attr("abs:href")) + + val external = hrefs + .map(_.trim) + .filterNot(href => href.isEmpty || href.contains(ourDomain)) + .toSet + + Future + .traverse(external) { link => println("getting " + link) http.get(link).map { home => val doc = Jsoup.parse(home) @@ -32,7 +35,8 @@ object PartnersBot { println(elem) elem } - }.foreach(println) + } + .foreach(println) } } } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/RedLinksBot.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/RedLinksBot.scala index f03cc506..060387fc 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/RedLinksBot.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/RedLinksBot.scala @@ -15,14 +15,21 @@ object RedLinksBot { def main(args: Array[String]): Unit = { for (page <- linkPages) { - val html = Await.result(http.get("https://commons.wikimedia.org/wiki/" + page), 1.minute) + val html = Await.result( + http.get("https://commons.wikimedia.org/wiki/" + page), + 1.minute + ) val doc = Jsoup.parse(html) val links = doc.select("a[href]").asScala - val missingFiles = links.filter { link => - link.attr("class") == "new" && link.attr("title").startsWith("File") - }.map(_.attr("title").replace(" (page does not exist)", "")) - val text = s"\n== [[$page]] ==\n" + missingFiles.map(title => s"#[[$title]]").mkString("\n") + val missingFiles = links + .filter { link => + link.attr("class") == "new" && link.attr("title").startsWith("File") + } + .map(_.attr("title").replace(" (page does not exist)", "")) + val text = s"\n== [[$page]] ==\n" + missingFiles + .map(title => s"#[[$title]]") + .mkString("\n") println(text) } } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/Replace.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/Replace.scala index e51f2ee9..081b42b6 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/Replace.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/Replace.scala @@ -3,11 +3,13 @@ package org.scalawiki.bots import java.util.regex.Pattern import scala.util.matching.{Regex, RegexFactory} -class ReplacementBase(old: String, - replacement: String, - summary: Option[String] = None, - useRegex: Boolean = true, - ignoreCase: Boolean = false) { +class ReplacementBase( + old: String, + replacement: String, + summary: Option[String] = None, + useRegex: Boolean = true, + ignoreCase: Boolean = false +) { val flags = (if (!useRegex) Pattern.LITERAL else 0) | (if (ignoreCase) Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE else 0) @@ -18,17 +20,20 @@ class ReplacementBase(old: String, } -case class Exceptions(titles: Seq[String], - requireTitles: Seq[String], - textContains: Seq[String], - inside: Seq[String], - insideTags: Seq[String]) { +case class Exceptions( + titles: Seq[String], + requireTitles: Seq[String], + textContains: Seq[String], + inside: Seq[String], + insideTags: Seq[String] +) { val (titlesR, requireTitlesR, textContainsR, insideR) = ( titles.map(_.r), requireTitles.map(_.r), textContains.map(_.r), - inside.map(_.r)) + inside.map(_.r) + ) def isTitleExcepted(title: String): Boolean = titlesR.exists(_.unapplySeq(title).isDefined) || @@ -39,17 +44,26 @@ case class Exceptions(titles: Seq[String], } -case class ReplaceConfig(regex: Boolean = false, - replacements: Map[String, String] = Map.empty, - pages: PageGenConfig = PageGenConfig()) +case class ReplaceConfig( + regex: Boolean = false, + replacements: Map[String, String] = Map.empty, + pages: PageGenConfig = PageGenConfig() +) class Replace(arguments: Seq[String]) extends PageGenerators(arguments) { - val regex = opt[Boolean]("regex", descr = "Make replacements using regular expressions.", default = Some(false)) - val replacements = trailArg[List[String]](descr = "Replacement pairs.", required = false, default = Some(Nil)) + val regex = opt[Boolean]( + "regex", + descr = "Make replacements using regular expressions.", + default = Some(false) + ) + val replacements = trailArg[List[String]]( + descr = "Replacement pairs.", + required = false, + default = Some(Nil) + ) verify() } - object Replace { def parse(args: Seq[String]): ReplaceConfig = { @@ -57,9 +71,13 @@ object Replace { val parsed = new Replace(args) ReplaceConfig( regex = parsed.regex(), - replacements = parsed.replacements().sliding(2, 2).toSeq - .map(s => s.head -> s.last).toMap, - pages = PageGenConfig( + replacements = parsed + .replacements() + .sliding(2, 2) + .toSeq + .map(s => s.head -> s.last) + .toMap, + pages = PageGenConfig( cat = parsed.category.toOption.toSeq, namespaces = parsed.namespace.toOption.toSeq ) @@ -73,30 +91,38 @@ object Replace { object TextLib { - /** - * - * @param text text to process - * @param old regex pattern string to replace - * @param replacement replacement - * @param exceptions text that matches these regex exceptions should not have replacements - * @param ignoreCase match both lower- and uppercase characters for replacement - * @param marker add this marker after the last replace or the end of text - * @return text with replacements + /** @param text + * text to process + * @param old + * regex pattern string to replace + * @param replacement + * replacement + * @param exceptions + * text that matches these regex exceptions should not have replacements + * @param ignoreCase + * match both lower- and uppercase characters for replacement + * @param marker + * add this marker after the last replace or the end of text + * @return + * text with replacements */ - def replaceExcept(text: String, - old: String, - replacement: String, - exceptions: Seq[Regex] = Seq.empty, - ignoreCase: Boolean = false, - marker: Option[String] = None) = { - - val regex = new ReplacementBase(old, replacement, None, true, ignoreCase).regex + def replaceExcept( + text: String, + old: String, + replacement: String, + exceptions: Seq[Regex] = Seq.empty, + ignoreCase: Boolean = false, + marker: Option[String] = None + ) = { + + val regex = + new ReplacementBase(old, replacement, None, true, ignoreCase).regex var markerPos = text.length - val exceptionMatches = exceptions.map(ex => ex.findAllMatchIn(text).buffered) + val exceptionMatches = + exceptions.map(ex => ex.findAllMatchIn(text).buffered) val it = RegexFactory.replacementIterator(text, regex) it foreach { m => - exceptionMatches.foreach(_.dropWhile(_.end < m.start)) val inException = exceptionMatches.exists(_.head.start <= m.start) @@ -114,4 +140,3 @@ object TextLib { } } - diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/ShortLinksBot.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/ShortLinksBot.scala index d27bfc99..747324b1 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/ShortLinksBot.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/ShortLinksBot.scala @@ -14,35 +14,41 @@ import scala.concurrent.Future import spray.util.pimpFuture import scala.io.Source -/** - * Converts links to Wikimedia Commons files to short links (with file id) and tries to add image author and license +/** Converts links to Wikimedia Commons files to short links (with file id) and + * tries to add image author and license */ object ShortLinksBot { val commons = MwBot.fromHost(MwBot.commons) val ukWiki = MwBot.fromHost(MwBot.ukWiki) def getPage(title: String): Future[Page] = { - val action = Action(Query( - TitlesParam(Seq(title)), - Prop( - Revisions(RvProp(Content)) + val action = Action( + Query( + TitlesParam(Seq(title)), + Prop( + Revisions(RvProp(Content)) + ) ) - )) - - commons.run(action).flatMap { commonsPages => - val commonsPage = commonsPages.head - if (commonsPage.missing) { - ukWiki.run(action).map(_.head) - } else Future.successful(commonsPage) - } + ) + + commons + .run(action) + .flatMap { commonsPages => + val commonsPage = commonsPages.head + if (commonsPage.missing) { + ukWiki.run(action).map(_.head) + } else Future.successful(commonsPage) + } .recoverWith { case e => Future.successful(Page(title = "Error! " + e)) } } def getPageLicense(page: Page): Option[String] = { - for (id <- page.id; - text <- page.revisions.headOption.flatMap(_.content)) yield { + for ( + id <- page.id; + text <- page.revisions.headOption.flatMap(_.content) + ) yield { val author = Image.getAuthorFromPage(text) @@ -51,8 +57,8 @@ object ShortLinksBot { .map(_.toLowerCase) .find { s => s.startsWith("cc-") || - s.startsWith("gfdl") || - s.startsWith("wikimapia") + s.startsWith("gfdl") || + s.startsWith("wikimapia") } .getOrElse("???") @@ -79,17 +85,17 @@ object ShortLinksBot { val start = replaced.indexOf("File:") if (start >= 0) { val decoded = URLDecoder.decode(replaced.substring(start), "UTF-8") - getPage(decoded.trim).map(page => getPageLicense(page).getOrElse(line)).recoverWith { case e => - Future.successful("Error! " + e) - } - } - else Future.successful(line) + getPage(decoded.trim) + .map(page => getPageLicense(page).getOrElse(line)) + .recoverWith { case e => + Future.successful("Error! " + e) + } + } else Future.successful(line) } def main(args: Array[String]) { val lines = Source.fromFile("arch.txt").getLines().toSeq - val parallel = false val updatedLines = if (parallel) { Future.sequence(lines.map(getFileSubstring)).await diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/Spain.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/Spain.scala index 7829c58f..cfc4f602 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/Spain.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/Spain.scala @@ -5,7 +5,12 @@ import java.util.TimeZone import org.scalawiki.MwBot import org.scalawiki.dto.cmd.Action -import org.scalawiki.dto.cmd.query.list.{CategoryMembers, CmLimit, CmNamespace, CmTitle} +import org.scalawiki.dto.cmd.query.list.{ + CategoryMembers, + CmLimit, + CmNamespace, + CmTitle +} import org.scalawiki.dto.cmd.query.prop._ import org.scalawiki.dto.cmd.query.{Generator, PageIdsParam, Query} import org.scalawiki.dto.{Namespace, Page} @@ -20,39 +25,43 @@ object Spain { def main(args: Array[String]) { bot = MwBot.fromHost(MwBot.commons) - val action = Action(Query( - Prop( - Info(), - ImageInfo( + val action = Action( + Query( + Prop( + Info(), + ImageInfo( // IiProp(Timestamp) + ) + // Revisions( + // RvProp(Ids) + // //,RvDir(Newer) + // ) + ), + Generator( + CategoryMembers( + CmTitle( + "Category:Evaluation of images from Wiki Loves Earth 2015 in Spain and Portugal - Round 1" + ), + CmLimit("500"), + CmNamespace(Seq(Namespace.FILE)) + ) ) - // Revisions( - // RvProp(Ids) - // //,RvDir(Newer) - // ) - ), - Generator(CategoryMembers( - CmTitle("Category:Evaluation of images from Wiki Loves Earth 2015 in Spain and Portugal - Round 1"), - CmLimit("500"), - CmNamespace(Seq(Namespace.FILE)))) - )) + ) + ) - bot.run(action) map { - pages => - val filteredPages = pages.filterNot { page => - val last = page.images.last - last.date.exists { - timestamp => - val start = ZonedDateTime.of(2015, 4, 30, 22, 0, 0, 0, ZoneOffset.UTC) - val end = ZonedDateTime.of(2015, 5, 31, 23, 0, 0, 0, ZoneOffset.UTC) - timestamp.isAfter(start) && timestamp.isBefore(end) - } + bot.run(action) map { pages => + val filteredPages = pages.filterNot { page => + val last = page.images.last + last.date.exists { timestamp => + val start = ZonedDateTime.of(2015, 4, 30, 22, 0, 0, 0, ZoneOffset.UTC) + val end = ZonedDateTime.of(2015, 5, 31, 23, 0, 0, 0, ZoneOffset.UTC) + timestamp.isAfter(start) && timestamp.isBefore(end) } - val size = filteredPages.size - filteredPages.foreach(p => println(p.title)) + } + val size = filteredPages.size + filteredPages.foreach(p => println(p.title)) } - // new DslQueryDbCache(new DslQuery(action, bot)).run() map { // pages => // Future.traverse(pages.sliding(50, 50)) { @@ -76,7 +85,6 @@ object Spain { // } } - def pagesRevisions(ids: Seq[Long]): Future[Seq[Page]] = { Future.traverse(ids)(pageRevisions).map(_.flatten) } @@ -84,17 +92,19 @@ object Spain { def pageRevisions(id: Long): Future[Option[Page]] = { import org.scalawiki.dto.cmd.query.prop.rvprop._ - val action = Action(Query( - PageIdsParam(Seq(id)), - Prop( - Info(), - Revisions( - RvProp(/*Content,*/ Ids, Size, User, UserId, rvprop.Timestamp), - RvLimit("max") + val action = Action( + Query( + PageIdsParam(Seq(id)), + Prop( + Info(), + Revisions( + RvProp( /*Content,*/ Ids, Size, User, UserId, rvprop.Timestamp), + RvLimit("max") + ) ) ) - )) + ) - bot.run(action).map (_.headOption) + bot.run(action).map(_.headOption) } } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/edu/InstitutionList.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/edu/InstitutionList.scala index 40e830c8..8fce4cd8 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/edu/InstitutionList.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/edu/InstitutionList.scala @@ -8,21 +8,24 @@ import org.scalawiki.http.HttpClient import scala.collection.JavaConverters._ import scala.concurrent.ExecutionContext.Implicits.global -case class Institution(id: Long, - name: String, - city: String, - country: String, - numCourses: Int, - numStudents: Int) +case class Institution( + id: Long, + name: String, + city: String, + country: String, + numCourses: Int, + numStudents: Int +) object InstitutionList { val http = HttpClient.get() val host = "uk.wikipedia.org" - val specialPage = "Спеціальна:Інституції" //"Special:Institutions" + val specialPage = "Спеціальна:Інституції" // "Special:Institutions" - val url = "https://" + host + "/wiki/" + URLEncoder.encode(specialPage, "UTF-8") + val url = + "https://" + host + "/wiki/" + URLEncoder.encode(specialPage, "UTF-8") def main(args: Array[String]): Unit = { @@ -36,18 +39,20 @@ object InstitutionList { val cols = row.select("td").asScala if (cols.nonEmpty) { - val institution = Institution(0, + val institution = Institution( + 0, name = cols(0).text, city = cols(1).text, country = cols(2).text, numCourses = cols(3).text.toInt, - numStudents = cols(4).text.toInt) + numStudents = cols(4).text.toInt + ) println(institution) } } - } recover { - case t: Throwable => println(t) + } recover { case t: Throwable => + println(t) } } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/Resolution.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/Resolution.scala index c138c4ef..30dfad8f 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/Resolution.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/Resolution.scala @@ -4,11 +4,16 @@ import better.files.{File => SFile} import org.scalawiki.dto.Page import org.scalawiki.bots.html -case class Resolution(number: Int, year: Int, month: String, day: String, - description: String, - text: String, - votes: Votes, - board: Board) extends Ordered[Resolution] { +case class Resolution( + number: Int, + year: Int, + month: String, + day: String, + description: String, + text: String, + votes: Votes, + board: Board +) extends Ordered[Resolution] { def date: String = s"$day $month $year" @@ -28,9 +33,8 @@ case class Resolution(number: Int, year: Int, month: String, day: String, def substRealNames(text: String): String = { var replaced = text - Resolution.realNames.foreach { - case (nick, name) => - replaced = replaced.replace(nick, name) + Resolution.realNames.foreach { case (nick, name) => + replaced = replaced.replace(nick, name) } replaced } @@ -41,7 +45,8 @@ case class Resolution(number: Int, year: Int, month: String, day: String, val linkR = "\\[\\[[^\\|\\]]*\\|" val noCats = text.replaceAll(catR, "") - val noLinks = noCats.replaceAll(linkR, "") + val noLinks = noCats + .replaceAll(linkR, "") .replace("]]", "") .replace("[[", "") .replace("{{u|", "") @@ -50,16 +55,21 @@ case class Resolution(number: Int, year: Int, month: String, day: String, .replace("{{text|1=", "") .replaceAll("^\\s*$", "") - val noEndingDate = noLinks.replaceAll("\\d+\\.\\d{2}\\.\\d{4}, м. Київ\\.?", "") + val noEndingDate = + noLinks.replaceAll("\\d+\\.\\d{2}\\.\\d{4}, м. Київ\\.?", "") val voteR = "''\\(«За»[^\\)]+\\)''".r - val withVotes = voteR.replaceAllIn(noEndingDate, m => { - "

" + m.matched.toString.replace("''", "") + "" - }) + val withVotes = voteR.replaceAllIn( + noEndingDate, + m => { + "

" + m.matched.toString.replace("''", "") + "" + } + ) val withRealNames = substRealNames(withVotes) - withRealNames.replaceFirst("#", "

  1. ") + withRealNames + .replaceFirst("#", "
    1. ") .replace("#:", "") .replace("#", "
    2. ") } catch { @@ -78,16 +88,25 @@ object Resolution { val title = page.title - val regex = "Рішення Правління №(\\d+)\\/(\\d{4}) від (\\d+) (\\p{L}+) (\\d{4})" - .r("num", "year1", "day", "month", "year2") - - regex.findFirstMatchIn(title).map { - m => - val number = m.group("num").toInt - val year = m.group("year1").toInt - val month = m.group("month") - val day = m.group("day") - Resolution(number, year, month, day, "", page.text.getOrElse(""), null, null) + val regex = + "Рішення Правління №(\\d+)\\/(\\d{4}) від (\\d+) (\\p{L}+) (\\d{4})" + .r("num", "year1", "day", "month", "year2") + + regex.findFirstMatchIn(title).map { m => + val number = m.group("num").toInt + val year = m.group("year1").toInt + val month = m.group("month") + val day = m.group("day") + Resolution( + number, + year, + month, + day, + "", + page.text.getOrElse(""), + null, + null + ) } } @@ -96,4 +115,4 @@ object Resolution { SFile(file).overwrite(output) println(s"written ${resolutions.size} resolutions") } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/SecretaryBot.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/SecretaryBot.scala index c7b293f4..e1c084df 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/SecretaryBot.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/SecretaryBot.scala @@ -11,9 +11,14 @@ object SecretaryBot extends WithBot with QueryLibrary { val host = "ua.wikimedia.org" def fetchResolutions(): Future[Seq[Resolution]] = { - for (all <- bot.run(pageLinksGenerator("Рішення Правління (список)"))) yield { - all.flatMap(Resolution.fromPage).filter(r => r.year == 2016 || r.date == "28 грудня 2015").toSeq.sorted - } + for (all <- bot.run(pageLinksGenerator("Рішення Правління (список)"))) + yield { + all + .flatMap(Resolution.fromPage) + .filter(r => r.year == 2016 || r.date == "28 грудня 2015") + .toSeq + .sorted + } } def main(args: Array[String]): Unit = { diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/UserGroup.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/UserGroup.scala index b95c6f7c..11eef27f 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/UserGroup.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/UserGroup.scala @@ -9,12 +9,11 @@ class UserGroup(val name: String, val users: Set[User]) { override def toString = { val list = if (users.nonEmpty) - s": ${SortedSet(users.map(_.login).toSeq:_*).mkString(", ")}" + s": ${SortedSet(users.map(_.login).toSeq: _*).mkString(", ")}" else "" s"${name.trim} — ${users.size}$list" } - } object UserGroup { @@ -22,6 +21,5 @@ object UserGroup { def empty(name: String) = new UserGroup(name, Set.empty[User]) } -class Board(val year: Int, users: Set[User]) extends UserGroup("Правління", users) - - +class Board(val year: Int, users: Set[User]) + extends UserGroup("Правління", users) diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/Votes.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/Votes.scala index 4471fdb3..8740fea3 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/Votes.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/finance/Votes.scala @@ -1,9 +1,10 @@ package org.scalawiki.bots.finance class Votes( - val support: UserGroup, - val oppose: UserGroup, - val abstain: UserGroup) { + val support: UserGroup, + val oppose: UserGroup, + val abstain: UserGroup +) { def passed = support.users.size > 3 @@ -23,7 +24,8 @@ object Votes { new Votes( byName.get("«за»").fold(UserGroup.empty("«За»"))(_.head), byName.get("«проти»").fold(UserGroup.empty("«Проти»"))(_.head), - byName.get("«утримались»").fold(UserGroup.empty("«Утримались»"))(_.head)) + byName.get("«утримались»").fold(UserGroup.empty("«Утримались»"))(_.head) + ) } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/Entry.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/Entry.scala index 21c97ebd..6cce3d3f 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/Entry.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/Entry.scala @@ -1,37 +1,58 @@ package org.scalawiki.bots.museum -import com.typesafe.config.{Config, ConfigFactory, ConfigRenderOptions, ConfigValueFactory} +import com.typesafe.config.{ + Config, + ConfigFactory, + ConfigRenderOptions, + ConfigValueFactory +} import net.ceedubs.ficus.FicusConfig case class Diff[T](name: String, before: T, after: T) -case class EntryImage(filePath: String, - sourceDescription: Option[String] = None, - uploadTitle: Option[String] = None, - wikiDescription: Option[String] = None, - wlmId: Option[String] = None, - size: Option[Long] = None, - uploaded: Option[Boolean] = None - ) { +case class EntryImage( + filePath: String, + sourceDescription: Option[String] = None, + uploadTitle: Option[String] = None, + wikiDescription: Option[String] = None, + wlmId: Option[String] = None, + size: Option[Long] = None, + uploaded: Option[Boolean] = None +) { def diff(other: EntryImage, prefix: String = ""): Seq[Diff[_]] = { (if (filePath != other.filePath) - Seq(Diff(prefix + "filePath", filePath, other.filePath)) - else Nil) ++ + Seq(Diff(prefix + "filePath", filePath, other.filePath)) + else Nil) ++ (if (uploadTitle != other.uploadTitle) - Seq(Diff(prefix + "uploadTitle", uploadTitle, other.uploadTitle)) - else Nil) ++ + Seq(Diff(prefix + "uploadTitle", uploadTitle, other.uploadTitle)) + else Nil) ++ (if (sourceDescription != other.sourceDescription) - Seq(Diff(prefix + "sourceDescription", sourceDescription, other.sourceDescription)) - else Nil) ++ + Seq( + Diff( + prefix + "sourceDescription", + sourceDescription, + other.sourceDescription + ) + ) + else Nil) ++ (if (wikiDescription != other.wikiDescription) - Seq(Diff(prefix + "wikiDescription", wikiDescription, other.wikiDescription)) - else Nil) ++ + Seq( + Diff( + prefix + "wikiDescription", + wikiDescription, + other.wikiDescription + ) + ) + else Nil) ++ (if (wlmId != other.wlmId) - Seq(Diff(prefix + "wlmId", wlmId, other.wlmId)) - else Nil) + Seq(Diff(prefix + "wlmId", wlmId, other.wlmId)) + else Nil) } - def updateFrom(other: EntryImage, names: Set[String] = Set.empty): EntryImage = { + def updateFrom( + other: EntryImage, + names: Set[String] = Set.empty + ): EntryImage = { this.copy( filePath = if (names.contains("filePath")) other.filePath @@ -52,20 +73,25 @@ case class EntryImage(filePath: String, } } -/** - * Upload entry +/** Upload entry * - * @param dir directory - * @param article wikipedia article - * @param wlmId Wiki Loves Monuments Id - * @param images images in the directory + * @param dir + * directory + * @param article + * wikipedia article + * @param wlmId + * Wiki Loves Monuments Id + * @param images + * images in the directory */ -case class Entry(dir: String, - article: Option[String] = None, - wlmId: Option[String] = None, - images: Seq[EntryImage] = Seq.empty, - text: Option[String] = None, - lang: String = "uk") { +case class Entry( + dir: String, + article: Option[String] = None, + wlmId: Option[String] = None, + images: Seq[EntryImage] = Seq.empty, + text: Option[String] = None, + lang: String = "uk" +) { val articleOrDir = article.getOrElse(dir) @@ -78,10 +104,13 @@ case class Entry(dir: String, def imageOrParentWlmId(image: EntryImage) = image.wlmId.orElse(wlmId) def genWikiDescription(image: EntryImage): String = - descriptionLang(image.sourceDescription.fold("")(_ + ", ") + interWikiLink(articleOrDir)) + + descriptionLang( + image.sourceDescription.fold("")(_ + ", ") + interWikiLink(articleOrDir) + ) + imageOrParentWlmId(image).fold("")(id => s" {{Monument Ukraine|$id}}") - def genUploadTitle(image: EntryImage, index: Int): String = s"$articleOrDir ${index + 1}" + def genUploadTitle(image: EntryImage, index: Int): String = + s"$articleOrDir ${index + 1}" def withWikiDescriptions: Entry = { copy(images = images.map { img => @@ -90,22 +119,20 @@ case class Entry(dir: String, } def withUploadTitles: Entry = { - copy(images = images.zipWithIndex.map { - case (img, index) => - img.copy(uploadTitle = Some(genUploadTitle(img, index))) + copy(images = images.zipWithIndex.map { case (img, index) => + img.copy(uploadTitle = Some(genUploadTitle(img, index))) }) } def genImageFields = this.withUploadTitles.withWikiDescriptions def imagesMaps: Seq[Map[String, String]] = { - images.map { - image => - Map("file" -> image.filePath) ++ - image.uploadTitle.map("title" -> _) ++ - image.wikiDescription.map("description" -> _) ++ - image.wlmId.map("wlm-id" -> _) ++ - image.sourceDescription.map("source-description" -> _) + images.map { image => + Map("file" -> image.filePath) ++ + image.uploadTitle.map("title" -> _) ++ + image.wikiDescription.map("description" -> _) ++ + image.wlmId.map("wlm-id" -> _) ++ + image.sourceDescription.map("source-description" -> _) } } @@ -130,25 +157,29 @@ case class Entry(dir: String, def diff(other: Entry): Seq[Diff[_]] = { (if (dir != other.dir) - Seq(Diff("dir", dir, other.dir)) - else Nil) ++ + Seq(Diff("dir", dir, other.dir)) + else Nil) ++ (if (article != other.article) - Seq(Diff("article", article, other.article)) - else Nil) ++ + Seq(Diff("article", article, other.article)) + else Nil) ++ (if (wlmId != other.wlmId) - Seq(Diff("wlmId", wlmId, other.wlmId)) - else Nil) ++ + Seq(Diff("wlmId", wlmId, other.wlmId)) + else Nil) ++ (if (images != other.images) { - images.zip(other.images).zipWithIndex - .flatMap { case ((img1, img2), index) => img1.diff(img2, s"images[$index].") - } - } - else Nil) + images + .zip(other.images) + .zipWithIndex + .flatMap { case ((img1, img2), index) => + img1.diff(img2, s"images[$index].") + } + } else Nil) } - def updateFrom(other: Entry, - names: Set[String] = Set.empty, - imageNames: Set[String] = Set.empty): Entry = { + def updateFrom( + other: Entry, + names: Set[String] = Set.empty, + imageNames: Set[String] = Set.empty + ): Entry = { this.copy( dir = if (names.contains("dir")) other.dir @@ -159,13 +190,11 @@ case class Entry(dir: String, wlmId = if (names.contains("wlmId")) other.wlmId else wlmId, - images = - if (imageNames.nonEmpty && images.size == other.images.size) { - images.zip(other.images) map { - case (img1, img2) => img1.updateFrom(img2, imageNames) - } + images = if (imageNames.nonEmpty && images.size == other.images.size) { + images.zip(other.images) map { case (img1, img2) => + img1.updateFrom(img2, imageNames) } - else images + } else images ) } } @@ -177,14 +206,17 @@ object Entry { val (dir, article, wlmId) = seq match { case Seq(dir, article, wlmId, _*) => (dir, Some(article), Some(wlmId)) - case Seq(dir, article) => (dir, Some(article), None) - case Seq(dir) => (dir, None, None) + case Seq(dir, article) => (dir, Some(article), None) + case Seq(dir) => (dir, None, None) } - Entry(dir, + Entry( + dir, article.filter(_.trim.nonEmpty), wlmId.filter(_.trim.nonEmpty), - Seq.empty, None) + Seq.empty, + None + ) } def fromConfig(javaConfig: Config, dir: String) = { @@ -204,11 +236,6 @@ object Entry { EntryImage(path, description, uploadTitle, wikiDescription, imageWlmId) } - Entry(dir, - Some(article), - wlmId, - images, - None - ) + Entry(dir, Some(article), wlmId, images, None) } } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/HtmlOutput.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/HtmlOutput.scala index 0420c1fd..c1bce2f7 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/HtmlOutput.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/HtmlOutput.scala @@ -20,7 +20,8 @@ object HtmlOutput { "\n", """
    - """) + """ + ) } val head = @@ -37,14 +38,18 @@ object HtmlOutput { makeEntryGallery(e) } - galleries.map(g => "" + head + "\n" + nav + g + "\n") + galleries.map(g => + "" + head + "\n" + nav + g + "\n" + ) } def makeEntryGallery(entry: Entry): String = { Gallery.asHtml( - entry.images.map { entry => new Image( - entry.filePath, - url = Some(SFile(entry.filePath).uri.toString)) + entry.images.map { entry => + new Image( + entry.filePath, + url = Some(SFile(entry.filePath).uri.toString) + ) }, entry.images.map(_.sourceDescription.getOrElse("")) ) diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/HtmlParser.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/HtmlParser.scala index 62540c83..670205f5 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/HtmlParser.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/HtmlParser.scala @@ -8,8 +8,8 @@ import scala.compat.Platform object HtmlParser { - /** - * Replaces special whitespace characters (short/nonbreaking space) with regular space + /** Replaces special whitespace characters (short/nonbreaking space) with + * regular space * @param s * @return */ @@ -21,25 +21,33 @@ object HtmlParser { def trimmedLines(html: String): Seq[String] = { val text = htmlText(html) - text.split("\n") + text + .split("\n") .map(replaceWs _ andThen (_.trim)) .filter(_.nonEmpty) .toList } - /** - * Tries to get page text from html code. + /** Tries to get page text from html code. * @param html * @return */ def htmlText(html: String): String = { - val tags2Nl = Jsoup.clean(html, "", + val tags2Nl = Jsoup.clean( + html, + "", Safelist.none().addTags("p"), new OutputSettings().prettyPrint(true) ) - Jsoup.clean(tags2Nl, "", - Safelist.none(), - new OutputSettings().prettyPrint(false) - ).split(Platform.EOL).map(_.trim).mkString(Platform.EOL) + Jsoup + .clean( + tags2Nl, + "", + Safelist.none(), + new OutputSettings().prettyPrint(false) + ) + .split(Platform.EOL) + .map(_.trim) + .mkString(Platform.EOL) } } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/ImageListParser.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/ImageListParser.scala index bca15626..e259e317 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/ImageListParser.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/ImageListParser.scala @@ -14,41 +14,44 @@ import better.files.{File => SFile} object ImageListParser { def getDocText(listFile: File): String = { - val extractor = if (listFile.getName.toLowerCase.endsWith(".docx")) - new XWPFWordExtractor(POIXMLDocument.openPackage(listFile.getAbsolutePath)) - else - new WordExtractor(new FileInputStream(listFile)) + val extractor = + if (listFile.getName.toLowerCase.endsWith(".docx")) + new XWPFWordExtractor( + POIXMLDocument.openPackage(listFile.getAbsolutePath) + ) + else + new WordExtractor(new FileInputStream(listFile)) extractor.getText } def docToHtml(docs: Seq[File]): Unit = { - docs.foreach { - doc => - val low = doc.getName.toLowerCase - val fullname = doc.getAbsolutePath - if (low.endsWith(".doc")) { - val html = fullname.replace(".doc", ".html") - if (!new File(html).exists()) { - WordToHtmlConverter.main(Array(fullname, html)) - } - } else if (low.endsWith(".docx")) { - val html = fullname.replace(".docx", ".html") - if (!new File(html).exists()) { - docxToHtml(fullname, html) - } + docs.foreach { doc => + val low = doc.getName.toLowerCase + val fullname = doc.getAbsolutePath + if (low.endsWith(".doc")) { + val html = fullname.replace(".doc", ".html") + if (!new File(html).exists()) { + WordToHtmlConverter.main(Array(fullname, html)) } + } else if (low.endsWith(".docx")) { + val html = fullname.replace(".docx", ".html") + if (!new File(html).exists()) { + docxToHtml(fullname, html) + } + } } } def docxToHtml(docFile: String, htmlFile: String) = { val in = new FileInputStream(new File(docFile)) val document = new XWPFDocument(in) - val options = XHTMLOptions.create().URIResolver(new FileURIResolver(new File("word/media"))) + val options = XHTMLOptions + .create() + .URIResolver(new FileURIResolver(new File("word/media"))) val out = new ByteArrayOutputStream() - XHTMLConverter.getInstance().convert(document, out, options) val text = out.toString() diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/ImageTemplate.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/ImageTemplate.scala index aa85477d..b12d79cb 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/ImageTemplate.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/ImageTemplate.scala @@ -4,17 +4,22 @@ import com.typesafe.config.{Config, ConfigFactory} import scala.collection.JavaConverters._ - object ImageTemplate { val parsed = ConfigFactory.parseResources("art-photo.conf") - def makeInfoPage(title: String, description: String, location: String): String = { - val cfg = resolve(Map( - "title" -> title, - "description" -> description, - "location" -> location - )) + def makeInfoPage( + title: String, + description: String, + location: String + ): String = { + val cfg = resolve( + Map( + "title" -> title, + "description" -> description, + "location" -> location + ) + ) cfg.getString("template") } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/Pereiaslav.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/Pereiaslav.scala index ea611def..cccfebca 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/Pereiaslav.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/museum/Pereiaslav.scala @@ -37,12 +37,12 @@ class Pereiaslav(conf: Config, fs: FileSystem = FileSystems.getDefault) { def getEntries: Future[Seq[Entry]] = { for (entries <- fromWikiTable(tablePage)) yield entries.map { entry => - val objDir = homeDir / entry.dir val files = list(objDir, isImage) val paths = files.map(_.pathAsString) - val descriptions = getImagesDescr(objDir, paths).map(Option.apply).padTo(paths.size, None) + val descriptions = + getImagesDescr(objDir, paths).map(Option.apply).padTo(paths.size, None) val images = files.zip(descriptions).map { case (f, d) => EntryImage(f.pathAsString, d, size = Some(f.size)) } @@ -64,7 +64,8 @@ class Pereiaslav(conf: Config, fs: FileSystem = FileSystems.getDefault) { descFile.flatMap { file => val content = file.contentAsString val lines = HtmlParser.trimmedLines(content) - val filenames = files.map(path => SFile(path).nameWithoutExtension.toLowerCase) + val filenames = + files.map(path => SFile(path).nameWithoutExtension.toLowerCase) lines.filter { line => filenames.exists(line.toLowerCase.startsWith) || line.head.isDigit } @@ -93,26 +94,24 @@ class Pereiaslav(conf: Config, fs: FileSystem = FileSystems.getDefault) { } def makeUploadFiles(entries: Seq[Entry]): Unit = { - entries.foreach { - entry => - val file = homeDir / entry.dir / "upload.conf" - val text = entry.toConfigString - if (!file.exists) { - println(s"Creating ${file.pathAsString}") - file.overwrite(text) - } + entries.foreach { entry => + val file = homeDir / entry.dir / "upload.conf" + val text = entry.toConfigString + if (!file.exists) { + println(s"Creating ${file.pathAsString}") + file.overwrite(text) + } } } def readUploadFiles(entries: Seq[Entry]): Seq[Entry] = { - entries.flatMap { - entry => - val file = homeDir / entry.dir / "upload.conf" - if (file.exists) { - val cfg = ConfigFactory.parseFile(file.toJava) - val loaded = Entry.fromConfig(cfg, entry.dir) - Some(loaded) - } else None + entries.flatMap { entry => + val file = homeDir / entry.dir / "upload.conf" + if (file.exists) { + val cfg = ConfigFactory.parseFile(file.toJava) + val loaded = Entry.fromConfig(cfg, entry.dir) + Some(loaded) + } else None } } @@ -143,7 +142,6 @@ class Pereiaslav(conf: Config, fs: FileSystem = FileSystems.getDefault) { def run() = { getEntries.map { original => - val genOrinal = original.map(_.genImageFields) val loaded = readUploadFiles(original) @@ -157,35 +155,38 @@ class Pereiaslav(conf: Config, fs: FileSystem = FileSystems.getDefault) { val images = genLoaded.flatMap(_.images) val count = images.size - images.zipWithIndex.foreach { - case (image, i) => - - val file = SFile(image.filePath) - val size = file.size + images.zipWithIndex.foreach { case (image, i) => + val file = SFile(image.filePath) + val size = file.size - val fileTitle = image.uploadTitle.get + file.extension.getOrElse("") - val fileFileTitle = "File:" + fileTitle - val url = commons.pageUrl(fileFileTitle, urlEncode = false) - val link = s"""
  2. $url""" - println(link) + val fileTitle = image.uploadTitle.get + file.extension.getOrElse("") + val fileFileTitle = "File:" + fileTitle + val url = commons.pageUrl(fileFileTitle, urlEncode = false) + val link = s"""
  3. $url""" + println(link) // print(s"Uploading image $i/$count $fileTitle ${size / 1024} KB ") - //val result = upload(image, bot).await + // val result = upload(image, bot).await - //println(result) + // println(result) } } } def upload(entry: EntryImage, bot: MwBot): Future[String] = { - val page = ImageTemplate.makeInfoPage(entry.uploadTitle.get, entry.wikiDescription.get, "") + val page = ImageTemplate.makeInfoPage( + entry.uploadTitle.get, + entry.wikiDescription.get, + "" + ) val file = SFile(entry.filePath) val fileTitle = entry.uploadTitle.get + file.extension.getOrElse("") - bot.page("File:" + fileTitle).upload(entry.filePath, Some(page), ignoreWarnings = true) + bot + .page("File:" + fileTitle) + .upload(entry.filePath, Some(page), ignoreWarnings = true) } - // genLoaded.zip(loaded).foreach { // case (o, l) => // val diffs = o.genImageFields.diff(l) @@ -195,7 +196,7 @@ class Pereiaslav(conf: Config, fs: FileSystem = FileSystems.getDefault) { // } // } - //val withSourceDescr = l.updateFrom(o, Set.empty, Set("sourceDescription")) + // val withSourceDescr = l.updateFrom(o, Set.empty, Set("sourceDescription")) // FileUtils.writeWithBackup(homeDir / l.dir / "upload.conf", withSourceDescr.toConfigString) // } @@ -229,5 +230,3 @@ object Pereiaslav { pereiaslav.run() } } - - diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/np/TTN.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/np/TTN.scala index a1c60555..e9087e4f 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/np/TTN.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/np/TTN.scala @@ -2,18 +2,20 @@ package org.scalawiki.bots.np import org.apache.poi.ss.usermodel.{Cell, CellType} -case class TTN(ttn: String, - date: String, - route: String, - sender: String, - senderContact: String, - receiver: String, - receiverContact: String, - description: String, - mass: Double, - places: Int, - value: Double, - cost: Double) { +case class TTN( + ttn: String, + date: String, + route: String, + sender: String, + senderContact: String, + receiver: String, + receiverContact: String, + description: String, + mass: Double, + places: Int, + value: Double, + cost: Double +) { def month: String = date.split("\\.").tail.reverse.mkString(".") def year: String = date.split("\\.").last def yyMmDd: String = date.split("\\.").reverse.mkString(".") diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/np/TTNReader.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/np/TTNReader.scala index cc004a04..c85b8d43 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/np/TTNReader.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/np/TTNReader.scala @@ -9,8 +9,8 @@ case class Person(contact: String, ttns: Seq[TTN]) object Person { def receivers(ttns: Seq[TTN]) = { - ttns.groupBy(_.receiverContact).map { - case (contact, personReceived) => Person(contact, personReceived) + ttns.groupBy(_.receiverContact).map { case (contact, personReceived) => + Person(contact, personReceived) } } @@ -20,8 +20,8 @@ case class TTNData(ttns: Seq[TTN]) { val byMonth = ttns.groupBy(_.month).view.mapValues(_.map(_.cost).sum).toSeq.sortBy(_._1) - val byYear = byMonth.groupMap { - case (month, ttns) => month.split("\\.").head + val byYear = byMonth.groupMap { case (month, ttns) => + month.split("\\.").head } { case (month, ttns) => ttns } def variousStat(): Unit = { @@ -33,8 +33,8 @@ case class TTNData(ttns: Seq[TTN]) { .groupBy(_.receiverContact) .view .mapValues(x => x.size) - .map { - case (contact, count) => (count, contact) + .map { case (contact, count) => + (count, contact) } .groupBy(_._1) .view @@ -47,8 +47,8 @@ case class TTNData(ttns: Seq[TTN]) { println(byMonth) println(byYearAvg) - lastYear.filter(_._1 >= 5) foreach { - case (count, people) => println(s"$count: $people") + lastYear.filter(_._1 >= 5) foreach { case (count, people) => + println(s"$count: $people") } // ttns.filter(x => x.receiverContact.contains("Мамон") && x.year == "2023").sortBy(_.yyMmDd).foreach { t => diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/Annotation.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/Annotation.scala index cbdbabf9..12c0cd56 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/Annotation.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/Annotation.scala @@ -13,25 +13,36 @@ class Annotation(val page: Page) { val annotation = createAnnotation(revisions :+ Annotation.revision0) import scala.collection.JavaConverters._ - val annotatedElements: Seq[AnnotatedElement[Revision, String]] = annotation.toSeq.flatMap(_.iterator().asScala.toSeq) - .filter(_.getRevision != null) - - val byRevisionContent: Map[Revision, Seq[String]] = annotatedElements.groupBy(_.getRevision).mapValues(_.map(_.getElement)).toMap - val byUserContent: Map[String, Seq[String]] = annotatedElements.groupBy(_.getRevision.user.flatMap(_.name).getOrElse("")).mapValues(_.map(_.getElement)).toMap - - val byRevisionSize = byRevisionContent.mapValues(_.map(_.length).sum) - val byUserSize = byUserContent.mapValues(_.map(_.length).sum) - - def createAnnotation(revisions: Seq[Revision]): Option[AnnotatedContent[Revision, String]] = { + val annotatedElements: Seq[AnnotatedElement[Revision, String]] = + annotation.toSeq + .flatMap(_.iterator().asScala.toSeq) + .filter(_.getRevision != null) + + val byRevisionContent: Map[Revision, Seq[String]] = annotatedElements + .groupBy(_.getRevision) + .mapValues(_.map(_.getElement)) + .toMap + val byUserContent: Map[String, Seq[String]] = annotatedElements + .groupBy(_.getRevision.user.flatMap(_.name).getOrElse("")) + .mapValues(_.map(_.getElement)) + .toMap + + val byRevisionSize = byRevisionContent.mapValues(_.map(_.length).sum) + val byUserSize = byUserContent.mapValues(_.map(_.length).sum) + + def createAnnotation( + revisions: Seq[Revision] + ): Option[AnnotatedContent[Revision, String]] = { val contentSize = revisions.count(_.content.isDefined) if (contentSize > 0) { val blameManager = new DefaultBlameManager() - val annotatedContent = revisions.foldLeft(null.asInstanceOf[AnnotatedContent[Revision, String]]) { - (annotation, revision) => - blameManager.blame(annotation, revision, splitByWords(revision.content)) + val annotatedContent = revisions.foldLeft( + null.asInstanceOf[AnnotatedContent[Revision, String]] + ) { (annotation, revision) => + blameManager.blame(annotation, revision, splitByWords(revision.content)) } Option(annotatedContent) } else { @@ -41,7 +52,7 @@ class Annotation(val page: Page) { def splitByWords(content: Option[String]): util.List[String] = { val array = content.fold(Array[String]())(_.split("[^\\pL_\\pN]+")) - util.Arrays.asList(array:_*) + util.Arrays.asList(array: _*) } } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/ArticleStat.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/ArticleStat.scala index e854fb6a..95f90626 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/ArticleStat.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/ArticleStat.scala @@ -2,7 +2,11 @@ package org.scalawiki.bots.stat import org.scalawiki.dto.filter.RevisionFilter -class ArticleStat(val filter: RevisionFilter, val revisionStats: Seq[RevisionStat], label: String) { +class ArticleStat( + val filter: RevisionFilter, + val revisionStats: Seq[RevisionStat], + label: String +) { println(s"making ArticleStat for label: $label") @@ -10,7 +14,8 @@ class ArticleStat(val filter: RevisionFilter, val revisionStats: Seq[RevisionSta val addedOrRewritten = revisionStats.map(_.addedOrRewritten) val added = new NumericArrayStat("Added", deltas) - val addedOrRewrittenStat = new NumericArrayStat("Added or rewritten", addedOrRewritten) + val addedOrRewrittenStat = + new NumericArrayStat("Added or rewritten", addedOrRewritten) val userStat = new UserStat(revisionStats) diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/ArticleStatBot.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/ArticleStatBot.scala index 3c1c53f2..1bef1699 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/ArticleStatBot.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/ArticleStatBot.scala @@ -10,7 +10,8 @@ import org.scalawiki.query.QueryLibrary import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -class ArticleStatBot(implicit val bot: MwBot = MwBot.fromHost(MwBot.ukWiki)) extends QueryLibrary { +class ArticleStatBot(implicit val bot: MwBot = MwBot.fromHost(MwBot.ukWiki)) + extends QueryLibrary { def pagesRevisions(ids: Seq[Long]): Future[Iterable[Option[Page]]] = { Future.traverse(ids)(pageRevisions) @@ -30,29 +31,32 @@ class ArticleStatBot(implicit val bot: MwBot = MwBot.fromHost(MwBot.ukWiki)) ext val newPagesIdsF = articlesWithTemplate(event.newTemplate) val improvedPagesIdsF = articlesWithTemplate(event.improvedTemplate) - Future.sequence(Seq(newPagesIdsF, improvedPagesIdsF)).flatMap { - ids => + Future.sequence(Seq(newPagesIdsF, improvedPagesIdsF)).flatMap { ids => + val newPagesIds = ids.head + val improvedPagesIds = ids.last + println(s"New ${newPagesIds.size} $newPagesIds") + println(s"Improved ${improvedPagesIds.size} $improvedPagesIds") - val newPagesIds = ids.head - val improvedPagesIds = ids.last - println(s"New ${newPagesIds.size} $newPagesIds") - println(s"Improved ${improvedPagesIds.size} $improvedPagesIds") + val allIds = newPagesIds.toSet ++ improvedPagesIds.toSet - val allIds = newPagesIds.toSet ++ improvedPagesIds.toSet - - pagesRevisions(allIds.toSeq).map { allPages => - - val revStats = allPages.iterator.flatten.toIndexedSeq.zipWithIndex.flatMap { + pagesRevisions(allIds.toSeq).map { allPages => + val revStats = allPages.iterator.flatten.toIndexedSeq.zipWithIndex + .flatMap { case (page, index) if page.history.editedIn(revisionFilter) => - bot.log.info(s"$index/${allIds.size} making revision stat for ${page.title}") + bot.log.info( + s"$index/${allIds.size} making revision stat for ${page.title}" + ) Some(RevisionStat.fromPage(page, revisionFilter)) case (page, index) => - bot.log.info(s"$index/${allIds.size} skipping ${page.title} because it's rejected by revision filter") + bot.log.info( + s"$index/${allIds.size} skipping ${page.title} because it's rejected by revision filter" + ) None - }.sortBy(-_.addedOrRewritten) + } + .sortBy(-_.addedOrRewritten) - EventStat(event, revStats) - } + EventStat(event, revStats) + } } } } @@ -90,9 +94,12 @@ object ArticleStatBot { val (contests, weeks) = Events.events() val sweden = weeks.find(_.newTemplate.startsWith("2019Sweden-week")).toSeq - Future.sequence(sweden.map(contestStat)).map(_.map(_.asWiki).mkString("\n")).map { wikitext => - FileUtils.write("articles.wiki", wikitext) - } + Future + .sequence(sweden.map(contestStat)) + .map(_.map(_.asWiki).mkString("\n")) + .map { wikitext => + FileUtils.write("articles.wiki", wikitext) + } } def eventSummary(stats: Seq[Long]): Unit = { @@ -101,4 +108,4 @@ object ArticleStatBot { println(s"!!!!!!!!! weeks: $events, bytes: $added") println(s"!!!!!!!!! weeks: $stats") } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/CatScan.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/CatScan.scala index 8cc8b66c..5be87fa8 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/CatScan.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/CatScan.scala @@ -20,7 +20,6 @@ import scala.collection.SortedSet import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future - object CatScan { val system = ActorSystem() @@ -61,11 +60,10 @@ object CatScan { val countFuture = getCountCached(bot, title) - countFuture.map { - count => - bot.system.log.info("Final count:" + count) + countFuture.map { count => + bot.system.log.info("Final count:" + count) - // SortedSet(total.asScala.toSeq: _*).foreach(t => println(s"* [[$t]]")) + // SortedSet(total.asScala.toSeq: _*).foreach(t => println(s"* [[$t]]")) // SortedSet(noUkWiki.asScala.toSeq: _*).foreach(t => println(s"* [[$t]]")) } } @@ -76,65 +74,92 @@ object CatScan { def getCount(bot: MwBot, title: String): Future[Set[String]] = { bot.system.log.info(s"Enter: $title, cache size: ${cache.size}") - cache(title, { () => - Future { - Set.empty[String] + cache( + title, + { () => + Future { + Set.empty[String] + } } - }) - val subCats = catsWithLinks(title, namespaces = Set(Namespace.CATEGORY), bot).flatMap { pages => - bot.system.log.info(s"Category $title has ${pages.size} subcats") - - val noUkCats = pages.filter(!_.langLinks.contains("uk")).map(_.title) - noUkWikiCats.addAll(noUkCats.toSeq.asJava) + ) + val subCats = + catsWithLinks(title, namespaces = Set(Namespace.CATEGORY), bot).flatMap { + pages => + bot.system.log.info(s"Category $title has ${pages.size} subcats") - allCats.addAll(pages.map(_.title).toSeq.asJava) + val noUkCats = pages.filter(!_.langLinks.contains("uk")).map(_.title) + noUkWikiCats.addAll(noUkCats.toSeq.asJava) - val futures = pages.map{ - page => getCountCached(bot, page.title) + allCats.addAll(pages.map(_.title).toSeq.asJava) + val futures = pages.map { page => + getCountCached(bot, page.title) + } + Future.reduce(futures)((s1, s2) => s1 ++ s2) } - Future.reduce(futures)((s1, s2) => s1 ++ s2) - } - val thisCat = catsWithLinks(title, namespaces = Set(Namespace.MAIN), bot).map { articles => + val thisCat = + catsWithLinks(title, namespaces = Set(Namespace.MAIN), bot).map { + articles => + import scala.collection.JavaConverters._ - import scala.collection.JavaConverters._ + val size = articles.size + val titles: Set[String] = articles.map(_.title).toSet + total.addAll(titles.asJava) - val size = articles.size - val titles: Set[String] = articles.map(_.title).toSet - total.addAll(titles.asJava) + val noUk = articles.filter(!_.langLinks.contains("uk")).map(_.title) + noUkWiki.addAll(noUk.toSeq.asJava) - val noUk = articles.filter(!_.langLinks.contains("uk")).map(_.title) - noUkWiki.addAll(noUk.toSeq.asJava) + val processed = categories.addAndGet(1) - val processed = categories.addAndGet(1) + bot.system.log.info( + s"Total is ${total.size}, processed = $processed (${processed * 100.0 / cache.size} %) Category $title has $size articles" + ) - bot.system.log.info(s"Total is ${total.size}, processed = $processed (${processed * 100.0 / cache.size} %) Category $title has $size articles") + if (processed == cache.size) { + println(cache.keys.mkString("All cats:\n", "\n", "\n")) - if (processed == cache.size) { - println(cache.keys.mkString("All cats:\n", "\n", "\n")) + println("== no UkWiki pages ==") + SortedSet(noUkWiki.asScala.toSeq: _*).foreach(t => + println(s"* [[$t]]") + ) - println("== no UkWiki pages ==") - SortedSet(noUkWiki.asScala.toSeq: _*).foreach(t => println(s"* [[$t]]")) - - println("== no UkWiki cats ==") - SortedSet(noUkWikiCats.asScala.toSeq: _*).foreach(t => println(s"* [[:$t]]")) + println("== no UkWiki cats ==") + SortedSet(noUkWikiCats.asScala.toSeq: _*).foreach(t => + println(s"* [[:$t]]") + ) + } + titles } - titles - } val sum = Seq(subCats, thisCat) Future.reduce(sum)((s1, s2) => s1 ++ s2) } - def catsWithLinks(title: String, namespaces: Set[Int], bot: MwBot): Future[Iterable[Page]] = { - val action = Action(Query( - Prop( - LangLinks(LlLimit("max")) - ), - Generator(ListArgs.toDsl("categorymembers", Some(title), None, namespaces, Some("max")).get) - )) + def catsWithLinks( + title: String, + namespaces: Set[Int], + bot: MwBot + ): Future[Iterable[Page]] = { + val action = Action( + Query( + Prop( + LangLinks(LlLimit("max")) + ), + Generator( + ListArgs + .toDsl( + "categorymembers", + Some(title), + None, + namespaces, + Some("max") + ) + .get + ) + ) + ) bot.run(action) } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/Event.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/Event.scala index 91fe31af..1f13a216 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/Event.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/Event.scala @@ -6,12 +6,14 @@ import com.typesafe.config.{Config, ConfigFactory} import scala.util.Try -case class ArticlesEvent(name: String, - start: ZonedDateTime, - end: ZonedDateTime, - newTemplate: String, - improvedTemplate: String, - id: Option[String] = None) +case class ArticlesEvent( + name: String, + start: ZonedDateTime, + end: ZonedDateTime, + newTemplate: String, + improvedTemplate: String, + id: Option[String] = None +) object Events { @@ -42,4 +44,4 @@ object Events { println(s"Contests: ${contests.mkString("\n")}") println(s"Weeks: ${weeks.mkString("\n")}") } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/NumericArrayStat.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/NumericArrayStat.scala index 9db4ae68..de7db48a 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/NumericArrayStat.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/NumericArrayStat.scala @@ -3,8 +3,9 @@ package org.scalawiki.bots.stat class NumericArrayStat(val name: String, val data: Seq[Long]) { val sum = data.sum val max = if (data.isEmpty) 0 else data.max - val min = if (data.isEmpty) 0 else data.min + val min = if (data.isEmpty) 0 else data.min val average = if (data.isEmpty) 0 else sum / data.size - override def toString = s"$name $sum (max: $max, min: $min, average: $average)" + override def toString = + s"$name $sum (max: $max, min: $min, average: $average)" } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/RevisionAnnotation.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/RevisionAnnotation.scala index 792e261a..fb059e35 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/RevisionAnnotation.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/RevisionAnnotation.scala @@ -4,19 +4,30 @@ import org.scalawiki.dto.filter.{AllRevisionsFilter, RevisionFilter} import org.scalawiki.dto.{Page, Revision} import org.xwiki.blame.AnnotatedElement -class RevisionAnnotation(val page: Page, revFilter: RevisionFilter = AllRevisionsFilter) { +class RevisionAnnotation( + val page: Page, + revFilter: RevisionFilter = AllRevisionsFilter +) { val revisions = revFilter(page.revisions) val annotation: Option[Annotation] = Annotation.create(page) def pageAnnotatedElements: Seq[AnnotatedElement[Revision, String]] = - annotation.fold(Seq.empty[AnnotatedElement[Revision, String]])(_.annotatedElements) + annotation.fold(Seq.empty[AnnotatedElement[Revision, String]])( + _.annotatedElements + ) val annotatedElements = pageAnnotatedElements .filter(element => revFilter.predicate(element.getRevision)) - val byRevisionContent: Map[Revision, Seq[String]] = annotatedElements.groupBy(_.getRevision).mapValues(_.map(_.getElement)).toMap - val byUserContent: Map[String, Seq[String]] = annotatedElements.groupBy(_.getRevision.user.flatMap(_.name) getOrElse "").mapValues(_.map(_.getElement)).toMap + val byRevisionContent: Map[Revision, Seq[String]] = annotatedElements + .groupBy(_.getRevision) + .mapValues(_.map(_.getElement)) + .toMap + val byUserContent: Map[String, Seq[String]] = annotatedElements + .groupBy(_.getRevision.user.flatMap(_.name) getOrElse "") + .mapValues(_.map(_.getElement)) + .toMap } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/RevisionStat.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/RevisionStat.scala index 7a18ad92..d641e02f 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/RevisionStat.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/RevisionStat.scala @@ -6,9 +6,11 @@ import org.scalawiki.dto.filter.{AllRevisionsFilter, RevisionFilter} import org.scalawiki.dto.history.History import org.scalawiki.dto.{Page, Revision} -class RevisionStat(val page: Page, - val byRevisionSize: Map[Revision, Long], - byUserSize: Map[String, Long]) { +class RevisionStat( + val page: Page, + val byRevisionSize: Map[Revision, Long], + byUserSize: Map[String, Long] +) { val history = new History(page.revisions) @@ -19,36 +21,44 @@ class RevisionStat(val page: Page, def byUserSize(user: String): Long = byUserSize.getOrElse(user, 0L) - val usersSorted = byUserSize.toSeq.sortBy { - case (user, size) => -size + val usersSorted = byUserSize.toSeq.sortBy { case (user, size) => + -size } - val usersSortedString = usersSorted.map { - case (user, size) => s"[[User:$user|$user]]: $size" - }.mkString("
    ") + val usersSortedString = usersSorted + .map { case (user, size) => + s"[[User:$user|$user]]: $size" + } + .mkString("
    ") - def stat = s"| [[${page.title}]] || $addedOrRewritten || $delta || ${byUserSize.keySet.size} || $usersSortedString" + def stat = + s"| [[${page.title}]] || $addedOrRewritten || $delta || ${byUserSize.keySet.size} || $usersSortedString" } object RevisionStat { - def statHeader = s"! title !! added or rewritten !! size increase !! users !! users by added or rewritten text size\n" + def statHeader = + s"! title !! added or rewritten !! size increase !! users !! users by added or rewritten text size\n" def fromPage(page: Page, revFilter: RevisionFilter = AllRevisionsFilter) = { val annotation = new RevisionAnnotation(page, revFilter) - val byRevisionSize = annotation.byRevisionContent.map { - case (rev, words) => - rev.withoutContent -> words.map(_.getBytes(StandardCharsets.UTF_8).length.toLong).sum + val byRevisionSize = annotation.byRevisionContent.map { case (rev, words) => + rev.withoutContent -> words + .map(_.getBytes(StandardCharsets.UTF_8).length.toLong) + .sum } - val byUserSize = annotation.byUserContent.mapValues(_.map(_.getBytes(StandardCharsets.UTF_8).length.toLong).sum).toMap + val byUserSize = annotation.byUserContent + .mapValues(_.map(_.getBytes(StandardCharsets.UTF_8).length.toLong).sum) + .toMap new RevisionStat( page.copy(revisions = revFilter(page.revisions.map(_.withoutContent))), - byRevisionSize, byUserSize + byRevisionSize, + byUserSize ) } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/UserContribList.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/UserContribList.scala index 39b23457..bbf90ca9 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/UserContribList.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/UserContribList.scala @@ -19,30 +19,46 @@ object UserContribList extends WithBot with QueryLibrary { val minEdits = 300 val minRecentEdits = 20 val minEditsTo = ZonedDateTime.of(2017, 4, 15, 0, 0, 0, 0, ZoneOffset.UTC) - val recentEditsFrom = ZonedDateTime.of(2016, 10, 15, 0, 0, 0, 0, ZoneOffset.UTC) + val recentEditsFrom = + ZonedDateTime.of(2016, 10, 15, 0, 0, 0, 0, ZoneOffset.UTC) - for (votedUsers <- VoteList.votedUsers; - allUsers <- getUsers(activeUsersQuery)) { + for ( + votedUsers <- VoteList.votedUsers; + allUsers <- getUsers(activeUsersQuery) + ) { val users = allUsers.filter { u => u.editCount.exists(_ > minEdits) && u.blocked.forall(_ == false) } val contribsFuture = Future.traverse(users) { user => - bot.run( - userContribs(user.name.get, TimeRange(Some(minEditsTo), None), minEdits.toString), limit = Some(minEdits)).map(_.toSeq) + bot + .run( + userContribs( + user.name.get, + TimeRange(Some(minEditsTo), None), + minEdits.toString + ), + limit = Some(minEdits) + ) + .map(_.toSeq) } - val eligible = for (pagesSeq <- contribsFuture) yield - for (pages <- pagesSeq if - pages.size >= minEdits && - pages(minRecentEdits - 1).revisions.headOption.exists(_.timestamp.exists(_.isAfter(recentEditsFrom))); - user <- pages.head.lastRevisionUser.flatMap(_.name) - ) yield user + val eligible = + for (pagesSeq <- contribsFuture) + yield for ( + pages <- pagesSeq if pages.size >= minEdits && + pages(minRecentEdits - 1).revisions.headOption + .exists(_.timestamp.exists(_.isAfter(recentEditsFrom))); + user <- pages.head.lastRevisionUser.flatMap(_.name) + ) yield user - eligible.map { users => - users.filterNot(votedUsers.contains).foreach(println) - }.failed.map(println) + eligible + .map { users => + users.filterNot(votedUsers.contains).foreach(println) + } + .failed + .map(println) } } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/UserStat.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/UserStat.scala index c7b36f21..9cd8c163 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/UserStat.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/UserStat.scala @@ -6,23 +6,30 @@ class UserStat(revisionStats: Seq[RevisionStat]) { } val byUser: Map[String, Seq[RevisionStat]] = users.map { user => - (user, revisionStats.filter(_.users.contains(user)).sortBy(-_.byUserSize(user))) + ( + user, + revisionStats.filter(_.users.contains(user)).sortBy(-_.byUserSize(user)) + ) }.toMap - val byUserAddedOrRemoved: Seq[(String, Long)] = byUser.toSeq.map { - case (user, statSeq) => (user, statSeq.map(_.byUserSize(user)).sum) - }.sortBy { - case (user, size) => -size - } + val byUserAddedOrRemoved: Seq[(String, Long)] = byUser.toSeq + .map { case (user, statSeq) => + (user, statSeq.map(_.byUserSize(user)).sum) + } + .sortBy { case (user, size) => + -size + } override def toString = { val header = "{| class='wikitable sortable'\n" + "|+ users\n" + "! user !! added or rewritten !! articles number !! article list\n" - header + byUserAddedOrRemoved.map { - case (user, size) => - s"| [[User:$user|$user]] || $size || ${byUser(user).size} || ${byUser(user).map(rs => s"[[${rs.page.title}]] ${rs.byUserSize(user)}").mkString("
    ")}" - } + header + byUserAddedOrRemoved + .map { case (user, size) => + s"| [[User:$user|$user]] || $size || ${byUser(user).size} || ${byUser(user) + .map(rs => s"[[${rs.page.title}]] ${rs.byUserSize(user)}") + .mkString("
    ")}" + } .mkString("\n|-\n", "\n|-\n", "") + "\n|}" } } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/zachte/UkWikiStat.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/zachte/UkWikiStat.scala index d317b8f2..7a0f55e0 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/zachte/UkWikiStat.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/stat/zachte/UkWikiStat.scala @@ -18,14 +18,13 @@ object UkWikiStat { val site = "Wikipedia" val url = s"https://stats.wikimedia.org/EN/Tables$site$lang.htm" - http.get(url).foreach { - html => - val doc = Jsoup.parse(html) - val table = doc.select("table#table1").first() - val rows = table.select("tr").asScala - for (row <- rows) { - val cols = row.select("td") - } + http.get(url).foreach { html => + val doc = Jsoup.parse(html) + val table = doc.select("table#table1").first() + val rows = table.select("tr").asScala + for (row <- rows) { + val cols = row.select("td") + } } } } diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/bots/vote/VoteList.scala b/scalawiki-bots/src/main/scala/org/scalawiki/bots/vote/VoteList.scala index 462fd9d1..53795695 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/bots/vote/VoteList.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/bots/vote/VoteList.scala @@ -15,7 +15,8 @@ object VoteList { val specialPage = "Special:SecurePoll/list/637" - val url = "https://vote.wikimedia.org/w/index.php?title=Special:SecurePoll/list/637&limit=5000&sort=vote_voter_name" + val url = + "https://vote.wikimedia.org/w/index.php?title=Special:SecurePoll/list/637&limit=5000&sort=vote_voter_name" def main(args: Array[String]): Unit = { @@ -44,10 +45,10 @@ object VoteList { names.flatten.distinct.toSeq - } recover { - case t: Throwable => println(t) - Seq.empty + } recover { case t: Throwable => + println(t) + Seq.empty } names } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/copyvio/CopyVio.scala b/scalawiki-bots/src/main/scala/org/scalawiki/copyvio/CopyVio.scala index 3276fd0e..b8f1bfb8 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/copyvio/CopyVio.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/copyvio/CopyVio.scala @@ -19,17 +19,25 @@ class CopyVio(val http: HttpClient) { ((__ \ "url").read[String] ~ (__ \ "confidence").read[Double] ~ (__ \ "violation").read[String] ~ - (__ \ "skipped").read[Boolean]) (CopyVioSource.apply _) + (__ \ "skipped").read[Boolean])(CopyVioSource.apply _) val sourcesReads = (__ \ "sources").read[Seq[CopyVioSource]] def baseUrl(project: String = "wikipedia", lang: String = "uk") = s"https://tools.wmflabs.org/copyvios/api.json?version=1&action=search&project=$project&lang=$lang" - def search(title: String, lang: String = "uk", project: String = "wikipedia") = + def search( + title: String, + lang: String = "uk", + project: String = "wikipedia" + ) = http.get(baseUrl(project, lang) + s"&title=$title") map parseResponse - def searchByRevId(revId: Long, lang: String = "uk", project: String = "wikipedia") = + def searchByRevId( + revId: Long, + lang: String = "uk", + project: String = "wikipedia" + ) = http.get(baseUrl(project, lang) + s"&oldid=$revId") map parseResponse def searchByPage(page: Page) = { @@ -48,7 +56,12 @@ object CopyVio extends WithBot with QueryLibrary { def pagesByIds(ids: Iterable[Long]): Future[Iterable[Page]] = { import org.scalawiki.dto.cmd.query.prop.rvprop._ - val action = Action(Query(PageIdsParam(ids), Prop(Info(), Revisions(RvProp(Ids) /*,RvLimit("max")*/)))) + val action = Action( + Query( + PageIdsParam(ids), + Prop(Info(), Revisions(RvProp(Ids) /*,RvLimit("max")*/ )) + ) + ) bot.run(action) } @@ -56,12 +69,16 @@ object CopyVio extends WithBot with QueryLibrary { def main(args: Array[String]) { val copyVio = new CopyVio(HttpClient.get()) - for (revIds <- articlesWithTemplate("Вікіпедія любить пам'ятки"); - pages <- pagesByIds(revIds); - page <- pages; - sources <- copyVio.searchByPage(page); - suspected <- sources.filter(_.isPossible)) { - println(s"## url: [${suspected.url}], violation ${suspected.violation}, confidence ${suspected.confidence}") + for ( + revIds <- articlesWithTemplate("Вікіпедія любить пам'ятки"); + pages <- pagesByIds(revIds); + page <- pages; + sources <- copyVio.searchByPage(page); + suspected <- sources.filter(_.isPossible) + ) { + println( + s"## url: [${suspected.url}], violation ${suspected.violation}, confidence ${suspected.confidence}" + ) } } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/main/scala/org/scalawiki/copyvio/CopyVioSource.scala b/scalawiki-bots/src/main/scala/org/scalawiki/copyvio/CopyVioSource.scala index fb99cc28..71c828d8 100644 --- a/scalawiki-bots/src/main/scala/org/scalawiki/copyvio/CopyVioSource.scala +++ b/scalawiki-bots/src/main/scala/org/scalawiki/copyvio/CopyVioSource.scala @@ -1,8 +1,10 @@ package org.scalawiki.copyvio -case class CopyVioSource(url: String, - confidence: Double, - violation: String, - skipped: Boolean) { +case class CopyVioSource( + url: String, + confidence: Double, + violation: String, + skipped: Boolean +) { def isPossible = violation != "none" } diff --git a/scalawiki-bots/src/main/scala/scala/util/matching/RegexFactory.scala b/scalawiki-bots/src/main/scala/scala/util/matching/RegexFactory.scala index c9681aea..daaffec9 100644 --- a/scalawiki-bots/src/main/scala/scala/util/matching/RegexFactory.scala +++ b/scalawiki-bots/src/main/scala/scala/util/matching/RegexFactory.scala @@ -7,4 +7,4 @@ object RegexFactory { def replacementIterator(text: String, old: Regex) = new Regex.MatchIterator(text, old, Seq.empty).replacementData -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/FileUtilsSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/FileUtilsSpec.scala index 8cf63d0e..0a329fa4 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/FileUtilsSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/FileUtilsSpec.scala @@ -54,4 +54,4 @@ class FileUtilsSpec extends Specification with BeforeEach { (root / "test.txt.1").exists === false } } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/MessageBotSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/MessageBotSpec.scala index ddc8ae71..e0f8efcc 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/MessageBotSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/MessageBotSpec.scala @@ -51,7 +51,10 @@ class MessageBotSpec extends Specification { val bot = new MessageBot(config) - bot.range === TimeRange(Some(ZonedDateTime.of(2016, 1, 29, 0, 0, 0, 0, ZoneOffset.UTC)), None) + bot.range === TimeRange( + Some(ZonedDateTime.of(2016, 1, 29, 0, 0, 0, 0, ZoneOffset.UTC)), + None + ) } } diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/PageFromFileBotSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/PageFromFileBotSpec.scala index 524532d6..5f45bf88 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/PageFromFileBotSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/PageFromFileBotSpec.scala @@ -38,7 +38,8 @@ class PageFromFileBotSpec extends Specification { } "support windows newlines" in { - val pages = PageFromFileBot.pages(docExample.replaceAll("\n", "\r\n")).toBuffer + val pages = + PageFromFileBot.pages(docExample.replaceAll("\n", "\r\n")).toBuffer pages.size === 2 pages.map(_.title) === titles pages.flatMap(_.text) === texts.map(_.replaceAll("\n", "\r\n")) @@ -56,7 +57,12 @@ class PageFromFileBotSpec extends Specification { |Another text |yyyy""".stripMargin.replaceAll("\r?\n", "\n") - val pages = PageFromFileBot.pages(docExampleOwnDelimiter, PageFromFileFormat(start = "xxxx", end = "yyyy")).toBuffer + val pages = PageFromFileBot + .pages( + docExampleOwnDelimiter, + PageFromFileFormat(start = "xxxx", end = "yyyy") + ) + .toBuffer pages.size === 2 pages.map(_.title) === titles pages.flatMap(_.text) === texts @@ -70,17 +76,15 @@ class PageFromFileBotSpec extends Specification { } "join title in page text" in { - val pages = titles.zip(texts) map { - case (title, text) => - Page(title).withText(text) + val pages = titles.zip(texts) map { case (title, text) => + Page(title).withText(text) } PageFromFileBot.join(pages, includeTitle = false) === docExample } "join include title" in { - val pages = titles.zip(textsWithoutTitles) map { - case (title, text) => - Page(title).withText(text) + val pages = titles.zip(textsWithoutTitles) map { case (title, text) => + Page(title).withText(text) } PageFromFileBot.join(pages, includeTitle = true) === docExample } @@ -93,12 +97,16 @@ class PageFromFileBotSpec extends Specification { val words = 50 val lines = 10 val articles = Math.min(5000, 5000 * (maxMemMB / 400)).toInt - println(s"$maxMemMB MB heap should be enough to process at least $articles articles") + println( + s"$maxMemMB MB heap should be enough to process at least $articles articles" + ) new StringBuilder().append() - val text = Array.fill(lines) { - Array.fill(words)("wikitext").mkString(" ") - }.mkString("{{-start-}}\n'''PageName''' ", "\n", "\n{{-end-}}") + val text = Array + .fill(lines) { + Array.fill(words)("wikitext").mkString(" ") + } + .mkString("{{-start-}}\n'''PageName''' ", "\n", "\n{{-end-}}") val chunk = Array.fill(articles)(text).mkString("\n") diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/PageGeneratorsSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/PageGeneratorsSpec.scala index d95be4dc..59e7c028 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/PageGeneratorsSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/PageGeneratorsSpec.scala @@ -48,4 +48,4 @@ class PageGeneratorsSpec extends Specification { // parse(Seq("-namespaces", "14,8")).values("namespace") === ns // } } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/ReplaceSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/ReplaceSpec.scala index e937c3c8..b65586d1 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/ReplaceSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/ReplaceSpec.scala @@ -14,7 +14,10 @@ class ReplaceSpec extends Specification { "replacements" in { parse(Seq("old", "new")).replacements === Map("old" -> "new") - parse(Seq("old1", "new1", "old2", "new2")).replacements === Map("old1" -> "new1", "old2" -> "new2") + parse(Seq("old1", "new1", "old2", "new2")).replacements === Map( + "old1" -> "new1", + "old2" -> "new2" + ) } "replacement and cat" in { @@ -24,7 +27,8 @@ class ReplaceSpec extends Specification { } "replacements and cat" in { - val args = parse(Seq("--cat", "category name", "old1", "new1", "old2", "new2")) + val args = + parse(Seq("--cat", "category name", "old1", "new1", "old2", "new2")) args.pages.cat === Seq("category name") args.replacements === Map("old1" -> "new1", "old2" -> "new2") } @@ -61,7 +65,11 @@ class ReplaceSpec extends Specification { replaceExcept("A behindB C", "(?<=behind)\\w", "Z") === "A behindZ C" // test regex with lookbehind and groups. - replaceExcept("A behindB C D", "(?<=behind)\\w( )", "$1Z") === "A behind ZC D" + replaceExcept( + "A behindB C D", + "(?<=behind)\\w( )", + "$1Z" + ) === "A behind ZC D" // test regex with lookahead. replaceExcept("A Bahead C", "\\w(?=ahead)", "Z") === "A Zahead C" @@ -86,12 +94,17 @@ class ReplaceSpec extends Specification { replaceExcept("1111", "11", "21") === "2121" // allow overlap not supported - //replaceExcept("1111", "11", "21") === "2221" + // replaceExcept("1111", "11", "21") === "2221" } "replacing not inside a specific regex" in { replaceExcept("123x123", "123", "000") === "000x000" - replaceExcept("123x123", "123", "000", exceptions = Seq("\\w123".r)) === "000x123" + replaceExcept( + "123x123", + "123", + "000", + exceptions = Seq("\\w123".r) + ) === "000x123" } } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/EntrySpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/EntrySpec.scala index 018390d4..92bafe85 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/EntrySpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/EntrySpec.scala @@ -18,11 +18,19 @@ class EntrySpec extends Specification { } "get dir, article and wlmId" in { - Entry.fromRow(Seq("dir", "article", "wlmId")) === Entry("dir", Some("article"), Some("wlmId")) + Entry.fromRow(Seq("dir", "article", "wlmId")) === Entry( + "dir", + Some("article"), + Some("wlmId") + ) } "get dir, article, wlmId and extra column" in { - Entry.fromRow(Seq("dir", "article", "wlmId", "something else")) === Entry("dir", Some("article"), Some("wlmId")) + Entry.fromRow(Seq("dir", "article", "wlmId", "something else")) === Entry( + "dir", + Some("article"), + Some("wlmId") + ) } "get dir, empty article and empty wlmId" in { @@ -30,7 +38,10 @@ class EntrySpec extends Specification { } "get dir, article and empty wlmId" in { - Entry.fromRow(Seq("dir", "article", " ")) === Entry("dir", Some("article")) + Entry.fromRow(Seq("dir", "article", " ")) === Entry( + "dir", + Some("article") + ) } } @@ -40,7 +51,8 @@ class EntrySpec extends Specification { } "default article to dir" in { - Entry("dir", + Entry( + "dir", images = Seq( EntryImage("image", Some("description")) ) @@ -55,7 +67,8 @@ class EntrySpec extends Specification { } "use article" in { - Entry("dir", + Entry( + "dir", Some("article"), images = Seq( EntryImage("image", Some("description")) @@ -71,7 +84,8 @@ class EntrySpec extends Specification { } "use parent wlm id" in { - Entry("dir", + Entry( + "dir", Some("article"), images = Seq( EntryImage("image", Some("description")) @@ -88,11 +102,16 @@ class EntrySpec extends Specification { } "override parent wlm id" in { - val entry = Entry("dir", + val entry = Entry( + "dir", Some("article"), images = Seq( EntryImage("image1", Some("description1"), wlmId = None), - EntryImage("image2", Some("description2"), wlmId = Some("specific-wlm-id")) + EntryImage( + "image2", + Some("description2"), + wlmId = Some("specific-wlm-id") + ) ), wlmId = Some("parent-wlm-id") ).genImageFields @@ -136,39 +155,54 @@ class EntrySpec extends Specification { } "read image" in { - val entry = entry0.copy( - images = Seq( - EntryImage("image", Some("description")) - )).genImageFields + val entry = entry0 + .copy( + images = Seq( + EntryImage("image", Some("description")) + ) + ) + .genImageFields roundTrip(entry, "dir") === entry } "read parent wlmId" in { - val entry = entry0.copy(wlmId = Some("wlm-id"), - images = Seq( - EntryImage("image", Some("description")) - )).genImageFields + val entry = entry0 + .copy( + wlmId = Some("wlm-id"), + images = Seq( + EntryImage("image", Some("description")) + ) + ) + .genImageFields roundTrip(entry, "dir") === entry } "read entry and image wlmId" in { - val entry = entry0.copy(wlmId = Some("parent-wlm-id"), - images = Seq( - EntryImage("image", Some("description"), wlmId = Some("image-wlm")) - )).genImageFields + val entry = entry0 + .copy( + wlmId = Some("parent-wlm-id"), + images = Seq( + EntryImage("image", Some("description"), wlmId = Some("image-wlm")) + ) + ) + .genImageFields roundTrip(entry, "dir") === entry } "read uploadTitle" in { - val origEntry = Entry("dir", article = Some("article"), + val origEntry = Entry( + "dir", + article = Some("article"), images = Seq( EntryImage("image1", Some("description1")) - )).genImageFields + ) + ).genImageFields - val str = origEntry.toConfig.root().render().replace("article 1", "article 2") + val str = + origEntry.toConfig.root().render().replace("article 1", "article 2") val cfg = ConfigFactory.parseString(str) val entry = Entry.fromConfig(cfg, "dir") val image = entry.images.head @@ -176,12 +210,18 @@ class EntrySpec extends Specification { } "read wiki-description" in { - val origEntry = Entry("dir", article = Some("article"), + val origEntry = Entry( + "dir", + article = Some("article"), images = Seq( EntryImage("image1", Some("description1")) - )).genImageFields + ) + ).genImageFields - val str = origEntry.toConfig.root().render().replace("{{uk|description1", "{{uk|description2") + val str = origEntry.toConfig + .root() + .render() + .replace("{{uk|description1", "{{uk|description2") val cfg = ConfigFactory.parseString(str) val entry = Entry.fromConfig(cfg, "dir") val image = entry.images.head @@ -192,7 +232,8 @@ class EntrySpec extends Specification { "diff reporter" should { "be quite" in { - val image = EntryImage("image", Some("description1"), Some("wiki-description1")) + val image = + EntryImage("image", Some("description1"), Some("wiki-description1")) val entry = entry0.copy(wlmId = Some("wlm-id"), images = Seq(image)) entry.diff(entry) === Seq.empty @@ -200,26 +241,52 @@ class EntrySpec extends Specification { "tell image entry field change" in { - val image = EntryImage("image", Some("description1"), wikiDescription = Some("wiki-description1")) + val image = EntryImage( + "image", + Some("description1"), + wikiDescription = Some("wiki-description1") + ) val entry = entry0.copy(wlmId = Some("wlm-id"), images = Seq(image)) - val changed = entry.copy(images = Seq(image.copy(wikiDescription = Some("wiki-description2")))) + val changed = entry.copy(images = + Seq(image.copy(wikiDescription = Some("wiki-description2"))) + ) - entry.diff(changed) === Seq(Diff("images[0].wikiDescription", Some("wiki-description1"), Some("wiki-description2"))) + entry.diff(changed) === Seq( + Diff( + "images[0].wikiDescription", + Some("wiki-description1"), + Some("wiki-description2") + ) + ) } "tell entry field change" in { val entry = entry0.copy(wlmId = Some("wlm-id1")) val changed = entry.copy(wlmId = Some("wlm-id2")) - entry.diff(changed) === Seq(Diff("wlmId", Some("wlm-id1"), Some("wlm-id2"))) + entry.diff(changed) === Seq( + Diff("wlmId", Some("wlm-id1"), Some("wlm-id2")) + ) } "tell all fields change" in { - val image1 = EntryImage("image1", Some("description1"), Some("upload-title1"), Some("wiki-description1"), Some("wlm-id11")) + val image1 = EntryImage( + "image1", + Some("description1"), + Some("upload-title1"), + Some("wiki-description1"), + Some("wlm-id11") + ) val entry1 = Entry("dir1", Some("article1"), Some("wlm-id1"), Seq(image1)) - val image2 = EntryImage("image2", Some("description2"), Some("upload-title2"), Some("wiki-description2"), Some("wlm-id12")) + val image2 = EntryImage( + "image2", + Some("description2"), + Some("upload-title2"), + Some("wiki-description2"), + Some("wlm-id12") + ) val entry2 = Entry("dir2", Some("article2"), Some("wlm-id2"), Seq(image2)) entry1.diff(entry2) === Seq( @@ -227,9 +294,21 @@ class EntrySpec extends Specification { Diff("article", Some("article1"), Some("article2")), Diff("wlmId", Some("wlm-id1"), Some("wlm-id2")), Diff("images[0].filePath", "image1", "image2"), - Diff("images[0].uploadTitle", Some("upload-title1"), Some("upload-title2")), - Diff("images[0].sourceDescription", Some("description1"), Some("description2")), - Diff("images[0].wikiDescription", Some("wiki-description1"), Some("wiki-description2")), + Diff( + "images[0].uploadTitle", + Some("upload-title1"), + Some("upload-title2") + ), + Diff( + "images[0].sourceDescription", + Some("description1"), + Some("description2") + ), + Diff( + "images[0].wikiDescription", + Some("wiki-description1"), + Some("wiki-description2") + ), Diff("images[0].wlmId", Some("wlm-id11"), Some("wlm-id12")) ) } @@ -239,10 +318,22 @@ class EntrySpec extends Specification { "change nothing" in { - val image1 = EntryImage("image1", Some("description1"), Some("upload-title1"), Some("wiki-description1"), Some("wlm-id11")) + val image1 = EntryImage( + "image1", + Some("description1"), + Some("upload-title1"), + Some("wiki-description1"), + Some("wlm-id11") + ) val entry1 = Entry("dir1", Some("article1"), Some("wlm-id1"), Seq(image1)) - val image2 = EntryImage("image2", Some("description2"), Some("upload-title2"), Some("wiki-description2"), Some("wlm-id12")) + val image2 = EntryImage( + "image2", + Some("description2"), + Some("upload-title2"), + Some("wiki-description2"), + Some("wlm-id12") + ) val entry2 = Entry("dir2", Some("article2"), Some("wlm-id2"), Seq(image2)) val updated = entry1.updateFrom( @@ -255,16 +346,34 @@ class EntrySpec extends Specification { "change all" in { - val image1 = EntryImage("image1", Some("description1"), Some("upload-title1"), Some("wiki-description1"), Some("wlm-id11")) + val image1 = EntryImage( + "image1", + Some("description1"), + Some("upload-title1"), + Some("wiki-description1"), + Some("wlm-id11") + ) val entry1 = Entry("dir1", Some("article1"), Some("wlm-id1"), Seq(image1)) - val image2 = EntryImage("image2", Some("description2"), Some("upload-title2"), Some("wiki-description2"), Some("wlm-id12")) + val image2 = EntryImage( + "image2", + Some("description2"), + Some("upload-title2"), + Some("wiki-description2"), + Some("wlm-id12") + ) val entry2 = Entry("dir2", Some("article2"), Some("wlm-id2"), Seq(image2)) val updated = entry1.updateFrom( entry2, Set("dir", "article", "wlmId"), - Set("filePath", "uploadTitle", "sourceDescription", "wikiDescription", "wlmId") + Set( + "filePath", + "uploadTitle", + "sourceDescription", + "wikiDescription", + "wlmId" + ) ) updated === entry2 } diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/HtmlParserSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/HtmlParserSpec.scala index 7e5f489a..a483caf4 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/HtmlParserSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/HtmlParserSpec.scala @@ -24,8 +24,7 @@ class HtmlParserSpec extends Specification { "htmlText" should { "join new lines" in { - HtmlParser.htmlText( - """ line1 + HtmlParser.htmlText(""" line1 | line2 | | line3 @@ -33,15 +32,17 @@ class HtmlParserSpec extends Specification { } "preserve html paragraphs" in { - HtmlParser.htmlText( - """

    line1 + HtmlParser + .htmlText("""

    line1 |line2

    |

    line3

    - """.stripMargin).split("\r?\n").map(_.trim) === + """.stripMargin) + .split("\r?\n") + .map(_.trim) === """line1 line2 |line3""".stripMargin.split("\r?\n").map(_.trim) } } } -} \ No newline at end of file +} diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/ImageTemplateSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/ImageTemplateSpec.scala index 5081b924..93462849 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/ImageTemplateSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/ImageTemplateSpec.scala @@ -42,7 +42,7 @@ class ImageTemplateSpec extends Specification { val params = Map( "title" -> "Archaeological Museum interior", "description" -> "Archaeological Museum interior description", - "location" -> "Archaeological Museum" + "location" -> "Archaeological Museum" ) val resolved = ImageTemplate.resolve(params) resolved.isResolved === true @@ -52,7 +52,7 @@ class ImageTemplateSpec extends Specification { "should makeInfoPage" in { ImageTemplate.makeInfoPage( title = "Archaeological Museum interior", - description = "Archaeological Museum interior description", + description = "Archaeological Museum interior description", location = "Archaeological Museum" ) === expected } diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/PereiaslavSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/PereiaslavSpec.scala index 98708123..ebc21689 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/PereiaslavSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/museum/PereiaslavSpec.scala @@ -29,11 +29,13 @@ class PereiaslavSpec extends Specification with BeforeEach with Mockito { def pereiaslav(cfg: Config = config()) = new Pereiaslav(cfg, fs) - def config(host: String = ukWiki, - tablePage: String = "tablePage", - home: String = "." + fs.getSeparator + "data", - lang: String = "uk", - wlmPage: String = ""): Config = { + def config( + host: String = ukWiki, + tablePage: String = "tablePage", + home: String = "." + fs.getSeparator + "data", + lang: String = "uk", + wlmPage: String = "" + ): Config = { val map = Map( "host" -> host, "table-page" -> tablePage, @@ -82,7 +84,8 @@ class PereiaslavSpec extends Specification with BeforeEach with Mockito { val html = descriptions.mkString( "

    Image list

    ", "

    \n

    ", - "

    ") + "

    " + ) (root / listName).overwrite(html) @@ -108,7 +111,12 @@ class PereiaslavSpec extends Specification with BeforeEach with Mockito { val entries = pereiaslav().getEntries.await entries.size === 2 - entries.head === Entry("name1", Some("article1"), Some("wlmId1"), Seq.empty) + entries.head === Entry( + "name1", + Some("article1"), + Some("wlmId1"), + Seq.empty + ) entries.last === Entry("name2", Some("article2"), None, Seq.empty) } @@ -133,11 +141,29 @@ class PereiaslavSpec extends Specification with BeforeEach with Mockito { val entries = pereiaslav().getEntries.await entries.size === 2 - entries.head === Entry("name1", Some("article1"), Some("wlmId1"), - (1 to 3).map { i => EntryImage((root / "name1" / s"$i.jpg").toString, None, size = Some(0)) } + entries.head === Entry( + "name1", + Some("article1"), + Some("wlmId1"), + (1 to 3).map { i => + EntryImage( + (root / "name1" / s"$i.jpg").toString, + None, + size = Some(0) + ) + } ) - entries.last === Entry("name2", Some("article2"), None, - (11 to 13).map { i => EntryImage((root / "name2" / s"$i.jpg").toString, None, size = Some(0)) } + entries.last === Entry( + "name2", + Some("article2"), + None, + (11 to 13).map { i => + EntryImage( + (root / "name2" / s"$i.jpg").toString, + None, + size = Some(0) + ) + } ) } } diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/np/WlmContactsSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/np/WlmContactsSpec.scala index b61fdfb2..4ab97110 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/np/WlmContactsSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/np/WlmContactsSpec.scala @@ -2,26 +2,26 @@ package org.scalawiki.bots.np import org.specs2.mutable.Specification -class WlmContactsSpec extends Specification{ +class WlmContactsSpec extends Specification { - "WlmContacts" should { - val numbers = WlmContacts.getNumbers - "pick 10 digits" in { - numbers.contains("380931234567") === true - } + "WlmContacts" should { + val numbers = WlmContacts.getNumbers + "pick 10 digits" in { + numbers.contains("380931234567") === true + } - "pick 12 digits" in { - numbers.contains("380671234567") === true - } + "pick 12 digits" in { + numbers.contains("380671234567") === true + } - "pick dashed/spaced/braced digits" in { - numbers.contains("380631234567") === true - numbers.contains("380971234567") === true - numbers.contains("380951234567") === true - numbers.contains("380951234567") === true - numbers.contains("380971234567") === true - numbers.contains("380631234567") === true - numbers.contains("380501234567") === true - } + "pick dashed/spaced/braced digits" in { + numbers.contains("380631234567") === true + numbers.contains("380971234567") === true + numbers.contains("380951234567") === true + numbers.contains("380951234567") === true + numbers.contains("380971234567") === true + numbers.contains("380631234567") === true + numbers.contains("380501234567") === true } + } } diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/AnnotationSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/AnnotationSpec.scala index cbecc6d0..791a0d8e 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/AnnotationSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/AnnotationSpec.scala @@ -3,7 +3,6 @@ package org.scalawiki.bots.stat import org.scalawiki.dto.{Page, Revision} import org.specs2.mutable.Specification - class AnnotationSpec extends Specification { def fromRevs(revs: Revision*) = diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/ArticleStatBotSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/ArticleStatBotSpec.scala index 45e8b79e..1b2af9b8 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/ArticleStatBotSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/ArticleStatBotSpec.scala @@ -10,7 +10,13 @@ class ArticleStatBotSpec extends Specification with MockBotSpec { "stat " should { "no stat" in { - val event = new ArticlesEvent("Martians", ZonedDateTime.now, ZonedDateTime.now, "Martians-week-new", "Martians-week-improved") + val event = new ArticlesEvent( + "Martians", + ZonedDateTime.now, + ZonedDateTime.now, + "Martians-week-new", + "Martians-week-improved" + ) val commands = getCommands(emptyResponse, emptyResponse) @@ -30,7 +36,13 @@ class ArticleStatBotSpec extends Specification with MockBotSpec { } "created 1 article" in { - val event = new ArticlesEvent("Martians", ZonedDateTime.parse("2015-06-07T16:45:30Z"), ZonedDateTime.now, "Martians-week-new", "Martians-week-improved") + val event = new ArticlesEvent( + "Martians", + ZonedDateTime.parse("2015-06-07T16:45:30Z"), + ZonedDateTime.now, + "Martians-week-new", + "Martians-week-improved" + ) val createdResponse = """{ "query": { "pages": { @@ -80,8 +92,12 @@ class ArticleStatBotSpec extends Specification with MockBotSpec { } } - - def checkStat(stat: UserStat, users: Set[String], byUser: Map[String, Seq[RevisionStat]], byUserAddedOrRemoved: Seq[(String, Long)]) = { + def checkStat( + stat: UserStat, + users: Set[String], + byUser: Map[String, Seq[RevisionStat]], + byUserAddedOrRemoved: Seq[(String, Long)] + ) = { stat.users === users stat.byUser === byUser stat.byUserAddedOrRemoved === byUserAddedOrRemoved @@ -94,15 +110,30 @@ class ArticleStatBotSpec extends Specification with MockBotSpec { | } |}""".stripMargin - def getCommands(newResponse: String, improvedResponse: String): Seq[HttpStub] = { + def getCommands( + newResponse: String, + improvedResponse: String + ): Seq[HttpStub] = { val newParams = Map( - "format" -> "json", "generator" -> "embeddedin", "inprop" -> "subjectid", "geilimit" -> "500", - "geititle" -> "Template:Martians-week-new", "prop" -> "info|revisions", "action" -> "query", "continue" -> "" + "format" -> "json", + "generator" -> "embeddedin", + "inprop" -> "subjectid", + "geilimit" -> "500", + "geititle" -> "Template:Martians-week-new", + "prop" -> "info|revisions", + "action" -> "query", + "continue" -> "" ) val improvedParams = Map( - "format" -> "json", "generator" -> "embeddedin", "inprop" -> "subjectid", "geilimit" -> "500", - "geititle" -> "Template:Martians-week-improved", "prop" -> "info|revisions", "action" -> "query", "continue" -> "" + "format" -> "json", + "generator" -> "embeddedin", + "inprop" -> "subjectid", + "geilimit" -> "500", + "geititle" -> "Template:Martians-week-improved", + "prop" -> "info|revisions", + "action" -> "query", + "continue" -> "" ) Seq( diff --git a/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/RevisionStatSpec.scala b/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/RevisionStatSpec.scala index 02ff5e83..365a7ba8 100644 --- a/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/RevisionStatSpec.scala +++ b/scalawiki-bots/src/test/scala/org/scalawiki/bots/stat/RevisionStatSpec.scala @@ -5,7 +5,6 @@ import org.specs2.mutable.Specification class RevisionStatSpec extends Specification { - "RevisionStatSpec" should { "work with empty history" in { @@ -27,10 +26,14 @@ class RevisionStatSpec extends Specification { } "work with one revision" in { - val words = Seq("revision", "text") + val words = Seq("revision", "text") val text = words.mkString(" ") val user = "user" - val rev = Revision(revId = Some(1), user = Some(User(12, user)), content = Some(text)) + val rev = Revision( + revId = Some(1), + user = Some(User(12, user)), + content = Some(text) + ) val page = new Page(Some(1), Some(0), "page", revisions = Seq(rev)) val stat = RevisionStat.fromPage(page) @@ -49,7 +52,11 @@ class RevisionStatSpec extends Specification { val words = Seq("текст", "ревізії") val text = words.mkString(" ") val user = "user" - val rev = Revision(revId = Some(1), user = Some(User(12, user)), content = Some(text)) + val rev = Revision( + revId = Some(1), + user = Some(User(12, user)), + content = Some(text) + ) val page = new Page(Some(1), Some(0), "page", revisions = Seq(rev)) val stat = RevisionStat.fromPage(page) @@ -66,7 +73,7 @@ class RevisionStatSpec extends Specification { "two revisions add text" in { - val words1 = Seq("revision1", "text1") + val words1 = Seq("revision1", "text1") val words2 = Seq("before", "between", "after") val text2Words = Seq("before", "revision1", "between", "text1", "after") @@ -74,8 +81,16 @@ class RevisionStatSpec extends Specification { val text2 = text2Words.mkString(" ") val user = "user" - val rev1 = Revision(revId = Some(1), user = Some(User(12, user)), content = Some(text1)) - val rev2 = Revision(revId = Some(2), user = Some(User(12, user)), content = Some(text2)) + val rev1 = Revision( + revId = Some(1), + user = Some(User(12, user)), + content = Some(text1) + ) + val rev2 = Revision( + revId = Some(2), + user = Some(User(12, user)), + content = Some(text2) + ) val page = new Page(Some(1), Some(0), "page", revisions = Seq(rev2, rev1)) val stat = RevisionStat.fromPage(page) @@ -85,22 +100,33 @@ class RevisionStatSpec extends Specification { stat.page === page.withoutContent stat.history.revisions === Seq(rev2.withoutContent, rev1.withoutContent) - stat.byRevisionSize === Map(rev1.withoutContent -> words1.mkString.length, rev2.withoutContent -> words2.mkString.length) + stat.byRevisionSize === Map( + rev1.withoutContent -> words1.mkString.length, + rev2.withoutContent -> words2.mkString.length + ) stat.addedOrRewritten === (words1 ++ words2).mkString.length } "two revisions remove text" in { val text1Words = Seq("before", "revision1", "between", "text1", "after") - val words2 = Seq("revision1", "text1") + val words2 = Seq("revision1", "text1") val words1 = Seq("before", "between", "after") val text1 = text1Words.mkString(" ") val text2 = words2.mkString(" ") val user = "user" - val rev1 = Revision(revId = Some(1), user = Some(User(12, user)), content = Some(text1)) - val rev2 = Revision(revId = Some(2), user = Some(User(12, user)), content = Some(text2)) + val rev1 = Revision( + revId = Some(1), + user = Some(User(12, user)), + content = Some(text1) + ) + val rev2 = Revision( + revId = Some(2), + user = Some(User(12, user)), + content = Some(text2) + ) val page = new Page(Some(1), Some(0), "page", revisions = Seq(rev2, rev1)) val stat = RevisionStat.fromPage(page) @@ -117,25 +143,39 @@ class RevisionStatSpec extends Specification { "two revisions replace text" in { - val words1 = Seq("firstRevision", "text") - val words2 = Seq("secondRevision", "text") + val words1 = Seq("firstRevision", "text") + val words2 = Seq("secondRevision", "text") val text1 = words1.mkString(" ") val text2 = words2.mkString(" ") val user = "user" - val rev1 = Revision(revId = Some(1), user = Some(User(12, user)), content = Some(text1)) - val rev2 = Revision(revId = Some(2), user = Some(User(12, user)), content = Some(text2)) + val rev1 = Revision( + revId = Some(1), + user = Some(User(12, user)), + content = Some(text1) + ) + val rev2 = Revision( + revId = Some(2), + user = Some(User(12, user)), + content = Some(text2) + ) val page = new Page(Some(1), Some(0), "page", revisions = Seq(rev2, rev1)) val stat = RevisionStat.fromPage(page) val annotation = new RevisionAnnotation(page) - annotation.byRevisionContent === Map(rev1 -> Seq("text"), rev2 -> Seq("secondRevision")) + annotation.byRevisionContent === Map( + rev1 -> Seq("text"), + rev2 -> Seq("secondRevision") + ) annotation.byUserContent === Map(user -> words2) stat.page === page.withoutContent stat.history.revisions === Seq(rev2.withoutContent, rev1.withoutContent) - stat.byRevisionSize === Map(rev1.withoutContent -> "text".length, rev2.withoutContent -> "secondRevision".length) + stat.byRevisionSize === Map( + rev1.withoutContent -> "text".length, + rev2.withoutContent -> "secondRevision".length + ) stat.addedOrRewritten === words2.mkString.length } diff --git a/scalawiki-core/src/it/scala/org/scalawiki/BaseIntegrationSpec.scala b/scalawiki-core/src/it/scala/org/scalawiki/BaseIntegrationSpec.scala index c2f6e77b..ef45383e 100644 --- a/scalawiki-core/src/it/scala/org/scalawiki/BaseIntegrationSpec.scala +++ b/scalawiki-core/src/it/scala/org/scalawiki/BaseIntegrationSpec.scala @@ -24,9 +24,12 @@ class BaseIntegrationSpec extends Specification { await(wiki.login(username, passwd)) def login(wiki: MwBot): String = { - LoginInfo.fromEnv().map { loginInfo => - await(wiki.login(loginInfo.login, loginInfo.password)) - }.getOrElse("No LoginInfo") + LoginInfo + .fromEnv() + .map { loginInfo => + await(wiki.login(loginInfo.login, loginInfo.password)) + } + .getOrElse("No LoginInfo") } def await[T](future: Future[T]): T = Await.result(future, http.timeout) diff --git a/scalawiki-core/src/it/scala/org/scalawiki/EditTokensIntegrationTest.scala b/scalawiki-core/src/it/scala/org/scalawiki/EditTokensIntegrationTest.scala index d000b9f7..2e785433 100644 --- a/scalawiki-core/src/it/scala/org/scalawiki/EditTokensIntegrationTest.scala +++ b/scalawiki-core/src/it/scala/org/scalawiki/EditTokensIntegrationTest.scala @@ -10,7 +10,8 @@ class EditTokensIntegrationTest extends BaseIntegrationSpec { val result = login(bot) result === "Success" - val response = await(bot.page("User:IlyaBot/test").edit("text1", Some("comment"))) + val response = + await(bot.page("User:IlyaBot/test").edit("text1", Some("comment"))) response === "Success" } } diff --git a/scalawiki-core/src/it/scala/org/scalawiki/IntegrationSpec.scala b/scalawiki-core/src/it/scala/org/scalawiki/IntegrationSpec.scala index 98c35b60..47e1e6bc 100644 --- a/scalawiki-core/src/it/scala/org/scalawiki/IntegrationSpec.scala +++ b/scalawiki-core/src/it/scala/org/scalawiki/IntegrationSpec.scala @@ -3,16 +3,20 @@ package org.scalawiki class IntegrationSpec extends BaseIntegrationSpec { "categoryMembers" should { - "list files" in { - val result = login(getCommonsBot) - result === "Success" + "list files" in { + val result = login(getCommonsBot) + result === "Success" - val commons = getCommonsBot - val images = await(commons.page("Category:Images from Wiki Loves Earth 2014 in Serbia").categoryMembers()) + val commons = getCommonsBot + val images = await( + commons + .page("Category:Images from Wiki Loves Earth 2014 in Serbia") + .categoryMembers() + ) - images.size must be_>(500) + images.size must be_>(500) - } + } } } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/ActionLibrary.scala b/scalawiki-core/src/main/scala/org/scalawiki/ActionLibrary.scala index 4aae7a48..83838780 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/ActionLibrary.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/ActionLibrary.scala @@ -8,19 +8,20 @@ trait ActionLibrary { def bot: MwBot def message(user: String, section: String, text: String) = { - bot.page("User_talk:" + user).edit(text, - section = Some("new"), - summary = Some(section) - ) + bot + .page("User_talk:" + user) + .edit(text, section = Some("new"), summary = Some(section)) } def email(user: String, subject: String, text: String) = { - val cmd = Action(EmailUser( - Target(user), - Subject(subject), - Text(text), - Token(bot.token) - )) + val cmd = Action( + EmailUser( + Target(user), + Subject(subject), + Text(text), + Token(bot.token) + ) + ) bot.await(bot.post(cmd.pairs.toMap)) } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/LoginInfo.scala b/scalawiki-core/src/main/scala/org/scalawiki/LoginInfo.scala index 8077c834..f063dbfe 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/LoginInfo.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/LoginInfo.scala @@ -10,10 +10,14 @@ case class LoginInfoValue(login: String, password: String) extends LoginInfo object LoginInfo { - def fromEnv(loginProp: String = "SCALAWIKI_LOGIN", - passwordProp: String = "SCALAWIKI_PASSWORD"): Option[LoginInfo] = - for (login <- sys.env.get(loginProp); - password <- sys.env.get(passwordProp)) - yield LoginInfoValue(login, password) + def fromEnv( + loginProp: String = "SCALAWIKI_LOGIN", + passwordProp: String = "SCALAWIKI_PASSWORD" + ): Option[LoginInfo] = + for ( + login <- sys.env.get(loginProp); + password <- sys.env.get(passwordProp) + ) + yield LoginInfoValue(login, password) } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/MwBot.scala b/scalawiki-core/src/main/scala/org/scalawiki/MwBot.scala index ffeabae9..042918a2 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/MwBot.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/MwBot.scala @@ -19,9 +19,11 @@ import scala.language.postfixOps trait ActionBot { - def run(action: Action, - context: Map[String, String] = Map.empty, - limit: Option[Long] = None): Future[Iterable[Page]] + def run( + action: Action, + context: Map[String, String] = Map.empty, + limit: Option[Long] = None + ): Future[Iterable[Page]] } @@ -31,9 +33,11 @@ trait MwBot extends ActionBot { def login(user: String, password: String): Future[String] - def run(action: Action, - context: Map[String, String] = Map.empty, - limit: Option[Long] = None): Future[Iterable[Page]] + def run( + action: Action, + context: Map[String, String] = Map.empty, + limit: Option[Long] = None + ): Future[Iterable[Page]] def get(params: Map[String, String]): Future[String] @@ -47,10 +51,12 @@ trait MwBot extends ActionBot { def postMultiPart[T](reads: Reads[T], params: Map[String, String]): Future[T] - def postFile[T](reads: Reads[T], - params: Map[String, String], - fileParam: String, - filename: String): Future[T] + def postFile[T]( + reads: Reads[T], + params: Map[String, String], + fileParam: String, + filename: String + ): Future[T] def page(title: String): SinglePageQuery @@ -88,10 +94,11 @@ object MediaWikiVersion { } } -class MwBotImpl(val site: Site, - val http: HttpClient = HttpClient.get(MwBot.system), - val system: ActorSystem = MwBot.system) - extends MwBot { +class MwBotImpl( + val site: Site, + val http: HttpClient = HttpClient.get(MwBot.system), + val system: ActorSystem = MwBot.system +) extends MwBot { def this(host: String) = this(Site.host(host)) @@ -103,8 +110,8 @@ class MwBotImpl(val site: Site, override def host: String = site.domain - val baseUrl - : String = site.protocol + "://" + host + site.portStr + site.scriptPath + val baseUrl: String = + site.protocol + "://" + host + site.portStr + site.scriptPath val indexUrl: String = baseUrl + "/index.php" @@ -125,9 +132,11 @@ class MwBotImpl(val site: Site, } } - def tryLogin(user: String, - password: String, - token: Option[String] = None): Future[LoginResponse] = { + def tryLogin( + user: String, + password: String, + token: Option[String] = None + ): Future[LoginResponse] = { val loginParams = Map( "action" -> "login", "lgname" -> user, @@ -136,8 +145,10 @@ class MwBotImpl(val site: Site, "utf8" -> "" ) ++ token.map("lgtoken" -> _) - for (response <- http.post(apiUrl, loginParams); - body <- http.getBody(response)) yield { + for ( + response <- http.post(apiUrl, loginParams); + body <- http.getBody(response) + ) yield { val contentType = response.entity.contentType contentType match { @@ -160,10 +171,15 @@ class MwBotImpl(val site: Site, override lazy val token: String = await(getToken) override lazy val mediaWikiVersion: MediaWikiVersion = await( - getMediaWikiVersion) + getMediaWikiVersion + ) def getMediaWikiVersion: Future[MediaWikiVersion] = - get(siteInfoReads, "action" -> "query", "meta" -> "siteinfo") map MediaWikiVersion.fromGenerator + get( + siteInfoReads, + "action" -> "query", + "meta" -> "siteinfo" + ) map MediaWikiVersion.fromGenerator def getToken: Future[String] = { val isMW_1_24 = mediaWikiVersion >= MediaWikiVersion.MW_1_24 @@ -171,26 +187,29 @@ class MwBotImpl(val site: Site, if (isMW_1_24) { get(tokenReads, "action" -> "query", "meta" -> "tokens") } else { - get(editTokenReads, - "action" -> "query", - "prop" -> "info", - "intoken" -> "edit", - "titles" -> "foo") + get( + editTokenReads, + "action" -> "query", + "prop" -> "info", + "intoken" -> "edit", + "titles" -> "foo" + ) } } def getTokens: Future[String] = get(tokensReads, "action" -> "tokens") - override def run(action: Action, - context: Map[String, String] = Map.empty, - limit: Option[Long] = None): Future[Iterable[Page]] = { + override def run( + action: Action, + context: Map[String, String] = Map.empty, + limit: Option[Long] = None + ): Future[Iterable[Page]] = { new DslQuery(action, this, context) .run(limit = limit) .map(_.allPages) - .recover { - case t: Throwable => - log.error(t, s"Error ${t.getMessage} running action" + action) - throw t + .recover { case t: Throwable => + log.error(t, s"Error ${t.getMessage} running action" + action) + throw t } } @@ -202,8 +221,10 @@ class MwBotImpl(val site: Site, def parseJson[T](reads: Reads[T], body: String): JsResult[T] = Json.parse(body).validate(reads) - def parseResponse[T](reads: Reads[T], - response: Future[HttpResponse]): Future[T] = + def parseResponse[T]( + reads: Reads[T], + response: Future[HttpResponse] + ): Future[T] = response flatMap http.getBody map { body => parseJson(reads, body).getOrElse { val exception = @@ -221,25 +242,33 @@ class MwBotImpl(val site: Site, override def post[T](reads: Reads[T], params: (String, String)*): Future[T] = post(reads, params.toMap) - override def post[T](reads: Reads[T], - params: Map[String, String]): Future[T] = + override def post[T]( + reads: Reads[T], + params: Map[String, String] + ): Future[T] = parseResponse(reads, http.post(apiUrl, params)) - override def postMultiPart[T](reads: Reads[T], - params: Map[String, String]): Future[T] = + override def postMultiPart[T]( + reads: Reads[T], + params: Map[String, String] + ): Future[T] = parseResponse(reads, http.postMultiPart(apiUrl, params)) - override def postFile[T](reads: Reads[T], - params: Map[String, String], - fileParam: String, - filename: String): Future[T] = + override def postFile[T]( + reads: Reads[T], + params: Map[String, String], + fileParam: String, + filename: String + ): Future[T] = parseResponse(reads, http.postFile(apiUrl, params, fileParam, filename)) - def pagesByTitle(titles: Set[String]): PageQuery = PageQuery.byTitles(titles, this) + def pagesByTitle(titles: Set[String]): PageQuery = + PageQuery.byTitles(titles, this) def pagesById(ids: Set[Long]): PageQuery = PageQuery.byIds(ids, this) - override def page(title: String): SinglePageQuery = PageQuery.byTitle(title, this) + override def page(title: String): SinglePageQuery = + PageQuery.byTitle(title, this) override def page(id: Long): SinglePageQuery = PageQuery.byId(id, this) @@ -250,11 +279,13 @@ class MwBotImpl(val site: Site, def getIndexUri(params: (String, String)*): Uri = Uri(indexUrl) withQuery Query( - params ++ Seq("format" -> "json", "utf8" -> ""): _*) + params ++ Seq("format" -> "json", "utf8" -> ""): _* + ) def getUri(params: (String, String)*): Uri = Uri(apiUrl) withQuery Query( - params ++ Seq("format" -> "json", "utf8" -> ""): _*) + params ++ Seq("format" -> "json", "utf8" -> ""): _* + ) override def get(params: Map[String, String]): Future[String] = { val uri: Uri = getUri(params) @@ -272,7 +303,8 @@ class MwBotImpl(val site: Site, def getUri(params: Map[String, String]): Uri = Uri(apiUrl) withQuery Query(params ++ Map("format" -> "json")) - override def await[T](future: Future[T]): T = Await.result(future, http.timeout) + override def await[T](future: Future[T]): T = + Await.result(future, http.timeout) } object MwBot { @@ -284,9 +316,11 @@ object MwBot { val system: ActorSystem = ActorSystem() - def create(site: Site, - loginInfo: Option[LoginInfo], - http: HttpClient = HttpClient.get(MwBot.system)): MwBot = { + def create( + site: Site, + loginInfo: Option[LoginInfo], + http: HttpClient = HttpClient.get(MwBot.system) + ): MwBot = { val bot = new MwBotImpl(site, http) loginInfo.foreach { info => @@ -297,22 +331,32 @@ object MwBot { val cache: Cache[String, MwBot] = LfuCache(system) - def fromHost(host: String, - protocol: String = "https", - port: Option[Int] = None, - loginInfo: Option[LoginInfo] = LoginInfo.fromEnv(), - http: HttpClient = HttpClient.get(MwBot.system)): MwBot = { + def fromHost( + host: String, + protocol: String = "https", + port: Option[Int] = None, + loginInfo: Option[LoginInfo] = LoginInfo.fromEnv(), + http: HttpClient = HttpClient.get(MwBot.system) + ): MwBot = { fromSite(Site.host(host, protocol, port), loginInfo, http) } - def fromSite(site: Site, - loginInfo: Option[LoginInfo] = LoginInfo.fromEnv(), - http: HttpClient = HttpClient.get(MwBot.system)): MwBot = { - Await.result(cache(site.domain, { () => - Future { - create(site, loginInfo, http) - } - }), 1.minute) + def fromSite( + site: Site, + loginInfo: Option[LoginInfo] = LoginInfo.fromEnv(), + http: HttpClient = HttpClient.get(MwBot.system) + ): MwBot = { + Await.result( + cache( + site.domain, + { () => + Future { + create(site, loginInfo, http) + } + } + ), + 1.minute + ) } val commons: String = Site.commons.domain diff --git a/scalawiki-core/src/main/scala/org/scalawiki/MwUtils.scala b/scalawiki-core/src/main/scala/org/scalawiki/MwUtils.scala index 3fdf288b..324c3542 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/MwUtils.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/MwUtils.scala @@ -1,13 +1,15 @@ package org.scalawiki object MwUtils { - /** - * Convenience method for normalizing MediaWiki titles. (Converts all - * spaces to underscores and makes the first letter caps). - * - * @param s the string to normalize - * @return the normalized string - */ + + /** Convenience method for normalizing MediaWiki titles. (Converts all spaces + * to underscores and makes the first letter caps). + * + * @param s + * the string to normalize + * @return + * the normalized string + */ def normalize(s: String): String = s.capitalize.replaceAll(" ", "_") } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/WithBot.scala b/scalawiki-core/src/main/scala/org/scalawiki/WithBot.scala index 14de2586..4de55edf 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/WithBot.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/WithBot.scala @@ -14,4 +14,3 @@ trait WithBot extends HasBot { } } - diff --git a/scalawiki-core/src/main/scala/org/scalawiki/cache/CachedBot.scala b/scalawiki-core/src/main/scala/org/scalawiki/cache/CachedBot.scala index 0c2870eb..74baab31 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/cache/CachedBot.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/cache/CachedBot.scala @@ -13,13 +13,19 @@ import scala.concurrent.duration._ import scala.concurrent.{Await, Future} import scala.concurrent.ExecutionContext.Implicits.global -class Caller(fn: String => String) extends java.util.function.Function[String, String] { +class Caller(fn: String => String) + extends java.util.function.Function[String, String] { override def apply(t: String): String = { fn.apply(t) } } -class Cache(name: String, entries: Int = 12 * 1024, valueSize: Int = 128 * 1024, persistent: Boolean = true) { +class Cache( + name: String, + entries: Int = 12 * 1024, + valueSize: Int = 128 * 1024, + persistent: Boolean = true +) { private val builder: ChronicleMapBuilder[String, String] = ChronicleMap .of(classOf[String], classOf[String]) @@ -38,28 +44,34 @@ class Cache(name: String, entries: Int = 12 * 1024, valueSize: Int = 128 * 1024, def remove(key: String): String = cache.remove(key) - def computeIfAbsent(key: String, fn: String => String): String = cache.computeIfAbsent(key, new Caller(fn)) + def computeIfAbsent(key: String, fn: String => String): String = + cache.computeIfAbsent(key, new Caller(fn)) } // TODO async compute, do not enter twice -class CachedBot(site: Site, name: String, persistent: Boolean, http: HttpClient = HttpClient.get(MwBot.system), - entries: Int = 12 * 1024, - valueSize: Int = 128 * 1024) - extends MwBotImpl(site) { +class CachedBot( + site: Site, + name: String, + persistent: Boolean, + http: HttpClient = HttpClient.get(MwBot.system), + entries: Int = 12 * 1024, + valueSize: Int = 128 * 1024 +) extends MwBotImpl(site) { val cache = new Cache(name + ".cache", entries, valueSize, persistent) - override def run(action: Action, - context: Map[String, String] = Map.empty, - limit: Option[Long] = None): Future[Iterable[Page]] = { + override def run( + action: Action, + context: Map[String, String] = Map.empty, + limit: Option[Long] = None + ): Future[Iterable[Page]] = { val future = super.run(action, context, limit) - future recoverWith { - case ex: MwException => - val key = paramsKey(ex.params) - cache.remove(key) - super.run(action, context, limit) + future recoverWith { case ex: MwException => + val key = paramsKey(ex.params) + cache.remove(key) + super.run(action, context, limit) } } @@ -108,7 +120,9 @@ object CachedBot { println("keys: " + keys.size) if (valueSizes.nonEmpty) { - println(s"values: total size: ${valueSizes.sum / (1024 * 1024)} MB, avg size: ${valueSizes.sum / (valueSizes.size * 1024)} KB") + println( + s"values: total size: ${valueSizes.sum / (1024 * 1024)} MB, avg size: ${valueSizes.sum / (valueSizes.size * 1024)} KB" + ) } } } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/Contributor.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/Contributor.scala index 71c904a3..694fe8fb 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/Contributor.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/Contributor.scala @@ -14,11 +14,14 @@ trait Contributor { object Contributor { - def apply(idOpt: Option[Long], nameOpt: Option[String]): Option[Contributor] = { + def apply( + idOpt: Option[Long], + nameOpt: Option[String] + ): Option[Contributor] = { (idOpt, nameOpt) match { - case (None, None) => None + case (None, None) => None case (Some(id), Some(name)) => Some(User(idOpt, nameOpt)) - case (Some(id), None) => Some(User(idOpt, nameOpt)) + case (Some(id), None) => Some(User(idOpt, nameOpt)) case (None, Some(name)) => val chars = name.toSet if ((chars -- "123456789.".toSet).isEmpty) @@ -28,4 +31,4 @@ object Contributor { } } -} \ No newline at end of file +} diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/GlobalUserInfo.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/GlobalUserInfo.scala index e4c13e80..fbe50777 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/GlobalUserInfo.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/GlobalUserInfo.scala @@ -2,16 +2,20 @@ package org.scalawiki.dto import java.time.ZonedDateTime -case class GlobalUserInfo(home: String, - id: Long, - registration: ZonedDateTime, - name: String, - merged: Seq[SulAccount], - editCount: Long) +case class GlobalUserInfo( + home: String, + id: Long, + registration: ZonedDateTime, + name: String, + merged: Seq[SulAccount], + editCount: Long +) -case class SulAccount(wiki: String, - url: String, - timestamp: ZonedDateTime, - method: String, - editCount: Long, - registration: ZonedDateTime) \ No newline at end of file +case class SulAccount( + wiki: String, + url: String, + timestamp: ZonedDateTime, + method: String, + editCount: Long, + registration: ZonedDateTime +) diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/Image.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/Image.scala index d7fad7d7..f64f7cd0 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/Image.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/Image.scala @@ -23,22 +23,23 @@ object ImageMetadata { val df = DateTimeFormatter.ofPattern(pattern) } -case class Image(title: String, - url: Option[String] = None, - pageUrl: Option[String] = None, - size: Option[Long] = None, - width: Option[Int] = None, - height: Option[Int] = None, - author: Option[String] = None, - uploader: Option[User] = None, - year: Option[String] = None, - date: Option[ZonedDateTime] = None, - monumentIds: Seq[String] = Nil, - pageId: Option[Long] = None, - metadata: Option[ImageMetadata] = None, - categories: Set[String] = Set.empty, - specialNominations: Set[String] = Set.empty) - extends Ordered[Image] { +case class Image( + title: String, + url: Option[String] = None, + pageUrl: Option[String] = None, + size: Option[Long] = None, + width: Option[Int] = None, + height: Option[Int] = None, + author: Option[String] = None, + uploader: Option[User] = None, + year: Option[String] = None, + date: Option[ZonedDateTime] = None, + monumentIds: Seq[String] = Nil, + pageId: Option[Long] = None, + metadata: Option[ImageMetadata] = None, + categories: Set[String] = Set.empty, + specialNominations: Set[String] = Set.empty +) extends Ordered[Image] { def compare(that: Image): Int = title.compareTo(that.title) @@ -81,7 +82,8 @@ object Image { def fromPageRevision( page: Page, monumentIdTemplate: Option[String], - specialNominationTemplates: Seq[String] = Nil): Option[Image] = { + specialNominationTemplates: Seq[String] = Nil + ): Option[Image] = { page.revisions.headOption.map { revision => val content = revision.content.getOrElse("") val parsedPage = TemplateParser.parsePage(content) @@ -118,20 +120,29 @@ object Image { } } - def fromPage(page: Page, - monumentIdTemplate: Option[String], - specialNominationTemplates: Seq[String] = Nil): Option[Image] = { - for (fromImage <- Image.fromPageImages(page); - fromRev <- Image.fromPageRevision(page, - monumentIdTemplate, - specialNominationTemplates)) + def fromPage( + page: Page, + monumentIdTemplate: Option[String], + specialNominationTemplates: Seq[String] = Nil + ): Option[Image] = { + for ( + fromImage <- Image.fromPageImages(page); + fromRev <- Image.fromPageRevision( + page, + monumentIdTemplate, + specialNominationTemplates + ) + ) yield { val renamedAuthor = fromRev.author.map(author => - AuthorsMap.renames.getOrElse(author, author)) - fromImage.copy(monumentIds = fromRev.monumentIds, - author = renamedAuthor, - categories = fromRev.categories, - specialNominations = fromRev.specialNominations) + AuthorsMap.renames.getOrElse(author, author) + ) + fromImage.copy( + monumentIds = fromRev.monumentIds, + author = renamedAuthor, + categories = fromRev.categories, + specialNominations = fromRev.specialNominations + ) } } @@ -177,16 +188,18 @@ object Image { } } - def basic(title: String, - timestamp: Option[ZonedDateTime], - uploader: Option[String], - size: Option[Long], - width: Option[Int], - height: Option[Int], - url: Option[String], - pageUrl: Option[String], - pageId: Option[Long], - metadata: Option[Map[String, String]] = None) = + def basic( + title: String, + timestamp: Option[ZonedDateTime], + uploader: Option[String], + size: Option[Long], + width: Option[Int], + height: Option[Int], + url: Option[String], + pageUrl: Option[String], + pageId: Option[Long], + metadata: Option[Map[String, String]] = None + ) = new Image( title = title, date = timestamp, @@ -200,8 +213,10 @@ object Image { metadata = metadata.map(ImageMetadata.apply) ) - def gallery(images: Iterable[String], - descriptions: Iterable[String] = Seq.empty): String = + def gallery( + images: Iterable[String], + descriptions: Iterable[String] = Seq.empty + ): String = Gallery.asWiki(images, descriptions) def resizedWidth(w: Int, h: Int, resizeToX: Int, resizeToY: Int): Int = { diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/IpContributor.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/IpContributor.scala index ba497ab9..54502b5a 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/IpContributor.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/IpContributor.scala @@ -1,8 +1,10 @@ package org.scalawiki.dto -case class IpContributor(ip: String, - editCount: Option[Long] = None, - blocked: Option[Boolean] = None) extends Contributor { +case class IpContributor( + ip: String, + editCount: Option[Long] = None, + blocked: Option[Boolean] = None +) extends Contributor { override def name = Some(ip) override def id: Option[Long] = None diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/LoginResponse.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/LoginResponse.scala index 4b388563..f49aa619 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/LoginResponse.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/LoginResponse.scala @@ -1,11 +1,13 @@ package org.scalawiki.dto -case class LoginResponse(result: String, - token: Option[String], - lgToken: Option[String], - lguserid: Option[Int], - lgusername: Option[String], - cookieprefix: Option[String], - sessionid: Option[String]) +case class LoginResponse( + result: String, + token: Option[String], + lgToken: Option[String], + lguserid: Option[Int], + lgusername: Option[String], + cookieprefix: Option[String], + sessionid: Option[String] +) // {"login":{"result":"NeedToken","token":"a504e9507bb8e8d7d3bf839ef096f8f7","cookieprefix":"ukwiki","sessionid":"37b1d67422436e253f5554de23ae0064"}} diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/MwException.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/MwException.scala index 72665eb4..e5797673 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/MwException.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/MwException.scala @@ -1,9 +1,12 @@ package org.scalawiki.dto -case class MwException(code: String, info: String, params: Map[String, String] = Map.empty) - extends RuntimeException( - s"MediaWiki Error: code: $code, info: $info, params: $params" - ) +case class MwException( + code: String, + info: String, + params: Map[String, String] = Map.empty +) extends RuntimeException( + s"MediaWiki Error: code: $code, info: $info, params: $params" + ) //object MwException { // def noParams diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/Page.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/Page.scala index 1b349d20..a23b3e9b 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/Page.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/Page.scala @@ -48,50 +48,62 @@ case class Page( object Page { - def full(id: Option[Long], - ns: Option[Int], - title: String, - missing: Boolean, - subjectId: Option[Long], - talkId: Option[Long], - invalidReason: Option[String]): Page = { - new Page(id, - ns, - title, - missing = missing, - subjectId = subjectId, - talkId = talkId, - invalidReason = invalidReason) + def full( + id: Option[Long], + ns: Option[Int], + title: String, + missing: Boolean, + subjectId: Option[Long], + talkId: Option[Long], + invalidReason: Option[String] + ): Page = { + new Page( + id, + ns, + title, + missing = missing, + subjectId = subjectId, + talkId = talkId, + invalidReason = invalidReason + ) } - def noText(id: Long, - ns: Option[Int], - title: String, - missing: Option[String] = None) = + def noText( + id: Long, + ns: Option[Int], + title: String, + missing: Option[String] = None + ) = new Page(Some(id), ns, title, missing = missing.fold(false)(_ => true)) def withText(id: Long, ns: Option[Int], title: String, text: Option[String]) = new Page(Some(id), ns, title, revisionsFromText(text)) - def withRevisionsText(id: Long, - ns: Option[Int], - title: String, - texts: Seq[String]) = + def withRevisionsText( + id: Long, + ns: Option[Int], + title: String, + texts: Seq[String] + ) = new Page(Some(id), ns, title, Revision.many(texts: _*)) - def withRevisions(id: Long, - ns: Option[Int], - title: String, - editToken: Option[String], - revisions: Seq[Revision], - missing: Option[String]) = - new Page(Some(id), - ns, - title, - revisions, - Seq.empty, - editToken, - missing.fold(false)(_ => true)) + def withRevisions( + id: Long, + ns: Option[Int], + title: String, + editToken: Option[String], + revisions: Seq[Revision], + missing: Option[String] + ) = + new Page( + Some(id), + ns, + title, + revisions, + Seq.empty, + editToken, + missing.fold(false)(_ => true) + ) def withImages(id: Long, ns: Option[Int], title: String, images: Seq[Image]) = new Page(Some(id), ns, title, Seq.empty, images) @@ -103,10 +115,12 @@ object Page { def apply(id: Long, ns: Option[Int], title: String) = new Page(Some(id), ns, title) - def withEditToken(id: Option[Long], - ns: Option[Int], - title: String, - editToken: Option[String]) = { + def withEditToken( + id: Option[Long], + ns: Option[Int], + title: String, + editToken: Option[String] + ) = { new Page(id, ns, title, Seq.empty, Seq.empty, editToken) } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/PageList.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/PageList.scala index 060d3964..7614efe7 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/PageList.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/PageList.scala @@ -2,15 +2,18 @@ package org.scalawiki.dto import scala.collection.mutable -class PageList(existing: mutable.LinkedHashMap[Long, Page] = - mutable.LinkedHashMap.empty, - missing: mutable.Buffer[Page] = mutable.Buffer.empty) { +class PageList( + existing: mutable.LinkedHashMap[Long, Page] = mutable.LinkedHashMap.empty, + missing: mutable.Buffer[Page] = mutable.Buffer.empty +) { def addPages(newPages: Seq[Page]): Unit = { val (newMissing, newExisting) = newPages.partition(_.id.isEmpty) missing ++= newMissing - for (page <- newExisting; - id <- page.id) { + for ( + page <- newExisting; + id <- page.id + ) { existing(id) = existing .get(id) .map(_.appendLists(page)) diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/Revision.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/Revision.scala index 256f1d54..d9f219c3 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/Revision.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/Revision.scala @@ -5,17 +5,17 @@ import java.time.ZonedDateTime import org.apache.commons.codec.digest.DigestUtils case class Revision( - revId: Option[Long] = None, - pageId: Option[Long] = None, - parentId: Option[Long] = None, - user: Option[Contributor] = None, - timestamp: Option[ZonedDateTime] = None, - comment: Option[String] = None, - content: Option[String] = None, - size: Option[Long] = None, - sha1: Option[String] = None, - textId: Option[Long] = None - ) { + revId: Option[Long] = None, + pageId: Option[Long] = None, + parentId: Option[Long] = None, + user: Option[Contributor] = None, + timestamp: Option[ZonedDateTime] = None, + comment: Option[String] = None, + content: Option[String] = None, + size: Option[Long] = None, + sha1: Option[String] = None, + textId: Option[Long] = None +) { // def this(revId: Int, parentId: Option[Int] = None, user: Option[Contributor] = None, timestamp: Option[DateTime] = None, // comment: Option[String] = None, content: Option[String] = None, size: Option[Int] = None, sha1: Option[String] = None) = { @@ -26,17 +26,21 @@ case class Revision( def isNewPage = parentId.contains(0) - def withContent(content: String*) = copy(content = Some(content.mkString("\n"))) + def withContent(content: String*) = + copy(content = Some(content.mkString("\n"))) def withText(text: String*) = copy(content = Some(text.mkString("\n"))) - def withIds(revId: Long, parentId: Long = 0) = copy(revId = Some(revId), parentId = Some(parentId)) + def withIds(revId: Long, parentId: Long = 0) = + copy(revId = Some(revId), parentId = Some(parentId)) - def withUser(userId: Long, login: String) = copy(user = Some(new User(Some(userId), Some(login)))) + def withUser(userId: Long, login: String) = + copy(user = Some(new User(Some(userId), Some(login)))) def withComment(comment: String) = copy(comment = Some(comment)) - def withTimeStamp(timestamp: ZonedDateTime = ZonedDateTime.now) = copy(timestamp = Some(timestamp)) + def withTimeStamp(timestamp: ZonedDateTime = ZonedDateTime.now) = + copy(timestamp = Some(timestamp)) def withoutContent = copy(content = None) } @@ -45,16 +49,16 @@ object Revision { def many(texts: String*) = texts .zip(texts.size to 1 by -1) - .map{ case (text, index) => - new Revision( - revId = Some(index), - pageId = Some(1L), - parentId = Some(index - 1), - content = Some(text), - size = Some(text.length), - sha1 = Some(DigestUtils.sha1Hex(text)) - ) - } + .map { case (text, index) => + new Revision( + revId = Some(index), + pageId = Some(1L), + parentId = Some(index - 1), + content = Some(text), + size = Some(text.length), + sha1 = Some(DigestUtils.sha1Hex(text)) + ) + } def one(text: String) = new Revision( @@ -66,6 +70,6 @@ object Revision { sha1 = Some(DigestUtils.sha1Hex(text)) ) - - def apply(revId: Long, pageId: Long) = new Revision(Some(revId), Some(pageId), None) + def apply(revId: Long, pageId: Long) = + new Revision(Some(revId), Some(pageId), None) } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/Site.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/Site.scala index 58ca9572..73dec0ce 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/Site.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/Site.scala @@ -2,14 +2,16 @@ package org.scalawiki.dto import java.net.URLEncoder -case class Site(langCode: Option[String], - family: String, - domain: String, - protocol: String = "https", - port: Option[Int] = None, - scriptPath: String = "/w", - script: String = "/w/index.php", - articlePath: String = "/wiki") { +case class Site( + langCode: Option[String], + family: String, + domain: String, + protocol: String = "https", + port: Option[Int] = None, + scriptPath: String = "/w", + script: String = "/w/index.php", + articlePath: String = "/wiki" +) { val home = protocol + "://" + domain @@ -20,7 +22,7 @@ case class Site(langCode: Option[String], URLEncoder.encode(underscored, "UTF-8") else underscored - ) + ) } def portStr = port.fold("")(":" + _) @@ -41,7 +43,16 @@ object Site { def localhost = { val scriptPath = "/w" val script = scriptPath + "/index.php" - Site(None, "wikipedia", "localhost", "http", Some(8080), scriptPath, script, articlePath = script) + Site( + None, + "wikipedia", + "localhost", + "http", + Some(8080), + scriptPath, + script, + articlePath = script + ) } def project(langCode: String, family: String) = @@ -49,12 +60,16 @@ object Site { def wikimedia(code: String) = Site(None, code, s"$code.wikimedia.org") - def host(host: String, protocol: String = "https", port: Option[Int] = None): Site = { + def host( + host: String, + protocol: String = "https", + port: Option[Int] = None + ): Site = { val list = host.split("\\.").toList list match { case code :: "wikimedia" :: "org" :: Nil => wikimedia(code) - case code :: family :: "org" :: Nil => project(code, family) + case code :: family :: "org" :: Nil => project(code, family) case _ => Site(None, host, host, protocol, port) } } -} \ No newline at end of file +} diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/User.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/User.scala index 2ae7f311..539249f1 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/User.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/User.scala @@ -3,16 +3,15 @@ package org.scalawiki.dto import java.time.ZonedDateTime case class User( - id: Option[Long], - login: Option[String], - editCount: Option[Long] = None, - registration: Option[ZonedDateTime] = None, - blocked: Option[Boolean] = None, - emailable: Option[Boolean] = None, - missing: Boolean = false, - sulAccounts: Seq[SulAccount] = Seq.empty - ) - extends Contributor { + id: Option[Long], + login: Option[String], + editCount: Option[Long] = None, + registration: Option[ZonedDateTime] = None, + blocked: Option[Boolean] = None, + emailable: Option[Boolean] = None, + missing: Boolean = false, + sulAccounts: Seq[SulAccount] = Seq.empty +) extends Contributor { override def name = login @@ -22,5 +21,3 @@ object User { def apply(id: Long, login: String) = new User(Some(id), Option(login)) } - - diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/UserContrib.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/UserContrib.scala index ad160dab..2eb90681 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/UserContrib.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/UserContrib.scala @@ -2,20 +2,36 @@ package org.scalawiki.dto import java.time.ZonedDateTime -case class UserContrib(userId: Long, - user: String, - pageId: Long, - revId: Long, - parentId: Long, - ns: Option[Int], - title: String, - timestamp: ZonedDateTime, - // isNew: Boolean, - // isMinor: Boolean, - comment: Option[String], - size: Option[Long]) { +case class UserContrib( + userId: Long, + user: String, + pageId: Long, + revId: Long, + parentId: Long, + ns: Option[Int], + title: String, + timestamp: ZonedDateTime, + // isNew: Boolean, + // isMinor: Boolean, + comment: Option[String], + size: Option[Long] +) { - def toPage = new Page(Some(pageId), ns, title, revisions = - Seq(new Revision(Some(revId), Some(pageId), Some(parentId), Some(User(userId, user)), Option(timestamp), comment, None, size)) + def toPage = new Page( + Some(pageId), + ns, + title, + revisions = Seq( + new Revision( + Some(revId), + Some(pageId), + Some(parentId), + Some(User(userId, user)), + Option(timestamp), + comment, + None, + size + ) + ) ) } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/Cmd.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/Cmd.scala index 84faa3be..41d15dc9 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/Cmd.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/Cmd.scala @@ -15,7 +15,10 @@ trait Parameter[+T] extends Equals { def flatten: Seq[Parameter[Any]] = Seq(this) } -abstract class EnumParameter[ARG <: EnumArg[ARG]](val name: String, val summary: String) extends Parameter[EnumArg[ARG]] { +abstract class EnumParameter[ARG <: EnumArg[ARG]]( + val name: String, + val summary: String +) extends Parameter[EnumArg[ARG]] { var allArgs: Seq[EnumArg[ARG]] = Seq.empty @@ -51,37 +54,46 @@ abstract class SingleParameter[T] extends Parameter[T] { } } +abstract class StringListParameter(val name: String, val summary: String) + extends ListParameter[String] -abstract class StringListParameter(val name: String, val summary: String) extends ListParameter[String] +abstract class IntListParameter(val name: String, val summary: String) + extends ListParameter[Int] -abstract class IntListParameter(val name: String, val summary: String) extends ListParameter[Int] +abstract class LongListParameter(val name: String, val summary: String) + extends ListParameter[Long] -abstract class LongListParameter(val name: String, val summary: String) extends ListParameter[Long] +abstract class IdListParameter(val name: String, val summary: String) + extends ListParameter[Long] -abstract class IdListParameter(val name: String, val summary: String) extends ListParameter[Long] +abstract class StringParameter(val name: String, val summary: String) + extends SingleParameter[String] -abstract class StringParameter(val name: String, val summary: String) extends SingleParameter[String] +abstract class IntParameter(val name: String, val summary: String) + extends SingleParameter[Int] -abstract class IntParameter(val name: String, val summary: String) extends SingleParameter[Int] +abstract class LongParameter(val name: String, val summary: String) + extends SingleParameter[Long] -abstract class LongParameter(val name: String, val summary: String) extends SingleParameter[Long] +abstract class IdParameter(val name: String, val summary: String) + extends SingleParameter[Long] -abstract class IdParameter(val name: String, val summary: String) extends SingleParameter[Long] - -abstract class DateTimeParameter(val name: String, val summary: String) extends SingleParameter[ZonedDateTime] { +abstract class DateTimeParameter(val name: String, val summary: String) + extends SingleParameter[ZonedDateTime] { override def pairs: Seq[(String, String)] = Seq(name -> Timestamp.format(arg)) } +abstract class BooleanParameter(val name: String, val summary: String) + extends SingleParameter[Boolean] -abstract class BooleanParameter(val name: String, val summary: String) extends SingleParameter[Boolean] - -abstract class ByteArrayParameter(val name: String, val summary: String) extends SingleParameter[Array[Byte]] +abstract class ByteArrayParameter(val name: String, val summary: String) + extends SingleParameter[Array[Byte]] trait ArgWithParams[P <: Parameter[Any], T <: EnumArg[T]] extends EnumArg[T] { def params: Seq[P] = Seq.empty - def byType[X : Manifest]: Seq[X] = + def byType[X: Manifest]: Seq[X] = params.collect { case x if manifest[X].runtimeClass.isInstance(x) => x.asInstanceOf[X] } @@ -100,19 +112,17 @@ trait EnumArg[+T <: EnumArg[T]] { def pairs: Seq[(String, String)] = Seq.empty } -abstract class EnumArgument[T <: EnumArg[T]](val name: String, val summary: String = "") extends EnumArg[T] +abstract class EnumArgument[T <: EnumArg[T]]( + val name: String, + val summary: String = "" +) extends EnumArg[T] trait ActionArg extends EnumArg[ActionArg] { /*val param = ActionParam*/ } -case class Action(override val arg: ActionArg) extends EnumParameter[ActionArg]("action", "") { +case class Action(override val arg: ActionArg) + extends EnumParameter[ActionArg]("action", "") { def query: Option[Query] = args.collectFirst { case q: Query => q } override def toString = pairs.toString() } - - - - - - diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/WatchParam.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/WatchParam.scala index e28062b4..9035a597 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/WatchParam.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/WatchParam.scala @@ -2,21 +2,51 @@ package org.scalawiki.dto.cmd trait WatchParam[+T] extends Parameter[T] -case class Watch(override val arg: Boolean = true) extends BooleanParameter("watch", - "Add the page to your watchlist. Deprecated. Use the watchlist argument") with WatchParam[Boolean] - -case class UnWatch(override val arg: Boolean = true) extends BooleanParameter("unwatch", - "Remove the page from your watchlist. Deprecated. Use the watchlist argument") with WatchParam[Boolean] +case class Watch(override val arg: Boolean = true) + extends BooleanParameter( + "watch", + "Add the page to your watchlist. Deprecated. Use the watchlist argument" + ) + with WatchParam[Boolean] + +case class UnWatch(override val arg: Boolean = true) + extends BooleanParameter( + "unwatch", + "Remove the page from your watchlist. Deprecated. Use the watchlist argument" + ) + with WatchParam[Boolean] // TODO single Enum value arg -case class WatchList(override val args: WatchListArg*) extends EnumParameter[WatchListArg]("watchlist", - "Specify how the watchlist is affected by this edit") with WatchParam[WatchListArg] - - +case class WatchList(override val args: WatchListArg*) + extends EnumParameter[WatchListArg]( + "watchlist", + "Specify how the watchlist is affected by this edit" + ) + with WatchParam[WatchListArg] trait WatchListArg extends EnumArg[WatchListArg] { val param = WatchList } -object WLWatch extends EnumArgument[WatchListArg]("watch", "add the page to the watchlist.") with WatchListArg -object WLUnWatch extends EnumArgument[WatchListArg]("unwatch", "remove the page from the watchlist.") with WatchListArg -object WLPreferences extends EnumArgument[WatchListArg]("preferences", "use the preference settings (Default).") with WatchListArg -object WLNoChange extends EnumArgument[WatchListArg]("nochange", "don't change the watchlist.") with WatchListArg \ No newline at end of file +object WLWatch + extends EnumArgument[WatchListArg]( + "watch", + "add the page to the watchlist." + ) + with WatchListArg +object WLUnWatch + extends EnumArgument[WatchListArg]( + "unwatch", + "remove the page from the watchlist." + ) + with WatchListArg +object WLPreferences + extends EnumArgument[WatchListArg]( + "preferences", + "use the preference settings (Default)." + ) + with WatchListArg +object WLNoChange + extends EnumArgument[WatchListArg]( + "nochange", + "don't change the watchlist." + ) + with WatchListArg diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/edit/Edit.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/edit/Edit.scala index 09de9319..4e97133b 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/edit/Edit.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/edit/Edit.scala @@ -5,78 +5,150 @@ import java.time.ZonedDateTime import org.scalawiki.dto.cmd._ case class Edit(override val params: EditParam[Any]*) - extends EnumArgument[ActionArg]("edit", "Edit and create pages.") - with ActionArg - with ArgWithParams[EditParam[Any], ActionArg] { + extends EnumArgument[ActionArg]("edit", "Edit and create pages.") + with ActionArg + with ArgWithParams[EditParam[Any], ActionArg] { def watch(params: WatchParam[Any]*) = ??? - } +} trait EditParam[+T] extends Parameter[T] -case class Title(override val arg: String) extends StringParameter("title", - "Title of the page you want to edit. Cannot be used together with pageid.") with EditParam[String] - -case class PageId(override val arg: Long) extends IdParameter("pageid", - "Page ID of the page you want to edit. Cannot be used together with title") with EditParam[Long] - -case class Section(override val arg: String) extends StringParameter("section", - "Section number. 0 for the top section, 'new' for a new section. Omit to act on the entire page") with EditParam[String] - -case class SectionTitle(override val arg: String) extends StringParameter("sectiontitle", - "Title to use if creating a new section. If not specified, summary will be used instead") with EditParam[String] - -case class Text(override val arg: String) extends StringParameter("text", - "New page (or section) content") with EditParam[String] - -case class Token(override val arg: String) extends StringParameter("token", "Edit token") with EditParam[String] - -case class Summary(override val arg: String) extends StringParameter("summary", - "Edit summary. Also section title when section=new and sectiontitle is not set") with EditParam[String] - -case class Minor(override val arg: Boolean = true) extends BooleanParameter("minor", - "If set, mark the edit as minor") with EditParam[Boolean] - -case class NotMinor(override val arg: Boolean = true) extends BooleanParameter("notminor", - "If set, don't mark the edit as minor, even if you have the 'Mark all my edits minor by default' preference enabled") with EditParam[Boolean] - -case class Bot(override val arg: Boolean = true) extends BooleanParameter("bot", - "If set, mark the edit as bot; even if you are using a bot account the edits will not be marked unless you set this flag") with EditParam[Boolean] - -case class BaseTimestamp(override val arg: ZonedDateTime) extends DateTimeParameter("basetimestamp", - "Timestamp of the base revision (obtained through prop=revisions&rvprop=timestamp). " + - "Used to detect edit conflicts; leave unset to ignore conflicts") with EditParam[ZonedDateTime] - -case class StartTimestamp(override val arg: ZonedDateTime) extends DateTimeParameter("starttimestamp", - "Timestamp when you started editing the page (e.g., when you fetched the current revision's text to begin editing it or checked the (non-)existence of the page). " + - "Used to detect if the page has been deleted since you started editing; leave unset to ignore conflicts") with EditParam[ZonedDateTime] - -case class Recreate(override val arg: Boolean = true) extends BooleanParameter("recreate", - "Override any errors about the article having been deleted in the meantime") with EditParam[Boolean] - -case class CreateOnly(override val arg: Boolean = true) extends BooleanParameter("createonly", - "Don't edit the page if it exists already") with EditParam[Boolean] - -case class NoCreate(override val arg: Boolean = true) extends BooleanParameter("nocreate", - "Throw an error if the page doesn't exist") with EditParam[Boolean] - -case class Md5(override val arg: String) extends StringParameter("md5", - "MD5 hash (hex) of the text parameter or the prependtext and appendtext parameters concatenated. " + - "If this parameter is set and the hashes don't match, the edit is rejected.") with EditParam[String] - -case class PrependText(override val arg: String) extends StringParameter("prependtext", - "Add this text to the beginning of the page. Overrides text") with EditParam[String] - -case class AppendText(override val arg: String) extends StringParameter("appendtext", - "Add this text to the end of the page. Overrides text. Use section=new to append a new section") with EditParam[String] - -case class Undo(override val arg: Long) extends IdParameter("appendtext", - "Revision ID to undo. Overrides text, prependtext and appendtext") with EditParam[Long] - -case class UndoAfter(override val arg: Long) extends IdParameter("undoafter", - "Undo all revisions from undo up to but not including this one. If not set, just undo one revision") with EditParam[Long] - -case class Redirect(override val arg: Boolean = true) extends BooleanParameter("redirect", - "Automatically resolve redirects") with EditParam[Boolean] - +case class Title(override val arg: String) + extends StringParameter( + "title", + "Title of the page you want to edit. Cannot be used together with pageid." + ) + with EditParam[String] + +case class PageId(override val arg: Long) + extends IdParameter( + "pageid", + "Page ID of the page you want to edit. Cannot be used together with title" + ) + with EditParam[Long] + +case class Section(override val arg: String) + extends StringParameter( + "section", + "Section number. 0 for the top section, 'new' for a new section. Omit to act on the entire page" + ) + with EditParam[String] + +case class SectionTitle(override val arg: String) + extends StringParameter( + "sectiontitle", + "Title to use if creating a new section. If not specified, summary will be used instead" + ) + with EditParam[String] + +case class Text(override val arg: String) + extends StringParameter("text", "New page (or section) content") + with EditParam[String] + +case class Token(override val arg: String) + extends StringParameter("token", "Edit token") + with EditParam[String] + +case class Summary(override val arg: String) + extends StringParameter( + "summary", + "Edit summary. Also section title when section=new and sectiontitle is not set" + ) + with EditParam[String] + +case class Minor(override val arg: Boolean = true) + extends BooleanParameter("minor", "If set, mark the edit as minor") + with EditParam[Boolean] + +case class NotMinor(override val arg: Boolean = true) + extends BooleanParameter( + "notminor", + "If set, don't mark the edit as minor, even if you have the 'Mark all my edits minor by default' preference enabled" + ) + with EditParam[Boolean] + +case class Bot(override val arg: Boolean = true) + extends BooleanParameter( + "bot", + "If set, mark the edit as bot; even if you are using a bot account the edits will not be marked unless you set this flag" + ) + with EditParam[Boolean] + +case class BaseTimestamp(override val arg: ZonedDateTime) + extends DateTimeParameter( + "basetimestamp", + "Timestamp of the base revision (obtained through prop=revisions&rvprop=timestamp). " + + "Used to detect edit conflicts; leave unset to ignore conflicts" + ) + with EditParam[ZonedDateTime] + +case class StartTimestamp(override val arg: ZonedDateTime) + extends DateTimeParameter( + "starttimestamp", + "Timestamp when you started editing the page (e.g., when you fetched the current revision's text to begin editing it or checked the (non-)existence of the page). " + + "Used to detect if the page has been deleted since you started editing; leave unset to ignore conflicts" + ) + with EditParam[ZonedDateTime] + +case class Recreate(override val arg: Boolean = true) + extends BooleanParameter( + "recreate", + "Override any errors about the article having been deleted in the meantime" + ) + with EditParam[Boolean] + +case class CreateOnly(override val arg: Boolean = true) + extends BooleanParameter( + "createonly", + "Don't edit the page if it exists already" + ) + with EditParam[Boolean] + +case class NoCreate(override val arg: Boolean = true) + extends BooleanParameter( + "nocreate", + "Throw an error if the page doesn't exist" + ) + with EditParam[Boolean] + +case class Md5(override val arg: String) + extends StringParameter( + "md5", + "MD5 hash (hex) of the text parameter or the prependtext and appendtext parameters concatenated. " + + "If this parameter is set and the hashes don't match, the edit is rejected." + ) + with EditParam[String] + +case class PrependText(override val arg: String) + extends StringParameter( + "prependtext", + "Add this text to the beginning of the page. Overrides text" + ) + with EditParam[String] + +case class AppendText(override val arg: String) + extends StringParameter( + "appendtext", + "Add this text to the end of the page. Overrides text. Use section=new to append a new section" + ) + with EditParam[String] + +case class Undo(override val arg: Long) + extends IdParameter( + "appendtext", + "Revision ID to undo. Overrides text, prependtext and appendtext" + ) + with EditParam[Long] + +case class UndoAfter(override val arg: Long) + extends IdParameter( + "undoafter", + "Undo all revisions from undo up to but not including this one. If not set, just undo one revision" + ) + with EditParam[Long] + +case class Redirect(override val arg: Boolean = true) + extends BooleanParameter("redirect", "Automatically resolve redirects") + with EditParam[Boolean] diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/email/EmailUser.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/email/EmailUser.scala index 107590a2..97cac393 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/email/EmailUser.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/email/EmailUser.scala @@ -3,19 +3,31 @@ package org.scalawiki.dto.cmd.email import org.scalawiki.dto.cmd._ case class EmailUser(override val params: EmailParam[Any]*) - extends EnumArgument[ActionArg]("emailuser", "Email a user.") - with ActionArg - with ArgWithParams[EmailParam[Any], ActionArg] + extends EnumArgument[ActionArg]("emailuser", "Email a user.") + with ActionArg + with ArgWithParams[EmailParam[Any], ActionArg] trait EmailParam[+T] extends Parameter[T] -case class Target(override val arg: String) extends StringParameter("target", "User to send email to.") with EmailParam[String] +case class Target(override val arg: String) + extends StringParameter("target", "User to send email to.") + with EmailParam[String] -case class Subject(override val arg: String) extends StringParameter("subject", "The subject of the message.") with EmailParam[String] +case class Subject(override val arg: String) + extends StringParameter("subject", "The subject of the message.") + with EmailParam[String] -case class Text(override val arg: String) extends StringParameter("text", "The message.") with EmailParam[String] +case class Text(override val arg: String) + extends StringParameter("text", "The message.") + with EmailParam[String] -case class Token(override val arg: String) extends StringParameter("token", "The subject of the message.") with EmailParam[String] - -case class CcMe(override val arg: Boolean) extends BooleanParameter("ccme", "If set, a copy of the email will be sent to you.") with EmailParam[Boolean] +case class Token(override val arg: String) + extends StringParameter("token", "The subject of the message.") + with EmailParam[String] +case class CcMe(override val arg: Boolean) + extends BooleanParameter( + "ccme", + "If set, a copy of the email will be sent to you." + ) + with EmailParam[Boolean] diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Generator.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Generator.scala index 99461815..aa735344 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Generator.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Generator.scala @@ -2,20 +2,22 @@ package org.scalawiki.dto.cmd.query import org.scalawiki.dto.cmd.EnumArg -/** - * Get the list of pages to work on by executing the specified query module - * @param generator - */ +/** Get the list of pages to work on by executing the specified query module + * @param generator + */ trait GeneratorArg extends EnumArg[GeneratorArg] -case class Generator(generator: EnumArg[GeneratorArg]) extends /*ArgWithParams[G, ListArg] with */ QueryParam[EnumArg[GeneratorArg]] { +case class Generator(generator: EnumArg[GeneratorArg]) + extends /*ArgWithParams[G, ListArg] with */ QueryParam[ + EnumArg[GeneratorArg] + ] { override def name: String = "generator" override def summary: String = "" override def pairs: Seq[(String, String)] = - Seq(name -> generator.name) ++ generator.pairs.map { - case (k,v) => ("g" + k, v) + Seq(name -> generator.name) ++ generator.pairs.map { case (k, v) => + ("g" + k, v) } } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Module.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Module.scala index e767d7f9..ed59b760 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Module.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Module.scala @@ -2,8 +2,5 @@ package org.scalawiki.dto.cmd.query import org.scalawiki.dto.cmd.{EnumArg, EnumArgument} -class Module[T <: EnumArg[T]](val prefix: String, name: String, summary: String ) extends EnumArgument[T](name, summary) -{ - - -} +class Module[T <: EnumArg[T]](val prefix: String, name: String, summary: String) + extends EnumArgument[T](name, summary) {} diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Query.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Query.scala index ca5a53c7..3c71f806 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Query.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/Query.scala @@ -5,16 +5,13 @@ import org.scalawiki.dto.cmd.query.list.{ListArg, ListParam} import org.scalawiki.dto.cmd.query.meta.{MetaArg, MetaParam} import org.scalawiki.dto.cmd.query.prop.{Revisions, Prop, PropArg} - -/** - * ?action=query - * - */ +/** ?action=query + */ case class Query(override val params: QueryParam[Any]*) - extends EnumArgument[ActionArg]("query", "Various queries.") - with ActionArg - with ArgWithParams[QueryParam[Any], ActionArg] { + extends EnumArgument[ActionArg]("query", "Various queries.") + with ActionArg + with ArgWithParams[QueryParam[Any], ActionArg] { val lists: Seq[ListArg] = byPF({ case p: ListParam => p }).flatMap(_.args) val props: Seq[PropArg] = byPF({ case p: Prop => p }).flatMap(_.args) @@ -22,32 +19,32 @@ case class Query(override val params: QueryParam[Any]*) def revisions = props.seq.collectFirst { case r: Revisions => r } def revisionsWithoutContent: Query = - Query(params.filterNot(_.isInstanceOf[Prop]) :+ - Prop( - props.filterNot(_.isInstanceOf[Revisions]) ++ - revisions.map(_.withoutContent).toSeq: _* - ):_* + Query( + params.filterNot(_.isInstanceOf[Prop]) :+ + Prop( + props.filterNot(_.isInstanceOf[Revisions]) ++ + revisions.map(_.withoutContent).toSeq: _* + ): _* ) val metas: Seq[MetaArg] = byPF({ case p: MetaParam => p }).flatMap(_.args) } -/** - * Marker trait for parameters available together with ?action=query - * - */ +/** Marker trait for parameters available together with ?action=query + */ trait QueryParam[+T] extends Parameter[T] - case class TitlesParam(override val args: Seq[String]) - extends StringListParameter("titles", "A list of titles to work on") with QueryParam[String] + extends StringListParameter("titles", "A list of titles to work on") + with QueryParam[String] case class PageIdsParam(override val args: Iterable[Long]) - extends IdListParameter("pageids", "A list of page IDs to work on") with QueryParam[Long] + extends IdListParameter("pageids", "A list of page IDs to work on") + with QueryParam[Long] case class RevIdsParam(override val args: Seq[Long]) - extends IdListParameter("revids", "A list of revision IDs to work on") with QueryParam[Long] - + extends IdListParameter("revids", "A list of revision IDs to work on") + with QueryParam[Long] //indexpageids Include an additional pageids section listing all returned page IDs. //export Export the current revisions of all given or generated pages. @@ -57,4 +54,4 @@ case class RevIdsParam(override val args: Seq[Long]) //rawcontinue Currently ignored. In the future, continue will become the default and this will be needed to receive the raw query-continue data. //generator Get the list of pages to work on by executing the specified query module. Note: Generator parameter names must be prefixed with a "g", see examples. //redirects Automatically resolve redirects in titles, pageids, and revids, and in pages returned by generator. -//converttitles Convert titles to other variants if necessary. Only works if the wiki's content language supports variant conversion. Languages that support variant conversion include gan, iu, kk, ku, shi, sr, tg, uz, zh. \ No newline at end of file +//converttitles Convert titles to other variants if necessary. Only works if the wiki's content language supports variant conversion. Languages that support variant conversion include gan, iu, kk, ku, shi, sr, tg, uz, zh. diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/AllUsers.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/AllUsers.scala index 80a32558..d1f98826 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/AllUsers.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/AllUsers.scala @@ -3,40 +3,61 @@ package org.scalawiki.dto.cmd.query.list import org.scalawiki.dto.cmd.query.Module import org.scalawiki.dto.cmd._ -/** - * https://www.mediawiki.org/wiki/API:Allusers - * @param params - */ +/** https://www.mediawiki.org/wiki/API:Allusers + * @param params + */ case class AllUsers(override val params: AuParam[Any]*) - extends Module[ListArg]("au", "allusers", "Enumerate all registered users, ordered by username.") - with ListArg - with ArgWithParams[AuParam[Any], ListArg] + extends Module[ListArg]( + "au", + "allusers", + "Enumerate all registered users, ordered by username." + ) + with ListArg + with ArgWithParams[AuParam[Any], ListArg] trait AuParam[+T] extends Parameter[T] -case class AuFrom(override val arg: String) extends StringParameter("aufrom", - "The user name to start enumerating from.") with AuParam[String] +case class AuFrom(override val arg: String) + extends StringParameter( + "aufrom", + "The user name to start enumerating from." + ) + with AuParam[String] -case class AuTo(override val arg: String) extends StringParameter("auto", - "The user name to stop enumerating at.") with AuParam[String] +case class AuTo(override val arg: String) + extends StringParameter("auto", "The user name to stop enumerating at.") + with AuParam[String] // TODO enum -case class AuDir(override val arg: String) extends StringParameter("audir", - "Direction to sort in.") with AuParam[String] +case class AuDir(override val arg: String) + extends StringParameter("audir", "Direction to sort in.") + with AuParam[String] -case class AuPrefix(override val arg: String) extends StringParameter("auprefix", - "Search for all users that begin with this value.") with AuParam[String] +case class AuPrefix(override val arg: String) + extends StringParameter( + "auprefix", + "Search for all users that begin with this value." + ) + with AuParam[String] -case class AuGroup(override val args: Seq[String]) extends StringListParameter("augroup", - "Limit users to given group name(s). Possible values: bot, sysop, bureaucrat (+ any other group that is defined on the wiki)") -with AuParam[String] +case class AuGroup(override val args: Seq[String]) + extends StringListParameter( + "augroup", + "Limit users to given group name(s). Possible values: bot, sysop, bureaucrat (+ any other group that is defined on the wiki)" + ) + with AuParam[String] -case class AuExcludeGroup(override val args: Seq[String]) extends StringListParameter("auexcludegroup", - "Exclude users in given group name (s).Can not be used together with 'group '.") with AuParam[String] +case class AuExcludeGroup(override val args: Seq[String]) + extends StringListParameter( + "auexcludegroup", + "Exclude users in given group name (s).Can not be used together with 'group '." + ) + with AuParam[String] -case class AuRights(override val args: Seq[String]) extends StringListParameter("aurights", - "Limit users to given right (s)") with AuParam[String] +case class AuRights(override val args: Seq[String]) + extends StringListParameter("aurights", "Limit users to given right (s)") + with AuParam[String] // TODO enum //blockinfo: Adds the information about a current block on the user @@ -45,15 +66,28 @@ case class AuRights(override val args: Seq[String]) extends StringListParameter( //rights: Lists rights that the user has //editcount: Adds the edit count of the user //registration: Adds the timestamp of when the user registered if available (may be blank) 1.12 + -case class AuProp(override val args: Seq[String]) extends StringListParameter("auprop", - "What pieces of information to include.") with AuParam[String] +case class AuProp(override val args: Seq[String]) + extends StringListParameter( + "auprop", + "What pieces of information to include." + ) + with AuParam[String] -case class AuLimit(override val arg: String) extends StringParameter("aulimit", - "How many total user names to return.") with AuParam[String] +case class AuLimit(override val arg: String) + extends StringParameter("aulimit", "How many total user names to return.") + with AuParam[String] -// not a boolean actually -case class AuWithEditsOnly(override val arg: Boolean) extends BooleanParameter("auwitheditsonly", - "Only list users who have made edits.") with AuParam[Boolean] +// not a boolean actually +case class AuWithEditsOnly(override val arg: Boolean) + extends BooleanParameter( + "auwitheditsonly", + "Only list users who have made edits." + ) + with AuParam[Boolean] -case class AuActiveUsers(override val arg: Boolean) extends BooleanParameter("auactiveusers", - "Only list users active in the last 30 days.") with AuParam[Boolean] \ No newline at end of file +case class AuActiveUsers(override val arg: Boolean) + extends BooleanParameter( + "auactiveusers", + "Only list users active in the last 30 days." + ) + with AuParam[Boolean] diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/CategoryMembers.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/CategoryMembers.scala index 55ecc207..dcb117f9 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/CategoryMembers.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/CategoryMembers.scala @@ -4,20 +4,32 @@ import org.scalawiki.dto.cmd._ import org.scalawiki.dto.cmd.query.Module case class CategoryMembers(override val params: CmParam[Any]*) - extends Module[ListArg]("cm", "categorymembers", "List all pages in a given category.") + extends Module[ListArg]( + "cm", + "categorymembers", + "List all pages in a given category." + ) with ListArg with ArgWithParams[CmParam[Any], ListArg] { - def this(title: Option[String], pageId: Option[Long], namespaces: Set[Int], limit: Option[String], params: CmParam[Any]*) = { - this(Seq( - title.map(CmTitle).toSeq, - pageId.map(CmPageId).toSeq, - limit.map(CmLimit).toSeq - ).flatten ++ - (if (namespaces.nonEmpty) - Seq(CmNamespace(namespaces.toSeq)) - else Seq.empty) - ++ params: _*) + def this( + title: Option[String], + pageId: Option[Long], + namespaces: Set[Int], + limit: Option[String], + params: CmParam[Any]* + ) = { + this( + Seq( + title.map(CmTitle).toSeq, + pageId.map(CmPageId).toSeq, + limit.map(CmLimit).toSeq + ).flatten ++ + (if (namespaces.nonEmpty) + Seq(CmNamespace(namespaces.toSeq)) + else Seq.empty) + ++ params: _* + ) } } @@ -27,46 +39,94 @@ trait CmTypeArg extends EnumArg[CmTypeArg] { val param = CmType } - -case class CmTitle(override val arg: String) extends StringParameter("cmtitle", - "Title to search. Cannot be used together with cmpageid.") with CmParam[String] - -case class CmPageId(override val arg: Long) extends IdParameter("cmpageid", - "Page ID to search. Cannot be used together with cmtitle.") with CmParam[Long] - -case class CmNamespace(override val args: Seq[Int]) extends IntListParameter("cmnamespace", - "Only include pages in these namespaces.") with CmParam[Int] - -case class CmLimit(override val arg: String) extends StringParameter("cmlimit", - "How many total pages to return.") with CmParam[String] +case class CmTitle(override val arg: String) + extends StringParameter( + "cmtitle", + "Title to search. Cannot be used together with cmpageid." + ) + with CmParam[String] + +case class CmPageId(override val arg: Long) + extends IdParameter( + "cmpageid", + "Page ID to search. Cannot be used together with cmtitle." + ) + with CmParam[Long] + +case class CmNamespace(override val args: Seq[Int]) + extends IntListParameter( + "cmnamespace", + "Only include pages in these namespaces." + ) + with CmParam[Int] + +case class CmLimit(override val arg: String) + extends StringParameter("cmlimit", "How many total pages to return.") + with CmParam[String] case class CmType(override val args: CmTypeArg*) - extends EnumParameter[CmTypeArg]("cmtype", "Which type of category members to include") with CmParam[CmTypeArg] + extends EnumParameter[CmTypeArg]( + "cmtype", + "Which type of category members to include" + ) + with CmParam[CmTypeArg] -object CmTypePage extends EnumArgument[CmTypeArg]("page", "pages") with CmTypeArg +object CmTypePage + extends EnumArgument[CmTypeArg]("page", "pages") + with CmTypeArg -object CmTypeSubCat extends EnumArgument[CmTypeArg]("subcat", "subcats") with CmTypeArg +object CmTypeSubCat + extends EnumArgument[CmTypeArg]("subcat", "subcats") + with CmTypeArg -object CmTypeFile extends EnumArgument[CmTypeArg]("file", "files") with CmTypeArg +object CmTypeFile + extends EnumArgument[CmTypeArg]("file", "files") + with CmTypeArg trait CmPropArg extends EnumArg[CmPropArg] { val param = CmProp } case class CmProp(override val args: CmPropArg*) - extends EnumParameter[CmPropArg]("cmprop", "Which properties to get for each revision:") with CmParam[CmPropArg] + extends EnumParameter[CmPropArg]( + "cmprop", + "Which properties to get for each revision:" + ) + with CmParam[CmPropArg] package cmprop { - object Ids extends EnumArgument[CmPropArg]("ids", "Adds the page ID.") with CmPropArg - - object Title extends EnumArgument[CmPropArg]("title", "Adds the title and namespace ID of the page.") with CmPropArg - - object SortKey extends EnumArgument[CmPropArg]("sortkey", "Adds the sortkey used for sorting in the category (hexadecimal string).") with CmPropArg - - object SortKeyPrefix extends EnumArgument[CmPropArg]("sortkeyprefix", "Adds the sortkey prefix used for sorting in the category (human-readable part of the sortkey).") with CmPropArg - - object Timestamp extends EnumArgument[CmPropArg]("timestamp", "Adds the timestamp of when the page was included.") with CmPropArg + object Ids + extends EnumArgument[CmPropArg]("ids", "Adds the page ID.") + with CmPropArg + + object Title + extends EnumArgument[CmPropArg]( + "title", + "Adds the title and namespace ID of the page." + ) + with CmPropArg + + object SortKey + extends EnumArgument[CmPropArg]( + "sortkey", + "Adds the sortkey used for sorting in the category (hexadecimal string)." + ) + with CmPropArg + + object SortKeyPrefix + extends EnumArgument[CmPropArg]( + "sortkeyprefix", + "Adds the sortkey prefix used for sorting in the category (human-readable part of the sortkey)." + ) + with CmPropArg + + object Timestamp + extends EnumArgument[CmPropArg]( + "timestamp", + "Adds the timestamp of when the page was included." + ) + with CmPropArg } @@ -75,7 +135,8 @@ trait CmSortArg extends EnumArg[CmSortArg] { } case class CmSort(override val args: CmSortArg*) - extends EnumParameter[CmSortArg]("cmsort", "Property to sort by.") with CmParam[CmSortArg] + extends EnumParameter[CmSortArg]("cmsort", "Property to sort by.") + with CmParam[CmSortArg] package cmsort { @@ -90,7 +151,8 @@ trait CmDirArg extends EnumArg[CmDirArg] { } case class CmDir(override val args: CmDirArg*) - extends EnumParameter[CmDirArg]("cmdir", "In which direction to sort.") with CmParam[CmDirArg] + extends EnumParameter[CmDirArg]("cmdir", "In which direction to sort.") + with CmParam[CmDirArg] object Asc extends EnumArgument[CmDirArg]("asc") with CmDirArg @@ -103,4 +165,4 @@ object Desc extends EnumArgument[CmDirArg]("desc") with CmDirArg //cmend //Timestamp to end listing at. Can only be used with cmsort=timestamp. // -//Type: timestamp (allowed formats) \ No newline at end of file +//Type: timestamp (allowed formats) diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/EmbeddedIn.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/EmbeddedIn.scala index 15d43fd6..3fd1d5c7 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/EmbeddedIn.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/EmbeddedIn.scala @@ -3,39 +3,61 @@ package org.scalawiki.dto.cmd.query.list import org.scalawiki.dto.cmd._ import org.scalawiki.dto.cmd.query.Module -/** - * ?action=query&prop=revisions - * - */ +/** ?action=query&prop=revisions + */ case class EmbeddedIn(override val params: EiParam[Any]*) - extends Module[ListArg]("ei", "embeddedin", "Find all pages that embed (transclude) the given page.") - with ListArg - with ArgWithParams[EiParam[Any], ListArg] { - - def this(title: Option[String], pageId: Option[Long], namespaces:Set[Int], limit: Option[String], params: EiParam[Any]*) = { - this(Seq( - title.map(EiTitle).toSeq, - pageId.map(EiPageId).toSeq, - Seq(EiNamespace(namespaces.toSeq)), - limit.map(EiLimit).toSeq - ).flatten ++ params:_*) + extends Module[ListArg]( + "ei", + "embeddedin", + "Find all pages that embed (transclude) the given page." + ) + with ListArg + with ArgWithParams[EiParam[Any], ListArg] { + + def this( + title: Option[String], + pageId: Option[Long], + namespaces: Set[Int], + limit: Option[String], + params: EiParam[Any]* + ) = { + this( + Seq( + title.map(EiTitle).toSeq, + pageId.map(EiPageId).toSeq, + Seq(EiNamespace(namespaces.toSeq)), + limit.map(EiLimit).toSeq + ).flatten ++ params: _* + ) } - + } trait EiParam[+T] extends Parameter[T] -case class EiTitle(override val arg: String) extends StringParameter("eititle", - "Title to search. Cannot be used together with eipageid.") with EiParam[String] -case class EiPageId(override val arg: Long) extends IdParameter("eipageid", - "Page ID to search. Cannot be used together with eititle.") with EiParam[Long] - -case class EiNamespace(override val args: Seq[Int]) extends IntListParameter("einamespace", - "Only include pages in these namespaces.") with EiParam[Int] +case class EiTitle(override val arg: String) + extends StringParameter( + "eititle", + "Title to search. Cannot be used together with eipageid." + ) + with EiParam[String] +case class EiPageId(override val arg: Long) + extends IdParameter( + "eipageid", + "Page ID to search. Cannot be used together with eititle." + ) + with EiParam[Long] -case class EiLimit(override val arg: String) extends StringParameter("eilimit", - "How many total pages to return.") with EiParam[String] +case class EiNamespace(override val args: Seq[Int]) + extends IntListParameter( + "einamespace", + "Only include pages in these namespaces." + ) + with EiParam[Int] +case class EiLimit(override val arg: String) + extends StringParameter("eilimit", "How many total pages to return.") + with EiParam[String] //eicontinue //When more results are available, use this to continue. diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/ListArgs.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/ListArgs.scala index 563b702f..b670fba4 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/ListArgs.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/ListArgs.scala @@ -3,10 +3,15 @@ package org.scalawiki.dto.cmd.query.list import org.scalawiki.dto.cmd.query.GeneratorArg import org.scalawiki.dto.cmd.query.prop.{ImLimit, Images} - object ListArgs { - def toDsl(module: String, title: Option[String], pageId: Option[Long], namespaces:Set[Int], limit: Option[String]): Option[GeneratorArg] = { + def toDsl( + module: String, + title: Option[String], + pageId: Option[Long], + namespaces: Set[Int], + limit: Option[String] + ): Option[GeneratorArg] = { val f = Seq(title.toSeq, pageId.toSeq, namespaces).flatten diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/ListParam.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/ListParam.scala index 81b9f8f3..3551efad 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/ListParam.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/ListParam.scala @@ -3,15 +3,15 @@ package org.scalawiki.dto.cmd.query.list import org.scalawiki.dto.cmd.{EnumArg, EnumParameter} import org.scalawiki.dto.cmd.query.{GeneratorArg, QueryParam} -/** - * ?action=query&list= parameter - * - */ +/** ?action=query&list= parameter + */ -case class ListParam(override val args: ListArg*) extends EnumParameter[ListArg]("list", "") with QueryParam[ListArg] +case class ListParam(override val args: ListArg*) + extends EnumParameter[ListArg]("list", "") + with QueryParam[ListArg] -/** - * ?action=query&list=argument - * - */ -trait ListArg extends EnumArg[ListArg] with GeneratorArg { val param = ListParam } +/** ?action=query&list=argument + */ +trait ListArg extends EnumArg[ListArg] with GeneratorArg { + val param = ListParam +} diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/UserContribs.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/UserContribs.scala index a6fe7b32..aaa6b0e6 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/UserContribs.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/UserContribs.scala @@ -5,46 +5,76 @@ import java.time.ZonedDateTime import org.scalawiki.dto.cmd.query.Module import org.scalawiki.dto.cmd._ -/** - * https://www.mediawiki.org/wiki/API:Usercontribs +/** https://www.mediawiki.org/wiki/API:Usercontribs * @param params */ case class UserContribs(override val params: UcParam[Any]*) - extends Module[ListArg]("uc", "usercontribs", - "Gets a list of contributions made by a given user, ordered by modification time.") - with ListArg - with ArgWithParams[UcParam[Any], ListArg] + extends Module[ListArg]( + "uc", + "usercontribs", + "Gets a list of contributions made by a given user, ordered by modification time." + ) + with ListArg + with ArgWithParams[UcParam[Any], ListArg] trait UcParam[+T] extends Parameter[T] -case class UcStart(override val arg: ZonedDateTime) extends DateTimeParameter("ucstart", - "The timestamp to start listing from.") with UcParam[ZonedDateTime] +case class UcStart(override val arg: ZonedDateTime) + extends DateTimeParameter("ucstart", "The timestamp to start listing from.") + with UcParam[ZonedDateTime] -case class UcEnd(override val arg: ZonedDateTime) extends DateTimeParameter("ucend", - "The timestamp to end listing at.") with UcParam[ZonedDateTime] +case class UcEnd(override val arg: ZonedDateTime) + extends DateTimeParameter("ucend", "The timestamp to end listing at.") + with UcParam[ZonedDateTime] -case class UcUser(override val args: Seq[String]) extends StringListParameter("ucuser", - "Users to retrieve contributions for.") with UcParam[String] +case class UcUser(override val args: Seq[String]) + extends StringListParameter( + "ucuser", + "Users to retrieve contributions for." + ) + with UcParam[String] -case class UcUserPrefix(override val arg: String) extends StringParameter("ucuserprefix", - "List contributions of all users whose name starts with this string.") with UcParam[String] +case class UcUserPrefix(override val arg: String) + extends StringParameter( + "ucuserprefix", + "List contributions of all users whose name starts with this string." + ) + with UcParam[String] // TODO enum -case class UcDir(override val arg: String) extends StringParameter("ucdir", - "Direction to sort in.") with UcParam[String] +case class UcDir(override val arg: String) + extends StringParameter("ucdir", "Direction to sort in.") + with UcParam[String] -case class UcLimit(override val arg: String) extends StringParameter("uclimit", - "Maximum amount of contributions to list.") with UcParam[String] +case class UcLimit(override val arg: String) + extends StringParameter( + "uclimit", + "Maximum amount of contributions to list." + ) + with UcParam[String] -case class UcNamespace(override val args: Seq[Int]) extends IntListParameter("ucnamespace", - "Only list contributions in these namespaces.") with UcParam[Int] +case class UcNamespace(override val args: Seq[Int]) + extends IntListParameter( + "ucnamespace", + "Only list contributions in these namespaces." + ) + with UcParam[Int] // TODO enum -case class UcProp(override val args: Seq[String]) extends StringListParameter("ucprop", - "Which properties to get.") with UcParam[String] +case class UcProp(override val args: Seq[String]) + extends StringListParameter("ucprop", "Which properties to get.") + with UcParam[String] -case class UcShow(override val args: Seq[String]) extends StringListParameter("ucshow", - "Only list contributions that meet these criteria.") with UcParam[String] +case class UcShow(override val args: Seq[String]) + extends StringListParameter( + "ucshow", + "Only list contributions that meet these criteria." + ) + with UcParam[String] -case class UcTag(override val args: Seq[String]) extends StringListParameter("uctag", - "Only list revisions tagged with this tag.") with UcParam[String] \ No newline at end of file +case class UcTag(override val args: Seq[String]) + extends StringListParameter( + "uctag", + "Only list revisions tagged with this tag." + ) + with UcParam[String] diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/Users.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/Users.scala index e2a0cded..070521ec 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/Users.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/list/Users.scala @@ -3,60 +3,94 @@ package org.scalawiki.dto.cmd.query.list import org.scalawiki.dto.cmd._ import org.scalawiki.dto.cmd.query.Module -/** - * https://www.mediawiki.org/wiki/API:Users +/** https://www.mediawiki.org/wiki/API:Users * * @param params - */ + */ case class Users(override val params: UsParam*) - extends Module[ListArg]("us", "users", "Get information about a list of users.") - with ListArg - with ArgWithParams[UsParam, ListArg] + extends Module[ListArg]( + "us", + "users", + "Get information about a list of users." + ) + with ListArg + with ArgWithParams[UsParam, ListArg] trait UsParam extends Parameter[Any] -case class UsUsers(override val args: Iterable[String]) extends StringListParameter("ususers", - "A list of user names to get information for.") with UsParam - +case class UsUsers(override val args: Iterable[String]) + extends StringListParameter( + "ususers", + "A list of user names to get information for." + ) + with UsParam -/** - * ?action=query&list=users&usprop= - * +/** ?action=query&list=users&usprop= */ case class UsProp(override val args: UsPropArg*) - extends EnumParameter[UsPropArg]("usprop", "Which properties to get (Default: none).") + extends EnumParameter[UsPropArg]( + "usprop", + "Which properties to get (Default: none)." + ) with UsParam -/** - * Trait for usprop= arguments - * +/** Trait for usprop= arguments */ trait UsPropArg extends EnumArg[UsPropArg] { val param = UsProp } -/** - * usprop= arguments - * +/** usprop= arguments */ -object UsBlockInfo extends EnumArgument[UsPropArg]("blockinfo", "Whether the user is blocked, by whom and why") with UsPropArg - -object UsGroups extends EnumArgument[UsPropArg]("groups", "All groups the user belongs to") with UsPropArg - -object UsImplicitGroups extends EnumArgument[UsPropArg]("implicitgroups", "All groups a user is automatically a member of") with UsPropArg - -object UsRights extends EnumArgument[UsPropArg]("rights", "All rights the user has") with UsPropArg - -object UsEditCount extends EnumArgument[UsPropArg]("editcount", "The number of edits the user has made") with UsPropArg - -object UsRegistration extends EnumArgument[UsPropArg]("registration", "The time and date the user registered at") with UsPropArg - -object UsEmailable extends EnumArgument[UsPropArg]("emailable", "Whether the user can and wants to receive e-mail through Special:Emailuser") with UsPropArg - -object UsGender extends EnumArgument[UsPropArg]("gender", "Tags the gender of the user. Returns \"male\", \"female\", or \"unknown\"") with UsPropArg - - - +object UsBlockInfo + extends EnumArgument[UsPropArg]( + "blockinfo", + "Whether the user is blocked, by whom and why" + ) + with UsPropArg + +object UsGroups + extends EnumArgument[UsPropArg]("groups", "All groups the user belongs to") + with UsPropArg + +object UsImplicitGroups + extends EnumArgument[UsPropArg]( + "implicitgroups", + "All groups a user is automatically a member of" + ) + with UsPropArg + +object UsRights + extends EnumArgument[UsPropArg]("rights", "All rights the user has") + with UsPropArg + +object UsEditCount + extends EnumArgument[UsPropArg]( + "editcount", + "The number of edits the user has made" + ) + with UsPropArg + +object UsRegistration + extends EnumArgument[UsPropArg]( + "registration", + "The time and date the user registered at" + ) + with UsPropArg + +object UsEmailable + extends EnumArgument[UsPropArg]( + "emailable", + "Whether the user can and wants to receive e-mail through Special:Emailuser" + ) + with UsPropArg + +object UsGender + extends EnumArgument[UsPropArg]( + "gender", + "Tags the gender of the user. Returns \"male\", \"female\", or \"unknown\"" + ) + with UsPropArg diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/meta/GlobalUserInfo.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/meta/GlobalUserInfo.scala index 33c4f65d..b2e118a6 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/meta/GlobalUserInfo.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/meta/GlobalUserInfo.scala @@ -3,52 +3,70 @@ package org.scalawiki.dto.cmd.query.meta import org.scalawiki.dto.cmd._ import org.scalawiki.dto.cmd.query.Module -/** - * ?action=query&meta=globaluserinfo - * +/** ?action=query&meta=globaluserinfo */ case class GlobalUserInfo(override val params: GuiParam[Any]*) - extends Module[MetaArg]("gui", "globaluserinfo", "Show information about a global user.") - with MetaArg with ArgWithParams[GuiParam[Any], MetaArg] - + extends Module[MetaArg]( + "gui", + "globaluserinfo", + "Show information about a global user." + ) + with MetaArg + with ArgWithParams[GuiParam[Any], MetaArg] trait GuiParam[+T] extends Parameter[T] -case class GuiUser(override val arg: String) extends StringParameter("guiuser", - "User to get information about. Defaults to the current user.") with GuiParam[String] +case class GuiUser(override val arg: String) + extends StringParameter( + "guiuser", + "User to get information about. Defaults to the current user." + ) + with GuiParam[String] -/** - * Which properties to get: - * ?action=query&meta=globaluserinfo&guiprop= +/** Which properties to get: ?action=query&meta=globaluserinfo&guiprop= */ -case class GuiProp(override val args: GuiPropArg*) extends - EnumParameter[GuiPropArg]("guiprop", "Which properties to get.") with GuiParam[GuiPropArg] +case class GuiProp(override val args: GuiPropArg*) + extends EnumParameter[GuiPropArg]("guiprop", "Which properties to get.") + with GuiParam[GuiPropArg] -/** - * Trait for guiprop= arguments - * +/** Trait for guiprop= arguments */ trait GuiPropArg extends EnumArg[GuiPropArg] { val param = GuiProp } -/** - * guiprop= arguments - * +/** guiprop= arguments */ -object Groups extends EnumArgument[GuiPropArg]("groups", "Get a list of global groups this user belongs to.") with GuiPropArg - -object Rights extends EnumArgument[GuiPropArg]("rights", "Get a list of global rights this user has.") with GuiPropArg - -object Merged extends EnumArgument[GuiPropArg]("merged", "Get a list of merged accounts.") with GuiPropArg - -object Unattached extends EnumArgument[GuiPropArg]("unattached", "Get a list of unattached accounts.") with GuiPropArg - -object EditCount extends EnumArgument[GuiPropArg]("editcount", "Get the user's global editcount.") with GuiPropArg - - - - - +object Groups + extends EnumArgument[GuiPropArg]( + "groups", + "Get a list of global groups this user belongs to." + ) + with GuiPropArg + +object Rights + extends EnumArgument[GuiPropArg]( + "rights", + "Get a list of global rights this user has." + ) + with GuiPropArg + +object Merged + extends EnumArgument[GuiPropArg]("merged", "Get a list of merged accounts.") + with GuiPropArg + +object Unattached + extends EnumArgument[GuiPropArg]( + "unattached", + "Get a list of unattached accounts." + ) + with GuiPropArg + +object EditCount + extends EnumArgument[GuiPropArg]( + "editcount", + "Get the user's global editcount." + ) + with GuiPropArg diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/meta/MetaParam.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/meta/MetaParam.scala index f0c192de..e43f4cc2 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/meta/MetaParam.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/meta/MetaParam.scala @@ -3,5 +3,7 @@ package org.scalawiki.dto.cmd.query.meta import org.scalawiki.dto.cmd.{EnumArg, EnumParameter} import org.scalawiki.dto.cmd.query.QueryParam -case class MetaParam(override val args: MetaArg*) extends EnumParameter[MetaArg]("meta", "") with QueryParam[MetaArg] +case class MetaParam(override val args: MetaArg*) + extends EnumParameter[MetaArg]("meta", "") + with QueryParam[MetaArg] trait MetaArg extends EnumArg[MetaArg] { val param = MetaParam } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Categories.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Categories.scala index 628758ab..2537d95d 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Categories.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Categories.scala @@ -3,18 +3,21 @@ package org.scalawiki.dto.cmd.query.prop import org.scalawiki.dto.cmd._ import org.scalawiki.dto.cmd.query.Module -/** - * ?action=query&prop=categories - * - */ +/** ?action=query&prop=categories + */ case class Categories(override val params: CategoriesParam[Any]*) - extends Module[PropArg]("cl", "categories", "Gets a list of all categories used on the provided pages.") - with PropArg with ArgWithParams[CategoriesParam[Any], PropArg] + extends Module[PropArg]( + "cl", + "categories", + "Gets a list of all categories used on the provided pages." + ) + with PropArg + with ArgWithParams[CategoriesParam[Any], PropArg] -/** - * Marker trait for parameters used with prop=langlinks - */ +/** Marker trait for parameters used with prop=langlinks + */ trait CategoriesParam[+T] extends Parameter[T] -case class ClLimit(override val arg: String) extends StringParameter("cllimit", - "How many categories to return") with CategoriesParam[String] \ No newline at end of file +case class ClLimit(override val arg: String) + extends StringParameter("cllimit", "How many categories to return") + with CategoriesParam[String] diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/CategoryInfo.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/CategoryInfo.scala index 4d3edfb3..7b8ce123 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/CategoryInfo.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/CategoryInfo.scala @@ -2,12 +2,12 @@ package org.scalawiki.dto.cmd.query.prop import org.scalawiki.dto.cmd.query.Module -/** - * ?action=query&prop=categoryinfo - * - */ +/** ?action=query&prop=categoryinfo + */ object CategoryInfo - extends Module[PropArg]("ci", "categoryinfo", "Gets information about categories.") - with PropArg - - + extends Module[PropArg]( + "ci", + "categoryinfo", + "Gets information about categories." + ) + with PropArg diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/ImageInfo.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/ImageInfo.scala index 5f98ebb2..8a5fe179 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/ImageInfo.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/ImageInfo.scala @@ -5,128 +5,252 @@ import java.time.ZonedDateTime import org.scalawiki.dto.cmd._ import org.scalawiki.dto.cmd.query.Module -/** - * ?action=query&prop=imageinfo - * See more at https://www.mediawiki.org/wiki/API:Imageinfo - * - */ +/** ?action=query&prop=imageinfo See more at + * https://www.mediawiki.org/wiki/API:Imageinfo + */ case class ImageInfo(override val params: IiParam*) - extends Module[PropArg]("ii", "imageinfo", "Gets image information.") with PropArg with ArgWithParams[IiParam, PropArg] + extends Module[PropArg]("ii", "imageinfo", "Gets image information.") + with PropArg + with ArgWithParams[IiParam, PropArg] -/** - * Marker trait for parameters used with prop=imageinfo - */ +/** Marker trait for parameters used with prop=imageinfo + */ trait IiParam extends Parameter[Any] - package iiprop { -/** - * ?action=query&prop=imageinfo&iiprop= - * - */ -case class IiProp(override val args: IiPropArg*) extends EnumParameter[IiPropArg]("iiprop", "Which properties to get.") with IiParam - -/** - * Trait for iiprop= arguments - * - */ - -trait IiPropArg extends EnumArg[IiPropArg] { - val param = IiProp -} - -/** - * iiprop= arguments - * - */ - -object Timestamp extends EnumArgument[IiPropArg]("timestamp", "The time and date of the revision (default)") with IiPropArg - -object User extends EnumArgument[IiPropArg]("user", "The user who made the revision (default)") with IiPropArg - -object UserId extends EnumArgument[IiPropArg]("userid", "User ID of the revision creator.") with IiPropArg - -object Comment extends EnumArgument[IiPropArg]("comment", "Comment by the user for the revision.") with IiPropArg - -object ParsedComment extends EnumArgument[IiPropArg]("parsedcomment", "Parsed comment by the user for the revision.") with IiPropArg - -object CanonicalTitle extends EnumArgument[IiPropArg]("canonicaltitle", "The canonical title of the image file.") with IiPropArg - -object Url extends EnumArgument[IiPropArg]("url", "URLs of the image and its description page.") with IiPropArg - -object Size extends EnumArgument[IiPropArg]("size", - "The image's size in bytes, plus width and height. " + - "A page count is also returned if the image is in a format that supports multiple pages.") with IiPropArg - -object Dimensions extends EnumArgument[IiPropArg]("dimensions", "dimensions: (Alias for size)") with IiPropArg - -object Sha1 extends EnumArgument[IiPropArg]("sha1", "The image's SHA-1 hash.") with IiPropArg - -object Mime extends EnumArgument[IiPropArg]("mime", "The image's MIME type.") with IiPropArg - -object ThumbMime extends EnumArgument[IiPropArg]("thumbmime", "The image thumbnail's MIME type.") with IiPropArg - -object MediaType extends EnumArgument[IiPropArg]("mediatype", "The media type of the image.") with IiPropArg - -object Metadata extends EnumArgument[IiPropArg]("metadata", "Exif metadata for the image, if available.") with IiPropArg - -object CommonMetadata extends EnumArgument[IiPropArg]("commonmetadata", "Generic metadata for the file format, if available.") with IiPropArg + /** ?action=query&prop=imageinfo&iiprop= + */ + case class IiProp(override val args: IiPropArg*) + extends EnumParameter[IiPropArg]("iiprop", "Which properties to get.") + with IiParam -object ExtMetadata extends EnumArgument[IiPropArg]("extmetadata", "HTML metadata from extensions which implement the GetExtendedMetadata hook. " + - "Also contains most of metadata, but in a somewhat standardized format.") with IiPropArg + /** Trait for iiprop= arguments + */ -object ArchiveName extends EnumArgument[IiPropArg]("archivename", "Archive name (old images only).") with IiPropArg - -object BitDepth extends EnumArgument[IiPropArg]("bitdepth", "The bit depth of the image.") with IiPropArg - - -object IiPropArgs { - val args = Seq(Timestamp, User, UserId, Comment, ParsedComment, CanonicalTitle, Url, Size, Dimensions, Sha1, - ThumbMime, MediaType, Metadata, CommonMetadata, ExtMetadata, ArchiveName, BitDepth) - val argsByName: Map[String, IiPropArg] = args.groupBy(_.name).mapValues(_.head).toMap - - def byNames(names: Seq[String]): Seq[IiPropArg] = { - names.flatMap(argsByName.get) + trait IiPropArg extends EnumArg[IiPropArg] { + val param = IiProp } -} + /** iiprop= arguments + */ + + object Timestamp + extends EnumArgument[IiPropArg]( + "timestamp", + "The time and date of the revision (default)" + ) + with IiPropArg + + object User + extends EnumArgument[IiPropArg]( + "user", + "The user who made the revision (default)" + ) + with IiPropArg + + object UserId + extends EnumArgument[IiPropArg]( + "userid", + "User ID of the revision creator." + ) + with IiPropArg + + object Comment + extends EnumArgument[IiPropArg]( + "comment", + "Comment by the user for the revision." + ) + with IiPropArg + + object ParsedComment + extends EnumArgument[IiPropArg]( + "parsedcomment", + "Parsed comment by the user for the revision." + ) + with IiPropArg + + object CanonicalTitle + extends EnumArgument[IiPropArg]( + "canonicaltitle", + "The canonical title of the image file." + ) + with IiPropArg + + object Url + extends EnumArgument[IiPropArg]( + "url", + "URLs of the image and its description page." + ) + with IiPropArg + + object Size + extends EnumArgument[IiPropArg]( + "size", + "The image's size in bytes, plus width and height. " + + "A page count is also returned if the image is in a format that supports multiple pages." + ) + with IiPropArg + + object Dimensions + extends EnumArgument[IiPropArg]( + "dimensions", + "dimensions: (Alias for size)" + ) + with IiPropArg + + object Sha1 + extends EnumArgument[IiPropArg]("sha1", "The image's SHA-1 hash.") + with IiPropArg + + object Mime + extends EnumArgument[IiPropArg]("mime", "The image's MIME type.") + with IiPropArg + + object ThumbMime + extends EnumArgument[IiPropArg]( + "thumbmime", + "The image thumbnail's MIME type." + ) + with IiPropArg + + object MediaType + extends EnumArgument[IiPropArg]( + "mediatype", + "The media type of the image." + ) + with IiPropArg + + object Metadata + extends EnumArgument[IiPropArg]( + "metadata", + "Exif metadata for the image, if available." + ) + with IiPropArg + + object CommonMetadata + extends EnumArgument[IiPropArg]( + "commonmetadata", + "Generic metadata for the file format, if available." + ) + with IiPropArg + + object ExtMetadata + extends EnumArgument[IiPropArg]( + "extmetadata", + "HTML metadata from extensions which implement the GetExtendedMetadata hook. " + + "Also contains most of metadata, but in a somewhat standardized format." + ) + with IiPropArg + + object ArchiveName + extends EnumArgument[IiPropArg]( + "archivename", + "Archive name (old images only)." + ) + with IiPropArg + + object BitDepth + extends EnumArgument[IiPropArg]("bitdepth", "The bit depth of the image.") + with IiPropArg + + object IiPropArgs { + val args = Seq( + Timestamp, + User, + UserId, + Comment, + ParsedComment, + CanonicalTitle, + Url, + Size, + Dimensions, + Sha1, + ThumbMime, + MediaType, + Metadata, + CommonMetadata, + ExtMetadata, + ArchiveName, + BitDepth + ) + val argsByName: Map[String, IiPropArg] = + args.groupBy(_.name).mapValues(_.head).toMap + + def byNames(names: Seq[String]): Seq[IiPropArg] = { + names.flatMap(argsByName.get) + } + } } -case class IiLimit(override val arg: String) extends StringParameter("iilimit", "How many image revisions to return (1 by default)") with IiParam - -case class IiStart(override val arg: ZonedDateTime) extends DateTimeParameter("iistart", "Timestamp to start listing from") with IiParam - -case class IiEnd(override val arg: ZonedDateTime) extends DateTimeParameter("iistart", "Timestamp to stop listing at") with IiParam - -case class IiUrlWidth(override val arg: Int) extends IntParameter("iiurlwidth", - "If iiprop=url is set, a URL to an image scaled to this width will be returned as well in thumburl along with thumbwidth and thumbheight. " + - "Old versions of images can't be scaled") with IiParam - -case class IiUrlHeight(override val arg: Int) extends IntParameter("iiurlheight", "Similar to iiurlwidth") with IiParam - -case class IiMetadataVersion(override val arg: String) extends StringParameter("iimetadataversion", - "What version of metadata to use. Only affects JPEGs (as of this writing). You usually want this set to latest.") with IiParam - -case class IiExtMetadataLanguage(override val arg: String) extends StringParameter("iiextmetadatalanguage", - "What language to fetch extmetadata in. This affects both which translation to fetch, if multiple are available, " + - "as well as how things like numbers and various values are formatted.") with IiParam - -case class IiExtMetadataMultilang(override val arg: Boolean = true) extends BooleanParameter("iiextmetadatamultilang", - "If translations for extmetadata property are available, fetch all of them. " + - "The values of the multi-languaged metadata items will be in the multi-language array format. " + - "(Which items are multilanguaged might change from image to image.)") with IiParam - -case class IiExtMetadataFilter(override val arg: Boolean = true) extends BooleanParameter("iiextmetadatafilter", - "If specified and non-empty, only these keys will be returned for prop=extmetadata") with IiParam - -case class IiUrlParam(override val arg: String) extends StringParameter("iiurlparam", - "The thumb parameter string. Allows user to optionally specify other parameters than width and height (like page number for pdfs). " + - "Format of the field varies with image format. PDF uses a format like page-px (e.g. page3-140px ). " + - "Its generally the part before the filename in the url of a thumbnail.") with IiParam - -case class IiLocalOnly(override val arg: Boolean = true) extends BooleanParameter("iilocalonly", "Only show local images.") with IiParam - - - +case class IiLimit(override val arg: String) + extends StringParameter( + "iilimit", + "How many image revisions to return (1 by default)" + ) + with IiParam + +case class IiStart(override val arg: ZonedDateTime) + extends DateTimeParameter("iistart", "Timestamp to start listing from") + with IiParam + +case class IiEnd(override val arg: ZonedDateTime) + extends DateTimeParameter("iistart", "Timestamp to stop listing at") + with IiParam + +case class IiUrlWidth(override val arg: Int) + extends IntParameter( + "iiurlwidth", + "If iiprop=url is set, a URL to an image scaled to this width will be returned as well in thumburl along with thumbwidth and thumbheight. " + + "Old versions of images can't be scaled" + ) + with IiParam + +case class IiUrlHeight(override val arg: Int) + extends IntParameter("iiurlheight", "Similar to iiurlwidth") + with IiParam + +case class IiMetadataVersion(override val arg: String) + extends StringParameter( + "iimetadataversion", + "What version of metadata to use. Only affects JPEGs (as of this writing). You usually want this set to latest." + ) + with IiParam + +case class IiExtMetadataLanguage(override val arg: String) + extends StringParameter( + "iiextmetadatalanguage", + "What language to fetch extmetadata in. This affects both which translation to fetch, if multiple are available, " + + "as well as how things like numbers and various values are formatted." + ) + with IiParam + +case class IiExtMetadataMultilang(override val arg: Boolean = true) + extends BooleanParameter( + "iiextmetadatamultilang", + "If translations for extmetadata property are available, fetch all of them. " + + "The values of the multi-languaged metadata items will be in the multi-language array format. " + + "(Which items are multilanguaged might change from image to image.)" + ) + with IiParam + +case class IiExtMetadataFilter(override val arg: Boolean = true) + extends BooleanParameter( + "iiextmetadatafilter", + "If specified and non-empty, only these keys will be returned for prop=extmetadata" + ) + with IiParam + +case class IiUrlParam(override val arg: String) + extends StringParameter( + "iiurlparam", + "The thumb parameter string. Allows user to optionally specify other parameters than width and height (like page number for pdfs). " + + "Format of the field varies with image format. PDF uses a format like page-px (e.g. page3-140px ). " + + "Its generally the part before the filename in the url of a thumbnail." + ) + with IiParam + +case class IiLocalOnly(override val arg: Boolean = true) + extends BooleanParameter("iilocalonly", "Only show local images.") + with IiParam diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Images.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Images.scala index 799863f3..90e6e265 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Images.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Images.scala @@ -3,22 +3,24 @@ package org.scalawiki.dto.cmd.query.prop import org.scalawiki.dto.cmd._ import org.scalawiki.dto.cmd.query.Module -/** - * ?action=query&prop=images - * - */ +/** ?action=query&prop=images + */ case class Images(override val params: ImagesParam[Any]*) - extends Module[PropArg]("im", "images", "Gets a list of all images used on the provided pages.") - with PropArg with ArgWithParams[ImagesParam[Any], PropArg] + extends Module[PropArg]( + "im", + "images", + "Gets a list of all images used on the provided pages." + ) + with PropArg + with ArgWithParams[ImagesParam[Any], PropArg] -/** - * Marker trait for parameters used with prop=langlinks - */ +/** Marker trait for parameters used with prop=langlinks + */ trait ImagesParam[+T] extends Parameter[T] -case class ImLimit(override val arg: String) extends StringParameter("imlimit", - "How many images to return") with ImagesParam[String] - +case class ImLimit(override val arg: String) + extends StringParameter("imlimit", "How many images to return") + with ImagesParam[String] ///** // * ?action=query&prop=info&llprop= @@ -53,4 +55,4 @@ case class ImLimit(override val arg: String) extends StringParameter("imlimit", //lllang: Language code MW 1.18+ //lltitle: Link to search for. Must be used with lllang MW 1.18+ //lldir: The direction in which to list MW 1.19+ -//llinlanguagecode: Language code for localised language names MW 1.23+ \ No newline at end of file +//llinlanguagecode: Language code for localised language names MW 1.23+ diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Info.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Info.scala index 11e4a538..74778f9b 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Info.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Info.scala @@ -3,44 +3,90 @@ package org.scalawiki.dto.cmd.query.prop import org.scalawiki.dto.cmd._ import org.scalawiki.dto.cmd.query.Module -/** - * ?action=query&prop=info - * - */ -case class Info(override val params:InfoParam*) extends Module[PropArg]("in", "info", "Get basic page information.") with PropArg with ArgWithParams[InfoParam, PropArg] +/** ?action=query&prop=info + */ +case class Info(override val params: InfoParam*) + extends Module[PropArg]("in", "info", "Get basic page information.") + with PropArg + with ArgWithParams[InfoParam, PropArg] - -/** - * Marker trait for parameters used with prop=info - */ +/** Marker trait for parameters used with prop=info + */ trait InfoParam extends Parameter[AnyRef] +/** ?action=query&prop=info&inprop= + */ +case class InProp(override val args: InPropArg*) + extends EnumParameter[InPropArg]( + "inprop", + "Which additional properties to get:" + ) + with InfoParam -/** - * ?action=query&prop=info&inprop= - * - */ -case class InProp(override val args: InPropArg*) extends EnumParameter[InPropArg]("inprop", "Which additional properties to get:") with InfoParam - - -/** - * Trait for inprop= arguments - * - */ +/** Trait for inprop= arguments + */ trait InPropArg extends EnumArg[InPropArg] { val param = InProp } -/** - * inprop= arguments - * - */ -object Protection extends EnumArgument[InPropArg]("protection", "List the protection level of each page.") with InPropArg -object TalkId extends EnumArgument[InPropArg]("talkid", "The page ID of the talk page for each non-talk page.") with InPropArg -object Watched extends EnumArgument[InPropArg]("watched", "List the watched status of each page.") with InPropArg -object Watchers extends EnumArgument[InPropArg]("watchers", "The page ID of the talk page for each non-talk page.") with InPropArg -object NotificationTimestamp extends EnumArgument[InPropArg]("notificationtimestamp", "The watchlist notification timestamp of each page.") with InPropArg -object SubjectId extends EnumArgument[InPropArg]("subjectid", "The page ID of the parent page for each talk page.") with InPropArg -object Url extends EnumArgument[InPropArg]("url", "Gives a full URL, an edit URL, and the canonical URL for each page.") with InPropArg -object Readable extends EnumArgument[InPropArg]("readable", "Whether the user can read this page.") with InPropArg -object Preload extends EnumArgument[InPropArg]("preload", "Gives the text returned by EditFormPreloadText.") with InPropArg -object DisplayTitle extends EnumArgument[InPropArg]("displaytitle", "Gives the way the page title is actually displayed.") with InPropArg +/** inprop= arguments + */ +object Protection + extends EnumArgument[InPropArg]( + "protection", + "List the protection level of each page." + ) + with InPropArg +object TalkId + extends EnumArgument[InPropArg]( + "talkid", + "The page ID of the talk page for each non-talk page." + ) + with InPropArg +object Watched + extends EnumArgument[InPropArg]( + "watched", + "List the watched status of each page." + ) + with InPropArg +object Watchers + extends EnumArgument[InPropArg]( + "watchers", + "The page ID of the talk page for each non-talk page." + ) + with InPropArg +object NotificationTimestamp + extends EnumArgument[InPropArg]( + "notificationtimestamp", + "The watchlist notification timestamp of each page." + ) + with InPropArg +object SubjectId + extends EnumArgument[InPropArg]( + "subjectid", + "The page ID of the parent page for each talk page." + ) + with InPropArg +object Url + extends EnumArgument[InPropArg]( + "url", + "Gives a full URL, an edit URL, and the canonical URL for each page." + ) + with InPropArg +object Readable + extends EnumArgument[InPropArg]( + "readable", + "Whether the user can read this page." + ) + with InPropArg +object Preload + extends EnumArgument[InPropArg]( + "preload", + "Gives the text returned by EditFormPreloadText." + ) + with InPropArg +object DisplayTitle + extends EnumArgument[InPropArg]( + "displaytitle", + "Gives the way the page title is actually displayed." + ) + with InPropArg diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/LangLinks.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/LangLinks.scala index f5acc790..e932c53d 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/LangLinks.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/LangLinks.scala @@ -3,24 +3,27 @@ package org.scalawiki.dto.cmd.query.prop import org.scalawiki.dto.cmd._ import org.scalawiki.dto.cmd.query.Module -/** - * ?action=query&prop=langlinks - * - */ -case class LangLinks(override val params:LangLinksParam[Any]*) - extends Module[PropArg]("ll", "langlinks", "Gets a list of all language links from the provided pages to other languages.") - with PropArg with ArgWithParams[LangLinksParam[Any], PropArg] - - -/** - * Marker trait for parameters used with prop=langlinks - */ +/** ?action=query&prop=langlinks + */ +case class LangLinks(override val params: LangLinksParam[Any]*) + extends Module[PropArg]( + "ll", + "langlinks", + "Gets a list of all language links from the provided pages to other languages." + ) + with PropArg + with ArgWithParams[LangLinksParam[Any], PropArg] + +/** Marker trait for parameters used with prop=langlinks + */ trait LangLinksParam[+T] extends Parameter[T] -case class LlLimit(override val arg: String) extends StringParameter("lllimit", - " How many langlinks to return. Default: 10. No more than 500 (5000 for bots) allowed.") with LangLinksParam[String] - - +case class LlLimit(override val arg: String) + extends StringParameter( + "lllimit", + " How many langlinks to return. Default: 10. No more than 500 (5000 for bots) allowed." + ) + with LangLinksParam[String] ///** // * ?action=query&prop=info&llprop= @@ -55,4 +58,4 @@ case class LlLimit(override val arg: String) extends StringParameter("lllimit", //lllang: Language code MW 1.18+ //lltitle: Link to search for. Must be used with lllang MW 1.18+ //lldir: The direction in which to list MW 1.19+ -//llinlanguagecode: Language code for localised language names MW 1.23+ \ No newline at end of file +//llinlanguagecode: Language code for localised language names MW 1.23+ diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Links.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Links.scala index 6ff0c6ef..515934c6 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Links.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Links.scala @@ -3,27 +3,31 @@ package org.scalawiki.dto.cmd.query.prop import org.scalawiki.dto.cmd._ import org.scalawiki.dto.cmd.query.Module -/** - * ?action=query&prop=links - * - */ -case class Links(override val params:LinksParam[Any]*) - extends Module[PropArg]("pl", "links", "Gets a list of all links on the provided pages.") - with PropArg with ArgWithParams[LinksParam[Any], PropArg] - - -/** - * Marker trait for parameters used with prop=links - */ +/** ?action=query&prop=links + */ +case class Links(override val params: LinksParam[Any]*) + extends Module[PropArg]( + "pl", + "links", + "Gets a list of all links on the provided pages." + ) + with PropArg + with ArgWithParams[LinksParam[Any], PropArg] + +/** Marker trait for parameters used with prop=links + */ trait LinksParam[+T] extends Parameter[T] -case class PlLimit(override val arg: String) extends StringParameter("pllimit", - "How many links to return. Default: 10. No more than 500 (5000 for bots) allowed.") with LinksParam[String] - -case class PlNamespace(override val args: Seq[Int]) extends IntListParameter("plnamespace", - "Only include pages in these namespaces.") with LinksParam[Int] - - - - - +case class PlLimit(override val arg: String) + extends StringParameter( + "pllimit", + "How many links to return. Default: 10. No more than 500 (5000 for bots) allowed." + ) + with LinksParam[String] + +case class PlNamespace(override val args: Seq[Int]) + extends IntListParameter( + "plnamespace", + "Only include pages in these namespaces." + ) + with LinksParam[Int] diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Prop.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Prop.scala index 59a6c2f9..1ee75597 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Prop.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Prop.scala @@ -3,14 +3,13 @@ package org.scalawiki.dto.cmd.query.prop import org.scalawiki.dto.cmd.query.{GeneratorArg, QueryParam} import org.scalawiki.dto.cmd.{EnumArg, EnumParameter} -/** - * ?action=query&prop= argument - * - */ +/** ?action=query&prop= argument + */ trait PropArg extends EnumArg[PropArg] with GeneratorArg { val param = Prop } -/** - * ?action=query&prop= parameter - */ -case class Prop(override val args: PropArg*) extends EnumParameter[PropArg]("prop", "") with QueryParam[PropArg] +/** ?action=query&prop= parameter + */ +case class Prop(override val args: PropArg*) + extends EnumParameter[PropArg]("prop", "") + with QueryParam[PropArg] diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Revisions.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Revisions.scala index 06b124c8..a03b6842 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Revisions.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/Revisions.scala @@ -6,133 +6,257 @@ import org.scalawiki.dto.cmd._ import org.scalawiki.dto.cmd.query.Module import org.scalawiki.dto.cmd.query.prop.rvprop.RvProp -/** - * ?action=query&prop=revisions - * See more at https://www.mediawiki.org/wiki/API:Revisions - * - */ +/** ?action=query&prop=revisions See more at + * https://www.mediawiki.org/wiki/API:Revisions + */ case class Revisions(override val params: RvParam*) - extends Module[PropArg]("rv", "revisions", "Get revision information.") - with PropArg - with ArgWithParams[RvParam, PropArg] { + extends Module[PropArg]("rv", "revisions", "Get revision information.") + with PropArg + with ArgWithParams[RvParam, PropArg] { def withoutContent = { val filtered: Seq[RvParam] = params.map { case p: RvProp => p.withoutContent - case other => other + case other => other } - Revisions(filtered:_*) + Revisions(filtered: _*) } def hasContent: Boolean = prop.exists(_.hasContent) - def prop = byPF({case p: RvProp => p }).headOption + def prop = byPF({ case p: RvProp => p }).headOption } -/** - * Marker trait for parameters used with prop=revisions - */ +/** Marker trait for parameters used with prop=revisions + */ trait RvParam extends Parameter[Any] package rvprop { -/** - * ?action=query&prop=revisions&rvprop= - * - */ -case class RvProp(override val args: RvPropArg*) - extends EnumParameter[RvPropArg]("rvprop", "Which properties to get for each revision:") with RvParam { + /** ?action=query&prop=revisions&rvprop= + */ + case class RvProp(override val args: RvPropArg*) + extends EnumParameter[RvPropArg]( + "rvprop", + "Which properties to get for each revision:" + ) + with RvParam { - def hasContent: Boolean = args.contains(Content) + def hasContent: Boolean = args.contains(Content) - def withoutContent: RvProp = RvProp(args.filter(_ != Content):_*) -} - -/** - * Trait for rvprop= arguments - * - */ - -trait RvPropArg extends EnumArg[RvPropArg] { - val param = RvProp -} - -/** - * rvprop= arguments - * - */ -object Ids extends EnumArgument[RvPropArg]("ids", "The ID of the revision.") with RvPropArg - -object Flags extends EnumArgument[RvPropArg]("flags", "Revision flags (minor).") with RvPropArg - -object Timestamp extends EnumArgument[RvPropArg]("timestamp", "The timestamp of the revision.") with RvPropArg - -object User extends EnumArgument[RvPropArg]("user", "User that made the revision.") with RvPropArg - -object UserId extends EnumArgument[RvPropArg]("userid", "User ID of the revision creator.") with RvPropArg - -object Size extends EnumArgument[RvPropArg]("size", "Length (bytes) of the revision.") with RvPropArg - -object Sha1 extends EnumArgument[RvPropArg]("sha1", "SHA-1 (base 16) of the revision.") with RvPropArg - -object ContentModel extends EnumArgument[RvPropArg]("contentmodel", "Content model ID of the revision.") with RvPropArg - -object Comment extends EnumArgument[RvPropArg]("comment", "Comment by the user for the revision.") with RvPropArg + def withoutContent: RvProp = RvProp(args.filter(_ != Content): _*) + } -object ParsedComment extends EnumArgument[RvPropArg]("parsedcomment", "Parsed comment by the user for the revision.") with RvPropArg + /** Trait for rvprop= arguments + */ -object Content extends EnumArgument[RvPropArg]("content", "Text of the revision.") with RvPropArg + trait RvPropArg extends EnumArg[RvPropArg] { + val param = RvProp + } -object Tags extends EnumArgument[RvPropArg]("tags", "Tags for the revision.") with RvPropArg + /** rvprop= arguments + */ + object Ids + extends EnumArgument[RvPropArg]("ids", "The ID of the revision.") + with RvPropArg + + object Flags + extends EnumArgument[RvPropArg]("flags", "Revision flags (minor).") + with RvPropArg + + object Timestamp + extends EnumArgument[RvPropArg]( + "timestamp", + "The timestamp of the revision." + ) + with RvPropArg + + object User + extends EnumArgument[RvPropArg]("user", "User that made the revision.") + with RvPropArg + + object UserId + extends EnumArgument[RvPropArg]( + "userid", + "User ID of the revision creator." + ) + with RvPropArg + + object Size + extends EnumArgument[RvPropArg]("size", "Length (bytes) of the revision.") + with RvPropArg + + object Sha1 + extends EnumArgument[RvPropArg]( + "sha1", + "SHA-1 (base 16) of the revision." + ) + with RvPropArg + + object ContentModel + extends EnumArgument[RvPropArg]( + "contentmodel", + "Content model ID of the revision." + ) + with RvPropArg + + object Comment + extends EnumArgument[RvPropArg]( + "comment", + "Comment by the user for the revision." + ) + with RvPropArg + + object ParsedComment + extends EnumArgument[RvPropArg]( + "parsedcomment", + "Parsed comment by the user for the revision." + ) + with RvPropArg + + object Content + extends EnumArgument[RvPropArg]("content", "Text of the revision.") + with RvPropArg + + object Tags + extends EnumArgument[RvPropArg]("tags", "Tags for the revision.") + with RvPropArg } -case class RvLimit(override val arg: String) extends StringParameter("rvlimit", "The maximum number of revisions to return.") with RvParam - -case class RvStartId(override val arg: Long) extends LongParameter("rvstartid", "Revision ID to start listing from.") with RvParam -case class RvEndId(override val arg: Long) extends LongParameter("rvendid", "Revision ID to stop listing at.") with RvParam - -case class RvStart(override val arg: ZonedDateTime) extends DateTimeParameter("rvstart", "Timestamp to start listing from.") with RvParam -case class RvEnd(override val arg: ZonedDateTime) extends DateTimeParameter("rvend", "Timestamp to end listing at.") with RvParam - -case class RvDir(override val args: RvDirArg*) extends EnumParameter[RvDirArg]("rvdir", "Which properties to get for each revision:") with RvParam +case class RvLimit(override val arg: String) + extends StringParameter( + "rvlimit", + "The maximum number of revisions to return." + ) + with RvParam + +case class RvStartId(override val arg: Long) + extends LongParameter("rvstartid", "Revision ID to start listing from.") + with RvParam +case class RvEndId(override val arg: Long) + extends LongParameter("rvendid", "Revision ID to stop listing at.") + with RvParam + +case class RvStart(override val arg: ZonedDateTime) + extends DateTimeParameter("rvstart", "Timestamp to start listing from.") + with RvParam +case class RvEnd(override val arg: ZonedDateTime) + extends DateTimeParameter("rvend", "Timestamp to end listing at.") + with RvParam + +case class RvDir(override val args: RvDirArg*) + extends EnumParameter[RvDirArg]( + "rvdir", + "Which properties to get for each revision:" + ) + with RvParam trait RvDirArg extends EnumArg[RvDirArg] { val param = RvDir } -object Older extends EnumArgument[RvDirArg]("older", "List newest revisions first.") with RvDirArg -object Newer extends EnumArgument[RvDirArg]("newer", "List oldest revisions first.") with RvDirArg - -case class RvUser(override val arg: String) extends StringParameter("rvuser", "Only list revisions made by this user.") with RvParam -case class RvExcludeUser(override val arg: String) extends StringParameter("rvexcludeuser", "Do not list revisions made by this user.") with RvParam - -case class RvExpandTemplates(override val arg: Boolean = true) extends BooleanParameter("rvexpandtemplates", "Expand templates in rvprop=content output.") with RvParam -case class RvGenerateXml(override val arg: Boolean = true) extends BooleanParameter("rvgeneratexml", "Generate XML parse tree for revision content.") with RvParam -case class RvParse(override val arg: Boolean = true) extends BooleanParameter("rvparse", "Parse revision content.") with RvParam -case class RvSection(override val arg: Int) extends IntParameter("rvsection", "If rvprop=content is set, only retrieve the contents of this section.") with RvParam -case class RvSlots(override val arg: String) extends StringParameter("rvslots", "rvslots") with RvParam +object Older + extends EnumArgument[RvDirArg]("older", "List newest revisions first.") + with RvDirArg +object Newer + extends EnumArgument[RvDirArg]("newer", "List oldest revisions first.") + with RvDirArg + +case class RvUser(override val arg: String) + extends StringParameter("rvuser", "Only list revisions made by this user.") + with RvParam +case class RvExcludeUser(override val arg: String) + extends StringParameter( + "rvexcludeuser", + "Do not list revisions made by this user." + ) + with RvParam + +case class RvExpandTemplates(override val arg: Boolean = true) + extends BooleanParameter( + "rvexpandtemplates", + "Expand templates in rvprop=content output." + ) + with RvParam +case class RvGenerateXml(override val arg: Boolean = true) + extends BooleanParameter( + "rvgeneratexml", + "Generate XML parse tree for revision content." + ) + with RvParam +case class RvParse(override val arg: Boolean = true) + extends BooleanParameter("rvparse", "Parse revision content.") + with RvParam +case class RvSection(override val arg: Int) + extends IntParameter( + "rvsection", + "If rvprop=content is set, only retrieve the contents of this section." + ) + with RvParam +case class RvSlots(override val arg: String) + extends StringParameter("rvslots", "rvslots") + with RvParam // TODO single Enum value arg -case class RvDiffTo(override val args: RvDiffToArg*) extends EnumParameter[RvDiffToArg]("rvdiffto", "Revision ID to diff each revision to.") with RvParam +case class RvDiffTo(override val args: RvDiffToArg*) + extends EnumParameter[RvDiffToArg]( + "rvdiffto", + "Revision ID to diff each revision to." + ) + with RvParam trait RvDiffToArg extends EnumArg[RvDiffToArg] { val param = RvDiffTo } -object Prev extends EnumArgument[RvDiffToArg]("prev", "previous revision.") with RvDiffToArg -object Next extends EnumArgument[RvDiffToArg]("next", "next revision.") with RvDiffToArg -object Cur extends EnumArgument[RvDiffToArg]("cur", "current revision.") with RvDiffToArg - -case class RvDiffToText(override val arg: String) extends StringParameter("rvdifftotext", "Text to diff each revision to.") with RvParam -case class RvTag(override val arg: String) extends StringParameter("rvtag", "Only list revisions tagged with this tag.") with RvParam +object Prev + extends EnumArgument[RvDiffToArg]("prev", "previous revision.") + with RvDiffToArg +object Next + extends EnumArgument[RvDiffToArg]("next", "next revision.") + with RvDiffToArg +object Cur + extends EnumArgument[RvDiffToArg]("cur", "current revision.") + with RvDiffToArg + +case class RvDiffToText(override val arg: String) + extends StringParameter("rvdifftotext", "Text to diff each revision to.") + with RvParam +case class RvTag(override val arg: String) + extends StringParameter( + "rvtag", + "Only list revisions tagged with this tag." + ) + with RvParam // TODO single Enum value arg case class RvContentFormat(override val args: RvContentFormatArg*) - extends EnumParameter[RvContentFormatArg]("rvcontentformat", "Serialization format used for difftotext and expected for output of content.") with RvParam - -trait RvContentFormatArg extends EnumArg[RvContentFormatArg] { val param = RvContentFormat } - -object TextXWiki extends EnumArgument[RvContentFormatArg]("text/x-wiki", "text/x-wiki") with RvContentFormatArg -object TextJavaScript extends EnumArgument[RvContentFormatArg]("text/javascript", "text/javascript") with RvContentFormatArg -object ApplicationJson extends EnumArgument[RvContentFormatArg]("application/json", "application/json") with RvContentFormatArg -object TextCss extends EnumArgument[RvContentFormatArg]("text/css", "text/css") with RvContentFormatArg -object TextPlain extends EnumArgument[RvContentFormatArg]("text/plain", "text/plain") with RvContentFormatArg + extends EnumParameter[RvContentFormatArg]( + "rvcontentformat", + "Serialization format used for difftotext and expected for output of content." + ) + with RvParam + +trait RvContentFormatArg extends EnumArg[RvContentFormatArg] { + val param = RvContentFormat +} +object TextXWiki + extends EnumArgument[RvContentFormatArg]("text/x-wiki", "text/x-wiki") + with RvContentFormatArg +object TextJavaScript + extends EnumArgument[RvContentFormatArg]( + "text/javascript", + "text/javascript" + ) + with RvContentFormatArg +object ApplicationJson + extends EnumArgument[RvContentFormatArg]( + "application/json", + "application/json" + ) + with RvContentFormatArg +object TextCss + extends EnumArgument[RvContentFormatArg]("text/css", "text/css") + with RvContentFormatArg +object TextPlain + extends EnumArgument[RvContentFormatArg]("text/plain", "text/plain") + with RvContentFormatArg diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/RvPropArgs.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/RvPropArgs.scala index 357caea1..d4866b63 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/RvPropArgs.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/query/prop/RvPropArgs.scala @@ -3,8 +3,22 @@ package org.scalawiki.dto.cmd.query.prop import org.scalawiki.dto.cmd.query.prop.rvprop._ object RvPropArgs { - val args = Seq(Ids, Flags, Timestamp, User, UserId, Size, Sha1, ContentModel, Comment, ParsedComment, Content, Tags) - val argsByName: Map[String, RvPropArg] = args.groupBy(_.name).mapValues(_.head).toMap + val args = Seq( + Ids, + Flags, + Timestamp, + User, + UserId, + Size, + Sha1, + ContentModel, + Comment, + ParsedComment, + Content, + Tags + ) + val argsByName: Map[String, RvPropArg] = + args.groupBy(_.name).mapValues(_.head).toMap def byNames(names: Seq[String]): Seq[RvPropArg] = { names.flatMap(argsByName.get) diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/upload/Upload.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/upload/Upload.scala index 4940a06d..c3fee081 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/upload/Upload.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/cmd/upload/Upload.scala @@ -2,11 +2,10 @@ package org.scalawiki.dto.cmd.upload import org.scalawiki.dto.cmd._ - case class Upload(override val params: UploadParam[Any]*) - extends EnumArgument[ActionArg]("upload", "upload files.") - with ActionArg - with ArgWithParams[UploadParam[Any], ActionArg] { + extends EnumArgument[ActionArg]("upload", "upload files.") + with ActionArg + with ArgWithParams[UploadParam[Any], ActionArg] { def watch(params: WatchParam[Any]*) = ??? @@ -14,49 +13,102 @@ case class Upload(override val params: UploadParam[Any]*) trait UploadParam[+T] extends Parameter[T] -case class Filename(override val arg: String) extends StringParameter("filename", "Target filename.") with UploadParam[String] - -case class Comment(override val arg: String) extends StringParameter("comment", - "Upload comment. Also used as the initial page text for new files if text parameter not provided") with UploadParam[String] - -case class Text(override val arg: String) extends StringParameter("text", "Initial page text for new files.") with UploadParam[String] - -case class Token(override val arg: String) extends StringParameter("token", "Initial page text for new files.") with UploadParam[String] - -case class IgnoreWarnings(override val arg: Boolean = true) extends BooleanParameter("ignorewarnings", - "Ignore any warnings. This must be set to upload a new version of an existing image.") with UploadParam[Boolean] - -case class File(override val arg: Array[Byte]) extends ByteArrayParameter("file", - "File contents") with UploadParam[Array[Byte]] - -case class Url(override val arg: String) extends StringParameter("url", "Url to fetch the file from") with UploadParam[String] - -case class FileKey(override val arg: String) extends StringParameter("filekey", - "/Key returned by a previous upload that failed due to warnings, or (with httpstatus) The upload_session_key of an asynchronous upload. " + - "Key that identifies a previous upload that was stashed temporarily.") with UploadParam[String] - -case class SessionKey(override val arg: String) extends StringParameter("sessionkey", - "Same as filekey, maintained for backward compatibility") with UploadParam[String] - -case class Stash(override val arg: String) extends StringParameter("stash", - "If set, the server will not add the file to the repository and stash it temporarily") with UploadParam[String] - -case class Chunk(override val arg: Array[Byte]) extends ByteArrayParameter("chunk", "Chunk contents") with UploadParam[Array[Byte]] - -case class Offset(override val arg: Long) extends LongParameter("offset", "Offset of chunk in bytes") with UploadParam[Long] - -case class FileSize(override val arg: Long) extends LongParameter("filesize", "Filesize of entire upload") with UploadParam[Long] - -case class Async(override val arg: Boolean = true) extends BooleanParameter("async", -"Make potentially large file operations asynchronous when possible") with UploadParam[Boolean] - -case class AsyncDownload(override val arg: Boolean = true) extends BooleanParameter("asyncdownload", "Filesize of entire upload") with UploadParam[Boolean] - -case class LeaveMessage(override val arg: Boolean = true) extends BooleanParameter("leavemessage", - "If asyncdownload is used, leave a message on the user talk page if finished ") with UploadParam[Boolean] - -case class StatusKey(override val arg: String) extends StringParameter("statuskey", - "Fetch the upload status for this file key (upload by URL)") with UploadParam[String] - -case class Checkstatus(override val arg: Boolean = true) extends BooleanParameter("checkstatus", - "Only fetch the upload status for the given file key") with UploadParam[Boolean] +case class Filename(override val arg: String) + extends StringParameter("filename", "Target filename.") + with UploadParam[String] + +case class Comment(override val arg: String) + extends StringParameter( + "comment", + "Upload comment. Also used as the initial page text for new files if text parameter not provided" + ) + with UploadParam[String] + +case class Text(override val arg: String) + extends StringParameter("text", "Initial page text for new files.") + with UploadParam[String] + +case class Token(override val arg: String) + extends StringParameter("token", "Initial page text for new files.") + with UploadParam[String] + +case class IgnoreWarnings(override val arg: Boolean = true) + extends BooleanParameter( + "ignorewarnings", + "Ignore any warnings. This must be set to upload a new version of an existing image." + ) + with UploadParam[Boolean] + +case class File(override val arg: Array[Byte]) + extends ByteArrayParameter("file", "File contents") + with UploadParam[Array[Byte]] + +case class Url(override val arg: String) + extends StringParameter("url", "Url to fetch the file from") + with UploadParam[String] + +case class FileKey(override val arg: String) + extends StringParameter( + "filekey", + "/Key returned by a previous upload that failed due to warnings, or (with httpstatus) The upload_session_key of an asynchronous upload. " + + "Key that identifies a previous upload that was stashed temporarily." + ) + with UploadParam[String] + +case class SessionKey(override val arg: String) + extends StringParameter( + "sessionkey", + "Same as filekey, maintained for backward compatibility" + ) + with UploadParam[String] + +case class Stash(override val arg: String) + extends StringParameter( + "stash", + "If set, the server will not add the file to the repository and stash it temporarily" + ) + with UploadParam[String] + +case class Chunk(override val arg: Array[Byte]) + extends ByteArrayParameter("chunk", "Chunk contents") + with UploadParam[Array[Byte]] + +case class Offset(override val arg: Long) + extends LongParameter("offset", "Offset of chunk in bytes") + with UploadParam[Long] + +case class FileSize(override val arg: Long) + extends LongParameter("filesize", "Filesize of entire upload") + with UploadParam[Long] + +case class Async(override val arg: Boolean = true) + extends BooleanParameter( + "async", + "Make potentially large file operations asynchronous when possible" + ) + with UploadParam[Boolean] + +case class AsyncDownload(override val arg: Boolean = true) + extends BooleanParameter("asyncdownload", "Filesize of entire upload") + with UploadParam[Boolean] + +case class LeaveMessage(override val arg: Boolean = true) + extends BooleanParameter( + "leavemessage", + "If asyncdownload is used, leave a message on the user talk page if finished " + ) + with UploadParam[Boolean] + +case class StatusKey(override val arg: String) + extends StringParameter( + "statuskey", + "Fetch the upload status for this file key (upload by URL)" + ) + with UploadParam[String] + +case class Checkstatus(override val arg: Boolean = true) + extends BooleanParameter( + "checkstatus", + "Only fetch the upload status for the given file key" + ) + with UploadParam[Boolean] diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala index 3a62d29f..3930efc7 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/PageFilter.scala @@ -4,14 +4,14 @@ import org.scalawiki.dto.Page object PageFilter { - def titles(titles: Set[String]) = (p:Page) => titles.contains(p.title) + def titles(titles: Set[String]) = (p: Page) => titles.contains(p.title) - def ids(ids: Set[Long]) = (p:Page) => p.id.exists(ids.contains) + def ids(ids: Set[Long]) = (p: Page) => p.id.exists(ids.contains) - def ns(namespaces: Set[Int]) = (p:Page) => p.ns.exists(namespaces.contains) + def ns(namespaces: Set[Int]) = (p: Page) => p.ns.exists(namespaces.contains) - val all = (p:Page) => true + val all = (p: Page) => true - val none = (p:Page) => false + val none = (p: Page) => false } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/RevisionFilter.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/RevisionFilter.scala index afecdaed..5740cb19 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/RevisionFilter.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/filter/RevisionFilter.scala @@ -14,26 +14,26 @@ trait RevisionFilter { } class RevisionFilterDateAndUser( - val from: Option[ZonedDateTime] = None, - val to: Option[ZonedDateTime] = None, - val userName: Option[String] = None, - val userId: Option[Long] = None) extends RevisionFilter { + val from: Option[ZonedDateTime] = None, + val to: Option[ZonedDateTime] = None, + val userName: Option[String] = None, + val userId: Option[Long] = None +) extends RevisionFilter { override def predicate(rev: Revision): Boolean = { - rev.timestamp.forall { - ts => - from.forall(f => ts.isAfter(f) || ts.isEqual(f)) && - to.forall(t => ts.isBefore(t) || ts.isEqual(t)) + rev.timestamp.forall { ts => + from.forall(f => ts.isAfter(f) || ts.isEqual(f)) && + to.forall(t => ts.isBefore(t) || ts.isEqual(t)) } && - rev.user.forall { // TODO tests - case user: User => - userName.forall(user.login.equals) && - userId.forall(user.id.equals) - case _ => false - } + rev.user.forall { // TODO tests + case user: User => + userName.forall(user.login.equals) && + userId.forall(user.id.equals) + case _ => false + } } } object AllRevisionsFilter extends RevisionFilter { override def predicate(rev: Revision): Boolean = true -} \ No newline at end of file +} diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala index 6999428e..bad165af 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/history/History.scala @@ -16,12 +16,14 @@ class History(val revisions: Seq[Revision]) { def delta(revisionFilter: RevisionFilter): Option[Long] = { val filtered = revisionFilter.apply(revisions) - val sum = for ( - oldest <- filtered.lastOption; - newest <- filtered.headOption; - d1 <- delta(oldest); - d2 <- delta(oldest, newest)) - yield d1 + d2 + val sum = + for ( + oldest <- filtered.lastOption; + newest <- filtered.headOption; + d1 <- delta(oldest); + d2 <- delta(oldest, newest) + ) + yield d1 + d2 sum } @@ -30,19 +32,21 @@ class History(val revisions: Seq[Revision]) { if (parentId == 0) revision.size else - revisions.find(_.revId.contains(parentId)).flatMap { - parent => delta(parent, revision) + revisions.find(_.revId.contains(parentId)).flatMap { parent => + delta(parent, revision) } } def delta(from: Revision, to: Revision): Option[Long] = for (fromSize <- from.size; toSize <- to.size) yield toSize - fromSize - def created: Option[ZonedDateTime] = revisions.lastOption.filter(_.parentId.forall(_ == 0)).flatMap(_.timestamp) + def created: Option[ZonedDateTime] = + revisions.lastOption.filter(_.parentId.forall(_ == 0)).flatMap(_.timestamp) def updated: Option[ZonedDateTime] = revisions.headOption.flatMap(_.timestamp) - def createdAfter(from: Option[ZonedDateTime]) = created.exists(rev => from.forall(rev.isAfter)) + def createdAfter(from: Option[ZonedDateTime]) = + created.exists(rev => from.forall(rev.isAfter)) def editedIn(revisionFilter: RevisionFilter) = revisionFilter.apply(revisions).nonEmpty diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Gallery.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Gallery.scala index c658f0bf..c69f0fb4 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Gallery.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Gallery.scala @@ -4,25 +4,31 @@ import org.scalawiki.dto.Image object Gallery { - def asWiki(images: Iterable[String], descriptions: Iterable[String] = Seq.empty): String = { + def asWiki( + images: Iterable[String], + descriptions: Iterable[String] = Seq.empty + ): String = { val fill = if (descriptions.size < images.size) { Seq.fill(images.size - descriptions.size)("") } else Seq.empty - images.zip(descriptions ++ fill) - .map { - case (image, description) => - val filed = (if (!image.startsWith("File:")) "File:" else "") + image + images + .zip(descriptions ++ fill) + .map { case (image, description) => + val filed = (if (!image.startsWith("File:")) "File:" else "") + image - val piped = if (description.nonEmpty) " | " + description else "" + val piped = if (description.nonEmpty) " | " + description else "" - filed + piped + filed + piped } .mkString("\n", "\n", "\n") } - def asHtml(images: Seq[Image], descriptions: Seq[String] = Seq.empty): String = { + def asHtml( + images: Seq[Image], + descriptions: Seq[String] = Seq.empty + ): String = { val width = 300 val height = 200 val fill = if (descriptions.size < images.size) { @@ -30,33 +36,43 @@ object Gallery { } else Seq.empty - images.zip(descriptions ++ fill) - .map { - case (image, description) => - val thumb = thumbUrl(image, width, height, false) - val imgUrl = image.pageUrl.getOrElse(image.url.get) + images + .zip(descriptions ++ fill) + .map { case (image, description) => + val thumb = thumbUrl(image, width, height, false) + val imgUrl = image.pageUrl.getOrElse(image.url.get) - val imageHtml = - s"""|
  4. + val imageHtml = + s"""|
  5. |
    | | ${image.title} | |
    """.stripMargin - val descrHtml = if (description.nonEmpty) { - "\n" + - s"""|
    + val descrHtml = if (description.nonEmpty) { + "\n" + + s"""|
    |

    $description

    |
    """.stripMargin - } else "" + } else "" - imageHtml + descrHtml + "\n
    6. " + imageHtml + descrHtml + "\n" - }.mkString("") + } + .mkString( + "" + ) } - def thumbUrl(image: Image, resizeToWidth: Int, resizeToHeight: Int, thumbUrl: Boolean = true): String = { + def thumbUrl( + image: Image, + resizeToWidth: Int, + resizeToHeight: Int, + thumbUrl: Boolean = true + ): String = { val url = image.url.get if (!thumbUrl) { url @@ -77,11 +93,14 @@ object Gallery { } else { url.substring(lastSlash + 1) } - url.replace("//upload.wikimedia.org/wikipedia/commons/", "//upload.wikimedia.org/wikipedia/commons/thumb/") + "/" + + url.replace( + "//upload.wikimedia.org/wikipedia/commons/", + "//upload.wikimedia.org/wikipedia/commons/thumb/" + ) + "/" + (if (isPdf) "page1-" - else if (isTif) "lossy-page1-" - else - "") + + else if (isTif) "lossy-page1-" + else + "") + px + "px-" + thumbStr + (if (isPdf || isTif) ".jpg" else "") } else { url diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/NodeFactory.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/NodeFactory.scala index b7056dd7..a4ee08e9 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/NodeFactory.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/NodeFactory.scala @@ -3,4 +3,5 @@ package org.scalawiki.dto.markup import org.sweble.wikitext.engine.utils.DefaultConfigEnWp import org.sweble.wikitext.parser.nodes.WikitextNodeFactoryImpl -object NodeFactory extends WikitextNodeFactoryImpl(DefaultConfigEnWp.generate.getParserConfig) \ No newline at end of file +object NodeFactory + extends WikitextNodeFactoryImpl(DefaultConfigEnWp.generate.getParserConfig) diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwNode.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwNode.scala index d09bf139..53177232 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwNode.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwNode.scala @@ -11,5 +11,4 @@ trait SwNode { def getText(wtNodeP: WtNode): String = WtRtDataPrinter.print(wtNodeP) - } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala index 00d98fca..4802e862 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/SwTemplate.scala @@ -8,22 +8,21 @@ import scala.collection.mutable case class SwTemplate(wtNode: WtTemplate) extends SwNode { val name: String = getText(wtNode.getName).trim - val args: mutable.Buffer[WtTemplateArgument] = wtNode.getArgs.asScala.collect { case arg: WtTemplateArgument => arg } + val args: mutable.Buffer[WtTemplateArgument] = + wtNode.getArgs.asScala.collect { case arg: WtTemplateArgument => arg } val template = getTemplate def getTemplate = { - val argsMap = args.zipWithIndex.map { - case (arg, index) => + val argsMap = args.zipWithIndex.map { case (arg, index) => + val name = + if (arg.hasName) + getText(arg.getName).trim + else + (index + 1).toString - val name = - if (arg.hasName) - getText(arg.getName).trim - else - (index + 1).toString + val value = getText(arg.getValue).trim - val value = getText(arg.getValue).trim - - name -> value + name -> value }.toMap new Template(name, argsMap) @@ -33,19 +32,19 @@ case class SwTemplate(wtNode: WtTemplate) extends SwNode { args.find(p => p.getName.isResolved && p.getName.getAsString.trim == name) def setTemplateParam(name: String, value: String) = { - getArg(name).foreach { - arg => - val orig = getText(arg.getValue) - - val padded = if (orig.startsWith(" ") && !value.startsWith(" ")) " " + value else value - val withNl = if (orig.endsWith("\n") && !value.endsWith("\n")) padded + "\n" else padded - - val f = NodeFactory - val node = f.value(f.list(f.text(withNl))) - arg.setValue(node) + getArg(name).foreach { arg => + val orig = getText(arg.getValue) + + val padded = + if (orig.startsWith(" ") && !value.startsWith(" ")) " " + value + else value + val withNl = + if (orig.endsWith("\n") && !value.endsWith("\n")) padded + "\n" + else padded + + val f = NodeFactory + val node = f.value(f.list(f.text(withNl))) + arg.setValue(node) } } } - - - diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala index e0f6cb27..99392855 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Table.scala @@ -1,11 +1,11 @@ package org.scalawiki.dto.markup - case class Table( - headers: Iterable[String], - data: Iterable[Iterable[String]], - title: String = "", - cssClass: String = "wikitable sortable") { + headers: Iterable[String], + data: Iterable[Iterable[String]], + title: String = "", + cssClass: String = "wikitable sortable" +) { def asWiki = { "{|" + @@ -13,11 +13,12 @@ case class Table( (if (title.nonEmpty) "\n|+ " + title else "") + (if (headers.nonEmpty) headers.mkString("\n! ", " !! ", "") else "") + (if (data.nonEmpty) - data.map { - row => - row.mkString("| ", " || ", "") - }.mkString("\n|-\n", "\n|-\n", "") - else "") + + data + .map { row => + row.mkString("| ", " || ", "") + } + .mkString("\n|-\n", "\n|-\n", "") + else "") + "\n|}" } @@ -26,17 +27,19 @@ case class Table( (if (cssClass.nonEmpty) s" class='$cssClass'" else "") + (if (title.nonEmpty) s"\n $title " else "") + (if (headers.nonEmpty) { - headers - .map(h => s" $h ") - .mkString("\n\n\n", "\n", "\n\n") - } - else "") + + headers + .map(h => s" $h ") + .mkString("\n\n\n", "\n", "\n\n") + } else "") + (if (data.nonEmpty) - data.map { row => row - .map(c => s" $c ") - .mkString("\n", "\n", "\n") - }.mkString("\n\n", "\n", "\n") - else "") + + data + .map { row => + row + .map(c => s" $c ") + .mkString("\n", "\n", "\n") + } + .mkString("\n\n", "\n", "\n") + else "") + "\n" } } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Template.scala b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Template.scala index d863b486..6cd99b8c 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Template.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/dto/markup/Template.scala @@ -1,8 +1,9 @@ package org.scalawiki.dto.markup case class Template( - templateName: String, - params: Map[String, String] = Map.empty) { + templateName: String, + params: Map[String, String] = Map.empty +) { def hasTemplateParam(name: String): Boolean = params.contains(name) @@ -10,11 +11,12 @@ case class Template( def getParamOpt(name: String): Option[String] = params.get(name) - def text: String = "{{" + templateName + params.map { case (k, v) => s"\n|$k = $v" }.mkString + "\n}}" + def text: String = "{{" + templateName + params.map { case (k, v) => + s"\n|$k = $v" + }.mkString + "\n}}" def setTemplateParam(name: String, value: String): Template = { copy(params = params + (name -> value)) } } - diff --git a/scalawiki-core/src/main/scala/org/scalawiki/edit/PageUpdater.scala b/scalawiki-core/src/main/scala/org/scalawiki/edit/PageUpdater.scala index ad1fa071..9a8d9469 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/edit/PageUpdater.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/edit/PageUpdater.scala @@ -11,10 +11,9 @@ class PageUpdater(task: PageUpdateTask) extends WithBot { def update() = { val titles = task.titles - val results = titles.zipWithIndex.map { - case (title, index) => - println(s"Processing page: $title, $index of ${titles.size}") - updatePage(title) + val results = titles.zipWithIndex.map { case (title, index) => + println(s"Processing page: $title, $index of ${titles.size}") + updatePage(title) } for (seq <- Future.sequence(results.toSeq)) { diff --git a/scalawiki-core/src/main/scala/org/scalawiki/http/HttpClient.scala b/scalawiki-core/src/main/scala/org/scalawiki/http/HttpClient.scala index 444ba194..4b31842a 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/http/HttpClient.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/http/HttpClient.scala @@ -17,16 +17,25 @@ trait HttpClient { def getResponse(url: Uri): Future[HttpResponse] def getResponse(url: String): Future[HttpResponse] - def post(url: String, params: (String, String)*): Future[HttpResponse] = post(url, params.toMap) + def post(url: String, params: (String, String)*): Future[HttpResponse] = + post(url, params.toMap) def post(url: String, params: Map[String, String]): Future[HttpResponse] def postUri(url: Uri, params: Map[String, String]): Future[HttpResponse] - def postMultiPart(url: String, params: Map[String, String]): Future[HttpResponse] + def postMultiPart( + url: String, + params: Map[String, String] + ): Future[HttpResponse] def postMultiPart(url: Uri, params: Map[String, String]): Future[HttpResponse] - def postFile(url: String, params: Map[String, String], fileParam: String, filename: String): Future[HttpResponse] + def postFile( + url: String, + params: Map[String, String], + fileParam: String, + filename: String + ): Future[HttpResponse] def getBody(response: HttpResponse): Future[String] @@ -35,5 +44,7 @@ trait HttpClient { object HttpClient { val JSON_UTF8 = ContentType(MediaTypes.`application/json`) - def get(system: ActorSystem = MwBot.system): HttpClient = new HttpClientAkka(system) -} \ No newline at end of file + def get(system: ActorSystem = MwBot.system): HttpClient = new HttpClientAkka( + system + ) +} diff --git a/scalawiki-core/src/main/scala/org/scalawiki/http/HttpClientAkka.scala b/scalawiki-core/src/main/scala/org/scalawiki/http/HttpClientAkka.scala index 0632e734..520def2e 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/http/HttpClientAkka.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/http/HttpClientAkka.scala @@ -5,7 +5,11 @@ import akka.http.scaladsl.Http import akka.http.scaladsl.client.RequestBuilding._ import akka.http.scaladsl.coding.{Deflate, Gzip, NoCoding} import akka.http.scaladsl.model.Multipart.FormData.BodyPart -import akka.http.scaladsl.model.headers.{HttpEncodings, `Accept-Encoding`, `User-Agent`} +import akka.http.scaladsl.model.headers.{ + HttpEncodings, + `Accept-Encoding`, + `User-Agent` +} import akka.http.scaladsl.model._ import akka.util.Timeout import net.spraycookies.tldlist.DefaultEffectiveTldList @@ -17,7 +21,8 @@ import scala.concurrent.Future import scala.concurrent.duration.{Duration, _} import scala.language.postfixOps -class HttpClientAkka(val system: ActorSystem = MwBot.system) extends HttpClient { +class HttpClientAkka(val system: ActorSystem = MwBot.system) + extends HttpClient { implicit val sys: ActorSystem = system @@ -38,7 +43,8 @@ class HttpClientAkka(val system: ActorSystem = MwBot.system) extends HttpClient decoder.decodeMessage(response) } - def sendReceive: HttpRequest => Future[HttpResponse] = req => Http().singleRequest(req) + def sendReceive: HttpRequest => Future[HttpResponse] = req => + Http().singleRequest(req) override implicit val timeout: Duration = 15.minutes @@ -49,14 +55,16 @@ class HttpClientAkka(val system: ActorSystem = MwBot.system) extends HttpClient implicit val timeout: Timeout = 5.minutes addHeaders( `Accept-Encoding`(HttpEncodings.gzip), - `User-Agent`(userAgent)) ~> + `User-Agent`(userAgent) + ) ~> cookied( sendReceive ~> decodeResponse ) } - override def get(url: String): Future[String] = submit(Get(url)) flatMap getBody + override def get(url: String): Future[String] = + submit(Get(url)) flatMap getBody override def get(url: Uri): Future[String] = submit(Get(url)) flatMap getBody @@ -64,33 +72,52 @@ class HttpClientAkka(val system: ActorSystem = MwBot.system) extends HttpClient override def getResponse(url: String): Future[HttpResponse] = submit(Get(url)) - override def post(url: String, params: Map[String, String]): Future[HttpResponse] = { + override def post( + url: String, + params: Map[String, String] + ): Future[HttpResponse] = { submit(Post(url, FormData(params))) } - override def postUri(url: Uri, params: Map[String, String]): Future[HttpResponse] = { + override def postUri( + url: Uri, + params: Map[String, String] + ): Future[HttpResponse] = { submit(Post(url, FormData(params))) } - override def postMultiPart(url: String, params: Map[String, String]): Future[HttpResponse] = { + override def postMultiPart( + url: String, + params: Map[String, String] + ): Future[HttpResponse] = { postMultiPart(Uri(url), params) } - override def postMultiPart(url: Uri, params: Map[String, String]): Future[HttpResponse] = { + override def postMultiPart( + url: Uri, + params: Map[String, String] + ): Future[HttpResponse] = { val bodyParts = params.map { case (key, value) => BodyPart(key, HttpEntity(value)) }.toList submit(Post(url, Multipart.FormData(bodyParts: _*))) } - override def postFile(url: String, params: Map[String, String], fileParam: String, filename: String): Future[HttpResponse] = { + override def postFile( + url: String, + params: Map[String, String], + fileParam: String, + filename: String + ): Future[HttpResponse] = { val bodyParts = params.map { case (key, value) => BodyPart(key, HttpEntity(value)) - } ++ Seq(BodyPart.fromPath(fileParam, MediaTypes.`image/jpeg`, Paths.get(filename))) + } ++ Seq( + BodyPart.fromPath(fileParam, MediaTypes.`image/jpeg`, Paths.get(filename)) + ) submit(Post(Uri(url), Multipart.FormData(bodyParts.toList: _*))) } def getBody(response: HttpResponse): Future[String] = response.entity.toStrict(5 minutes).map(_.data.utf8String) -} \ No newline at end of file +} diff --git a/scalawiki-core/src/main/scala/org/scalawiki/json/MwReads.scala b/scalawiki-core/src/main/scala/org/scalawiki/json/MwReads.scala index 24b3b4c7..a08091ef 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/json/MwReads.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/json/MwReads.scala @@ -17,27 +17,25 @@ object MwReads { (__ \ "login" \ "lgusername").readNullable[String] and (__ \ "login" \ "cookieprefix").readNullable[String] and (__ \ "login" \ "sessionid").readNullable[String] - )(LoginResponse.apply _) + )(LoginResponse.apply _) - def tokenReads: Reads[String] = (__ \ "query" \ "tokens" \ "csrftoken").read[String] + def tokenReads: Reads[String] = + (__ \ "query" \ "tokens" \ "csrftoken").read[String] def tokensReads: Reads[String] = (__ \ "tokens" \ "edittoken").read[String] def editResponseReads: Reads[String] = (__ \ "edit" \ "result").read[String] - def uploadResponseReads: Reads[String] = (__ \ "upload" \ "result").read[String] + def uploadResponseReads: Reads[String] = + (__ \ "upload" \ "result").read[String] - def siteInfoReads: Reads[String] = (__ \ "query" \ "general" \ "generator").read[String] + def siteInfoReads: Reads[String] = + (__ \ "query" \ "general" \ "generator").read[String] def editTokenReads: Reads[String] = (__ \\ "edittoken").read[String] def errorReads: Reads[MwException] = ( (__ \ "error" \ "code").read[String] and (__ \ "error" \ "info").read[String] - )( - (code, info) => MwException.apply(code, info) - ) + )((code, info) => MwException.apply(code, info)) } - - - diff --git a/scalawiki-core/src/main/scala/org/scalawiki/json/Parser.scala b/scalawiki-core/src/main/scala/org/scalawiki/json/Parser.scala index 97d70e8e..9bd19aeb 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/json/Parser.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/json/Parser.scala @@ -29,7 +29,8 @@ class Parser(val action: Action) { continue = getContinue(json) jsonObj.value match { - case value if value.contains("query") => parseQueryAction(json, queryChild) + case value if value.contains("query") => + parseQueryAction(json, queryChild) case _ => Seq.empty } } @@ -41,17 +42,17 @@ class Parser(val action: Action) { val jsons = (queryChild match { case "pages" => pagesJson.asInstanceOf[JsObject].values case "allusers" | "usercontribs" => pagesJson.asInstanceOf[JsArray].value - case "globaluserinfo" => Seq(pagesJson) - case _ => pagesJson.asInstanceOf[JsArray].value + case "globaluserinfo" => Seq(pagesJson) + case _ => pagesJson.asInstanceOf[JsArray].value }).map(_.asInstanceOf[JsObject]) jsons.map { j => queryChild match { - case "pages" => parsePage(j) + case "pages" => parsePage(j) case "allusers" | "users" => parseUser(j, queryChild) - case "usercontribs" => parseUserContrib(j) - case "globaluserinfo" => parseGlobalUserInfo(j) - case _ => parsePage(j) + case "usercontribs" => parseUserContrib(j) + case "globaluserinfo" => parseGlobalUserInfo(j) + case _ => parsePage(j) } }.toSeq } @@ -72,39 +73,62 @@ class Parser(val action: Action) { val links = getLinks(pageJson) val categoryInfo = getCategoryInfo(pageJson) - page.copy(revisions = revisions, images = images, langLinks = langLinks, links = links, categoryInfo = categoryInfo) + page.copy( + revisions = revisions, + images = images, + langLinks = langLinks, + links = links, + categoryInfo = categoryInfo + ) } def getImages(pageJson: JsObject, page: Page): Seq[Image] = { - pageJson.validate { - if (pageJson.value.contains("imageinfo")) { - Parser.imageInfoReads(page.id, Some(page.title)) - } else { - // if (pageJson.value.contains("images")) { - Parser.imageReads() + pageJson + .validate { + if (pageJson.value.contains("imageinfo")) { + Parser.imageInfoReads(page.id, Some(page.title)) + } else { + // if (pageJson.value.contains("images")) { + Parser.imageReads() + } } - }.getOrElse(Seq.empty) + .getOrElse(Seq.empty) } // hacky wrapping into page // TODO refactor return types def parseUser(userJson: JsObject, queryChild: String): Page = { - val hasEmptyRegistration = userJson.value.get("registration") + val hasEmptyRegistration = userJson.value + .get("registration") .collect({ case jsStr: JsString => jsStr.value.isEmpty }) .getOrElse(false) - val mappedJson = if (hasEmptyRegistration) userJson - "registration" else userJson + val mappedJson = + if (hasEmptyRegistration) userJson - "registration" else userJson // TODO move out of loop or get from request? val prefix = queryChild match { case "allusers" => "au" - case "users" => "us" + case "users" => "us" } - val props = params.get(prefix + "prop").map(_.split("\\|")).getOrElse(Array.empty[String]).toSet - - val blocked = if (props.contains("blockinfo")) Some(userJson.keys.contains("blockid")) else None - val emailable = if (props.contains("emailable")) Some(userJson.keys.contains("emailable")) else None + val props = params + .get(prefix + "prop") + .map(_.split("\\|")) + .getOrElse(Array.empty[String]) + .toSet + + val blocked = + if (props.contains("blockinfo")) Some(userJson.keys.contains("blockid")) + else None + val emailable = + if (props.contains("emailable")) Some(userJson.keys.contains("emailable")) + else None val jsResult = mappedJson.validate(Parser.userReads) val user = jsResult.get.copy(blocked = blocked, emailable = emailable) - new Page(id = None, title = user.name.get, ns = Some(Namespace.USER), revisions = Seq(Revision(user = Some(user)))) + new Page( + id = None, + title = user.name.get, + ns = Some(Namespace.USER), + revisions = Seq(Revision(user = Some(user))) + ) } def parseUserContrib(userJson: JsObject): Page = { @@ -113,28 +137,43 @@ class Parser(val action: Action) { } def getContinue(json: JsValue): Map[String, String] = { - (json \ "continue").asOpt[JsObject].map(_.value.mapValues[String] { - case JsNumber(n) => n.toString() - case JsString(s) => s - }.toMap) + (json \ "continue") + .asOpt[JsObject] + .map( + _.value + .mapValues[String] { + case JsNumber(n) => n.toString() + case JsString(s) => s + } + .toMap + ) .getOrElse(Map.empty[String, String]) } def getLangLinks(pageJson: JsObject): Map[String, String] = { - (pageJson \ "langlinks").asOpt[Seq[Map[String, String]]].map { - _.map(l => l("lang") -> l("*")).toMap - }.getOrElse(Map.empty[String, String]) + (pageJson \ "langlinks") + .asOpt[Seq[Map[String, String]]] + .map { + _.map(l => l("lang") -> l("*")).toMap + } + .getOrElse(Map.empty[String, String]) } def getLinks(pageJson: JsObject): Seq[Page] = { - (pageJson \ "links").asOpt[JsArray].map { - _.value.map { l => - new Page(id = None, - ns = (l \ "ns").asOpt[Int], - title = (l \ "title").as[String] - ) - }.toSeq - }.getOrElse(Nil) + (pageJson \ "links") + .asOpt[JsArray] + .map { + _.value + .map { l => + new Page( + id = None, + ns = (l \ "ns").asOpt[Int], + title = (l \ "title").as[String] + ) + } + .toSeq + } + .getOrElse(Nil) } def getCategoryInfo(pageJson: JsObject): Option[CategoryInfo] = @@ -153,9 +192,19 @@ class Parser(val action: Action) { sulAccounts = gui.merged ) - new Page(id = None, title = gui.name, ns = Some(Namespace.USER), revisions = Seq(Revision(user = Some(user)))) + new Page( + id = None, + title = gui.name, + ns = Some(Namespace.USER), + revisions = Seq(Revision(user = Some(user))) + ) } else { - new Page(id = None, title = "missing", ns = Some(Namespace.USER), revisions = Seq.empty) + new Page( + id = None, + title = "missing", + ns = Some(Namespace.USER), + revisions = Seq.empty + ) } } @@ -182,8 +231,12 @@ object Parser { (__ \ "revisions").read[Seq[Revision]] } - private def imageInfoReads(pageId: Option[Long], title: Option[String]): Reads[Seq[Image]] = { - implicit val imageReads: Reads[Image] = ImageReads(title = title, pageId = pageId) + private def imageInfoReads( + pageId: Option[Long], + title: Option[String] + ): Reads[Seq[Image]] = { + implicit val imageReads: Reads[Image] = + ImageReads(title = title, pageId = pageId) (__ \ "imageinfo").read[Seq[Image]] } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/json/WikiReads.scala b/scalawiki-core/src/main/scala/org/scalawiki/json/WikiReads.scala index a3326aa6..26617347 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/json/WikiReads.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/json/WikiReads.scala @@ -5,17 +5,26 @@ import java.time.format.DateTimeFormatter import org.scalawiki.dto._ import play.api.libs.json.JsonValidationError -import play.api.libs.json.{JsError, JsNumber, JsPath, JsResult, JsString, JsSuccess, JsValue, Reads, _} +import play.api.libs.json.{ + JsError, + JsNumber, + JsPath, + JsResult, + JsString, + JsSuccess, + JsValue, + Reads, + _ +} -/** - * Created by francisco on 03/11/16. +/** Created by francisco on 03/11/16. */ -trait WikiReads[T] extends Reads[T] { -} +trait WikiReads[T] extends Reads[T] {} abstract class WikiResponseReads[T]() { val TIMESTAMP_PATTERN = "yyyy-MM-dd'T'HH:mm:ss'Z'" - val df = DateTimeFormatter.ofPattern(TIMESTAMP_PATTERN).withZone(ZoneOffset.UTC) + val df = + DateTimeFormatter.ofPattern(TIMESTAMP_PATTERN).withZone(ZoneOffset.UTC) lazy val zonedDateTimeReads = zonedDateReads(TIMESTAMP_PATTERN) @@ -24,15 +33,38 @@ abstract class WikiResponseReads[T]() { def parseDateOpt(input: String): Option[ZonedDateTime] = scala.util.control.Exception.allCatch[ZonedDateTime] opt parseDate(input) - def zonedDateReads(pattern: String, corrector: String => String = identity): Reads[ZonedDateTime] = + def zonedDateReads( + pattern: String, + corrector: String => String = identity + ): Reads[ZonedDateTime] = new Reads[ZonedDateTime] { def reads(json: JsValue): JsResult[ZonedDateTime] = json match { - case JsNumber(d) => JsSuccess(ZonedDateTime.ofInstant(Instant.ofEpochSecond(d.toLong), ZoneOffset.UTC)) - case JsString(s) => parseDateOpt(corrector(s)) match { - case Some(d) => JsSuccess(d) - case None => JsError(Seq(JsPath() -> Seq(JsonValidationError("error.expected.jodadate.format", pattern)))) - } - case _ => JsError(Seq(JsPath() -> Seq(JsonValidationError("error.expected.date")))) + case JsNumber(d) => + JsSuccess( + ZonedDateTime.ofInstant( + Instant.ofEpochSecond(d.toLong), + ZoneOffset.UTC + ) + ) + case JsString(s) => + parseDateOpt(corrector(s)) match { + case Some(d) => JsSuccess(d) + case None => + JsError( + Seq( + JsPath() -> Seq( + JsonValidationError( + "error.expected.jodadate.format", + pattern + ) + ) + ) + ) + } + case _ => + JsError( + Seq(JsPath() -> Seq(JsonValidationError("error.expected.date"))) + ) } } } @@ -57,7 +89,7 @@ case class PageReads() extends WikiResponseReads with WikiReads[Page] { (__ \ "subjectid").readNullable[Long] ~ (__ \ "talkid").readNullable[Long] ~ (__ \ "invalidreason").readNullable[String] - ) (Page.full _) + )(Page.full _) override def reads(json: JsValue): JsResult[Page] = pageRead.reads(json) } @@ -74,22 +106,25 @@ case class UserReads() extends WikiResponseReads with WikiReads[User] { (__ \ "blockid").readNullable[Long].map(_.map(_ => true)) ~ (__ \ "emailable").readNullable[String].map(_.map(_ => true)) ~ (__ \ "missing").readNullable[String].map(_.isDefined) - ) (User.apply( - _: Option[Long], - _: Option[String], - _: Option[Long], - _: Option[ZonedDateTime], - _: Option[Boolean], - _: Option[Boolean], - _: Boolean - ) + )( + User.apply( + _: Option[Long], + _: Option[String], + _: Option[Long], + _: Option[ZonedDateTime], + _: Option[Boolean], + _: Option[Boolean], + _: Boolean + ) ) override def reads(json: JsValue): JsResult[User] = userRead.reads(json) } case class RevisionRead(override val pageId: Option[Long]) - extends WikiResponseReads with WikiReads[Revision] with PageIdParameter { + extends WikiResponseReads + with WikiReads[Revision] + with PageIdParameter { import play.api.libs.functional.syntax._ @@ -100,27 +135,34 @@ case class RevisionRead(override val pageId: Option[Long]) ( (__ \ "userid").readNullable[Long] ~ (__ \ "user").readNullable[String] - ) (Contributor.apply _) ~ + )(Contributor.apply _) ~ (__ \ "timestamp").readNullable[ZonedDateTime](zonedDateTimeReads) ~ (__ \ "comment").readNullable[String] ~ (__ \\ "*").readNullable[String] ~ (__ \ "size").readNullable[Long] ~ - (__ \ "sha1").readNullable[String] //~ - //Reads.pure[Option[Long]](None) // textId - ) (Revision.apply(_: Option[Long], - _: Option[Long], - _: Option[Long], - _: Option[Contributor], - _: Option[ZonedDateTime], - _: Option[String], - _: Option[String], - _: Option[Long], - _: Option[String])) - - override def reads(json: JsValue): JsResult[Revision] = revisionRead.reads(json) + (__ \ "sha1").readNullable[String] // ~ + // Reads.pure[Option[Long]](None) // textId + )( + Revision.apply( + _: Option[Long], + _: Option[Long], + _: Option[Long], + _: Option[Contributor], + _: Option[ZonedDateTime], + _: Option[String], + _: Option[String], + _: Option[Long], + _: Option[String] + ) + ) + + override def reads(json: JsValue): JsResult[Revision] = + revisionRead.reads(json) } -case class GlobalUserInfoReads() extends WikiResponseReads with WikiReads[GlobalUserInfo] { +case class GlobalUserInfoReads() + extends WikiResponseReads + with WikiReads[GlobalUserInfo] { import play.api.libs.functional.syntax._ @@ -131,7 +173,7 @@ case class GlobalUserInfoReads() extends WikiResponseReads with WikiReads[Global (__ \ "method").read[String] ~ (__ \ "editcount").read[Long] ~ (__ \ "registration").read[ZonedDateTime](zonedDateTimeReads) - ) (SulAccount.apply _) + )(SulAccount.apply _) private val globalInfoReads: Reads[GlobalUserInfo] = ( @@ -141,31 +183,34 @@ case class GlobalUserInfoReads() extends WikiResponseReads with WikiReads[Global (__ \ "name").read[String] ~ (__ \ "merged").read[Seq[SulAccount]] ~ (__ \ "editcount").read[Long] - ) (GlobalUserInfo.apply _) - + )(GlobalUserInfo.apply _) - override def reads(json: JsValue): JsResult[GlobalUserInfo] = globalInfoReads.reads(json) + override def reads(json: JsValue): JsResult[GlobalUserInfo] = + globalInfoReads.reads(json) } -case class ImageReads(override val pageId: Option[Long] = None, override val title: Option[String] = None) - extends WikiResponseReads +case class ImageReads( + override val pageId: Option[Long] = None, + override val title: Option[String] = None +) extends WikiResponseReads with WikiReads[Image] - with PageIdParameter with TitleParameter { + with PageIdParameter + with TitleParameter { import play.api.libs.functional.syntax._ def readKv(js: JsObject): (String, String) = { (js \ "name").as[String] -> (js \ "value" match { - case JsDefined(JsString(str)) => str + case JsDefined(JsString(str)) => str case JsDefined(JsNumber(n)) if n.isValidInt => n.toString - case x => x.toString + case x => x.toString }) } private val imagesRead: Reads[Image] = ( (title match { case Some(t) => Reads.pure[String](t) - case None => (__ \ "title").read[String] + case None => (__ \ "title").read[String] }) ~ (__ \ "timestamp").readNullable[ZonedDateTime](zonedDateTimeReads) ~ (__ \ "user").readNullable[String] ~ @@ -175,13 +220,17 @@ case class ImageReads(override val pageId: Option[Long] = None, override val tit (__ \ "url").readNullable[String] ~ (__ \ "descriptionurl").readNullable[String] ~ Reads.pure[Option[Long]](pageId) ~ - (__ \ "metadata").readNullable[Seq[JsObject]].map(_.map(_.map(readKv).toMap)) - ) (Image.basic _) + (__ \ "metadata") + .readNullable[Seq[JsObject]] + .map(_.map(_.map(readKv).toMap)) + )(Image.basic _) override def reads(json: JsValue): JsResult[Image] = imagesRead.reads(json) } -case class CategoryInfoReads() extends WikiResponseReads with WikiReads[CategoryInfo] { +case class CategoryInfoReads() + extends WikiResponseReads + with WikiReads[CategoryInfo] { import play.api.libs.functional.syntax._ @@ -190,12 +239,15 @@ case class CategoryInfoReads() extends WikiResponseReads with WikiReads[Category (__ \ "pages").read[Long] ~ (__ \ "files").read[Long] ~ (__ \ "subcats").read[Long] - ) (CategoryInfo.apply _) + )(CategoryInfo.apply _) - override def reads(json: JsValue): JsResult[CategoryInfo] = categoryRead.reads(json) + override def reads(json: JsValue): JsResult[CategoryInfo] = + categoryRead.reads(json) } -case class UserContributorReads() extends WikiResponseReads with WikiReads[UserContrib] { +case class UserContributorReads() + extends WikiResponseReads + with WikiReads[UserContrib] { import play.api.libs.functional.syntax._ @@ -212,7 +264,8 @@ case class UserContributorReads() extends WikiResponseReads with WikiReads[UserC // (__ \ "minor").read[Boolea] ~ (__ \ "comment").readNullable[String] ~ // can be hidden (__ \ "size").readNullable[Long] - ) (UserContrib.apply _) + )(UserContrib.apply _) - override def reads(json: JsValue): JsResult[UserContrib] = userContribRead.reads(json) + override def reads(json: JsValue): JsResult[UserContrib] = + userContribRead.reads(json) } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/query/DslQuery.scala b/scalawiki-core/src/main/scala/org/scalawiki/query/DslQuery.scala index a76f4429..db0fee6c 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/query/DslQuery.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/query/DslQuery.scala @@ -8,25 +8,32 @@ import org.scalawiki.json.Parser import scala.concurrent.Future import scala.util.{Failure, Success} -case class QueryProgress(pages: Long, - done: Boolean, - action: Action, - bot: MwBot, - context: Map[String, String] = Map.empty) - -class DslQuery(val action: Action, - val bot: MwBot, - context: Map[String, String] = Map.empty) { +case class QueryProgress( + pages: Long, + done: Boolean, + action: Action, + bot: MwBot, + context: Map[String, String] = Map.empty +) + +class DslQuery( + val action: Action, + val bot: MwBot, + context: Map[String, String] = Map.empty +) { import scala.concurrent.ExecutionContext.Implicits.global var startTime: Long = 0 - def run(continue: Map[String, String] = Map("continue" -> ""), - pages: PageList = new PageList(), - limit: Option[Long] = None): Future[PageList] = { + def run( + continue: Map[String, String] = Map("continue" -> ""), + pages: PageList = new PageList(), + limit: Option[Long] = None + ): Future[PageList] = { - val params = action.pairs ++ Seq("format" -> "json", "utf8" -> "") ++ continue + val params = + action.pairs ++ Seq("format" -> "json", "utf8" -> "") ++ continue if (startTime == 0) startTime = System.nanoTime() @@ -36,31 +43,32 @@ class DslQuery(val action: Action, implicit val success: retry.Success[String] = retry.Success[String](_ => true) - retry.Backoff()(odelay.Timer.default)(() => bot.post(params.toMap)) flatMap { - body => - val parser = new Parser(action) - - parser.parse(body) match { - case Success(newPages) => - pages.addPages(newPages) - - val newContinue = parser.continue - if (newContinue.isEmpty || limit.exists(_ <= pages.size)) { - - onProgress(pages.size, done = true) - Future.successful(pages) - } else { - run(newContinue, pages, limit) - } - - case Failure(mwEx: MwException) => - val withParams = mwEx.copy(params = params.toMap) - bot.log.error(s"${bot.host} exception $withParams, body: $body") - Future.failed(withParams) - case Failure(ex) => - bot.log.error(s"${bot.host} exception $ex, body: $body") - Future.failed(MwException(ex.getMessage, ex.getMessage, params.toMap)) - } + retry.Backoff()(odelay.Timer.default)(() => + bot.post(params.toMap) + ) flatMap { body => + val parser = new Parser(action) + + parser.parse(body) match { + case Success(newPages) => + pages.addPages(newPages) + + val newContinue = parser.continue + if (newContinue.isEmpty || limit.exists(_ <= pages.size)) { + + onProgress(pages.size, done = true) + Future.successful(pages) + } else { + run(newContinue, pages, limit) + } + + case Failure(mwEx: MwException) => + val withParams = mwEx.copy(params = params.toMap) + bot.log.error(s"${bot.host} exception $withParams, body: $body") + Future.failed(withParams) + case Failure(ex) => + bot.log.error(s"${bot.host} exception $ex, body: $body") + Future.failed(MwException(ex.getMessage, ex.getMessage, params.toMap)) + } } } @@ -69,7 +77,8 @@ class DslQuery(val action: Action, val estimatedTime = (System.nanoTime() - startTime) / Math.pow(10, 9) bot.log.info( - s"${bot.host} Action completed with $pages pages in $estimatedTime seconds, $action.pairs") + s"${bot.host} Action completed with $pages pages in $estimatedTime seconds, $action.pairs" + ) } else { bot.log.info(s"${bot.host} pages: $pages action: $action.pairs") } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/query/PageQuery.scala b/scalawiki-core/src/main/scala/org/scalawiki/query/PageQuery.scala index b6c31a03..656349bc 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/query/PageQuery.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/query/PageQuery.scala @@ -10,7 +10,8 @@ trait PageQuery { def revisions( namespaces: Set[Int] = Set.empty, props: Set[String] = Set.empty, - continueParam: Option[(String, String)] = None): Future[Iterable[Page]] + continueParam: Option[(String, String)] = None + ): Future[Iterable[Page]] } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/query/PageQueryImplDsl.scala b/scalawiki-core/src/main/scala/org/scalawiki/query/PageQueryImplDsl.scala index f7b4bf12..b35ad784 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/query/PageQueryImplDsl.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/query/PageQueryImplDsl.scala @@ -15,10 +15,11 @@ import retry.Success import scala.concurrent.Future -class PageQueryImplDsl(query: Either[Set[Long], Set[String]], - bot: MwBot, - context: Map[String, String] = Map.empty) - extends PageQuery +class PageQueryImplDsl( + query: Either[Set[Long], Set[String]], + bot: MwBot, + context: Map[String, String] = Map.empty +) extends PageQuery with SinglePageQuery { override def withContext(context: Map[String, String]) = @@ -27,7 +28,8 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], override def revisions( namespaces: Set[Int], props: Set[String], - continueParam: Option[(String, String)]): Future[Iterable[Page]] = { + continueParam: Option[(String, String)] + ): Future[Iterable[Page]] = { import org.scalawiki.dto.cmd.query.prop.rvprop._ @@ -46,7 +48,8 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], RvLimit("max") ) ) - )) + ) + ) bot.run(action, context) } @@ -58,7 +61,8 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], props: Set[String], continueParam: Option[(String, String)], limit: String, - titlePrefix: Option[String]): Future[Iterable[Page]] = { + titlePrefix: Option[String] + ): Future[Iterable[Page]] = { val pageId: Option[Long] = query.left.toOption.map(_.head) val title: Option[String] = query.right.toOption.map(_.head) @@ -71,7 +75,8 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], Revisions(RvProp(RvPropArgs.byNames(props.toSeq): _*)) ), Generator( - ListArgs.toDsl(generator, title, pageId, namespaces, Some(limit)).get) + ListArgs.toDsl(generator, title, pageId, namespaces, Some(limit)).get + ) ) val action = Action(Query(queryParams: _*)) @@ -85,7 +90,8 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], props: Set[String], continueParam: Option[(String, String)], limit: String, - titlePrefix: Option[String]): Future[Iterable[Page]] = { + titlePrefix: Option[String] + ): Future[Iterable[Page]] = { import org.scalawiki.dto.cmd.query.prop.iiprop._ val pageId: Option[Long] = query.left.toOption.map(_.head) @@ -106,13 +112,16 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], bot.run(action, context) } - private def pagesParam(pageId: Option[Long], - title: Option[String], - generatorArg: Option[GeneratorArg]) = { + private def pagesParam( + pageId: Option[Long], + title: Option[String], + generatorArg: Option[GeneratorArg] + ) = { val pagesInGenerator = generatorArg.exists( _.pairs .map(_._1) - .exists(p => p.endsWith("title") || p.endsWith("pageid"))) + .exists(p => p.endsWith("title") || p.endsWith("pageid")) + ) if (pagesInGenerator) Seq.empty[QueryParam[String]] else { title.map(t => TitlesParam(Seq(t))).toSeq ++ pageId @@ -121,11 +130,13 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], } } - override def edit(text: String, - summary: Option[String] = None, - section: Option[String] = None, - token: Option[String] = None, - multi: Boolean = false) = { + override def edit( + text: String, + summary: Option[String] = None, + section: Option[String] = None, + token: Option[String] = None, + multi: Boolean = false + ) = { val page = query.fold( ids => PageId(ids.head), @@ -137,17 +148,20 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], page, Text(text), Token(token.fold(bot.token)(identity)) - )) + ) + ) val params = action.pairs.toMap ++ - Map("action" -> "edit", - "format" -> "json", - "utf8" -> "", - "bot" -> "x", - "assert" -> "user", - "assert" -> "bot") ++ section - .map(s => "section" -> s) - .toSeq ++ summary.map(s => "summary" -> s).toSeq + Map( + "action" -> "edit", + "format" -> "json", + "utf8" -> "", + "bot" -> "x", + "assert" -> "user", + "assert" -> "bot" + ) ++ section + .map(s => "section" -> s) + .toSeq ++ summary.map(s => "summary" -> s).toSeq import scala.concurrent.ExecutionContext.Implicits.global def performEdit(): Future[String] = { @@ -165,10 +179,12 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], retry.Backoff()(odelay.Timer.default)(() => performEdit()) } - override def upload(filename: String, - text: Option[String] = None, - comment: Option[String] = None, - ignoreWarnings: Boolean = false): Future[String] = { + override def upload( + filename: String, + text: Option[String] = None, + comment: Option[String] = None, + ignoreWarnings: Boolean = false + ): Future[String] = { val page = query.right.toOption.fold(filename)(_.head) val token = bot.token val fileContents = Files.readAllBytes(Paths.get(filename)) @@ -191,7 +207,8 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], override def whatTranscludesHere( namespaces: Set[Int], - continueParam: Option[(String, String)]): Future[Iterable[Page]] = { + continueParam: Option[(String, String)] + ): Future[Iterable[Page]] = { val pages = query.fold( ids => EiPageId(ids.head), titles => EiTitle(titles.head) @@ -206,14 +223,16 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], EiNamespace(namespaces.toSeq) ) ) - )) + ) + ) bot.run(action, context) } override def categoryMembers( namespaces: Set[Int], - continueParam: Option[(String, String)]): Future[Iterable[Page]] = { + continueParam: Option[(String, String)] + ): Future[Iterable[Page]] = { val pages = query.fold( ids => CmPageId(ids.head), titles => CmTitle(titles.head) @@ -224,18 +243,22 @@ class PageQueryImplDsl(query: Either[Set[Long], Set[String]], .map(_ => CmTypeSubCat) ++ namespaces.filter(_ == Namespace.FILE).map(_ => CmTypeFile) - val cmParams = Seq(pages, CmLimit("max"), CmNamespace(namespaces.toSeq)) ++ (if (cmTypes.nonEmpty) - Seq(CmType( - cmTypes.toSeq: _*)) - else - Seq.empty) + val cmParams = Seq( + pages, + CmLimit("max"), + CmNamespace(namespaces.toSeq) + ) ++ (if (cmTypes.nonEmpty) + Seq(CmType(cmTypes.toSeq: _*)) + else + Seq.empty) val action = Action( Query( ListParam( CategoryMembers(cmParams: _*) ) - )) + ) + ) bot.run(action, context) } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/query/QueryLibrary.scala b/scalawiki-core/src/main/scala/org/scalawiki/query/QueryLibrary.scala index 7de43ddb..892130e9 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/query/QueryLibrary.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/query/QueryLibrary.scala @@ -16,41 +16,75 @@ import scala.concurrent.Future trait QueryLibrary { - def imagesByGenerator(generator: Generator, withUrl: Boolean = false, withMetadata: Boolean = false, rvSlots: Option[String] = None): Action = { + def imagesByGenerator( + generator: Generator, + withUrl: Boolean = false, + withMetadata: Boolean = false, + rvSlots: Option[String] = None + ): Action = { import org.scalawiki.dto.cmd.query.prop._ val iiProps = Seq(Timestamp, iiprop.User, iiprop.Size) ++ (if (withUrl) Seq(iiprop.Url) else Seq.empty) ++ (if (withMetadata) Seq(iiprop.Metadata) else Seq.empty) - Action(Query( - Prop( - Info(), - Revisions( - Seq(RvProp(rvprop.Ids, rvprop.Content, rvprop.Timestamp, rvprop.User, rvprop.UserId)) ++ rvSlots.map(s => RvSlots(s)).toSeq: _* + Action( + Query( + Prop( + Info(), + Revisions( + Seq( + RvProp( + rvprop.Ids, + rvprop.Content, + rvprop.Timestamp, + rvprop.User, + rvprop.UserId + ) + ) ++ rvSlots.map(s => RvSlots(s)).toSeq: _* + ), + ImageInfo(IiProp(iiProps: _*)) ), - ImageInfo(IiProp(iiProps: _*)) - ), - generator - )) + generator + ) + ) } val allUsersQuery = - Action(Query(ListParam( - AllUsers( - AuProp(Seq("registration", "editcount", "blockinfo")), - AuWithEditsOnly(true), AuLimit("max"), AuExcludeGroup(Seq("bot"))) - ))) + Action( + Query( + ListParam( + AllUsers( + AuProp(Seq("registration", "editcount", "blockinfo")), + AuWithEditsOnly(true), + AuLimit("max"), + AuExcludeGroup(Seq("bot")) + ) + ) + ) + ) val activeUsersQuery = - Action(Query(ListParam( - AllUsers( - AuActiveUsers(true), - AuProp(Seq("registration", "editcount", "blockinfo")), - AuWithEditsOnly(true), AuLimit("max"), AuExcludeGroup(Seq("bot"))) - ))) - - def userContribs(username: String, range: TimeRange, limit: String = "max", dir: String = "older"): Action = { + Action( + Query( + ListParam( + AllUsers( + AuActiveUsers(true), + AuProp(Seq("registration", "editcount", "blockinfo")), + AuWithEditsOnly(true), + AuLimit("max"), + AuExcludeGroup(Seq("bot")) + ) + ) + ) + ) + + def userContribs( + username: String, + range: TimeRange, + limit: String = "max", + dir: String = "older" + ): Action = { val ucParams = Seq( UcUser(Seq(username)), UcLimit(limit), @@ -60,58 +94,77 @@ trait QueryLibrary { Action(Query(ListParam(UserContribs(ucParams: _*)))) } - def userCreatedPages(user: String, range: TimeRange)(implicit bot: ActionBot): Future[(String, Set[String])] = { - bot.run(userContribs(user, range, dir = "newer")).map { - pages => - user -> pages - .filter(p => p.isArticle && p.history.hasPageCreation) - .map(_.title) - .toSet + def userCreatedPages(user: String, range: TimeRange)(implicit + bot: ActionBot + ): Future[(String, Set[String])] = { + bot.run(userContribs(user, range, dir = "newer")).map { pages => + user -> pages + .filter(p => p.isArticle && p.history.hasPageCreation) + .map(_.title) + .toSet } } - def userProps(users: Iterable[String]) = Action(Query( - ListParam(Users( - UsUsers(users), - UsProp(UsEmailable, UsGender) - )) - )) - - def globalUserInfo(username: String) = Action(Query(MetaParam( - GlobalUserInfo( - GuiProp(Merged, Unattached, EditCount), - GuiUser(username) - )))) + def userProps(users: Iterable[String]) = Action( + Query( + ListParam( + Users( + UsUsers(users), + UsProp(UsEmailable, UsGender) + ) + ) + ) + ) + + def globalUserInfo(username: String) = Action( + Query( + MetaParam( + GlobalUserInfo( + GuiProp(Merged, Unattached, EditCount), + GuiUser(username) + ) + ) + ) + ) - def pageLinks(title: String, ns: Int) = Action(Query( - Prop(Links(PlNamespace(Seq(ns)), PlLimit("max"))), - TitlesParam(Seq(title)) - )) + def pageLinks(title: String, ns: Int) = Action( + Query( + Prop(Links(PlNamespace(Seq(ns)), PlLimit("max"))), + TitlesParam(Seq(title)) + ) + ) def pageRevisionsQuery(id: Long): Action = { import org.scalawiki.dto.cmd.query.prop.rvprop._ - Action(Query( - PageIdsParam(Seq(id)), - Prop( - Info(), - Revisions( - RvProp(Content, Ids, Size, User, UserId, rvprop.Timestamp), - RvLimit("max") + Action( + Query( + PageIdsParam(Seq(id)), + Prop( + Info(), + Revisions( + RvProp(Content, Ids, Size, User, UserId, rvprop.Timestamp), + RvLimit("max") + ) ) ) - )) + ) } def pageLinksGenerator(title: String, nsSeq: Seq[Int] = Seq.empty) = - Action(Query( - TitlesParam(Seq(title)), - Generator( - Links(PlNamespace(nsSeq), PlLimit("max")) - ), - Prop(Revisions(RvProp(rvprop.Ids, rvprop.Content))) - )) - - def generatorWithTemplate(template: String, ns: Set[Int] = Set.empty): Generator = { + Action( + Query( + TitlesParam(Seq(title)), + Generator( + Links(PlNamespace(nsSeq), PlLimit("max")) + ), + Prop(Revisions(RvProp(rvprop.Ids, rvprop.Content))) + ) + ) + + def generatorWithTemplate( + template: String, + ns: Set[Int] = Set.empty + ): Generator = { val params = Seq( EiTitle("Template:" + template), EiLimit("500") @@ -121,33 +174,43 @@ trait QueryLibrary { } def categoryMembersGenerator(category: String, ns: Set[Int] = Set.empty) = - Generator(CategoryMembers( - (Seq(CmTitle("Category:" + category)) - ++ (if (ns.nonEmpty) Seq(CmNamespace(ns.toSeq)) else Seq.empty)): _* - )) + Generator( + CategoryMembers( + (Seq(CmTitle("Category:" + category)) + ++ (if (ns.nonEmpty) Seq(CmNamespace(ns.toSeq)) else Seq.empty)): _* + ) + ) def pagesWithTemplate(template: String): Action = pagesByGenerator(generatorWithTemplate(template)) def pagesByGenerator(generator: Generator): Action = { - Action(Query( - Prop( - Info(InProp(SubjectId)), - Revisions() - ), - generator)) + Action( + Query( + Prop( + Info(InProp(SubjectId)), + Revisions() + ), + generator + ) + ) } - def articlesWithTemplate(template: String)(implicit bot: ActionBot): Future[Iterable[Long]] = { + def articlesWithTemplate( + template: String + )(implicit bot: ActionBot): Future[Iterable[Long]] = { bot.run(pagesWithTemplate(template)).map { pages => pages.map(p => p.subjectId.getOrElse(p.id.get)) } } - def pagesToUsers(pages: Iterable[Page]): Iterable[Contributor] = pages.flatMap(_.lastRevisionUser) + def pagesToUsers(pages: Iterable[Page]): Iterable[Contributor] = + pages.flatMap(_.lastRevisionUser) - def getUsers(action: Action)(implicit bot: ActionBot): Future[Iterable[Contributor]] = + def getUsers(action: Action)(implicit + bot: ActionBot + ): Future[Iterable[Contributor]] = bot.run(action).map(pagesToUsers) } -object QueryLibrary extends QueryLibrary \ No newline at end of file +object QueryLibrary extends QueryLibrary diff --git a/scalawiki-core/src/main/scala/org/scalawiki/query/SinglePageQuery.scala b/scalawiki-core/src/main/scala/org/scalawiki/query/SinglePageQuery.scala index 201f3c0e..f1480545 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/query/SinglePageQuery.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/query/SinglePageQuery.scala @@ -8,16 +8,19 @@ trait SinglePageQuery { def whatTranscludesHere( namespaces: Set[Int] = Set.empty, - continueParam: Option[(String, String)] = None): Future[Iterable[Page]] + continueParam: Option[(String, String)] = None + ): Future[Iterable[Page]] def categoryMembers( namespaces: Set[Int] = Set.empty, - continueParam: Option[(String, String)] = None): Future[Iterable[Page]] + continueParam: Option[(String, String)] = None + ): Future[Iterable[Page]] def revisions( namespaces: Set[Int] = Set.empty, props: Set[String] = Set.empty, - continueParam: Option[(String, String)] = None): Future[Iterable[Page]] + continueParam: Option[(String, String)] = None + ): Future[Iterable[Page]] def revisionsByGenerator( generator: String, @@ -26,7 +29,8 @@ trait SinglePageQuery { props: Set[String] = Set.empty, continueParam: Option[(String, String)] = None, limit: String = "max", - titlePrefix: Option[String] = None): Future[Iterable[Page]] + titlePrefix: Option[String] = None + ): Future[Iterable[Page]] def imageInfoByGenerator( generator: String, @@ -36,18 +40,23 @@ trait SinglePageQuery { Set("timestamp", "user", "size", "url" /*, "extmetadata"*/ ), continueParam: Option[(String, String)] = None, limit: String = "max", - titlePrefix: Option[String] = None): Future[Iterable[Page]] - - def edit(text: String, - summary: Option[String] = None, - section: Option[String] = None, - token: Option[String] = None, - multi: Boolean = true): Future[Any] // TODO specific result - - def upload(filename: String, - text: Option[String] = None, - comment: Option[String] = None, - ignoreWarnings: Boolean = true): Future[String] + titlePrefix: Option[String] = None + ): Future[Iterable[Page]] + + def edit( + text: String, + summary: Option[String] = None, + section: Option[String] = None, + token: Option[String] = None, + multi: Boolean = true + ): Future[Any] // TODO specific result + + def upload( + filename: String, + text: Option[String] = None, + comment: Option[String] = None, + ignoreWarnings: Boolean = true + ): Future[String] def withContext(context: Map[String, String]): SinglePageQuery } diff --git a/scalawiki-core/src/main/scala/org/scalawiki/time/TimeRange.scala b/scalawiki-core/src/main/scala/org/scalawiki/time/TimeRange.scala index 4e788171..ec95ecce 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/time/TimeRange.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/time/TimeRange.scala @@ -2,4 +2,4 @@ package org.scalawiki.time import java.time.ZonedDateTime -case class TimeRange(start: Option[ZonedDateTime], end: Option[ZonedDateTime]) \ No newline at end of file +case class TimeRange(start: Option[ZonedDateTime], end: Option[ZonedDateTime]) diff --git a/scalawiki-core/src/main/scala/org/scalawiki/wikitext/SwebleParser.scala b/scalawiki-core/src/main/scala/org/scalawiki/wikitext/SwebleParser.scala index c4792a74..9c2b322a 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/wikitext/SwebleParser.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/wikitext/SwebleParser.scala @@ -31,23 +31,35 @@ trait SwebleParser { node.asScala.view.flatMap(child => findNode(child, pf)).headOption } - def collectNodes[T](node: WtNode, pf: PartialFunction[WtNode, T]): mutable.Buffer[T] = { + def collectNodes[T]( + node: WtNode, + pf: PartialFunction[WtNode, T] + ): mutable.Buffer[T] = { if (pf.isDefinedAt(node)) mutable.Buffer(pf(node)) else node.asScala.flatMap(child => collectNodes(child, pf)) } - def nodesToText[T <: AstNode[WtNode]](node: WtNode, pf: PartialFunction[WtNode, T]): mutable.Buffer[String] = + def nodesToText[T <: AstNode[WtNode]]( + node: WtNode, + pf: PartialFunction[WtNode, T] + ): mutable.Buffer[String] = collectNodes(node, pf).map(c => getText(c.get(1)).trim) def getText(node: WtNode): String = { WtRtDataPrinter.print(node) } - def getTemplateName(template: WtTemplate): String = getText(template.getName).trim + def getTemplateName(template: WtTemplate): String = getText( + template.getName + ).trim - def replace[T <: WtNode](wiki: String, pf: PartialFunction[WtNode, T], mapper: (T => Unit)): String = { + def replace[T <: WtNode]( + wiki: String, + pf: PartialFunction[WtNode, T], + mapper: (T => Unit) + ): String = { val page = parsePage("Some title", wiki).getPage replaceNodeWithText(page, pf, mapper) @@ -55,7 +67,11 @@ trait SwebleParser { getText(page) } - def replaceNodeWithText[T <: WtNode](node: WtNode, pf: PartialFunction[WtNode, T], mapper: (T => Unit)): Unit = { + def replaceNodeWithText[T <: WtNode]( + node: WtNode, + pf: PartialFunction[WtNode, T], + mapper: (T => Unit) + ): Unit = { if (pf.isDefinedAt(node)) mapper(node.asInstanceOf[T]) else diff --git a/scalawiki-core/src/main/scala/org/scalawiki/wikitext/TableParser.scala b/scalawiki-core/src/main/scala/org/scalawiki/wikitext/TableParser.scala index fe97e21c..9878049f 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/wikitext/TableParser.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/wikitext/TableParser.scala @@ -13,8 +13,8 @@ object TableParser extends SwebleParser { case h: WtTableHeader => h } - val cellFunc: PartialFunction[WtNode, WtTableCell] = { - case c: WtTableCell => c + val cellFunc: PartialFunction[WtNode, WtTableCell] = { case c: WtTableCell => + c } val headerOrCell = cellFunc orElse headerFunc @@ -23,24 +23,26 @@ object TableParser extends SwebleParser { val page = parsePage("Some title", wiki).getPage - findNode(page, { case t: WtTableImplicitTableBody => t }).map { - tableBody => + findNode(page, { case t: WtTableImplicitTableBody => t }) + .map { tableBody => val rows = collectNodes(tableBody, { case r: WtTableRow => r }) - val headers = rows.headOption.toSeq.flatMap { - head => nodesToText(head, headerFunc) + val headers = rows.headOption.toSeq.flatMap { head => + nodesToText(head, headerFunc) } val dataRows = if (headers.isEmpty) rows else rows.drop(1) - val items = dataRows.map { - row => nodesToText(row, headerOrCell) - }.filter(_.nonEmpty) + val items = dataRows + .map { row => + nodesToText(row, headerOrCell) + } + .filter(_.nonEmpty) new Table(headers, items, "", "") - }.getOrElse(new Table(Seq.empty, Seq.empty, "", "")) + } + .getOrElse(new Table(Seq.empty, Seq.empty, "", "")) } } - diff --git a/scalawiki-core/src/main/scala/org/scalawiki/wikitext/TemplateParser.scala b/scalawiki-core/src/main/scala/org/scalawiki/wikitext/TemplateParser.scala index 0e36303c..7794be73 100644 --- a/scalawiki-core/src/main/scala/org/scalawiki/wikitext/TemplateParser.scala +++ b/scalawiki-core/src/main/scala/org/scalawiki/wikitext/TemplateParser.scala @@ -16,15 +16,36 @@ object TemplateParser extends SwebleParser { parsePage("Some title", wiki).getPage } - def getTemplate(page: EngPage, templateName: Option[String] = None): Option[Template] = { - findNode(page, { case t: WtTemplate if templateName.forall(getTemplateName(t).equals) => nodeToTemplate(t) }) + def getTemplate( + page: EngPage, + templateName: Option[String] = None + ): Option[Template] = { + findNode( + page, + { + case t: WtTemplate if templateName.forall(getTemplateName(t).equals) => + nodeToTemplate(t) + } + ) } - def collectTemplates(page: EngPage, templateName: String): mutable.Buffer[Template] = { - collectNodes(page, { case t: WtTemplate if getTemplateName(t) == templateName => nodeToTemplate(t) }) + def collectTemplates( + page: EngPage, + templateName: String + ): mutable.Buffer[Template] = { + collectNodes( + page, + { + case t: WtTemplate if getTemplateName(t) == templateName => + nodeToTemplate(t) + } + ) } - def parseOne(wiki: String, templateName: Option[String] = None): Option[Template] = { + def parseOne( + wiki: String, + templateName: Option[String] = None + ): Option[Template] = { getTemplate(parsePage(wiki), templateName) } @@ -33,6 +54,8 @@ object TemplateParser extends SwebleParser { collectTemplates(page, templateName) } - def nodeToTemplate(wtTemplate: WtTemplate): Template = new SwTemplate(wtTemplate).getTemplate + def nodeToTemplate(wtTemplate: WtTemplate): Template = new SwTemplate( + wtTemplate + ).getTemplate } diff --git a/scalawiki-core/src/main/scala/spray/util.scala b/scalawiki-core/src/main/scala/spray/util.scala index a1e37e69..a3bf0afa 100644 --- a/scalawiki-core/src/main/scala/spray/util.scala +++ b/scalawiki-core/src/main/scala/spray/util.scala @@ -6,5 +6,6 @@ import scala.concurrent.Future package object util { - implicit def pimpFuture[T](fut: Future[T]): PimpedFuture[T] = new PimpedFuture[T](fut) + implicit def pimpFuture[T](fut: Future[T]): PimpedFuture[T] = + new PimpedFuture[T](fut) } diff --git a/scalawiki-core/src/main/scala/spray/util/pimps/PimpedFuture.scala b/scalawiki-core/src/main/scala/spray/util/pimps/PimpedFuture.scala index 55d10b6f..93d7b50e 100644 --- a/scalawiki-core/src/main/scala/spray/util/pimps/PimpedFuture.scala +++ b/scalawiki-core/src/main/scala/spray/util/pimps/PimpedFuture.scala @@ -30,4 +30,4 @@ class PimpedFuture[+A](underlying: Future[A]) { def ready(implicit timeout: Timeout = 1.minute): Future[A] = Await.ready(underlying, timeout.duration) -} \ No newline at end of file +} diff --git a/scalawiki-core/src/test/scala/org/scalawiki/DockerSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/DockerSpec.scala index e2bf2d4d..9fc64de5 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/DockerSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/DockerSpec.scala @@ -20,8 +20,17 @@ trait WithDocker extends BeforeAfterAll { "--server http://localhost:8080 --scriptpath=" def checkMysql() = { - Seq("docker", "exec", "scalawiki_database_1", - "mysql", "--user=root", "--password=root_pass", "-s", "-e", "use my_wiki") ! ProcessLogger(_ => (), _ => ()) + Seq( + "docker", + "exec", + "scalawiki_database_1", + "mysql", + "--user=root", + "--password=root_pass", + "-s", + "-e", + "use my_wiki" + ) ! ProcessLogger(_ => (), _ => ()) } override def beforeAll(): Unit = { diff --git a/scalawiki-core/src/test/scala/org/scalawiki/EditSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/EditSpec.scala index ad6cf934..173f7dc9 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/EditSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/EditSpec.scala @@ -6,36 +6,88 @@ import spray.util.pimpFuture class EditSpec extends Specification with MockBotSpec { - val tokenResponse = """{"batchcomplete":"","query":{"tokens":{"csrftoken":"cafebabe+\\"}}}""" - val successResponse = """{"edit":{"result":"Success","pageid":1776370,"title":"pageTitle","contentmodel":"wikitext"}}""" - val errorResponse = """{"edit":{"result":"Error","pageid":1776370,"title":"pageTitle","contentmodel":"wikitext"}}""" + val tokenResponse = + """{"batchcomplete":"","query":{"tokens":{"csrftoken":"cafebabe+\\"}}}""" + val successResponse = + """{"edit":{"result":"Success","pageid":1776370,"title":"pageTitle","contentmodel":"wikitext"}}""" + val errorResponse = + """{"edit":{"result":"Error","pageid":1776370,"title":"pageTitle","contentmodel":"wikitext"}}""" "bot" should { "edit successfully" in { - val siteInfo = TestUtils.resourceAsString("/org/scalawiki/ukwiki_siteinfo.json") + val siteInfo = + TestUtils.resourceAsString("/org/scalawiki/ukwiki_siteinfo.json") val bot: MwBot = getBot( - HttpStub(Map("action" -> "query", "meta" -> "siteinfo", "format" -> "json"), siteInfo), - HttpStub(Map("action" -> "query", "meta" -> "tokens", "format" -> "json"), tokenResponse), - HttpStub(Map("assert" -> "bot", "format" -> "json", "text" -> "pageText", "token" -> "cafebabe+\\", "bot" -> "x", - "title" -> "pageTitle", "action" -> "edit", "summary" -> "editsummary"), successResponse) + HttpStub( + Map("action" -> "query", "meta" -> "siteinfo", "format" -> "json"), + siteInfo + ), + HttpStub( + Map("action" -> "query", "meta" -> "tokens", "format" -> "json"), + tokenResponse + ), + HttpStub( + Map( + "assert" -> "bot", + "format" -> "json", + "text" -> "pageText", + "token" -> "cafebabe+\\", + "bot" -> "x", + "title" -> "pageTitle", + "action" -> "edit", + "summary" -> "editsummary" + ), + successResponse + ) ) - val result = bot.page("pageTitle").edit("pageText", Some("editsummary")).await + val result = + bot.page("pageTitle").edit("pageText", Some("editsummary")).await result === "Success" } "edit retry error once" in { - val siteInfo = TestUtils.resourceAsString("/org/scalawiki/ukwiki_siteinfo.json") + val siteInfo = + TestUtils.resourceAsString("/org/scalawiki/ukwiki_siteinfo.json") val bot: MwBot = getBot( - HttpStub(Map("action" -> "query", "meta" -> "siteinfo", "format" -> "json"), siteInfo), - HttpStub(Map("action" -> "query", "meta" -> "tokens", "format" -> "json"), tokenResponse), - HttpStub(Map("assert" -> "bot", "format" -> "json", "text" -> "pageText", "token"-> "cafebabe+\\", "bot" -> "x", - "title" -> "pageTitle", "action" -> "edit", "summary" -> "editsummary"), errorResponse), - HttpStub(Map("assert" -> "bot", "format" -> "json", "text" -> "pageText", "token"-> "cafebabe+\\", "bot" -> "x", - "title" -> "pageTitle", "action" -> "edit", "summary" -> "editsummary"), successResponse) + HttpStub( + Map("action" -> "query", "meta" -> "siteinfo", "format" -> "json"), + siteInfo + ), + HttpStub( + Map("action" -> "query", "meta" -> "tokens", "format" -> "json"), + tokenResponse + ), + HttpStub( + Map( + "assert" -> "bot", + "format" -> "json", + "text" -> "pageText", + "token" -> "cafebabe+\\", + "bot" -> "x", + "title" -> "pageTitle", + "action" -> "edit", + "summary" -> "editsummary" + ), + errorResponse + ), + HttpStub( + Map( + "assert" -> "bot", + "format" -> "json", + "text" -> "pageText", + "token" -> "cafebabe+\\", + "bot" -> "x", + "title" -> "pageTitle", + "action" -> "edit", + "summary" -> "editsummary" + ), + successResponse + ) ) - val result = bot.page("pageTitle").edit("pageText", Some("editsummary")).await + val result = + bot.page("pageTitle").edit("pageText", Some("editsummary")).await result === "Success" } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/LoginSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/LoginSpec.scala index d0feb5d4..22a6b478 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/LoginSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/LoginSpec.scala @@ -8,7 +8,10 @@ import org.specs2.matcher.ThrownExpectations import org.specs2.mutable.Specification import scala.concurrent.duration._ -class LoginSpec(implicit ee: ExecutionEnv) extends Specification with MockBotSpec with ThrownExpectations { +class LoginSpec(implicit ee: ExecutionEnv) + extends Specification + with MockBotSpec + with ThrownExpectations { type EE = ExecutionEnv @@ -43,13 +46,21 @@ class LoginSpec(implicit ee: ExecutionEnv) extends Specification with MockBotSpe val throttled = result("Throttled") val (user, password) = ("userName", "secret") - val loginAction = Map("action" -> "login", "format" -> "json", "lgname" -> user, "lgpassword" -> password) + val loginAction = Map( + "action" -> "login", + "format" -> "json", + "lgname" -> user, + "lgpassword" -> password + ) "login" should { "get token and login" >> { val bot = getBot( HttpStub(loginAction, needToken), - HttpStub(loginAction ++ Map("lgtoken" -> "token-value+\\"), loginSuccess) + HttpStub( + loginAction ++ Map("lgtoken" -> "token-value+\\"), + loginSuccess + ) ) bot.login(user, password).map(_ === "Success").awaitFor(timeout) @@ -75,18 +86,25 @@ class LoginSpec(implicit ee: ExecutionEnv) extends Specification with MockBotSpe "err503" >> { - val err = TestUtils.resourceAsString("/org/scalawiki/Wikimedia Error.html") + val err = + TestUtils.resourceAsString("/org/scalawiki/Wikimedia Error.html") val bot = getBot( - HttpStub(loginAction, err, contentType = ContentType(MediaTypes.`text/html`, HttpCharsets.`UTF-8`)) + HttpStub( + loginAction, + err, + contentType = + ContentType(MediaTypes.`text/html`, HttpCharsets.`UTF-8`) + ) ) val f = bot.login(user, password) - f.failed.map { - case e: MwException => + f.failed + .map { case e: MwException => e.info must contain("Error: 503, Service Unavailable") - }.awaitFor(timeout) + } + .awaitFor(timeout) } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/MwBotSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/MwBotSpec.scala index 327bdfec..1141941e 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/MwBotSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/MwBotSpec.scala @@ -11,7 +11,13 @@ class MwBotSpec extends Specification with MockBotSpec with Mockito { "return a page text" in { val pageText = "some vandalism" - val bot = getBot(HttpStub(Map("title" -> "PageTitle", "action" -> "raw"), pageText, "/w/index.php")) + val bot = getBot( + HttpStub( + Map("title" -> "PageTitle", "action" -> "raw"), + pageText, + "/w/index.php" + ) + ) bot.pageText("pageTitle").await === pageText } @@ -20,7 +26,13 @@ class MwBotSpec extends Specification with MockBotSpec with Mockito { "get missing page text" should { "return error" in { - val bot = getBot(HttpStub(Map("title" -> "PageTitle", "action" -> "raw"), null, "/w/index.php")) + val bot = getBot( + HttpStub( + Map("title" -> "PageTitle", "action" -> "raw"), + null, + "/w/index.php" + ) + ) bot.pageText("pageTitle").await === "" // TODO error } @@ -37,7 +49,9 @@ class MwBotSpec extends Specification with MockBotSpec with Mockito { } "parse out Debian suffix" in { - fromGenerator("MediaWiki 1.19.20+dfsg-0+deb7u3") === MediaWikiVersion("1.19") + fromGenerator("MediaWiki 1.19.20+dfsg-0+deb7u3") === MediaWikiVersion( + "1.19" + ) } "parse custom to unknow" in { @@ -51,4 +65,4 @@ class MwBotSpec extends Specification with MockBotSpec with Mockito { MediaWikiVersion("1.19") must be < MediaWikiVersion("1.24") } } -} \ No newline at end of file +} diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/ImageSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/ImageSpec.scala index c9255856..4e08598b 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/ImageSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/ImageSpec.scala @@ -6,7 +6,8 @@ import org.specs2.mutable.Specification class ImageSpec extends Specification { def makeTemplate(author: String, description: String = "") = - new Template("Information", + new Template( + "Information", Map( "description" -> description, "date" -> "", @@ -29,7 +30,9 @@ class ImageSpec extends Specification { } "get author from link" in { - val wiki = makeTemplate("[http://wikimapia.org/#show=/user/2246978/ Alexander Savitsky]") + val wiki = makeTemplate( + "[http://wikimapia.org/#show=/user/2246978/ Alexander Savitsky]" + ) Image.getAuthorFromPage(wiki) === "Alexander Savitsky" } @@ -37,9 +40,13 @@ class ImageSpec extends Specification { "fromPageRevision" should { "parse author and id" in { - val wiki = makeTemplate("[[User:PhotoMaster|PhotoMaster]]", "{{Monument|nature-park-id}}") + val wiki = makeTemplate( + "[[User:PhotoMaster|PhotoMaster]]", + "{{Monument|nature-park-id}}" + ) - val page = Page("File:Image.jpg").copy(revisions = Seq(Revision.one(wiki))) + val page = + Page("File:Image.jpg").copy(revisions = Seq(Revision.one(wiki))) val image = Image.fromPageRevision(page, Some("Monument")).get image.author === Some("PhotoMaster") @@ -47,9 +54,13 @@ class ImageSpec extends Specification { } "parse two monuments" in { - val wiki = makeTemplate("[[User:PhotoMaster|PhotoMaster]]", "{{Monument|id1}}{{Monument|id2}}") + val wiki = makeTemplate( + "[[User:PhotoMaster|PhotoMaster]]", + "{{Monument|id1}}{{Monument|id2}}" + ) - val page = Page("File:Image.jpg").copy(revisions = Seq(Revision.one(wiki))) + val page = + Page("File:Image.jpg").copy(revisions = Seq(Revision.one(wiki))) val image = Image.fromPageRevision(page, Some("Monument")).get image.author === Some("PhotoMaster") @@ -59,26 +70,37 @@ class ImageSpec extends Specification { "parse categories" in { - val page = Page("File:Image.jpg").copy(revisions = Seq(Revision.one(withCategories))) + val page = Page("File:Image.jpg").copy(revisions = + Seq(Revision.one(withCategories)) + ) val image = Image.fromPageRevision(page, Some("Monument")).get - image.categories === Set("Wiki loves monuments in Ukraine 2018 - Quality", + image.categories === Set( + "Wiki loves monuments in Ukraine 2018 - Quality", "Uploaded via Campaign:wlm-ua", "Obviously ineligible submissions for WLM 2018 in Ukraine", - "Saint Nicholas Church on Water") + "Saint Nicholas Church on Water" + ) } } "parse special nomination template" in { - val page1 = Page("File:Image.jpg").copy(revisions = Seq(Revision.one(withSpecialNominations))) - val image1 = Image.fromPageRevision(page1, Some("Monument"), Seq("WLM2021-UA-Aero")).get + val page1 = Page("File:Image.jpg").copy(revisions = + Seq(Revision.one(withSpecialNominations)) + ) + val image1 = Image + .fromPageRevision(page1, Some("Monument"), Seq("WLM2021-UA-Aero")) + .get image1.specialNominations === Set("WLM2021-UA-Aero") - val page2 = Page("File:Image.jpg").copy(revisions = Seq(Revision.one(withCategories))) - val image2 = Image.fromPageRevision(page2, Some("Monument"), Seq("WLM2021-UA-Aero")).get + val page2 = + Page("File:Image.jpg").copy(revisions = Seq(Revision.one(withCategories))) + val image2 = Image + .fromPageRevision(page2, Some("Monument"), Seq("WLM2021-UA-Aero")) + .get image2.specialNominations === Set.empty } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/SiteSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/SiteSpec.scala index c4f5f2b9..eee12d55 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/SiteSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/SiteSpec.scala @@ -6,21 +6,39 @@ class SiteSpec extends Specification { "host" should { "wikipedia" in { - Site.host("en.wikipedia.org") === Site(Some("en"), "wikipedia", "en.wikipedia.org", "https", None, "/w") + Site.host("en.wikipedia.org") === Site( + Some("en"), + "wikipedia", + "en.wikipedia.org", + "https", + None, + "/w" + ) } "wikimedia" in { - Site.host("commons.wikimedia.org") === Site(None, "commons", "commons.wikimedia.org", "https", None, "/w") + Site.host("commons.wikimedia.org") === Site( + None, + "commons", + "commons.wikimedia.org", + "https", + None, + "/w" + ) } } "pageUrl" should { "commons page url" in { - Site.commons.pageUrl("File:Image.jpg") === "https://commons.wikimedia.org/wiki/File:Image.jpg" + Site.commons.pageUrl( + "File:Image.jpg" + ) === "https://commons.wikimedia.org/wiki/File:Image.jpg" } "localhost page url" in { - Site.localhost.pageUrl("File:Image.jpg") === "http://localhost/w/index.php/File:Image.jpg" + Site.localhost.pageUrl( + "File:Image.jpg" + ) === "http://localhost/w/index.php/File:Image.jpg" } } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/DslSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/DslSpec.scala index c0e021a2..91fce1e4 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/DslSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/DslSpec.scala @@ -1,12 +1,17 @@ package org.scalawiki.dto.cmd -import org.scalawiki.dto.cmd.query.list.{CmTitle, CategoryMembers, EiTitle, EmbeddedIn} +import org.scalawiki.dto.cmd.query.list.{ + CmTitle, + CategoryMembers, + EiTitle, + EmbeddedIn +} import org.scalawiki.dto.cmd.query.{Generator, Query} import org.scalawiki.dto.cmd.query.prop._ import org.specs2.matcher.ThrownMessages import org.specs2.mutable.Specification -class DslSpec extends Specification with ThrownMessages { +class DslSpec extends Specification with ThrownMessages { "1" should { "23" in { @@ -19,8 +24,11 @@ class DslSpec extends Specification with ThrownMessages { ) ) - // action.flatten.map(_.name).toSet === Set("action", "prop", "inprop") - action.query.toSeq.flatMap(_.props).map(_.name).toSet === Set("info", "revisions") + // action.flatten.map(_.name).toSet === Set("action", "prop", "inprop") + action.query.toSeq.flatMap(_.props).map(_.name).toSet === Set( + "info", + "revisions" + ) } } @@ -38,19 +46,22 @@ class DslSpec extends Specification with ThrownMessages { action.pairs.toMap === Map( "action" -> "query", "prop" -> "info|revisions", - "inprop" -> "subjectid") + "inprop" -> "subjectid" + ) } } "2" should { "34" in { - val action = Action(Query( + val action = Action( + Query( Prop( Info(InProp(SubjectId)), Revisions() ), Generator(EmbeddedIn(EiTitle("Template:Name"))) - )) + ) + ) action.pairs.toMap === Map( "action" -> "query", diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/query/QuerySpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/query/QuerySpec.scala index c6fde0f1..8e8204f9 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/query/QuerySpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/query/QuerySpec.scala @@ -5,7 +5,6 @@ import org.scalawiki.dto.cmd.query.prop.rvprop.{Content, Ids, RvProp} import org.scalawiki.dto.cmd.query.prop.{Info, Prop, Revisions} import org.specs2.mutable.Specification - class QuerySpec extends Specification { "revision" should { @@ -26,7 +25,8 @@ class QuerySpec extends Specification { Prop( Info(), Revisions(RvProp(Ids, Content)) - )) + ) + ) query.props === Seq(Info(), Revisions(RvProp(Ids, Content))) val revisions = query.revisions.get @@ -65,4 +65,4 @@ class QuerySpec extends Specification { ) } } -} \ No newline at end of file +} diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/query/list/CategoryMembersSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/query/list/CategoryMembersSpec.scala index 1624ad3b..f5137c9b 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/query/list/CategoryMembersSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/cmd/query/list/CategoryMembersSpec.scala @@ -12,12 +12,14 @@ class CategoryMembersSpec extends Specification { ) def categoryMembers(args: CmParam[Any]*): Map[String, String] = { - Action(Query(ListParam(CategoryMembers(args:_*)))).pairs.toMap + Action(Query(ListParam(CategoryMembers(args: _*)))).pairs.toMap } "get Map" should { "get title" in { - categoryMembers(CmTitle("Category:Physics")) === baseParams + ("cmtitle" -> "Category:Physics") + categoryMembers( + CmTitle("Category:Physics") + ) === baseParams + ("cmtitle" -> "Category:Physics") } "get pageid" in { @@ -25,12 +27,19 @@ class CategoryMembersSpec extends Specification { } "get props" in { - categoryMembers(CmPageId(1234L), CmProp(Ids, Title, cmprop.Timestamp)) === baseParams + + categoryMembers( + CmPageId(1234L), + CmProp(Ids, Title, cmprop.Timestamp) + ) === baseParams + ("cmpageid" -> "1234", "cmprop" -> "ids|title|timestamp") } "get sort" in { - categoryMembers(CmPageId(1234L), CmSort(cmsort.Timestamp), CmDir(Asc)) === baseParams + + categoryMembers( + CmPageId(1234L), + CmSort(cmsort.Timestamp), + CmDir(Asc) + ) === baseParams + ("cmpageid" -> "1234", "cmsort" -> "timestamp", "cmdir" -> "asc") } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/history/RevisionFilterSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/history/RevisionFilterSpec.scala index 9ad7d0f8..723c3a25 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/history/RevisionFilterSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/history/RevisionFilterSpec.scala @@ -9,7 +9,7 @@ import jp.ne.opt.chronoscala.Imports._ class RevisionFilterSpec extends Specification { - "revision filter by date" should { + "revision filter by date" should { val now: ZonedDateTime = ZonedDateTime.now @@ -19,7 +19,7 @@ class RevisionFilterSpec extends Specification { val rf = new RevisionFilterDateAndUser(from = Some(now - 1.month)) - val filtered = rf(Seq(r1,r2)) + val filtered = rf(Seq(r1, r2)) filtered.size === 1 filtered.head.revId === Some(2) @@ -31,7 +31,7 @@ class RevisionFilterSpec extends Specification { val rf = new RevisionFilterDateAndUser(to = Some(now - 1.month)) - val filtered = rf(Seq(r1,r2)) + val filtered = rf(Seq(r1, r2)) filtered.size === 1 filtered.head.revId === Some(1) @@ -43,9 +43,12 @@ class RevisionFilterSpec extends Specification { val r3 = Revision(3, 1).withTimeStamp(now - 1.months) val r4 = Revision(4, 1).withTimeStamp(now) - val rf = new RevisionFilterDateAndUser(from = Some(now - 2.month), to = Some(now - 1.month)) + val rf = new RevisionFilterDateAndUser( + from = Some(now - 2.month), + to = Some(now - 1.month) + ) - val filtered = rf(Seq(r1,r2, r3, r4)) + val filtered = rf(Seq(r1, r2, r3, r4)) filtered.size === 2 filtered.map(_.revId) === Seq(2, 3).map(Option.apply) diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryHtmlSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryHtmlSpec.scala index 2554024b..e25a2aaf 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryHtmlSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryHtmlSpec.scala @@ -11,9 +11,8 @@ class GalleryHtmlSpec extends Specification { "be without descriptions" in { val title = "1.jpg" - val images = Seq(Image(s"File:$title", - url = Some(s"http://domain/path/$title") - )) + val images = + Seq(Image(s"File:$title", url = Some(s"http://domain/path/$title"))) Gallery.asHtml(images) must haveSameLinesAs( s"""|""".stripMargin) + |""".stripMargin + ) } "be with descriptions" in { val title = "1.jpg" val description = title + " description" - val images = Seq(Image(s"File:$title", - url = Some(s"http://domain/path/$title") - )) + val images = + Seq(Image(s"File:$title", url = Some(s"http://domain/path/$title"))) Gallery.asHtml(images, Seq(description)) must haveSameLinesAs( s"""|""".stripMargin) + |""".stripMargin + ) } } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryThumbSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryThumbSpec.scala index e0a6d68c..9cd5d7bc 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryThumbSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryThumbSpec.scala @@ -8,66 +8,134 @@ class GalleryThumbSpec extends Specification { "thumbUrl" should { "preserve url" in { val url = "https://host/image.jpg" - val image = new Image("image.jpg", url = Some(url), width = Some(1024), height = Some(768)) + val image = new Image( + "image.jpg", + url = Some(url), + width = Some(1024), + height = Some(768) + ) Gallery.thumbUrl(image, 800, 600, false) === url } "preserve large image url" in { val url = "https://host/image.jpg" - val image = new Image("image.jpg", url = Some(url), width = Some(1024), height = Some(768)) + val image = new Image( + "image.jpg", + url = Some(url), + width = Some(1024), + height = Some(768) + ) Gallery.thumbUrl(image, 1024, 768, true) === url } - "resize url" in { val url = "https://host/image.jpg" - val image = new Image("image.jpg", url = Some(url), width = Some(1024), height = Some(768)) - - Gallery.thumbUrl(image, 800, 600, true) === "https://host/image.jpg/800px-image.jpg" + val image = new Image( + "image.jpg", + url = Some(url), + width = Some(1024), + height = Some(768) + ) + + Gallery.thumbUrl( + image, + 800, + 600, + true + ) === "https://host/image.jpg/800px-image.jpg" } "resize long url" in { val title = Seq.fill(162)("a").mkString + ".jpg" val url = s"https://host/$title" - val image = new Image(title, url = Some(url), width = Some(1024), height = Some(768)) - - Gallery.thumbUrl(image, 800, 600, true) === s"https://host/$title/800px-thumbnail.jpg" + val image = new Image( + title, + url = Some(url), + width = Some(1024), + height = Some(768) + ) + + Gallery.thumbUrl( + image, + 800, + 600, + true + ) === s"https://host/$title/800px-thumbnail.jpg" } "resize sublong url" in { val title = Seq.fill(161)("a").mkString + ".jpg" val url = s"https://host/$title" - val image = new Image(title, url = Some(url), width = Some(1024), height = Some(768)) - - Gallery.thumbUrl(image, 800, 600, true) === s"https://host/$title/800px-$title" + val image = new Image( + title, + url = Some(url), + width = Some(1024), + height = Some(768) + ) + + Gallery.thumbUrl( + image, + 800, + 600, + true + ) === s"https://host/$title/800px-$title" } "resize utf-8 url" in { val title = Seq.fill(161)("я").mkString + ".jpg" val url = s"https://host/$title" - val image = new Image(title, url = Some(url), width = Some(1024), height = Some(768)) - - Gallery.thumbUrl(image, 800, 600, true) === s"https://host/$title/800px-thumbnail.jpg" + val image = new Image( + title, + url = Some(url), + width = Some(1024), + height = Some(768) + ) + + Gallery.thumbUrl( + image, + 800, + 600, + true + ) === s"https://host/$title/800px-thumbnail.jpg" } "resize pdf url" in { val title = "document.pdf" val url = s"https://host/$title" - val image = new Image(title, url = Some(url), width = Some(1024), height = Some(768)) - - Gallery.thumbUrl(image, 800, 600, true) === s"https://host/$title/page1-800px-$title.jpg" + val image = new Image( + title, + url = Some(url), + width = Some(1024), + height = Some(768) + ) + + Gallery.thumbUrl( + image, + 800, + 600, + true + ) === s"https://host/$title/page1-800px-$title.jpg" } "resize tiff url" in { val title = "image.tif" val url = s"https://host/$title" - val image = new Image(title, url = Some(url), width = Some(1024), height = Some(768)) - - Gallery.thumbUrl(image, 800, 600, true) === s"https://host/$title/lossy-page1-800px-$title.jpg" + val image = new Image( + title, + url = Some(url), + width = Some(1024), + height = Some(768) + ) + + Gallery.thumbUrl( + image, + 800, + 600, + true + ) === s"https://host/$title/lossy-page1-800px-$title.jpg" } } } - diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryWikiSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryWikiSpec.scala index a44b1ba1..4c5ea80d 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryWikiSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/GalleryWikiSpec.scala @@ -12,8 +12,7 @@ class GalleryWikiSpec extends Specification { "be without descriptions" in { val images = (1 to 3).map(i => s"File:$i.jpg") - Image.gallery(images) must haveSameLinesAs( - """ + Image.gallery(images) must haveSameLinesAs(""" |File:1.jpg |File:2.jpg |File:3.jpg @@ -24,8 +23,7 @@ class GalleryWikiSpec extends Specification { val images = (1 to 3).map(i => s"File:$i.jpg") val descriptions = (1 to 3).map("Description " + _) - Image.gallery(images, descriptions) must haveSameLinesAs( - """ + Image.gallery(images, descriptions) must haveSameLinesAs(""" |File:1.jpg | Description 1 |File:2.jpg | Description 2 |File:3.jpg | Description 3 @@ -36,8 +34,7 @@ class GalleryWikiSpec extends Specification { val images = (1 to 3).map(_ + ".jpg") val descriptions = (1 to 3).map("Description " + _) - Image.gallery(images, descriptions) must haveSameLinesAs( - """ + Image.gallery(images, descriptions) must haveSameLinesAs(""" |File:1.jpg | Description 1 |File:2.jpg | Description 2 |File:3.jpg | Description 3 diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/LineUtil.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/LineUtil.scala index ed38b14d..45d67125 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/LineUtil.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/LineUtil.scala @@ -5,8 +5,8 @@ import org.specs2.text.LinesContent object LineUtil { implicit def linesForString: LinesContent[String] = new LinesContent[String] { - def name (s: String) = "lines of text" - def lines (s: String): Seq[String] = s.split("\r?\n") + def name(s: String) = "lines of text" + def lines(s: String): Seq[String] = s.split("\r?\n") } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/TableHtmlSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/TableHtmlSpec.scala index ee994ff1..b29fabf8 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/TableHtmlSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/TableHtmlSpec.scala @@ -24,8 +24,7 @@ class TableHtmlSpec extends Specification { "make table with 1 column header" in { val table = new Table(Seq("header1"), Seq.empty, "", "") - table.asHtml must haveSameLinesAs( - """ + table.asHtml must haveSameLinesAs("""
    | | | @@ -36,8 +35,7 @@ class TableHtmlSpec extends Specification { "make table with 2 columns header" in { val table = new Table(Seq("header1", "header2"), Seq.empty, "", "") - table.asHtml must haveSameLinesAs( - """
    header1
    + table.asHtml must haveSameLinesAs("""
    | | | @@ -48,9 +46,9 @@ class TableHtmlSpec extends Specification { } "make table with 3 columns headers" in { - val table = new Table(Seq("header1", "header2", "header3"), Seq.empty, "", "") - table.asHtml must haveSameLinesAs( - """
    header1
    + val table = + new Table(Seq("header1", "header2", "header3"), Seq.empty, "", "") + table.asHtml must haveSameLinesAs("""
    | | | @@ -63,8 +61,7 @@ class TableHtmlSpec extends Specification { "make table with 1 data column" in { val table = new Table(Seq.empty, Seq(Seq("data11")), "", "") - table.asHtml must haveSameLinesAs( - """
    header1
    + table.asHtml must haveSameLinesAs("""
    | | | @@ -75,8 +72,7 @@ class TableHtmlSpec extends Specification { "make table with 2 data columns" in { val table = new Table(Seq.empty, Seq(Seq("data11", "data12")), "", "") - table.asHtml must haveSameLinesAs( - """
    data11
    + table.asHtml must haveSameLinesAs("""
    | | | @@ -87,9 +83,9 @@ class TableHtmlSpec extends Specification { } "make table with 1 data column and 2 rows" in { - val table = new Table(Seq.empty, Seq(Seq("data11"), Seq("data21")), "", "") - table.asHtml must haveSameLinesAs( - """
    data11
    + val table = + new Table(Seq.empty, Seq(Seq("data11"), Seq("data21")), "", "") + table.asHtml must haveSameLinesAs("""
    | | | @@ -103,16 +99,14 @@ class TableHtmlSpec extends Specification { "make table with css class and title" in { val table = new Table(Seq.empty, Seq.empty, "title", "wikitable") - table.asHtml must haveSameLinesAs( - """
    data11
    class='wikitable' + table.asHtml must haveSameLinesAs("""
    class='wikitable' | |
    title
    """.stripMargin) } "make table with css class and 1 column header" in { val table = new Table(Seq("header1"), Seq.empty, "", "wikitable") - table.asHtml must haveSameLinesAs( - """ class='wikitable' + table.asHtml must haveSameLinesAs("""
    class='wikitable' | | | @@ -123,8 +117,7 @@ class TableHtmlSpec extends Specification { "make table with css class and 1 data column" in { val table = new Table(Seq.empty, Seq(Seq("data11")), "", "wikitable") - table.asHtml must haveSameLinesAs( - """
    header1
    class='wikitable' + table.asHtml must haveSameLinesAs("""
    class='wikitable' | | | @@ -135,8 +128,7 @@ class TableHtmlSpec extends Specification { "make table with title and 1 column header" in { val table = new Table(Seq("header1"), Seq.empty, "title", "") - table.asHtml must haveSameLinesAs( - """
    data11
    + table.asHtml must haveSameLinesAs("""
    | | | @@ -148,8 +140,7 @@ class TableHtmlSpec extends Specification { "make table with title and 1 data column" in { val table = new Table(Seq.empty, Seq(Seq("data11")), "title", "") - table.asHtml must haveSameLinesAs( - """
    title
    + table.asHtml must haveSameLinesAs("""
    | | | @@ -161,8 +152,7 @@ class TableHtmlSpec extends Specification { "make table with 1 column header and 1 data row" in { val table = new Table(Seq("header1"), Seq(Seq("data11")), "", "") - table.asHtml must haveSameLinesAs( - """
    title
    + table.asHtml must haveSameLinesAs("""
    | | | diff --git a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/TableWikiSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/TableWikiSpec.scala index f5a8b898..9b22af97 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/TableWikiSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/dto/markup/TableWikiSpec.scala @@ -31,7 +31,8 @@ class TableWikiSpec extends Specification { } "make table with 3 columns headers" in { - val table = new Table(Seq("header1", "header2", "header3"), Seq.empty, "", "") + val table = + new Table(Seq("header1", "header2", "header3"), Seq.empty, "", "") table.asWiki === "{|\n! header1 !! header2 !! header3\n|}" } @@ -46,7 +47,8 @@ class TableWikiSpec extends Specification { } "make table with 1 data column and 2 rows" in { - val table = new Table(Seq.empty, Seq(Seq("data11"), Seq("data21")), "", "") + val table = + new Table(Seq.empty, Seq(Seq("data11"), Seq("data21")), "", "") table.asWiki === "{|\n|-\n| data11\n|-\n| data21\n|}" } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/json/ParserSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/json/ParserSpec.scala index 6af0b501..1631fd07 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/json/ParserSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/json/ParserSpec.scala @@ -38,7 +38,8 @@ class ParserSpec extends Specification { } "New Multipage query" should { - val queryContinueStr = s"""{$queryStr, "continue": {"continue": "-||", "eicontinue": "qcValue"}}""" + val queryContinueStr = + s"""{$queryStr, "continue": {"continue": "-||", "eicontinue": "qcValue"}}""" val parser = new Parser(emptyAction) val pagesOpt = parser.parse(queryContinueStr).toOption @@ -53,7 +54,8 @@ class ParserSpec extends Specification { "Legacy Multipage query" should { // TODO query-continue is not supported, but we should report it then - val queryContinueStr = s"""{"query-continue": {"$queryType": {"$queryContinue": "qcValue" }}, $queryStr}""" + val queryContinueStr = + s"""{"query-continue": {"$queryType": {"$queryContinue": "qcValue" }}, $queryStr}""" val parser = new Parser(emptyAction) val pagesOpt = parser.parse(queryContinueStr).toOption @@ -68,10 +70,11 @@ class ParserSpec extends Specification { } def checkPages(pagesOpt: Option[Seq[Page]]): MatchResult[Any] = { - pagesOpt.map { pages => - pages must have size 1 - checkPage(pages(0)) - } + pagesOpt + .map { pages => + pages must have size 1 + checkPage(pages(0)) + } .getOrElse(pagesOpt must beSome) } @@ -89,12 +92,14 @@ class ParserSpec extends Specification { "parse page with lang links" in { val s = resourceAsString("/org/scalawiki/query/langLinks.json") - val action = Action(Query( - Prop( - LangLinks(LlLimit("max")) - ), - TitlesParam(Seq("Article")) - )) + val action = Action( + Query( + Prop( + LangLinks(LlLimit("max")) + ), + TitlesParam(Seq("Article")) + ) + ) val parser = new Parser(action) val page = parser.parse(s).get.head @@ -108,17 +113,21 @@ class ParserSpec extends Specification { "fr" -> "Article", "it" -> "Articolo", "nl" -> "Artikel", - "pt" -> "Artigo") + "pt" -> "Artigo" + ) } "parse mwerror" in { - val action = Action(Query(Prop(Revisions()), - TitlesParam(Seq("PageTitle")), - PageIdsParam(Seq(123456)) - )) + val action = Action( + Query( + Prop(Revisions()), + TitlesParam(Seq("PageTitle")), + PageIdsParam(Seq(123456)) + ) + ) val json = - """{ + """{ "servedby": "mw1202", "error": { "code": "multisource", @@ -131,7 +140,7 @@ class ParserSpec extends Specification { val result = parser.parse(json) result.isFailure === true - val mw = result match { case Failure(mw:MwException) => mw } + val mw = result match { case Failure(mw: MwException) => mw } mw.code === "multisource" mw.info === "Cannot use 'pageids' at the same time as 'titles'" } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/json/WikiReadsSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/json/WikiReadsSpec.scala index 26347b27..2c1ccfce 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/json/WikiReadsSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/json/WikiReadsSpec.scala @@ -6,8 +6,7 @@ import org.scalawiki.dto._ import org.specs2.mutable.Specification import play.api.libs.json._ -/** - * Created by francisco on 03/11/16. +/** Created by francisco on 03/11/16. */ class WikiReadsSpec extends Specification { @@ -44,7 +43,9 @@ class WikiReadsSpec extends Specification { revisionObject.revId === Some(23) revisionObject.parentId === Some(5) revisionObject.user.get.name === Some("wikiUser") - revisionObject.timestamp === Some(ZonedDateTime.parse("2016-11-04T08:20:40Z")) + revisionObject.timestamp === Some( + ZonedDateTime.parse("2016-11-04T08:20:40Z") + ) } val globalUserInfoRead: WikiReads[GlobalUserInfo] = GlobalUserInfoReads() @@ -63,14 +64,18 @@ class WikiReadsSpec extends Specification { "registration":"2016-11-07T19:27:40Z", "editcount":5}""" "transform to GlobalUserInfo object" in { - val globalUserInfo = globalUserInfoRead.reads(Json.parse(globalUserInfoJson)).get + val globalUserInfo = + globalUserInfoRead.reads(Json.parse(globalUserInfoJson)).get globalUserInfo.name === "name" globalUserInfo.merged(0).method === "post" - globalUserInfo.merged(0).registration === ZonedDateTime.parse("2016-11-07T19:30:40Z") + globalUserInfo.merged(0).registration === ZonedDateTime.parse( + "2016-11-07T19:30:40Z" + ) } val imageRead: WikiReads[Image] = ImageReads() - val imageJson: String = """{"user":"user", "size":50, "width": 60, "height":60,"title":"page title"}""" + val imageJson: String = + """{"user":"user", "size":50, "width": 60, "height":60,"title":"page title"}""" "transform to Image without pageId either title to object" in { val image: Image = imageRead.reads(Json.parse(imageJson)).get image.author === None @@ -79,9 +84,11 @@ class WikiReadsSpec extends Specification { image.height === Some(60) } - val imageReadWithParameters: WikiReads[Image] = ImageReads(Some(1), Some("title")) + val imageReadWithParameters: WikiReads[Image] = + ImageReads(Some(1), Some("title")) "transform to Image with pageId and title to object" in { - val image: Image = imageReadWithParameters.reads(Json.parse(imageJson)).get + val image: Image = + imageReadWithParameters.reads(Json.parse(imageJson)).get image.author === None image.title === "title" image.pageId === Some(1) @@ -89,9 +96,11 @@ class WikiReadsSpec extends Specification { } val categoryInfoRead: WikiReads[CategoryInfo] = CategoryInfoReads() - val categoryInfoJson = """{"size": 45, "pages": 5, "files": 7, "subcats": 3}""" + val categoryInfoJson = + """{"size": 45, "pages": 5, "files": 7, "subcats": 3}""" "transform to CategoryInfo to object" in { - val categoryInfo: CategoryInfo = categoryInfoRead.reads(Json.parse(categoryInfoJson)).get + val categoryInfo: CategoryInfo = + categoryInfoRead.reads(Json.parse(categoryInfoJson)).get categoryInfo.pages === 5 categoryInfo.files === 7 categoryInfo.subCats === 3 @@ -109,7 +118,8 @@ class WikiReadsSpec extends Specification { "timestamp":"2016-11-09T07:02:23Z"}""" "transform to UserContrib to object" in { - val userContrib: UserContrib = userContributorRead.reads(Json.parse(userContribJson)).get + val userContrib: UserContrib = + userContributorRead.reads(Json.parse(userContribJson)).get userContrib.userId === 1 userContrib.pageId === 34 userContrib.revId === 5 diff --git a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/BaseMockServerSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/BaseMockServerSpec.scala index 8c018763..2b1dfa38 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/BaseMockServerSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/BaseMockServerSpec.scala @@ -31,33 +31,52 @@ class BaseMockServerSpec extends Specification with StubServer { def await[T](future: Future[T]): T = Await.result(future, http.timeout) def stubResponse(path: String, code: Int, body: String) = { - mockServer.when(request() - .withMethod("POST") - .withPath(path) - ).respond(response() - .withStatusCode(code) - .withHeaders( - header(CONTENT_TYPE, MediaType.JSON_UTF_8.toString) - ).withBody(body) - ) + mockServer + .when( + request() + .withMethod("POST") + .withPath(path) + ) + .respond( + response() + .withStatusCode(code) + .withHeaders( + header(CONTENT_TYPE, MediaType.JSON_UTF_8.toString) + ) + .withBody(body) + ) } - def stubOk(parameters: Map[String, String], body: String): Unit = stubResponse(parameters, 200, body) + def stubOk(parameters: Map[String, String], body: String): Unit = + stubResponse(parameters, 200, body) - def stubResponse(parameters: Map[String, String], code: Int, body: String): Unit = { - mockServer.when(request() - .withMethod("POST") - .withPath(apiUrl) - .withBody( - params( - parameters.map { case (name, value) => new Parameter(name, value) }.toBuffer.asJava) + def stubResponse( + parameters: Map[String, String], + code: Int, + body: String + ): Unit = { + mockServer + .when( + request() + .withMethod("POST") + .withPath(apiUrl) + .withBody( + params( + parameters + .map { case (name, value) => new Parameter(name, value) } + .toBuffer + .asJava + ) + ) + ) + .respond( + response() + .withStatusCode(code) + .withHeaders( + header(CONTENT_TYPE, MediaType.JSON_UTF_8.toString) + ) + .withBody(body) ) - ).respond(response() - .withStatusCode(code) - .withHeaders( - header(CONTENT_TYPE, MediaType.JSON_UTF_8.toString) - ).withBody(body) - ) } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/ImagesEmbeddedInMockServerSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/ImagesEmbeddedInMockServerSpec.scala index f7ecc581..960a7d0c 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/ImagesEmbeddedInMockServerSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/ImagesEmbeddedInMockServerSpec.scala @@ -4,11 +4,14 @@ import org.scalawiki.query.QueryLibrary import org.scalawiki.util.TestUtils.resourceAsString import spray.util.pimpFuture -class ImagesEmbeddedInMockServerSpec extends BaseMockServerSpec with QueryLibrary { +class ImagesEmbeddedInMockServerSpec + extends BaseMockServerSpec + with QueryLibrary { "images" should { "successfully get images" in { - val response = resourceAsString("/org/scalawiki/query/embeddedinIiPropRvProp.json") + val response = + resourceAsString("/org/scalawiki/query/embeddedinIiPropRvProp.json") val action = Map( "action" -> "query", @@ -24,16 +27,22 @@ class ImagesEmbeddedInMockServerSpec extends BaseMockServerSpec with QueryLibrar val bot = getBot - val future = bot.run(imagesByGenerator(generatorWithTemplate("UkrainianNaturalHeritageSite"))) + val future = bot.run( + imagesByGenerator(generatorWithTemplate("UkrainianNaturalHeritageSite")) + ) val info = future.await info should not(beEmpty) info.size === 50 - info.map(_.text.exists(_.contains("UkrainianNaturalHeritageSite"))) === List.fill(50)(true) + info.map( + _.text.exists(_.contains("UkrainianNaturalHeritageSite")) + ) === List.fill(50)(true) } "successfully get images new rvslots format" in { - val response = resourceAsString("/org/scalawiki/query/embeddedinIiPropRvPropRvSlots.json") + val response = resourceAsString( + "/org/scalawiki/query/embeddedinIiPropRvPropRvSlots.json" + ) val action = Map( "action" -> "query", "geititle" -> "Template:UkrainianNaturalHeritageSite", @@ -49,12 +58,19 @@ class ImagesEmbeddedInMockServerSpec extends BaseMockServerSpec with QueryLibrar val bot = getBot - val future = bot.run(imagesByGenerator(generatorWithTemplate("UkrainianNaturalHeritageSite"), rvSlots = Some("main"))) + val future = bot.run( + imagesByGenerator( + generatorWithTemplate("UkrainianNaturalHeritageSite"), + rvSlots = Some("main") + ) + ) val info = future.await info should not(beEmpty) info.size === 50 - info.map(_.text.exists(_.contains("UkrainianNaturalHeritageSite"))) === List.fill(50)(true) + info.map( + _.text.exists(_.contains("UkrainianNaturalHeritageSite")) + ) === List.fill(50)(true) info.map(_.images.nonEmpty) === List.fill(50)(true) } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/ImagesMockServerSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/ImagesMockServerSpec.scala index 7e4ab696..1fad1478 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/ImagesMockServerSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/ImagesMockServerSpec.scala @@ -7,7 +7,9 @@ class ImagesMockServerSpec extends BaseMockServerSpec { "images" should { "succesfully get image infos" in { - val response = resourceAsString("/org/scalawiki/query/Commons_Wiki_Loves_Earth_2017_Winners_imageInfos.json") + val response = resourceAsString( + "/org/scalawiki/query/Commons_Wiki_Loves_Earth_2017_Winners_imageInfos.json" + ) val action = Map( "action" -> "query", "titles" -> "Commons:Wiki_Loves_Earth_2017/Winners", @@ -21,9 +23,13 @@ class ImagesMockServerSpec extends BaseMockServerSpec { val bot = getBot - val future = bot.page("Commons:Wiki_Loves_Earth_2017/Winners") - .imageInfoByGenerator("images", "im", - props = Set("timestamp", "user", "size", "url")) + val future = bot + .page("Commons:Wiki_Loves_Earth_2017/Winners") + .imageInfoByGenerator( + "images", + "im", + props = Set("timestamp", "user", "size", "url") + ) val info = future.await info should not(beEmpty) info.size === 382 @@ -31,7 +37,9 @@ class ImagesMockServerSpec extends BaseMockServerSpec { } "succesfully get image revisions" in { - val response = resourceAsString("/org/scalawiki/query/Commons_Wiki_Loves_Earth_2019_Winners_revisions.json") + val response = resourceAsString( + "/org/scalawiki/query/Commons_Wiki_Loves_Earth_2019_Winners_revisions.json" + ) val action = Map( "action" -> "query", "titles" -> "Commons:Wiki_Loves_Earth_2019/Winners", @@ -45,14 +53,21 @@ class ImagesMockServerSpec extends BaseMockServerSpec { val bot = getBot - val future = bot.page("Commons:Wiki_Loves_Earth_2019/Winners") - .revisionsByGenerator("images", "im", - props = Set("ids", "content", "user", "comment"), limit = "50") + val future = bot + .page("Commons:Wiki_Loves_Earth_2019/Winners") + .revisionsByGenerator( + "images", + "im", + props = Set("ids", "content", "user", "comment"), + limit = "50" + ) val info = future.await info should not(beEmpty) info.size === 50 info.map(_.revisions.size) === List.fill(50)(1) - info.map(_.revisions.head.content.exists(_.nonEmpty)) === List.fill(50)(true) + info.map(_.revisions.head.content.exists(_.nonEmpty)) === List.fill(50)( + true + ) } } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/LoginMockServerSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/LoginMockServerSpec.scala index 2b7235b8..b79d1afe 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/LoginMockServerSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/LoginMockServerSpec.scala @@ -6,7 +6,12 @@ class LoginMockServerSpec extends BaseMockServerSpec { val (password, wrongPassword) = ("secretPassword", "wrongPassword") def withCredentials(user: String, password: String) = - Map("action" -> "login", "format" -> "json", "lgname" -> user, "lgpassword" -> password) + Map( + "action" -> "login", + "format" -> "json", + "lgname" -> user, + "lgpassword" -> password + ) val needToken = """{ "login": { diff --git a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/LongRevisionsMockServerSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/LongRevisionsMockServerSpec.scala index cae0d3a9..901872bf 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/LongRevisionsMockServerSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/LongRevisionsMockServerSpec.scala @@ -9,7 +9,8 @@ import scala.concurrent.ExecutionContext.Implicits.global class LongRevisionsMockServerSpec extends BaseMockServerSpec with QueryLibrary { val response = resourceAsString( - "/org/scalawiki/query/longRevisionHistory.json") + "/org/scalawiki/query/longRevisionHistory.json" + ) "revisions" should { "succesfully get long response" in { diff --git a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/StubServer.scala b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/StubServer.scala index 294013f3..f18fa4a2 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/mockserver/StubServer.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/mockserver/StubServer.scala @@ -3,7 +3,7 @@ package org.scalawiki.mockserver import org.mockserver.integration.ClientAndServer.startClientAndServer import org.specs2.specification.BeforeAfterEach -trait StubServer extends BeforeAfterEach { +trait StubServer extends BeforeAfterEach { val Port = 8080 val Host = "localhost" val Protocol = "http" @@ -21,4 +21,4 @@ trait StubServer extends BeforeAfterEach { } def url(path: String) = s"$Protocol://$Host:$Port$path" -} \ No newline at end of file +} diff --git a/scalawiki-core/src/test/scala/org/scalawiki/parser/TableParserSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/parser/TableParserSpec.scala index c9a0e432..80523820 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/parser/TableParserSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/parser/TableParserSpec.scala @@ -41,7 +41,12 @@ class TableParserSpec extends Specification { "parse table with 2 data columns" in { val wiki = "{|\n|-\n| data11 || data12\n|}" - parser.parse(wiki) === new Table(Seq.empty, Seq(Seq("data11", "data12")), "", "") + parser.parse(wiki) === new Table( + Seq.empty, + Seq(Seq("data11", "data12")), + "", + "" + ) } "parse table with formatted data" in { @@ -60,12 +65,22 @@ class TableParserSpec extends Specification { "parse table with empty columns" in { val wiki = "{|\n|-\n| data11 || || data13\n|}" - parser.parse(wiki) === new Table(Seq.empty, Seq(Seq("data11", "", "data13")), "", "") + parser.parse(wiki) === new Table( + Seq.empty, + Seq(Seq("data11", "", "data13")), + "", + "" + ) } "parse table with 1 data column and 2 rows" in { val wiki = "{|\n|-\n| data11\n|-\n| data21\n|}" - parser.parse(wiki) === new Table(Seq.empty, Seq(Seq("data11"), Seq("data21")), "", "") + parser.parse(wiki) === new Table( + Seq.empty, + Seq(Seq("data11"), Seq("data21")), + "", + "" + ) } "parse table with header and data" in { @@ -75,17 +90,32 @@ class TableParserSpec extends Specification { "parse table with 2 columns header and data" in { val wiki = "{|\n! header1 !! header2\n|-\n| data11 || data12\n|}" - parser.parse(wiki) === Table(Seq("header1", "header2"), Seq(Seq("data11", "data12")), "", "") + parser.parse(wiki) === Table( + Seq("header1", "header2"), + Seq(Seq("data11", "data12")), + "", + "" + ) } "parse table with 2 columns header and 1 column data" in { val wiki = "{|\n! header1 !! header2\n|-\n| data11 ||\n|}" - parser.parse(wiki) === Table(Seq("header1", "header2"), Seq(Seq("data11", "")), "", "") + parser.parse(wiki) === Table( + Seq("header1", "header2"), + Seq(Seq("data11", "")), + "", + "" + ) } "parse table with 2 rows header" in { val wiki = "{|\n! header1 !! header2\n|-\n! data11 || data12\n|}" - parser.parse(wiki) === Table(Seq("header1", "header2"), Seq(Seq("data11", "data12")), "", "") + parser.parse(wiki) === Table( + Seq("header1", "header2"), + Seq(Seq("data11", "data12")), + "", + "" + ) } } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/parser/TemplateParserSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/parser/TemplateParserSpec.scala index 7656b4fa..1a1e9b5f 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/parser/TemplateParserSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/parser/TemplateParserSpec.scala @@ -30,20 +30,26 @@ class TemplateParserSpec extends Specification { } "parse positional parameter article link" in { - val template = parser.parseOne("{{TemplateName | [[ article | link ]] }}").head + val template = + parser.parseOne("{{TemplateName | [[ article | link ]] }}").head template.templateName === "TemplateName" template.params === Map("1" -> "[[ article | link ]]") } "parse positional parameter article two article links" in { - val template = parser.parseOne("{{TemplateName | [[ article1 | link1 ]], [[ article2 | link2 ]] }}").head + val template = parser + .parseOne( + "{{TemplateName | [[ article1 | link1 ]], [[ article2 | link2 ]] }}" + ) + .head template.templateName === "TemplateName" - template.params === Map("1" -> "[[ article1 | link1 ]], [[ article2 | link2 ]]") + template.params === Map( + "1" -> "[[ article1 | link1 ]], [[ article2 | link2 ]]" + ) } - "parse positional parameter template" in { val template = parser.parseOne("{{TemplateName | {{child}} }}").head template.templateName === "TemplateName" @@ -73,7 +79,8 @@ class TemplateParserSpec extends Specification { } "parse two positional parameters newline" in { - val template = parser.parseOne("{{TemplateName\n|param1\n|param2\n}}").head + val template = + parser.parseOne("{{TemplateName\n|param1\n|param2\n}}").head template.templateName === "TemplateName" template.params === Map("1" -> "param1", "2" -> "param2") @@ -129,7 +136,8 @@ class TemplateParserSpec extends Specification { } "parse two named positional parameters newline" in { - val template = parser.parseOne("{{TemplateName\n|1=param1\n|2=param2\n}}").head + val template = + parser.parseOne("{{TemplateName\n|1=param1\n|2=param2\n}}").head template.templateName === "TemplateName" template.params === Map("1" -> "param1", "2" -> "param2") @@ -163,8 +171,6 @@ class TemplateParserSpec extends Specification { template.params === Map("name" -> "value") } - } - } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/ListAllUsersSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/ListAllUsersSpec.scala index 93cf9d69..edee3f6e 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/ListAllUsersSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/ListAllUsersSpec.scala @@ -32,14 +32,17 @@ class ListAllUsersSpec extends Specification with MockBotSpec { }}""" val commands = Seq( - HttpStub(Map("action" -> "query", - "list" -> queryType, - "continue" -> "", - "format" -> "json", - "utf8" -> "", + HttpStub( + Map( + "action" -> "query", + "list" -> queryType, + "continue" -> "", + "format" -> "json", + "utf8" -> "" // "auprop" -> "registration|editcount|blockinfo" - ), - response1) + ), + response1 + ) ) val bot = getBot(commands: _*) @@ -88,10 +91,12 @@ class ListAllUsersSpec extends Specification with MockBotSpec { ] }}""" - val query = Map("action" -> "query", - "list" -> queryType, - "auprop" -> "registration|editcount|blockinfo", - "continue" -> "") + val query = Map( + "action" -> "query", + "list" -> queryType, + "auprop" -> "registration|editcount|blockinfo", + "continue" -> "" + ) val bot = getBot(Seq(HttpStub(query, response1)): _*) @@ -108,16 +113,20 @@ class ListAllUsersSpec extends Specification with MockBotSpec { result must have size 2 val users = result.flatMap(_.lastRevisionUser) - users(0) === new User(Some(3634417), - Some("Y"), - Some(13892), - Some(Timestamp.parse("2007-02-22T03:19:08Z")), - Some(true)) - users(1) === new User(Some(53928), - Some("Y (usurped)"), - Some(0), - None, - Some(false)) + users(0) === new User( + Some(3634417), + Some("Y"), + Some(13892), + Some(Timestamp.parse("2007-02-22T03:19:08Z")), + Some(true) + ) + users(1) === new User( + Some(53928), + Some("Y (usurped)"), + Some(0), + None, + Some(false) + ) } "return users with continue" in { @@ -160,8 +169,10 @@ class ListAllUsersSpec extends Specification with MockBotSpec { val commands = Seq( HttpStub(query + ("continue" -> ""), response1), - HttpStub(query ++ Map("aufrom" -> "! ! !", "continue" -> "-||"), - response2) + HttpStub( + query ++ Map("aufrom" -> "! ! !", "continue" -> "-||"), + response2 + ) ) val bot = getBot(commands: _*) diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/ListCategoryMembersSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/ListCategoryMembersSpec.scala index ce3ca37b..947bd80e 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/ListCategoryMembersSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/ListCategoryMembersSpec.scala @@ -26,16 +26,35 @@ class ListCategoryMembersSpec extends Specification with MockBotSpec { | { "categorymembers": [{"pageid": 4571809, "ns": 2, "title": "User:Formator"}] }}""".stripMargin val commands = Seq( - HttpStub(Map("action" -> "query", "list" -> queryType, "cmlimit" -> "max", - "cmtitle" -> "Category:SomeCategory", "cmnamespace" -> "", "continue" -> ""), response1), - HttpStub(Map("action" -> "query", "list" -> queryType, "cmlimit" -> "max", - "cmtitle" -> "Category:SomeCategory", "cmnamespace" -> "", - "continue" -> "-||", "cmcontinue" -> "10|Stub|6674690"), response2) + HttpStub( + Map( + "action" -> "query", + "list" -> queryType, + "cmlimit" -> "max", + "cmtitle" -> "Category:SomeCategory", + "cmnamespace" -> "", + "continue" -> "" + ), + response1 + ), + HttpStub( + Map( + "action" -> "query", + "list" -> queryType, + "cmlimit" -> "max", + "cmtitle" -> "Category:SomeCategory", + "cmnamespace" -> "", + "continue" -> "-||", + "cmcontinue" -> "10|Stub|6674690" + ), + response2 + ) ) val bot = getBot(commands: _*) - val result = bot.page("Category:SomeCategory").categoryMembers().map(_.toSeq).await + val result = + bot.page("Category:SomeCategory").categoryMembers().map(_.toSeq).await result must have size 2 result.head === Page(569559, Some(1), "Talk:Welfare reform") result(1) === Page(4571809, Some(2), "User:Formator") @@ -85,16 +104,33 @@ class ListCategoryMembersSpec extends Specification with MockBotSpec { | } |}""".stripMargin - val commands = Seq( - HttpStub(Map("action" -> "query", "prop" -> "categoryinfo", - "titles" -> "Albert Einstein|Category:Foo|Category:Infobox_templates|NoSuchTitle", "continue" -> ""), response1) + HttpStub( + Map( + "action" -> "query", + "prop" -> "categoryinfo", + "titles" -> "Albert Einstein|Category:Foo|Category:Infobox_templates|NoSuchTitle", + "continue" -> "" + ), + response1 + ) ) val bot = getBot(commands: _*) - val query = Action(Query(TitlesParam(Seq("Albert Einstein", "Category:Foo", "Category:Infobox_templates", "NoSuchTitle")), - Prop(CategoryInfo))) + val query = Action( + Query( + TitlesParam( + Seq( + "Albert Einstein", + "Category:Foo", + "Category:Infobox_templates", + "NoSuchTitle" + ) + ), + Prop(CategoryInfo) + ) + ) val result = bot.run(query).map(_.toSeq).await result must have size 4 diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/ListUserContribsSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/ListUserContribsSpec.scala index 8cd567af..8e40640d 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/ListUserContribsSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/ListUserContribsSpec.scala @@ -38,7 +38,12 @@ class ListUserContribsSpec extends Specification with MockBotSpec { } }""" - val query = Map("action" -> "query", "list" -> queryType, "ucuser" -> "Catrope", "continue" -> "") + val query = Map( + "action" -> "query", + "list" -> queryType, + "ucuser" -> "Catrope", + "continue" -> "" + ) val commands = Seq( HttpStub(query, response1) ) @@ -59,9 +64,22 @@ class ListUserContribsSpec extends Specification with MockBotSpec { result must have size 1 val user = User(4587601, "Catrope") - val rev = new Revision(Some(136629050), Some(11650099), Some(0), Some(user), - Some(Timestamp.parse("2007-06-07T16:45:30Z")), Some("Creation; directing to BW"), None, Some(119)) - result.head === new Page(Some(11650099), Some(3), "User talk:Catrope", revisions = Seq(rev)) + val rev = new Revision( + Some(136629050), + Some(11650099), + Some(0), + Some(user), + Some(Timestamp.parse("2007-06-07T16:45:30Z")), + Some("Creation; directing to BW"), + None, + Some(119) + ) + result.head === new Page( + Some(11650099), + Some(3), + "User talk:Catrope", + revisions = Seq(rev) + ) } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/ListUsersSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/ListUsersSpec.scala index 0ae173e2..150da124 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/ListUsersSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/ListUsersSpec.scala @@ -23,7 +23,10 @@ class ListUsersSpec extends Specification with MockBotSpec { |}""".stripMargin val commands = Seq( - HttpStub(Map("action" -> "query", "list" -> queryType, "continue" -> ""), response1) + HttpStub( + Map("action" -> "query", "list" -> queryType, "continue" -> ""), + response1 + ) ) val bot = getBot(commands: _*) @@ -56,9 +59,11 @@ class ListUsersSpec extends Specification with MockBotSpec { |}""".stripMargin val expectedParams = Map( - "action" -> "query", "list" -> queryType, + "action" -> "query", + "list" -> queryType, "ususers" -> "MissingUser", - "continue" -> "") + "continue" -> "" + ) val commands = Seq(HttpStub(expectedParams, response1)) @@ -82,7 +87,6 @@ class ListUsersSpec extends Specification with MockBotSpec { ) } - "with missing user" in { val queryType = "users" @@ -101,9 +105,12 @@ class ListUsersSpec extends Specification with MockBotSpec { | } |}""".stripMargin - val expectedParams = Map("action" -> "query", "list" -> queryType, + val expectedParams = Map( + "action" -> "query", + "list" -> queryType, "ususers" -> "MissingUser|ExistingUser", - "continue" -> "") + "continue" -> "" + ) val commands = Seq(HttpStub(expectedParams, response1)) @@ -149,10 +156,13 @@ class ListUsersSpec extends Specification with MockBotSpec { | } |}""".stripMargin - val expectedParams = Map("action" -> "query", "list" -> queryType, + val expectedParams = Map( + "action" -> "query", + "list" -> queryType, "usprop" -> "editcount|emailable", "ususers" -> "Y|Y (usurped)", - "continue" -> "") + "continue" -> "" + ) val commands = Seq( HttpStub(expectedParams, response1) @@ -166,7 +176,8 @@ class ListUsersSpec extends Specification with MockBotSpec { ListParam( Users( UsUsers(Seq("Y", "Y (usurped)")), - UsProp(UsEditCount, UsEmailable)) + UsProp(UsEditCount, UsEmailable) + ) ) ) ) @@ -176,8 +187,22 @@ class ListUsersSpec extends Specification with MockBotSpec { val result = bot.run(action).map(_.toSeq).await result must have size 2 val users = result.flatMap(_.lastRevisionUser) - users(0) === new User(Some(3634417), Some("Y"), Some(13892), None, None, Some(true)) - users(1) === new User(Some(53928), Some("Y (usurped)"), Some(0), None, None, Some(false)) + users(0) === new User( + Some(3634417), + Some("Y"), + Some(13892), + None, + None, + Some(true) + ) + users(1) === new User( + Some(53928), + Some("Y (usurped)"), + Some(0), + None, + None, + Some(false) + ) } "return users with continue" in { @@ -218,8 +243,24 @@ class ListUsersSpec extends Specification with MockBotSpec { |}""".stripMargin val commands = Seq( - HttpStub(Map("action" -> "query", "ususers" -> "!|! !|! ! !|! ! ! !", "list" -> queryType, "continue" -> ""), response1), - HttpStub(Map("action" -> "query", "ususers" -> "!|! !|! ! !|! ! ! !", "list" -> queryType, "continue" -> "-||"), response2) + HttpStub( + Map( + "action" -> "query", + "ususers" -> "!|! !|! ! !|! ! ! !", + "list" -> queryType, + "continue" -> "" + ), + response1 + ), + HttpStub( + Map( + "action" -> "query", + "ususers" -> "!|! !|! ! !|! ! ! !", + "list" -> queryType, + "continue" -> "-||" + ), + response2 + ) ) val bot = getBot(commands: _*) diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/MetaGlobalUserInfoSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/MetaGlobalUserInfoSpec.scala index 1aa7b49d..4a31aadb 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/MetaGlobalUserInfoSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/MetaGlobalUserInfoSpec.scala @@ -21,16 +21,24 @@ class MetaGlobalUserInfoSpec extends Specification with MockBotSpec { EditCount ), GuiUser("Ilya") - )))) + ) + ) + ) + ) def commands(response: String) = Seq( - HttpStub(Map("action" -> "query", - "meta" -> "globaluserinfo", - "guiuser" -> "Ilya", - "guiprop" -> "merged|unattached|editcount", - "continue" -> ""), - response)) + HttpStub( + Map( + "action" -> "query", + "meta" -> "globaluserinfo", + "guiuser" -> "Ilya", + "guiprop" -> "merged|unattached|editcount", + "continue" -> "" + ), + response + ) + ) "globaluserinfo" should { diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/PageQuerySpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/PageQuerySpec.scala index ef83c8bc..75af00ff 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/PageQuerySpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/PageQuerySpec.scala @@ -30,19 +30,48 @@ class PageQuerySpec extends Specification { "revisions": [{ "revid": 2, "userid": 2, "user": "u2", "comment": "c2", "*": "$pageText2"}]} }}}""" - val bot = getBot(HttpStub( - Map( - "pageids" -> "569559|4571809", - "action" -> "query", - "prop" -> "info|revisions", - "continue" -> "", "rvlimit" -> "max", - "rvprop" -> "ids|content|user|comment"), response)) - - val future = bot.pagesById(Set(569559L, 4571809L)).revisions(Set.empty[Int], Set("ids", "content", "user", "comment")).map(_.toSeq) + val bot = getBot( + HttpStub( + Map( + "pageids" -> "569559|4571809", + "action" -> "query", + "prop" -> "info|revisions", + "continue" -> "", + "rvlimit" -> "max", + "rvprop" -> "ids|content|user|comment" + ), + response + ) + ) + + val future = bot + .pagesById(Set(569559L, 4571809L)) + .revisions(Set.empty[Int], Set("ids", "content", "user", "comment")) + .map(_.toSeq) val result = future.await result must have size 2 - result(0) === Page(Some(569559), Some(1), "Talk:Welfare reform", Seq(Revision(1, 569559).withUser(1, "u1").withComment("c1").withText(pageText1))) - result(1) === Page(Some(4571809), Some(2), "User:Formator", Seq(Revision(2, 4571809).withUser(2, "u2").withComment("c2").withText(pageText2))) + result(0) === Page( + Some(569559), + Some(1), + "Talk:Welfare reform", + Seq( + Revision(1, 569559) + .withUser(1, "u1") + .withComment("c1") + .withText(pageText1) + ) + ) + result(1) === Page( + Some(4571809), + Some(2), + "User:Formator", + Seq( + Revision(2, 4571809) + .withUser(2, "u2") + .withComment("c2") + .withText(pageText2) + ) + ) } } @@ -61,28 +90,44 @@ class PageQuerySpec extends Specification { | } |}""".stripMargin - val bot = getBot(HttpStub( - Map( - "titles" -> "Absent", - "action" -> "query", - "prop" -> "info|revisions", - "continue" -> "", "rvlimit" -> "max", - "rvprop" -> "ids|content|user|comment"), response)) - - val future = bot.pagesByTitle(Set("Absent")).revisions(Set.empty[Int], Set("ids", "content", "user", "comment")) + val bot = getBot( + HttpStub( + Map( + "titles" -> "Absent", + "action" -> "query", + "prop" -> "info|revisions", + "continue" -> "", + "rvlimit" -> "max", + "rvprop" -> "ids|content|user|comment" + ), + response + ) + ) + + val future = bot + .pagesByTitle(Set("Absent")) + .revisions(Set.empty[Int], Set("ids", "content", "user", "comment")) val result = future.await result must have size 1 result.head === Page(None, Some(0), "Absent", missing = true) } "invalid page title" in { - val response = """{"batchcomplete":"","query":{"pages":{"-1":{"title":"[[Page]]","invalidreason":"The requested page title contains invalid characters: \"[\".","invalid":""}}}}""".stripMargin + val response = + """{"batchcomplete":"","query":{"pages":{"-1":{"title":"[[Page]]","invalidreason":"The requested page title contains invalid characters: \"[\".","invalid":""}}}}""".stripMargin val action = Action(Query(TitlesParam(Seq("[[Page]]")))) - val bot = getBot(HttpStub((action.pairs ++ Seq("continue" -> "")).toMap, response)) + val bot = + getBot(HttpStub((action.pairs ++ Seq("continue" -> "")).toMap, response)) val result = bot.run(action).await result must have size 1 - result.head === Page(None, None, "[[Page]]", invalidReason = Some("""The requested page title contains invalid characters: "[".""")) + result.head === Page( + None, + None, + "[[Page]]", + invalidReason = + Some("""The requested page title contains invalid characters: "[".""") + ) } def getBot(commands: HttpStub*) = { diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/PropEmbeddedInSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/PropEmbeddedInSpec.scala index 21f05055..6271c8f4 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/PropEmbeddedInSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/PropEmbeddedInSpec.scala @@ -16,8 +16,12 @@ class PropEmbeddedInSpec extends Specification with MockBotSpec { val response = """{ "query": { "embeddedin": [] }}""" val query = Map( - "action" -> "query", "list" -> queryType, "eilimit" -> "max", - "eititle" -> "Template:SomeTemplate", "einamespace" -> "", "continue" -> "" + "action" -> "query", + "list" -> queryType, + "eilimit" -> "max", + "eititle" -> "Template:SomeTemplate", + "einamespace" -> "", + "continue" -> "" ) val bot = getBot(HttpStub(query, response)) @@ -36,8 +40,12 @@ class PropEmbeddedInSpec extends Specification with MockBotSpec { }}""" val query = Map( - "action" -> "query", "list" -> queryType, "eilimit" -> "max", - "eititle" -> "Template:SomeTemplate", "einamespace" -> "", "continue" -> "" + "action" -> "query", + "list" -> queryType, + "eilimit" -> "max", + "eititle" -> "Template:SomeTemplate", + "einamespace" -> "", + "continue" -> "" ) val bot = getBot(HttpStub(query, response)) @@ -59,14 +67,21 @@ class PropEmbeddedInSpec extends Specification with MockBotSpec { ]}}""" val query = Map( - "action" -> "query", "list" -> queryType, "eilimit" -> "max", + "action" -> "query", + "list" -> queryType, + "eilimit" -> "max", "eititle" -> "Template:SomeTemplate", - "einamespace" -> "", "continue" -> "" + "einamespace" -> "", + "continue" -> "" ) val bot = getBot(HttpStub(query, response)) - val result = bot.page("Template:SomeTemplate").whatTranscludesHere().map(_.toSeq).await + val result = bot + .page("Template:SomeTemplate") + .whatTranscludesHere() + .map(_.toSeq) + .await result must have size 2 result(0) === Page(569559, Some(1), "Talk:Welfare reform") result(1) === Page(4571809, Some(2), "User:Formator") @@ -84,19 +99,32 @@ class PropEmbeddedInSpec extends Specification with MockBotSpec { "continue": { "continue":"-||", "eicontinue": "10|Stub|6674690" } }""" - val response2 = """{ "query": { "embeddedin": [{"pageid": 4571809, "ns": 2, "title": "User:Formator" }] }}""" + val response2 = + """{ "query": { "embeddedin": [{"pageid": 4571809, "ns": 2, "title": "User:Formator" }] }}""" - val query = Map("action" -> "query", "list" -> queryType, "eilimit" -> "max", - "eititle" -> "Template:SomeTemplate", "einamespace" -> "") + val query = Map( + "action" -> "query", + "list" -> queryType, + "eilimit" -> "max", + "eititle" -> "Template:SomeTemplate", + "einamespace" -> "" + ) val commands = Seq( HttpStub(query + ("continue" -> ""), response1), - HttpStub(query ++ Map("continue" -> "-||", "eicontinue" -> "10|Stub|6674690"), response2) + HttpStub( + query ++ Map("continue" -> "-||", "eicontinue" -> "10|Stub|6674690"), + response2 + ) ) val bot = getBot(commands: _*) - val result = bot.page("Template:SomeTemplate").whatTranscludesHere().map(_.toSeq).await + val result = bot + .page("Template:SomeTemplate") + .whatTranscludesHere() + .map(_.toSeq) + .await result must have size 2 result(0) === Page(569559, Some(1), "Talk:Welfare reform") result(1) === Page(4571809, Some(2), "User:Formator") diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/PropImageInfoSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/PropImageInfoSpec.scala index f9c42ee3..f8722a6d 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/PropImageInfoSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/PropImageInfoSpec.scala @@ -62,36 +62,92 @@ class PropImageInfoSpec extends Specification with MockBotSpec { | }]}}}} | """.stripMargin - val page1 = Page(Some(32885574), Some(6), "File:Dovbush-rocks 01.JPG", Seq.empty, - Seq(Image.basic("File:Dovbush-rocks 01.JPG", Some(Timestamp.parse("2014-05-20T20:54:33Z")), Some("Taras r"), Some(4270655), Some(3648), Some(2736), - Some("https://upload.wikimedia.org/wikipedia/commons/e/ea/Dovbush-rocks_01.JPG"), - Some("https://commons.wikimedia.org/wiki/File:Dovbush-rocks_01.JPG"), Some(32885574)))) - - val page2 = Page(Some(32885597), Some(6), "File:Dovbush-rocks 02.JPG", Seq.empty, - Seq(Image.basic("File:Dovbush-rocks 02.JPG", Some(Timestamp.parse("2014-05-20T20:55:12Z")), Some("Taras r"), Some(4537737), Some(2736), Some(3648), - Some("https://upload.wikimedia.org/wikipedia/commons/2/26/Dovbush-rocks_02.JPG"), - Some("https://commons.wikimedia.org/wiki/File:Dovbush-rocks_02.JPG"), Some(32885597)))) + val page1 = Page( + Some(32885574), + Some(6), + "File:Dovbush-rocks 01.JPG", + Seq.empty, + Seq( + Image.basic( + "File:Dovbush-rocks 01.JPG", + Some(Timestamp.parse("2014-05-20T20:54:33Z")), + Some("Taras r"), + Some(4270655), + Some(3648), + Some(2736), + Some( + "https://upload.wikimedia.org/wikipedia/commons/e/ea/Dovbush-rocks_01.JPG" + ), + Some("https://commons.wikimedia.org/wiki/File:Dovbush-rocks_01.JPG"), + Some(32885574) + ) + ) + ) + + val page2 = Page( + Some(32885597), + Some(6), + "File:Dovbush-rocks 02.JPG", + Seq.empty, + Seq( + Image.basic( + "File:Dovbush-rocks 02.JPG", + Some(Timestamp.parse("2014-05-20T20:55:12Z")), + Some("Taras r"), + Some(4537737), + Some(2736), + Some(3648), + Some( + "https://upload.wikimedia.org/wikipedia/commons/2/26/Dovbush-rocks_02.JPG" + ), + Some("https://commons.wikimedia.org/wiki/File:Dovbush-rocks_02.JPG"), + Some(32885597) + ) + ) + ) "get image info in generator" should { "query by category members" in { val commands = Seq( - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "imageinfo", "iiprop" -> "timestamp|user|comment", - "continue" -> ""), response1("cm")), - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "imageinfo", "iiprop" -> "timestamp|user|comment", - "continue" -> "gcmcontinue||", - "gcmcontinue" -> "file|44454d45524749373631312e4a50470a44454d45524749373631312e4a5047|32763876"), - response2) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "imageinfo", + "iiprop" -> "timestamp|user|comment", + "continue" -> "" + ), + response1("cm") + ), + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "imageinfo", + "iiprop" -> "timestamp|user|comment", + "continue" -> "gcmcontinue||", + "gcmcontinue" -> "file|44454d45524749373631312e4a50470a44454d45524749373631312e4a5047|32763876" + ), + response2 + ) ) val bot = getBot(commands: _*) - val future = bot.page("Category:SomeCategory") - .imageInfoByGenerator("categorymembers", "cm", Set.empty, Set("timestamp", "user", "comment")).map(_.toSeq) + val future = bot + .page("Category:SomeCategory") + .imageInfoByGenerator( + "categorymembers", + "cm", + Set.empty, + Set("timestamp", "user", "comment") + ) + .map(_.toSeq) val result = future.await @@ -104,22 +160,44 @@ class PropImageInfoSpec extends Specification with MockBotSpec { "query by page images" in { val commands = Seq( - HttpStub(Map("action" -> "query", - "generator" -> "images", "titles" -> "Commons:SomePage", "gimlimit" -> "max", - "prop" -> "imageinfo", "iiprop" -> "timestamp|user|comment", - "continue" -> ""), response1("im")), - HttpStub(Map("action" -> "query", - "generator" -> "images", "titles" -> "Commons:SomePage", "gimlimit" -> "max", - "prop" -> "imageinfo", "iiprop" -> "timestamp|user|comment", - "continue" -> "gimcontinue||", - "gimcontinue" -> "file|44454d45524749373631312e4a50470a44454d45524749373631312e4a5047|32763876"), - response2) + HttpStub( + Map( + "action" -> "query", + "generator" -> "images", + "titles" -> "Commons:SomePage", + "gimlimit" -> "max", + "prop" -> "imageinfo", + "iiprop" -> "timestamp|user|comment", + "continue" -> "" + ), + response1("im") + ), + HttpStub( + Map( + "action" -> "query", + "generator" -> "images", + "titles" -> "Commons:SomePage", + "gimlimit" -> "max", + "prop" -> "imageinfo", + "iiprop" -> "timestamp|user|comment", + "continue" -> "gimcontinue||", + "gimcontinue" -> "file|44454d45524749373631312e4a50470a44454d45524749373631312e4a5047|32763876" + ), + response2 + ) ) val bot = getBot(commands: _*) - val future = bot.page("Commons:SomePage") - .imageInfoByGenerator("images", "im", Set.empty, Set("timestamp", "user", "comment")).map(_.toSeq) + val future = bot + .page("Commons:SomePage") + .imageInfoByGenerator( + "images", + "im", + Set.empty, + Set("timestamp", "user", "comment") + ) + .map(_.toSeq) val result = future.await @@ -133,13 +211,26 @@ class PropImageInfoSpec extends Specification with MockBotSpec { val s = resourceAsString("/org/scalawiki/query/imageMetadata.json") val commands = Seq( - HttpStub(Map("action" -> "query", "format" -> "json", "prop" -> "imageinfo", "pageids" -> "58655318", - "iiprop" -> "metadata", "continue" -> ""), s) + HttpStub( + Map( + "action" -> "query", + "format" -> "json", + "prop" -> "imageinfo", + "pageids" -> "58655318", + "iiprop" -> "metadata", + "continue" -> "" + ), + s + ) ) val bot = getBot(commands: _*) - val future = bot.run(Action(Query(PageIdsParam(Seq(58655318)), Prop(ImageInfo(IiProp(Metadata)))))) + val future = bot.run( + Action( + Query(PageIdsParam(Seq(58655318)), Prop(ImageInfo(IiProp(Metadata)))) + ) + ) val result = future.await result must have size 1 @@ -151,7 +242,9 @@ class PropImageInfoSpec extends Specification with MockBotSpec { image.metadata.isDefined === true val metadata = image.metadata.get metadata.camera === Some("Canon EOS 450D") - metadata.date === Some(ZonedDateTime.of(2017, 4, 22, 12, 26, 44, 0, ZoneOffset.UTC)) + metadata.date === Some( + ZonedDateTime.of(2017, 4, 22, 12, 26, 44, 0, ZoneOffset.UTC) + ) } } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/PropImagesSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/PropImagesSpec.scala index 5139342b..8333cce0 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/PropImagesSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/PropImagesSpec.scala @@ -37,25 +37,42 @@ class PropImagesSpec extends Specification with MockBotSpec { | } |}""".stripMargin - val commands = Seq( - HttpStub(Map("action" -> "query", "titles" -> "Albert_Einstein", "prop" -> "images", "format" -> "json", "continue" -> ""), response) + HttpStub( + Map( + "action" -> "query", + "titles" -> "Albert_Einstein", + "prop" -> "images", + "format" -> "json", + "continue" -> "" + ), + response + ) ) val bot = getBot(commands: _*) - val action = Action(Query( - TitlesParam(Seq("Albert_Einstein")), - Prop(Images()) - )) + val action = Action( + Query( + TitlesParam(Seq("Albert_Einstein")), + Prop(Images()) + ) + ) val result = bot.run(action).map(_.toSeq).await result must have size 1 val page = result(0) - page === new Page(Some(736), Some(0), "Albert Einstein", images = Seq( - new Image("File:1919 eclipse positive.jpg"), - new Image("File:Albert Einstein's exam of maturity grades (color2).jpg") - )) + page === new Page( + Some(736), + Some(0), + "Albert Einstein", + images = Seq( + new Image("File:1919 eclipse positive.jpg"), + new Image( + "File:Albert Einstein's exam of maturity grades (color2).jpg" + ) + ) + ) } } @@ -93,18 +110,28 @@ class PropImagesSpec extends Specification with MockBotSpec { | } |}""".stripMargin - val commands = Seq( - HttpStub(Map("action" -> "query", "titles" -> "Commons:Wiki_Loves_Earth_2015/Winners", - "prop" -> "imageinfo", "generator" -> "images", "format" -> "json", "continue" -> ""), response) + HttpStub( + Map( + "action" -> "query", + "titles" -> "Commons:Wiki_Loves_Earth_2015/Winners", + "prop" -> "imageinfo", + "generator" -> "images", + "format" -> "json", + "continue" -> "" + ), + response + ) ) val bot = getBot(commands: _*) - val action = Action(Query( - TitlesParam(Seq("Commons:Wiki_Loves_Earth_2015/Winners")), - Prop(ImageInfo()), - Generator(Images()) - )) + val action = Action( + Query( + TitlesParam(Seq("Commons:Wiki_Loves_Earth_2015/Winners")), + Prop(ImageInfo()), + Generator(Images()) + ) + ) val result = bot.run(action).await result must have size 2 diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/PropLangLinksSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/PropLangLinksSpec.scala index 1086e2c1..8e33fe2b 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/PropLangLinksSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/PropLangLinksSpec.scala @@ -77,26 +77,47 @@ class PropLangLinksSpec extends Specification with MockBotSpec { } """ - val query = Map("action" -> "query", "prop" -> "langlinks", - "generator" -> "categorymembers", "gcmtitle" -> category, "gcmnamespace" -> "0", - "gcmlimit" -> "2", "lllimit" -> "2") + val query = Map( + "action" -> "query", + "prop" -> "langlinks", + "generator" -> "categorymembers", + "gcmtitle" -> category, + "gcmnamespace" -> "0", + "gcmlimit" -> "2", + "lllimit" -> "2" + ) val commands = Seq( HttpStub(query + ("continue" -> ""), response1), - HttpStub(query ++ Map( - "continue" -> "||", - "llcontinue" -> "6863578|cs" - ), response2) + HttpStub( + query ++ Map( + "continue" -> "||", + "llcontinue" -> "6863578|cs" + ), + response2 + ) ) val bot = getBot(commands: _*) - val action = Action(Query( - Prop( - LangLinks(LlLimit("2")) - ), - Generator(ListArgs.toDsl("categorymembers", Some(category), None, Set(Namespace.MAIN), Some("2")).get) - )) + val action = Action( + Query( + Prop( + LangLinks(LlLimit("2")) + ), + Generator( + ListArgs + .toDsl( + "categorymembers", + Some(category), + None, + Set(Namespace.MAIN), + Some("2") + ) + .get + ) + ) + ) val result = bot.run(action).await diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/PropLinksSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/PropLinksSpec.scala index 9b5f506f..aec30e3c 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/PropLinksSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/PropLinksSpec.scala @@ -1,6 +1,5 @@ package org.scalawiki.query - import org.scalawiki.dto.cmd.Action import org.scalawiki.dto.cmd.query.prop.{Links, Prop} import org.scalawiki.dto.cmd.query.{Query, TitlesParam} @@ -13,7 +12,8 @@ class PropLinksSpec extends Specification with MockBotSpec { "get links" should { "merge pages" in { - val title = "Reactive Streams" // https://en.wikipedia.org/wiki/Reactive_Streams + val title = + "Reactive Streams" // https://en.wikipedia.org/wiki/Reactive_Streams val response1 = """{ @@ -67,26 +67,37 @@ class PropLinksSpec extends Specification with MockBotSpec { |}""".stripMargin val commands = Seq( - HttpStub(Map("action" -> "query", - "titles" -> title, - "prop" -> "links", - "continue" -> ""), response1), - HttpStub(Map("action" -> "query", - "titles" -> title, - "prop" -> "links", - "continue" -> "||", - "plcontinue" -> "48896716|0|Domain-specific_language"), - response2) + HttpStub( + Map( + "action" -> "query", + "titles" -> title, + "prop" -> "links", + "continue" -> "" + ), + response1 + ), + HttpStub( + Map( + "action" -> "query", + "titles" -> title, + "prop" -> "links", + "continue" -> "||", + "plcontinue" -> "48896716|0|Domain-specific_language" + ), + response2 + ) ) val bot = getBot(commands: _*) - val action = Action(Query( - Prop( - Links() - ), - TitlesParam(Seq(title)) - )) + val action = Action( + Query( + Prop( + Links() + ), + TitlesParam(Seq(title)) + ) + ) val result = bot.run(action).await @@ -95,7 +106,12 @@ class PropLinksSpec extends Specification with MockBotSpec { p1.id === Some(48896716) p1.title === "Reactive Streams" p1.links.size === 4 - p1.links.map(_.title) === Seq("API", "Data buffer", "Implementation", "Interoperability") + p1.links.map(_.title) === Seq( + "API", + "Data buffer", + "Implementation", + "Interoperability" + ) p1.links.flatMap(_.ns).toSet === Set(0) } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/query/PropRevisionsSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/query/PropRevisionsSpec.scala index fcebba3a..fa31d06a 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/query/PropRevisionsSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/query/PropRevisionsSpec.scala @@ -26,30 +26,80 @@ class PropRevisionsSpec extends Specification with MockBotSpec { |"revisions": [{"revid": 2, "user": "u2", "comment": "c2", "*": "$pageText2"}]} }}}""".stripMargin val commands = Seq( - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|comment", - "continue" -> ""), response1), - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|comment", - "continue" -> "gcmcontinue||", "gcmcontinue" -> "10|Stub|6674690"), response2) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|comment", + "continue" -> "" + ), + response1 + ), + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|comment", + "continue" -> "gcmcontinue||", + "gcmcontinue" -> "10|Stub|6674690" + ), + response2 + ) ) val bot = getBot(commands: _*) - val future = bot.page("Category:SomeCategory") - .revisionsByGenerator("categorymembers", "cm", Set.empty, Set("ids", "content", "user", "comment")).map(_.toSeq) + val future = bot + .page("Category:SomeCategory") + .revisionsByGenerator( + "categorymembers", + "cm", + Set.empty, + Set("ids", "content", "user", "comment") + ) + .map(_.toSeq) val result = future.await result must have size 2 - result(0) === Page(Some(569559L), Some(1), "Talk:Welfare reform", - Seq(Revision(Some(1L), Some(569559L), None, Some(User(None, Some("u1"))), None, Some("c1"), Some(pageText1))) + result(0) === Page( + Some(569559L), + Some(1), + "Talk:Welfare reform", + Seq( + Revision( + Some(1L), + Some(569559L), + None, + Some(User(None, Some("u1"))), + None, + Some("c1"), + Some(pageText1) + ) + ) ) - result(1) === Page(Some(4571809L), Some(2), "User:Formator", - Seq(Revision(Some(2L), Some(4571809L), None, Some(User(None, Some("u2"))), None, Some("c2"), Some(pageText2))) + result(1) === Page( + Some(4571809L), + Some(2), + "User:Formator", + Seq( + Revision( + Some(2L), + Some(4571809L), + None, + Some(User(None, Some("u2"))), + None, + Some("c2"), + Some(pageText2) + ) + ) ) } } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/time/DateTimeReadersSpec.scala b/scalawiki-core/src/test/scala/org/scalawiki/time/DateTimeReadersSpec.scala index 36c6c6f3..0b01a33a 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/time/DateTimeReadersSpec.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/time/DateTimeReadersSpec.scala @@ -1,6 +1,5 @@ package org.scalawiki.time - import java.time.format.DateTimeParseException import java.time.{Instant, ZoneOffset, ZonedDateTime} @@ -9,8 +8,7 @@ import net.ceedubs.ficus.Ficus._ import org.specs2.mutable.Specification class DateTimeReadersSpec extends Specification { - val config = ConfigFactory.parseString( - """ + val config = ConfigFactory.parseString(""" num = 123 str = "2013-01-05T12:00:00Z" invalid = "invalid" @@ -24,9 +22,13 @@ class DateTimeReadersSpec extends Specification { "DateTimeReader" should { "read iso-8601 string value to ReadableInstant (DateTime)" in { - config.as[ZonedDateTime]("str") === ZonedDateTime.parse("2013-01-05T12:00:00Z") - config.as[Option[ZonedDateTime]]("str") === Option(ZonedDateTime.parse("2013-01-05T12:00:00Z")) + config.as[ZonedDateTime]("str") === ZonedDateTime.parse( + "2013-01-05T12:00:00Z" + ) + config.as[Option[ZonedDateTime]]("str") === Option( + ZonedDateTime.parse("2013-01-05T12:00:00Z") + ) config.as[ZonedDateTime]("invalid") must throwA[DateTimeParseException] } } -} \ No newline at end of file +} diff --git a/scalawiki-core/src/test/scala/org/scalawiki/util/HttpStub.scala b/scalawiki-core/src/test/scala/org/scalawiki/util/HttpStub.scala index d4c055d4..527c5603 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/util/HttpStub.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/util/HttpStub.scala @@ -3,9 +3,11 @@ package org.scalawiki.util import akka.http.scaladsl.model.ContentType import org.scalawiki.http.HttpClient -case class HttpStub(queryParam: Map[String, String], - response: String, - path: String = "/w/api.php", - contentType: ContentType = HttpClient.JSON_UTF8) { +case class HttpStub( + queryParam: Map[String, String], + response: String, + path: String = "/w/api.php", + contentType: ContentType = HttpClient.JSON_UTF8 +) { val query = queryParam + ("format" -> "json") + ("utf8" -> "") } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/util/TestHttpClient.scala b/scalawiki-core/src/test/scala/org/scalawiki/util/TestHttpClient.scala index ef6e6cdf..fc483ed0 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/util/TestHttpClient.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/util/TestHttpClient.scala @@ -28,10 +28,14 @@ class TestHttpClient(val host: String, commandsParam: Seq[HttpStub]) override def getResponse(url: Uri): Future[HttpResponse] = getResponse(url, url.query().toMap) - def getResponse(url: Uri, - params: Map[String, String]): Future[HttpResponse] = { - require(commands.nonEmpty, - "Unexpected query: " + url.toString() + " with params:\n" + params) + def getResponse( + url: Uri, + params: Map[String, String] + ): Future[HttpResponse] = { + require( + commands.nonEmpty, + "Unexpected query: " + url.toString() + " with params:\n" + params + ) val command = commands.dequeue() @@ -43,35 +47,43 @@ class TestHttpClient(val host: String, commandsParam: Seq[HttpStub]) require(matchResult.isSuccess, matchResult.message) val pageResponse = Option(command.response) - .fold(HttpResponse(StatusCodes.NotFound))( - text => - HttpResponse( - StatusCodes.OK, - entity = HttpEntity(command.contentType, - text.getBytes(StandardCharsets.UTF_8)) + .fold(HttpResponse(StatusCodes.NotFound))(text => + HttpResponse( + StatusCodes.OK, + entity = HttpEntity( + command.contentType, + text.getBytes(StandardCharsets.UTF_8) + ) ) ) Promise.successful(pageResponse).future } - override def post(url: String, - params: Map[String, String]): Future[HttpResponse] = + override def post( + url: String, + params: Map[String, String] + ): Future[HttpResponse] = getResponse(url, params) - override def postUri(url: Uri, - params: Map[String, String]): Future[HttpResponse] = + override def postUri( + url: Uri, + params: Map[String, String] + ): Future[HttpResponse] = getResponse(url, params) override def postMultiPart( url: String, - params: Map[String, String]): Future[HttpResponse] = + params: Map[String, String] + ): Future[HttpResponse] = getResponse(url, params) - override def postFile(url: String, - params: Map[String, String], - fileParam: String, - filename: String): Future[HttpResponse] = ??? + override def postFile( + url: String, + params: Map[String, String], + fileParam: String, + filename: String + ): Future[HttpResponse] = ??? override def get(url: String): Future[String] = getResponse(url) flatMap getBody @@ -83,6 +95,7 @@ class TestHttpClient(val host: String, commandsParam: Seq[HttpStub]) override def postMultiPart( url: Uri, - params: Map[String, String]): Future[HttpResponse] = ??? + params: Map[String, String] + ): Future[HttpResponse] = ??? } diff --git a/scalawiki-core/src/test/scala/org/scalawiki/util/TestUtils.scala b/scalawiki-core/src/test/scala/org/scalawiki/util/TestUtils.scala index 3f71811b..74636b7e 100644 --- a/scalawiki-core/src/test/scala/org/scalawiki/util/TestUtils.scala +++ b/scalawiki-core/src/test/scala/org/scalawiki/util/TestUtils.scala @@ -3,8 +3,8 @@ package org.scalawiki.util import scala.io.{Codec, Source} object TestUtils { - def resourceAsString(resource: String): String = { - val is = getClass.getResourceAsStream(resource) - Source.fromInputStream(is)(Codec.UTF8).mkString - } + def resourceAsString(resource: String): String = { + val is = getClass.getResourceAsStream(resource) + Source.fromInputStream(is)(Codec.UTF8).mkString + } } diff --git a/scalawiki-dumps/src/main/scala/org/scalawiki/compress/Compression.scala b/scalawiki-dumps/src/main/scala/org/scalawiki/compress/Compression.scala index 424b8fab..3d3068c3 100644 --- a/scalawiki-dumps/src/main/scala/org/scalawiki/compress/Compression.scala +++ b/scalawiki-dumps/src/main/scala/org/scalawiki/compress/Compression.scala @@ -3,8 +3,14 @@ package org.scalawiki.compress import java.io.{InputStream, OutputStream} import java.nio.file.{Files, Path} -import org.apache.commons.compress.compressors.bzip2.{BZip2CompressorInputStream, BZip2CompressorOutputStream} -import org.apache.commons.compress.compressors.gzip.{GzipCompressorInputStream, GzipCompressorOutputStream} +import org.apache.commons.compress.compressors.bzip2.{ + BZip2CompressorInputStream, + BZip2CompressorOutputStream +} +import org.apache.commons.compress.compressors.gzip.{ + GzipCompressorInputStream, + GzipCompressorOutputStream +} abstract class Compression(val ext: String) { def inputStream(in: InputStream): InputStream @@ -15,13 +21,16 @@ abstract class Compression(val ext: String) { object Bz2 extends Compression("bz2") { override def inputStream(in: InputStream) = new BZip2CompressorInputStream(in) - override def outputStream(out: OutputStream) = new BZip2CompressorOutputStream(out) + override def outputStream(out: OutputStream) = + new BZip2CompressorOutputStream(out) } object Gz extends Compression("gz") { override def inputStream(in: InputStream) = new GzipCompressorInputStream(in) - override def outputStream(out: OutputStream) = new GzipCompressorOutputStream(out) + override def outputStream(out: OutputStream) = new GzipCompressorOutputStream( + out + ) } object SevenZ extends Compression("7z") { @@ -46,8 +55,10 @@ object Compression { compressions.find(_.ext == ext).getOrElse(NoCompression) } - def inputStream(path: Path) = get(path).inputStream(Files.newInputStream(path)) + def inputStream(path: Path) = + get(path).inputStream(Files.newInputStream(path)) - def outputStream(path: Path) = get(path).outputStream(Files.newOutputStream(path)) + def outputStream(path: Path) = + get(path).outputStream(Files.newOutputStream(path)) } diff --git a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/Block.scala b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/Block.scala index cfa83e07..5125a42f 100644 --- a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/Block.scala +++ b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/Block.scala @@ -15,5 +15,4 @@ case class Block(offset: Long, size: Long) { // def intersection(offsetParam: Long, sizeParam: Long): Block = // contains(offsetParam) && contains(offsetParam + sizeParam) - } diff --git a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/DumpFile.scala b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/DumpFile.scala index f2d6bed2..7da392e3 100644 --- a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/DumpFile.scala +++ b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/DumpFile.scala @@ -13,18 +13,20 @@ class DumpFileInfo(val path: Path) { val dotParts = filename split "\\." val withoutExtension = dotParts(0) - val extensions = dotParts slice(1, dotParts.length) + val extensions = dotParts slice (1, dotParts.length) val hyphensParts = withoutExtension split "-" - val database: Option[String] = if (hyphensParts(0) == "pagecounts") - None - else - Some(hyphensParts(0)) - - val dumpType = if (hyphensParts(0) == "pagecounts") - "pagecounts" - else - hyphensParts.slice(2, hyphensParts.length).mkString("-") + val database: Option[String] = + if (hyphensParts(0) == "pagecounts") + None + else + Some(hyphensParts(0)) + + val dumpType = + if (hyphensParts(0) == "pagecounts") + "pagecounts" + else + hyphensParts.slice(2, hyphensParts.length).mkString("-") val date = hyphensParts(1) @@ -39,11 +41,13 @@ class DumpFileInfo(val path: Path) { def isPagesXml = isXml && dumpType.startsWith("pages-") - def isPageIndex = isTxt && dumpType.startsWith("pages-") && dumpType.endsWith("-index") + def isPageIndex = + isTxt && dumpType.startsWith("pages-") && dumpType.endsWith("-index") def isPageCounts = dumpType == "pagecounts" - def isIndex(path: Path) = path.getFileName.toString.startsWith(withoutExtension + "-index.txt") + def isIndex(path: Path) = + path.getFileName.toString.startsWith(withoutExtension + "-index.txt") def dumpFile: Option[DumpFile] = { if (isPagesXml) @@ -74,7 +78,6 @@ class SqlDump(info: DumpFileInfo) extends DumpFile(info) class PageViewCountDump(info: DumpFileInfo) extends DumpFile(info) - object DumpFile { def get(path: Path): Option[DumpFile] = { diff --git a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/PageIndex.scala b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/PageIndex.scala index e7eadf19..b6b06582 100644 --- a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/PageIndex.scala +++ b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/PageIndex.scala @@ -1,8 +1,7 @@ package org.scalawiki.xml - case class PageIndex(offset: Long, id: Long, title: String) { - + override def toString = s"$offset:$id:$title" } diff --git a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/SkippingInputStream.scala b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/SkippingInputStream.scala index 3cfda43c..8b419827 100644 --- a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/SkippingInputStream.scala +++ b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/SkippingInputStream.scala @@ -2,12 +2,15 @@ package org.scalawiki.xml import java.io.{IOException, FilterInputStream, InputStream} -/** - * Reads just the specified blocks from the underlying input stream, skipping over everything else. - * @param in underlying input stream - * @param blocks blocks to read - */ -class SkippingInputStream(in: InputStream, val blocks: Seq[Block]) extends FilterInputStream(in) { +/** Reads just the specified blocks from the underlying input stream, skipping + * over everything else. + * @param in + * underlying input stream + * @param blocks + * blocks to read + */ +class SkippingInputStream(in: InputStream, val blocks: Seq[Block]) + extends FilterInputStream(in) { private var offset: Long = 0L private var mark: Long = -1 @@ -52,8 +55,7 @@ class SkippingInputStream(in: InputStream, val blocks: Seq[Block]) extends Filte if (block.contains(offset, len - bytesRead)) { // read fully lastRead = in.read(b, off + bytesRead, len - bytesRead) - } - else if (block.contains(offset)) { + } else if (block.contains(offset)) { // read partially lastRead = in.read(b, off + bytesRead, (block.end - offset + 1).toInt) } else if (offset < block.offset) { diff --git a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlIndex.scala b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlIndex.scala index 9aa3255d..2e3cb51a 100644 --- a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlIndex.scala +++ b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlIndex.scala @@ -12,7 +12,9 @@ class XmlIndex(val pages: Seq[PageIndex]) { val nl = System.lineSeparator() def save(os: OutputStream) = { - pages.map(pi => (pi.toString + nl).getBytes(StandardCharsets.UTF_8)).foreach(os.write) + pages + .map(pi => (pi.toString + nl).getBytes(StandardCharsets.UTF_8)) + .foreach(os.write) } } @@ -26,4 +28,4 @@ object XmlIndex { def fromInputStream(is: InputStream): Iterator[PageIndex] = Source.fromInputStream(is).getLines().map(PageIndex.fromString) -} \ No newline at end of file +} diff --git a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlParser.scala b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlParser.scala index 9d0b53df..0994e84a 100644 --- a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlParser.scala +++ b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlParser.scala @@ -11,11 +11,16 @@ import org.scalawiki.dto.Image import scala.collection.{Iterator, AbstractIterator} -case class SiteInfo(name: Option[String], db: Option[String], generator: Option[String]) +case class SiteInfo( + name: Option[String], + db: Option[String], + generator: Option[String] +) class XmlParser( - val parser: XMLStreamReader2 with LocationInfo, - val pageFilter: Page => Boolean = PageFilter.all) extends Iterable[Page] { + val parser: XMLStreamReader2 with LocationInfo, + val pageFilter: Page => Boolean = PageFilter.all +) extends Iterable[Page] { private var _siteInfo: Option[SiteInfo] = None private var _namespaces: Map[Int, String] = Map.empty @@ -52,7 +57,7 @@ class XmlParser( def close() = parser match { case stax2Parser: XMLStreamReader2 => stax2Parser.closeCompletely() - case _ => parser.close() + case _ => parser.close() } private def readSiteInfo() = { @@ -86,19 +91,18 @@ class XmlParser( private def readPage(): Option[Page] = if (findElementStart("page")) { _pageStartingByteOffset = parser.getStartingByteOffset - for (title <- readElement("title"); - ns <- readElement("ns").map(_.toInt); - id <- readElement("id").map(_.toLong) + for ( + title <- readElement("title"); + ns <- readElement("ns").map(_.toInt); + id <- readElement("id").map(_.toLong) ) yield { val revisions = readRevisions(id) val images = readImageInfo() new Page(Some(id), Some(ns), title, revisions.toSeq) } - } - else + } else None - private def readRevisions(pageId: Long): Seq[Revision] = { // TODO streaming and filtering var revisions = Seq.empty[Revision] @@ -137,10 +141,15 @@ class XmlParser( val user: Option[Contributor] = ( for ( - username <- readElement("username", next = "ip", parent = "contributor"); - userId <- readElement("id", "ip", "contributor").map(_.toInt)) - yield new User(Some(userId), Some(username)) + username <- readElement( + "username", + next = "ip", + parent = "contributor" + ); + userId <- readElement("id", "ip", "contributor").map(_.toInt) ) + yield new User(Some(userId), Some(username)) + ) .orElse( readElement("ip", parent = "contributor").map(new IpContributor(_)) ) @@ -156,25 +165,42 @@ class XmlParser( None } - private def readElement(name: String, next: String = "", parent: String = ""): Option[String] = { + private def readElement( + name: String, + next: String = "", + parent: String = "" + ): Option[String] = { if (findElementStart(name, next, parent)) Some(parser.getElementText) else None } - private def findElementStart(name: String, next: String = "", parent: String = ""): Boolean = { + private def findElementStart( + name: String, + next: String = "", + parent: String = "" + ): Boolean = { parser.getEventType match { - case XMLStreamConstants.START_ELEMENT if parser.getLocalName == name => true - case XMLStreamConstants.START_ELEMENT if parser.getLocalName == next => false - case XMLStreamConstants.END_ELEMENT if parser.getLocalName == parent => false + case XMLStreamConstants.START_ELEMENT if parser.getLocalName == name => + true + case XMLStreamConstants.START_ELEMENT if parser.getLocalName == next => + false + case XMLStreamConstants.END_ELEMENT if parser.getLocalName == parent => + false case _ => while (parser.hasNext) { val event = parser.next() event match { - case XMLStreamConstants.START_ELEMENT if parser.getLocalName == name => return true - case XMLStreamConstants.START_ELEMENT if parser.getLocalName == next => return false - case XMLStreamConstants.END_ELEMENT if parser.getLocalName == parent => return false + case XMLStreamConstants.START_ELEMENT + if parser.getLocalName == name => + return true + case XMLStreamConstants.START_ELEMENT + if parser.getLocalName == next => + return false + case XMLStreamConstants.END_ELEMENT + if parser.getLocalName == parent => + return false case _ => } } @@ -183,8 +209,13 @@ class XmlParser( } private def findElementEnd(name: String) = { - if (parser.getEventType != XMLStreamConstants.END_ELEMENT || parser.getLocalName != name) - while (parser.next() != XMLStreamConstants.END_ELEMENT || parser.getLocalName != name) {} + if ( + parser.getEventType != XMLStreamConstants.END_ELEMENT || parser.getLocalName != name + ) + while ( + parser + .next() != XMLStreamConstants.END_ELEMENT || parser.getLocalName != name + ) {} } } @@ -193,32 +224,49 @@ object XmlParser { val xmlInputFactory = newXmlInputFactory - def newXmlParser(xmlReader: XMLStreamReader, pageFilter: Page => Boolean = PageFilter.all) = - new XmlParser(xmlReader.asInstanceOf[XMLStreamReader2 with LocationInfo], pageFilter) - - def parseFile(filename: String, - pageFilter: Page => Boolean = PageFilter.all) = - newXmlParser(xmlInputFactory.createXMLStreamReader(new File(filename)), pageFilter) - - - def parseInputStream(is: InputStream, - pageFilter: Page => Boolean = PageFilter.all) = + def newXmlParser( + xmlReader: XMLStreamReader, + pageFilter: Page => Boolean = PageFilter.all + ) = + new XmlParser( + xmlReader.asInstanceOf[XMLStreamReader2 with LocationInfo], + pageFilter + ) + + def parseFile( + filename: String, + pageFilter: Page => Boolean = PageFilter.all + ) = + newXmlParser( + xmlInputFactory.createXMLStreamReader(new File(filename)), + pageFilter + ) + + def parseInputStream( + is: InputStream, + pageFilter: Page => Boolean = PageFilter.all + ) = newXmlParser(xmlInputFactory.createXMLStreamReader(is), pageFilter) - def parseReader(reader: Reader, - pageFilter: Page => Boolean = PageFilter.all) = + def parseReader( + reader: Reader, + pageFilter: Page => Boolean = PageFilter.all + ) = newXmlParser(xmlInputFactory.createXMLStreamReader(reader), pageFilter) - def parseString(data: String, - pageFilter: Page => Boolean = PageFilter.all) = + def parseString(data: String, pageFilter: Page => Boolean = PageFilter.all) = parseReader(new StringReader(data), pageFilter) // def parseUrl(url: URL) = new XmlParser(xmlInputFactory.createXMLStreamReader(url)) def newXmlInputFactory: XMLInputFactory2 = { - val xmlInputFactory = XMLInputFactory.newInstance().asInstanceOf[XMLInputFactory2] + val xmlInputFactory = + XMLInputFactory.newInstance().asInstanceOf[XMLInputFactory2] xmlInputFactory.configureForSpeed() - xmlInputFactory.setProperty(XMLInputFactory.IS_REPLACING_ENTITY_REFERENCES, false) + xmlInputFactory.setProperty( + XMLInputFactory.IS_REPLACING_ENTITY_REFERENCES, + false + ) xmlInputFactory } -} \ No newline at end of file +} diff --git a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlWriter.scala b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlWriter.scala index 80a190e1..8aed9643 100644 --- a/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlWriter.scala +++ b/scalawiki-dumps/src/main/scala/org/scalawiki/xml/XmlWriter.scala @@ -31,7 +31,7 @@ class XmlWriter(writer: XMLStreamWriter) { writer.flush() writer match { case stax2Writer: XMLStreamWriter2 => stax2Writer.closeCompletely() - case _ => writer.close() + case _ => writer.close() } } @@ -78,7 +78,11 @@ class XmlWriter(writer: XMLStreamWriter) { writeElement("model", "wikitext") writeElement("format", "text/x-wiki") - writeElement("text", rev.content.getOrElse(""), Seq("xml:space" -> "preserve")) + writeElement( + "text", + rev.content.getOrElse(""), + Seq("xml:space" -> "preserve") + ) rev.sha1.foreach(writeElement("sha1", _)) writer.writeEndElement() @@ -88,7 +92,11 @@ class XmlWriter(writer: XMLStreamWriter) { writer.writeEmptyElement(name) } - def writeElement(name: String, value: Any, attrs: Seq[(String, String)] = Seq.empty) = { + def writeElement( + name: String, + value: Any, + attrs: Seq[(String, String)] = Seq.empty + ) = { writer.writeStartElement(name) for ((k, v) <- attrs) writer.writeAttribute(k, v) @@ -101,20 +109,24 @@ class XmlWriter(writer: XMLStreamWriter) { object XmlWriter { - val outputFactory = XMLOutputFactory.newFactory().asInstanceOf[XMLOutputFactory2] + val outputFactory = + XMLOutputFactory.newFactory().asInstanceOf[XMLOutputFactory2] outputFactory.configureForSpeed() def create(w: Writer) = { - val writer = new IndentingXMLStreamWriter(XmlWriter.outputFactory.createXMLStreamWriter(w)) + val writer = new IndentingXMLStreamWriter( + XmlWriter.outputFactory.createXMLStreamWriter(w) + ) new XmlWriter(writer) } def create(os: OutputStream) = { - val writer = new IndentingXMLStreamWriter(XmlWriter.outputFactory.createXMLStreamWriter(os)) + val writer = new IndentingXMLStreamWriter( + XmlWriter.outputFactory.createXMLStreamWriter(os) + ) new XmlWriter(writer) } - // def createWriter(): XMLStreamWriter = // xmlOutputFactory.createXMLStreamWriter(System.out) @@ -130,8 +142,7 @@ object XmlWriter { "xmlns:xsi" -> "http://www.w3.org/2001/XMLSchema-instance", "xsi:schemaLocation" -> (ns + " " + schema), "version" -> version, - "xml:lang" -> "en" //TODO lang + "xml:lang" -> "en" // TODO lang ) - -} \ No newline at end of file +} diff --git a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/DataDumpSpec.scala b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/DataDumpSpec.scala index f90a3d10..7ae21e57 100644 --- a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/DataDumpSpec.scala +++ b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/DataDumpSpec.scala @@ -10,15 +10,13 @@ class DataDumpSpec extends Specification with BeforeEach { val filename = "ukwiki-20150311-pages-articles-multistream.xml.bz2" - override def before = { - - } + override def before = {} "data dump" should { "fill data from filesystem" in { val fs = Jimfs.newFileSystem(Configuration.unix()) val path = "/path/to" - val fullname = path + "/" + filename + val fullname = path + "/" + filename val dd = new DumpFileInfo(fs.getPath(fullname)) @@ -39,7 +37,7 @@ class DataDumpSpec extends Specification with BeforeEach { val dd = DumpFile.get(dumpPath).get.asInstanceOf[PagesDump] - dd.findIndexFile.map(_.toString) === Some(indexPath.toString) + dd.findIndexFile.map(_.toString) === Some(indexPath.toString) } "find uncompressed index" in { @@ -51,7 +49,7 @@ class DataDumpSpec extends Specification with BeforeEach { val dd = DumpFile.get(dumpPath).get.asInstanceOf[PagesDump] - dd.findIndexFile.map(_.toString) === Some(indexPath.toString) + dd.findIndexFile.map(_.toString) === Some(indexPath.toString) } "do not find index with wrong date" in { @@ -68,7 +66,10 @@ class DataDumpSpec extends Specification with BeforeEach { } - def newInMemoryDirectory(path: String = "/foo", configuration: Configuration = Configuration.unix()): Path = { + def newInMemoryDirectory( + path: String = "/foo", + configuration: Configuration = Configuration.unix() + ): Path = { val fs = Jimfs.newFileSystem(configuration) val dir = fs.getPath(path) Files.createDirectory(dir) diff --git a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/SkippingInputStreamSpec.scala b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/SkippingInputStreamSpec.scala index 0dded976..108df2a8 100644 --- a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/SkippingInputStreamSpec.scala +++ b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/SkippingInputStreamSpec.scala @@ -4,13 +4,16 @@ import java.io.ByteArrayInputStream import org.specs2.mutable.Specification -class SkippingInputStreamSpec extends Specification{ +class SkippingInputStreamSpec extends Specification { "SkippingInputStream" should { "read nothing" in { val input = "1234567890".getBytes - val sis = new SkippingInputStream(new ByteArrayInputStream(input), Seq(Block(0, 0))) + val sis = new SkippingInputStream( + new ByteArrayInputStream(input), + Seq(Block(0, 0)) + ) val buf = Array.fill[Byte](10)(0) sis.read(buf) === -1 buf === Array.fill[Byte](10)(0) @@ -18,7 +21,10 @@ class SkippingInputStreamSpec extends Specification{ "read all stream to large buf" in { val input = "1234567890".getBytes - val sis = new SkippingInputStream(new ByteArrayInputStream(input), Seq(Block(0, 10))) + val sis = new SkippingInputStream( + new ByteArrayInputStream(input), + Seq(Block(0, 10)) + ) val buf = Array.fill[Byte](10)(0) sis.read(buf) === 10 buf === input @@ -27,7 +33,10 @@ class SkippingInputStreamSpec extends Specification{ "read all stream with block larger than input" in { val input = "1234567890".getBytes - val sis = new SkippingInputStream(new ByteArrayInputStream(input), Seq(Block(0, 20))) + val sis = new SkippingInputStream( + new ByteArrayInputStream(input), + Seq(Block(0, 20)) + ) val buf = Array.fill[Byte](10)(0) sis.read(buf) === 10 buf === input @@ -36,7 +45,10 @@ class SkippingInputStreamSpec extends Specification{ "read all stream to small buf" in { val input = "1234567890" - val sis = new SkippingInputStream(new ByteArrayInputStream(input.getBytes), Seq(Block(0, 10))) + val sis = new SkippingInputStream( + new ByteArrayInputStream(input.getBytes), + Seq(Block(0, 10)) + ) val buf = Array.fill[Byte](5)(0) sis.read(buf) === 5 new String(buf) === input.slice(0, 5) @@ -47,17 +59,25 @@ class SkippingInputStreamSpec extends Specification{ } "read first two blocks to large buf" in { - val input = ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes - val sis = new SkippingInputStream(new ByteArrayInputStream(input), Seq(Block(0, 10), Block(10, 10))) + val input = + ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes + val sis = new SkippingInputStream( + new ByteArrayInputStream(input), + Seq(Block(0, 10), Block(10, 10)) + ) val buf = Array.fill[Byte](20)('-') sis.read(buf) === 20 - new String(buf) === "1234567890" + "qwertyuiop" + new String(buf) === "1234567890" + "qwertyuiop" sis.read(buf) === -1 } "read first two blocks to small buf" in { - val input = ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes - val sis = new SkippingInputStream(new ByteArrayInputStream(input), Seq(Block(0, 10), Block(10, 10))) + val input = + ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes + val sis = new SkippingInputStream( + new ByteArrayInputStream(input), + Seq(Block(0, 10), Block(10, 10)) + ) val buf = Array.fill[Byte](5)('-') sis.read(buf) === 5 @@ -72,26 +92,38 @@ class SkippingInputStreamSpec extends Specification{ } "read last two blocks" in { - val input = ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes - val sis = new SkippingInputStream(new ByteArrayInputStream(input), Seq(Block(20, 9), Block(29, 7))) + val input = + ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes + val sis = new SkippingInputStream( + new ByteArrayInputStream(input), + Seq(Block(20, 9), Block(29, 7)) + ) val buf = Array.fill[Byte](20)('-') sis.read(buf) === 16 - new String(buf) === "asdfghjkl" + "zxcvbnm" + "----" + new String(buf) === "asdfghjkl" + "zxcvbnm" + "----" sis.read(buf) === -1 } "read odd blocks" in { - val input = ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes - val sis = new SkippingInputStream(new ByteArrayInputStream(input), Seq(Block(0, 10), Block(20, 9))) + val input = + ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes + val sis = new SkippingInputStream( + new ByteArrayInputStream(input), + Seq(Block(0, 10), Block(20, 9)) + ) val buf = Array.fill[Byte](20)('-') sis.read(buf) === 19 - new String(buf) === "1234567890" + "asdfghjkl" + "-" + new String(buf) === "1234567890" + "asdfghjkl" + "-" sis.read(buf) === -1 } "read odd blocks in small buff" in { - val input = ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes - val sis = new SkippingInputStream(new ByteArrayInputStream(input), Seq(Block(0, 10), Block(20, 9))) + val input = + ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes + val sis = new SkippingInputStream( + new ByteArrayInputStream(input), + Seq(Block(0, 10), Block(20, 9)) + ) val buf = Array.fill[Byte](5)('-') sis.read(buf) === 5 new String(buf) === "12345" @@ -105,16 +137,24 @@ class SkippingInputStreamSpec extends Specification{ } "read even blocks" in { - val input = ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes - val sis = new SkippingInputStream(new ByteArrayInputStream(input), Seq(Block(10, 10), Block(29, 7))) + val input = + ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes + val sis = new SkippingInputStream( + new ByteArrayInputStream(input), + Seq(Block(10, 10), Block(29, 7)) + ) val buf = Array.fill[Byte](20)('-') sis.read(buf) === 17 new String(buf) === "qwertyuiop" + "zxcvbnm" + "---" } "read even blocks in small buf" in { - val input = ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes - val sis = new SkippingInputStream(new ByteArrayInputStream(input), Seq(Block(10, 10), Block(29, 7))) + val input = + ("1234567890" + "qwertyuiop" + "asdfghjkl" + "zxcvbnm").getBytes + val sis = new SkippingInputStream( + new ByteArrayInputStream(input), + Seq(Block(10, 10), Block(29, 7)) + ) val buf = Array.fill[Byte](5)('-') sis.read(buf) === 5 new String(buf) === "qwert" diff --git a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlHelper.scala b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlHelper.scala index 5b488eba..90a3dcbb 100644 --- a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlHelper.scala +++ b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlHelper.scala @@ -18,10 +18,12 @@ object XmlHelper { {content} .toString() - def pageXml(title: String = "Page title", - ns: Int = 0, - id: Long = 1, - revisions: NodeSeq): Node = + def pageXml( + title: String = "Page title", + ns: Int = 0, + id: Long = 1, + revisions: NodeSeq + ): Node = {title} {ns} @@ -29,7 +31,16 @@ object XmlHelper { {revisions} - def revisionXml(revId: Long, parentId: Long, timestamp: ZonedDateTime, user: String, userId: Int, comment: String, text: String, sha1: String): Node = + def revisionXml( + revId: Long, + parentId: Long, + timestamp: ZonedDateTime, + user: String, + userId: Int, + comment: String, + text: String, + sha1: String + ): Node = {revId} {parentId} diff --git a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlIndexSpec.scala b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlIndexSpec.scala index 93c83e4a..9b48f093 100644 --- a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlIndexSpec.scala +++ b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlIndexSpec.scala @@ -53,7 +53,8 @@ class XmlIndexSpec extends Specification { |5004:3:File:Some image.jpg |""".stripMargin - val pages = XmlIndex.fromInputStream(new ByteArrayInputStream(index.getBytes)).toSeq + val pages = + XmlIndex.fromInputStream(new ByteArrayInputStream(index.getBytes)).toSeq pages.size === 3 diff --git a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlParserSpec.scala b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlParserSpec.scala index 9e15b33d..093c98f4 100644 --- a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlParserSpec.scala +++ b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlParserSpec.scala @@ -27,11 +27,13 @@ class XmlParserSpec extends Specification { val seq = parser.iterator.toSeq seq.isEmpty === true - parser.siteInfo === Some(SiteInfo( - Some("Вікіпедія"), - Some("ukwiki"), - Some("MediaWiki 1.24wmf22") - )) + parser.siteInfo === Some( + SiteInfo( + Some("Вікіпедія"), + Some("ukwiki"), + Some("MediaWiki 1.24wmf22") + ) + ) val namespaces = parser.namespaces namespaces.size === 24 @@ -44,9 +46,27 @@ class XmlParserSpec extends Specification { val (title, ns, pageId) = ("Page title", 0, 123) val (revId, parentId, timestamp, user, userId, comment, text, sha1) = - (345, 456, ZonedDateTime.now, "user", 567, "revision comment", "revision text", "sha1") - - val revsXml = revisionXml(revId, parentId, timestamp, user, userId, comment, text, sha1) + ( + 345, + 456, + ZonedDateTime.now, + "user", + 567, + "revision comment", + "revision text", + "sha1" + ) + + val revsXml = revisionXml( + revId, + parentId, + timestamp, + user, + userId, + comment, + text, + sha1 + ) val xml = mediawiki(siteInfoXml ++ pageXml(title, ns, pageId, revsXml)) val parser = XmlParser.parseString(xml) @@ -61,16 +81,43 @@ class XmlParserSpec extends Specification { revs.size === 1 val rev = revs(0) - checkRevision(revId, parentId, timestamp, user, userId, comment, text, rev) + checkRevision( + revId, + parentId, + timestamp, + user, + userId, + comment, + text, + rev + ) } "parse a page without siteinfo" in { val (title, ns, pageId) = ("Page title", 0, 123) val (revId, parentId, timestamp, user, userId, comment, text, sha1) = - (345, 456, ZonedDateTime.now, "user", 567, "revision comment", "revision text", "sha1") - - val revsXml = revisionXml(revId, parentId, timestamp, user, userId, comment, text, sha1) + ( + 345, + 456, + ZonedDateTime.now, + "user", + 567, + "revision comment", + "revision text", + "sha1" + ) + + val revsXml = revisionXml( + revId, + parentId, + timestamp, + user, + userId, + comment, + text, + sha1 + ) val xml = mediawiki(pageXml(title, ns, pageId, revsXml)) val parser = XmlParser.parseString(xml) @@ -85,22 +132,87 @@ class XmlParserSpec extends Specification { revs.size === 1 val rev = revs(0) - checkRevision(revId, parentId, timestamp, user, userId, comment, text, rev) + checkRevision( + revId, + parentId, + timestamp, + user, + userId, + comment, + text, + rev + ) } "parse page with two revisions" in { val (title1, ns1, pageId1) = ("Page title1", 0, 123) - val (revId1, parentId1, timestamp1, user1, userId1, comment1, text1, sha1) = - (1345, 1456, ZonedDateTime.now - 1.month, "user", 1567, "revision comment1", "revision text1", "sha1") - val (revId2, parentId2, timestamp2, user2, userId2, comment2, text2, sha2) = - (2345, 2456, ZonedDateTime.now, "user2", 2567, "revision comment2", "revision text2", "sha2") - - val revsXml1 = revisionXml(revId1, parentId1, timestamp1, user1, userId1, comment1, text1, sha1) - val revsXml2 = revisionXml(revId2, parentId2, timestamp2, user2, userId2, comment2, text2, sha2) - - val xml = mediawiki(siteInfoXml ++ - pageXml(title1, ns1, pageId1, revsXml1 ++ revsXml2)) + val ( + revId1, + parentId1, + timestamp1, + user1, + userId1, + comment1, + text1, + sha1 + ) = + ( + 1345, + 1456, + ZonedDateTime.now - 1.month, + "user", + 1567, + "revision comment1", + "revision text1", + "sha1" + ) + val ( + revId2, + parentId2, + timestamp2, + user2, + userId2, + comment2, + text2, + sha2 + ) = + ( + 2345, + 2456, + ZonedDateTime.now, + "user2", + 2567, + "revision comment2", + "revision text2", + "sha2" + ) + + val revsXml1 = revisionXml( + revId1, + parentId1, + timestamp1, + user1, + userId1, + comment1, + text1, + sha1 + ) + val revsXml2 = revisionXml( + revId2, + parentId2, + timestamp2, + user2, + userId2, + comment2, + text2, + sha2 + ) + + val xml = mediawiki( + siteInfoXml ++ + pageXml(title1, ns1, pageId1, revsXml1 ++ revsXml2) + ) val parser = XmlParser.parseString(xml) @@ -112,25 +224,99 @@ class XmlParserSpec extends Specification { val revs1 = page1.revisions revs1.size === 2 - checkRevision(revId1, parentId1, timestamp1, user1, userId1, comment1, text1, revs1(0)) - checkRevision(revId2, parentId2, timestamp2, user2, userId2, comment2, text2, revs1(1)) + checkRevision( + revId1, + parentId1, + timestamp1, + user1, + userId1, + comment1, + text1, + revs1(0) + ) + checkRevision( + revId2, + parentId2, + timestamp2, + user2, + userId2, + comment2, + text2, + revs1(1) + ) } "parse two pages" in { val (title1, ns1, pageId1) = ("Page title1", 0, 123) val (title2, ns2, pageId2) = ("Page title2", 1, 234) - val (revId1, parentId1, timestamp1, user1, userId1, comment1, text1, sha1) = - (1345, 1456, ZonedDateTime.now - 1.month, "user", 1567, "revision comment1", "revision text1", "sha1") - val (revId2, parentId2, timestamp2, user2, userId2, comment2, text2, sha2) = - (2345, 2456, ZonedDateTime.now - 2.month, "user2", 2567, "revision comment2", "revision text2", "sha2") - - val revsXml1 = revisionXml(revId1, parentId1, timestamp1, user1, userId1, comment1, text1, sha1) - val revsXml2 = revisionXml(revId2, parentId2, timestamp2, user2, userId2, comment2, text2, sha2) - - val xml = mediawiki(siteInfoXml ++ - pageXml(title1, ns1, pageId1, revsXml1) ++ - pageXml(title2, ns2, pageId2, revsXml2)) + val ( + revId1, + parentId1, + timestamp1, + user1, + userId1, + comment1, + text1, + sha1 + ) = + ( + 1345, + 1456, + ZonedDateTime.now - 1.month, + "user", + 1567, + "revision comment1", + "revision text1", + "sha1" + ) + val ( + revId2, + parentId2, + timestamp2, + user2, + userId2, + comment2, + text2, + sha2 + ) = + ( + 2345, + 2456, + ZonedDateTime.now - 2.month, + "user2", + 2567, + "revision comment2", + "revision text2", + "sha2" + ) + + val revsXml1 = revisionXml( + revId1, + parentId1, + timestamp1, + user1, + userId1, + comment1, + text1, + sha1 + ) + val revsXml2 = revisionXml( + revId2, + parentId2, + timestamp2, + user2, + userId2, + comment2, + text2, + sha2 + ) + + val xml = mediawiki( + siteInfoXml ++ + pageXml(title1, ns1, pageId1, revsXml1) ++ + pageXml(title2, ns2, pageId2, revsXml2) + ) val parser = XmlParser.parseString(xml) @@ -142,14 +328,32 @@ class XmlParserSpec extends Specification { val revs1 = page1.revisions revs1.size === 1 - checkRevision(revId1, parentId1, timestamp1, user1, userId1, comment1, text1, revs1(0)) + checkRevision( + revId1, + parentId1, + timestamp1, + user1, + userId1, + comment1, + text1, + revs1(0) + ) val page2 = pages(1) (page2.id, page2.ns, page2.title) === (Some(pageId2), Some(ns2), title2) val revs2 = page2.revisions revs2.size === 1 - checkRevision(revId2, parentId2, timestamp2, user2, userId2, comment2, text2, revs2(0)) + checkRevision( + revId2, + parentId2, + timestamp2, + user2, + userId2, + comment2, + text2, + revs2(0) + ) } } @@ -157,9 +361,19 @@ class XmlParserSpec extends Specification { val (title, ns, pageId) = ("Page title", 0, 123) val (revId, parentId, timestamp, user, userId, comment, text, sha1) = - (345, 456, ZonedDateTime.now, "user", 567, "revision comment", "Main page", "sha1") - - val revsXml = revisionXml(revId, parentId, timestamp, user, userId, comment, text, sha1) + ( + 345, + 456, + ZonedDateTime.now, + "user", + 567, + "revision comment", + "Main page", + "sha1" + ) + + val revsXml = + revisionXml(revId, parentId, timestamp, user, userId, comment, text, sha1) val xml = mediawiki(pageXml(title, ns, pageId, revsXml)) val parser = XmlParser.parseString(xml) @@ -177,15 +391,16 @@ class XmlParserSpec extends Specification { checkRevision(revId, parentId, timestamp, user, userId, comment, text, rev) } - "parse mediawiki export-demo" in { val parser = XmlHelper.parseExportDemo - parser.siteInfo === Some(SiteInfo( - Some("DemoWiki"), - Some("demowiki"), - Some("MediaWiki 1.24") - )) + parser.siteInfo === Some( + SiteInfo( + Some("DemoWiki"), + Some("demowiki"), + Some("MediaWiki 1.24") + ) + ) val namespaces = parser.namespaces namespaces.size === 18 @@ -205,11 +420,17 @@ class XmlParserSpec extends Specification { val p1Revs = p1.revisions p1Revs.size === 2 val p1r1 = p1Revs(0) - (p1r1.revId, p1r1.parentId, p1r1.comment, p1r1.content) === (Some(100), Some(99), Some("I have just one thing to say!"), Some("A bunch of [[text]] here.")) + (p1r1.revId, p1r1.parentId, p1r1.comment, p1r1.content) === (Some( + 100 + ), Some(99), Some("I have just one thing to say!"), Some( + "A bunch of [[text]] here." + )) p1r1.user === Some(User(42, "Foobar")) val p1r2 = p1Revs(1) - (p1r2.revId, p1r2.parentId, p1r2.comment, p1r2.content) === (Some(99), None, Some("new!"), Some("An earlier [[revision]].")) + (p1r2.revId, p1r2.parentId, p1r2.comment, p1r2.content) === (Some( + 99 + ), None, Some("new!"), Some("An earlier [[revision]].")) p1r2.user === Some(IpContributor("10.0.0.2")) val p2 = pages(1) @@ -217,7 +438,11 @@ class XmlParserSpec extends Specification { val p2Revs = p2.revisions p2Revs.size === 1 val p2r1 = p2Revs(0) - (p2r1.revId, p2r1.parentId, p2r1.comment, p2r1.content) === (Some(101), None, Some("hey"), Some("WHYD YOU LOCK PAGE??!!! i was editing that jerk")) + (p2r1.revId, p2r1.parentId, p2r1.comment, p2r1.content) === (Some( + 101 + ), None, Some("hey"), Some( + "WHYD YOU LOCK PAGE??!!! i was editing that jerk" + )) p2r1.user === Some(IpContributor("10.0.0.2")) val p3 = pages(2) @@ -225,7 +450,11 @@ class XmlParserSpec extends Specification { val p3Revs = p3.revisions p3Revs.size === 1 val p3r1 = p3Revs(0) - (p3r1.revId, p3r1.parentId, p3r1.comment, p3r1.content) === (Some(102), None, Some("My awesomeest image!"), Some("This is an awesome little imgae. I lurves it. {{PD}}")) + (p3r1.revId, p3r1.parentId, p3r1.comment, p3r1.content) === (Some( + 102 + ), None, Some("My awesomeest image!"), Some( + "This is an awesome little imgae. I lurves it. {{PD}}" + )) p3r1.user === Some(User(42, "Foobar")) val ii = p3.images @@ -246,17 +475,40 @@ class XmlParserSpec extends Specification { val p1Revs = p1.revisions p1Revs.size === 2 val p1r1 = p1Revs(0) - (p1r1.revId, p1r1.parentId, p1r1.comment, p1r1.content) === (Some(100), Some(99), Some("I have just one thing to say!"), Some("A bunch of [[text]] here.")) + (p1r1.revId, p1r1.parentId, p1r1.comment, p1r1.content) === (Some( + 100 + ), Some(99), Some("I have just one thing to say!"), Some( + "A bunch of [[text]] here." + )) p1r1.user === Some(User(42, "Foobar")) val p1r2 = p1Revs(1) - (p1r2.revId, p1r2.parentId, p1r2.comment, p1r2.content) === (Some(99), None, Some("new!"), Some("An earlier [[revision]].")) + (p1r2.revId, p1r2.parentId, p1r2.comment, p1r2.content) === (Some( + 99 + ), None, Some("new!"), Some("An earlier [[revision]].")) p1r2.user === Some(IpContributor("10.0.0.2")) } - def checkRevision(revId: Long, parentId: Long, timestamp: ZonedDateTime, user: String, userId: Long, comment: String, text: String, revision: Revision): MatchResult[Any] = { - (revision.id, revision.parentId, revision.timestamp.map(Timestamp.format), revision.comment, revision.content) === - (Some(revId), Some(parentId), Some(Timestamp.format(timestamp)), Some(comment), Some(text)) + def checkRevision( + revId: Long, + parentId: Long, + timestamp: ZonedDateTime, + user: String, + userId: Long, + comment: String, + text: String, + revision: Revision + ): MatchResult[Any] = { + ( + revision.id, + revision.parentId, + revision.timestamp.map(Timestamp.format), + revision.comment, + revision.content + ) === + (Some(revId), Some(parentId), Some(Timestamp.format(timestamp)), Some( + comment + ), Some(text)) revision.user === Some(User(userId, user)) } diff --git a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlWriterSpec.scala b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlWriterSpec.scala index e0fdc7dd..529db0c3 100644 --- a/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlWriterSpec.scala +++ b/scalawiki-dumps/src/test/scala/org/scalawiki/xml/XmlWriterSpec.scala @@ -17,10 +17,39 @@ class XmlWriterSpec extends Specification with XmlMatchers { "serialize page" in { val (title, ns, pageId) = ("Page title", 0, 123) - val (revId, parentId, timestamp, user, userId, comment, text, minor, sha1) = - (345, 456, ZonedDateTime.now, "user", 567, "revision comment", "revision text", true, "sha1") + val ( + revId, + parentId, + timestamp, + user, + userId, + comment, + text, + minor, + sha1 + ) = + ( + 345, + 456, + ZonedDateTime.now, + "user", + 567, + "revision comment", + "revision text", + true, + "sha1" + ) - val rev = Revision(Some(revId), Some(pageId), Some(parentId), Some(User(userId, user)), Some(timestamp), Some(comment), Some(text), sha1 = Some(sha1)/*, minor = Some(minor)*/) + val rev = Revision( + Some(revId), + Some(pageId), + Some(parentId), + Some(User(userId, user)), + Some(timestamp), + Some(comment), + Some(text), + sha1 = Some(sha1) /*, minor = Some(minor)*/ + ) val page = Page(Some(pageId), Some(ns), title, Seq(rev)) val sw = new StringWriter() @@ -33,8 +62,18 @@ class XmlWriterSpec extends Specification with XmlMatchers { val actualXml = XML.loadString(str) - val revsXml = revisionXml(revId, parentId, timestamp, user, userId, comment, text, sha1) - val expectXml = XML.loadString(mediawiki(pageXml(title, ns, pageId, revsXml))) + val revsXml = revisionXml( + revId, + parentId, + timestamp, + user, + userId, + comment, + text, + sha1 + ) + val expectXml = + XML.loadString(mediawiki(pageXml(title, ns, pageId, revsXml))) trim(actualXml) === trim(expectXml) } diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Categories.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Categories.scala index 053f92d8..9a0f303e 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Categories.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Categories.scala @@ -2,48 +2,53 @@ package org.scalawiki.sql import slick.driver.H2Driver.api._ -/** - * https://www.mediawiki.org/wiki/Manual:Category_table - * Track all existing categories. Something is a category if - * it has an entry somewhere in categorylinks, or - * it once did (Task T28411). - * Categories might not have corresponding pages, so they need to be tracked separately. cat_pages, cat_subcats, and cat_files are signed to make underflow more obvious. - * Note The pages and sub-categories are stored in the categorylinks table. - * Note: Information regarding which categories are hidden is stored in the page_props table. - * @param tag - */ - -class Categories(tag: Tag, tableName: String) extends Table[Category](tag, tableName) { +/** https://www.mediawiki.org/wiki/Manual:Category_table Track all existing + * categories. Something is a category if it has an entry somewhere in + * categorylinks, or it once did (Task T28411). Categories might not have + * corresponding pages, so they need to be tracked separately. cat_pages, + * cat_subcats, and cat_files are signed to make underflow more obvious. Note + * The pages and sub-categories are stored in the categorylinks table. Note: + * Information regarding which categories are hidden is stored in the + * page_props table. + * @param tag + */ + +class Categories(tag: Tag, tableName: String) + extends Table[Category](tag, tableName) { def id = column[Long]("cat_id", O.PrimaryKey) - /** - * Name of the category, in the same form as page.page_title (with underscores). - * If there is a category page corresponding to this category, by definition, - * it has this name (in the Category namespace). - * @return - */ + /** Name of the category, in the same form as page.page_title (with + * underscores). If there is a category page corresponding to this category, + * by definition, it has this name (in the Category namespace). + * @return + */ def title = column[String]("cat_title") - /** - * Number of pages in the category. This number includes the number of subcategories and the number of files. - * @return - */ + /** Number of pages in the category. This number includes the number of + * subcategories and the number of files. + * @return + */ def pages = column[Int]("cat_pages") - /** - * Number of sub-categories in the category. - * @return - */ + /** Number of sub-categories in the category. + * @return + */ def subCats = column[Int]("cat_subcats") - /** - * Number of files (i.e. Image: namespace members) in the category. - * @return - */ + /** Number of files (i.e. Image: namespace members) in the category. + * @return + */ def files = column[Int]("cat_files") - def * = (id, title, pages, subCats, files) <>(Category.tupled, Category.unapply) + def * = + (id, title, pages, subCats, files) <> (Category.tupled, Category.unapply) } -case class Category(id: Long, title: String, pages: Int, subCats: Int, files: Int) +case class Category( + id: Long, + title: String, + pages: Int, + subCats: Int, + files: Int +) diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/DslQueryDbCache.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/DslQueryDbCache.scala index 9bcd5c69..b05a4a86 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/DslQueryDbCache.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/DslQueryDbCache.scala @@ -9,9 +9,13 @@ import org.scalawiki.query.DslQuery import scala.concurrent.Future class DbCachedBot(apiBot: MwBot, database: MwDatabase) - extends MwBotImpl(apiBot.host, apiBot.asInstanceOf[MwBotImpl].http) { + extends MwBotImpl(apiBot.host, apiBot.asInstanceOf[MwBotImpl].http) { - override def run(action: Action, context: Map[String, String] = Map.empty, limit: Option[Long] = None): Future[Iterable[Page]] = { + override def run( + action: Action, + context: Map[String, String] = Map.empty, + limit: Option[Long] = None + ): Future[Iterable[Page]] = { new DslQueryDbCache(new DslQuery(action, apiBot), database).run() } } @@ -31,38 +35,40 @@ class DslQueryDbCache(val dslQuery: DslQuery, val database: MwDatabase) { val pageDao = database.pageDao def run(): Future[Iterable[Page]] = { - dslQuery.action.query.map { - query => + dslQuery.action.query + .map { query => if (cachingProvided(query)) { runWithCaching(query) } else { dslQuery.run().map(_.allPages) } - }.getOrElse(Future.successful(Seq.empty)) + } + .getOrElse(Future.successful(Seq.empty)) } - def cachingProvided(query: Query): Boolean = query.revisions.exists(_.hasContent) + def cachingProvided(query: Query): Boolean = + query.revisions.exists(_.hasContent) def runWithCaching(query: Query): Future[Iterable[Page]] = { - idsOnlyApiQuery(query).run().map(_.allPages).flatMap { - idsOnlyPages => - - val pageIds = idsOnlyPages.flatMap(_.id).toSet - val revIds = idsOnlyPages.flatMap(_.revisions.headOption.flatMap(_.id)).toSet + idsOnlyApiQuery(query).run().map(_.allPages).flatMap { idsOnlyPages => + val pageIds = idsOnlyPages.flatMap(_.id).toSet + val revIds = + idsOnlyPages.flatMap(_.revisions.headOption.flatMap(_.id)).toSet - checkUnusualCases(idsOnlyPages) + checkUnusualCases(idsOnlyPages) - val dbPages = fromDb(revIds, pageIds) + val dbPages = fromDb(revIds, pageIds) - notInDb(query, pageIds, dbPages).map { - mergePages(dbPages, _) - } + notInDb(query, pageIds, dbPages).map { + mergePages(dbPages, _) + } } } def checkUnusualCases(pages: Iterable[Page]): Unit = { - val noRevs = pages.filter(p => p.revisions.isEmpty || p.revisions.head.id.isEmpty) + val noRevs = + pages.filter(p => p.revisions.isEmpty || p.revisions.head.id.isEmpty) if (noRevs.nonEmpty) { log.error("No revs pages" + noRevs.toBuffer) } @@ -74,9 +80,13 @@ class DslQueryDbCache(val dslQuery: DslQuery, val database: MwDatabase) { new DslQuery(idsAction, dslQuery.bot) } - def mergePages(dbPages: Iterable[Page], notInDbPages: Iterable[Page]): Iterable[Page] = { + def mergePages( + dbPages: Iterable[Page], + notInDbPages: Iterable[Page] + ): Iterable[Page] = { if (notInDbPages.nonEmpty) { - val needNewRevPageIds = dbPages.filter(_.revisions.isEmpty).flatMap(_.id).toSet + val needNewRevPageIds = + dbPages.filter(_.revisions.isEmpty).flatMap(_.id).toSet val duplicates = notInDbPages.groupBy(_.id.get).values.filter(_.size > 1) if (duplicates.nonEmpty) { @@ -92,7 +102,11 @@ class DslQueryDbCache(val dslQuery: DslQuery, val database: MwDatabase) { dbPages.filter(_.revisions.nonEmpty) ++ notInDbPages } - def notInDb(query: Query, ids: Set[Long], dbPages: Seq[Page]): Future[Iterable[Page]] = { + def notInDb( + query: Query, + ids: Set[Long], + dbPages: Seq[Page] + ): Future[Iterable[Page]] = { val dbIds = dbPages.filter(_.revisions.nonEmpty).flatMap(_.id).toSet val notInDbIds = ids -- dbIds @@ -107,22 +121,28 @@ class DslQueryDbCache(val dslQuery: DslQuery, val database: MwDatabase) { } else { notInDBQuery(query, notInDbIds.toSeq.sorted) } - val notInDbQueries = notInDbQueryDtos.map(dto => new DslQuery(Action(dto), dslQuery.bot)) + val notInDbQueries = + notInDbQueryDtos.map(dto => new DslQuery(Action(dto), dslQuery.bot)) - Future.traverse(notInDbQueries)(_.run()).map(seqs => seqs.flatMap(_.allPages)) + Future + .traverse(notInDbQueries)(_.run()) + .map(seqs => seqs.flatMap(_.allPages)) } } def notInDBQuery(query: Query, ids: Iterable[Long]): Seq[Query] = { - ids.sliding(50, 50).map { window => - val params = query.params.filterNot { p => - p.isInstanceOf[Generator] || + ids + .sliding(50, 50) + .map { window => + val params = query.params.filterNot { p => + p.isInstanceOf[Generator] || p.isInstanceOf[TitlesParam] || p.isInstanceOf[PageIdsParam] - } :+ PageIdsParam(window.toSeq) + } :+ PageIdsParam(window.toSeq) - Query(params: _*) - }.toSeq + Query(params: _*) + } + .toSeq } def fromDb(revIds: Set[Long], ids: Set[Long]): Seq[Page] = { @@ -133,22 +153,25 @@ class DslQueryDbCache(val dslQuery: DslQuery, val database: MwDatabase) { log.error(s"pageIds.size ${ids.size}, revIds.size: ${revIds.size}") } - val pagesFromDb = pageDao.listWithText.filter(p => ids.contains(p.id.get)).map { - p => - if (revIds.contains(p.revisions.head.id.get)) p else p.copy(revisions = Seq.empty) - } + val pagesFromDb = + pageDao.listWithText.filter(p => ids.contains(p.id.get)).map { p => + if (revIds.contains(p.revisions.head.id.get)) p + else p.copy(revisions = Seq.empty) + } // log.info(s"pagesFromDb pageIds ${pagesFromDb.flatMap(_.id)}") val estimatedTime = (System.nanoTime() - startTime) / Math.pow(10, 9) - log.info(s"${bot.host} Db query completed with ${pagesFromDb.size} pages in $estimatedTime seconds") + log.info( + s"${bot.host} Db query completed with ${pagesFromDb.size} pages in $estimatedTime seconds" + ) pagesFromDb } - /** - * - * @param toDbPages pages to save in Db. Can be either new pages or new page revisions - * @param needNewRevPageIds ids of pages that are already in db but have new revisions to add + /** @param toDbPages + * pages to save in Db. Can be either new pages or new page revisions + * @param needNewRevPageIds + * ids of pages that are already in db but have new revisions to add */ def toDb(toDbPages: Iterable[Page], needNewRevPageIds: Set[Long]) = { val startTime = System.nanoTime() @@ -157,8 +180,8 @@ class DslQueryDbCache(val dslQuery: DslQuery, val database: MwDatabase) { // log.info(s"toDbPages pageIds: ${toDbPages.flatMap(_.id).toBuffer}") // log.info(s"old rev inDb pageIds: $inDbPageIds") - - val (newRevPages, newPages) = toDbPages.partition(p => needNewRevPageIds.contains(p.id.get)) + val (newRevPages, newPages) = + toDbPages.partition(p => needNewRevPageIds.contains(p.id.get)) // log.info(s"newPages pageIds: ${newPages.flatMap(_.id).toBuffer}") // log.info(s"newRevPages pageIds: ${newRevPages.flatMap(_.id).toBuffer}") @@ -182,6 +205,8 @@ class DslQueryDbCache(val dslQuery: DslQuery, val database: MwDatabase) { } val estimatedTime = (System.nanoTime() - startTime) / Math.pow(10, 9) - log.info(s"${bot.host} Insert completed with ${toDbPages.size} pages in $estimatedTime seconds") + log.info( + s"${bot.host} Insert completed with ${toDbPages.size} pages in $estimatedTime seconds" + ) } } diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Images.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Images.scala index 03a2222e..c6c1f8c9 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Images.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Images.scala @@ -5,97 +5,87 @@ import org.scalawiki.dto.Image import slick.driver.H2Driver.api._ -/** - * https://www.mediawiki.org/wiki/Manual:Image_table - * The image table describes images and other uploaded files. - * However, the image description pages are stored like other pages. - * @param tag - */ +/** https://www.mediawiki.org/wiki/Manual:Image_table The image table describes + * images and other uploaded files. However, the image description pages are + * stored like other pages. + * @param tag + */ class Images(tag: Tag, tableName: String) extends Table[Image](tag, tableName) { - /** - * Filename using underscores. - * @return - */ + + /** Filename using underscores. + * @return + */ def name = column[String]("img_name", O.PrimaryKey) - /** - * File size in bytes. - * @return - */ + /** File size in bytes. + * @return + */ def size = column[Long]("img_size") - /** - * Image width, in pixels. - * @return - */ + /** Image width, in pixels. + * @return + */ def width = column[Int]("img_width") - /** - * Image height, in pixels. - * @return - */ + /** Image height, in pixels. + * @return + */ def height = column[Int]("img_height") - /** - * Serialized PHP array of the file's properties. - * @return - */ + /** Serialized PHP array of the file's properties. + * @return + */ def metadata = column[String]("img_metadata") - /** - * Bit-depth of GIF/PNG palette-based images (up to 8-bit). - * Non-palette images (JPEG/PNG/TIFF/SVG) are 0, 8, or 16. All other files default to 0. - * @return - */ + /** Bit-depth of GIF/PNG palette-based images (up to 8-bit). Non-palette + * images (JPEG/PNG/TIFF/SVG) are 0, 8, or 16. All other files default to 0. + * @return + */ def bits = column[Int]("img_bits") - /** - * Possibilities are UNKNOWN, BITMAP, DRAWING, AUDIO, VIDEO, MULTIMEDIA, OFFICE, TEXT, EXECUTABLE, and ARCHIVE. - * @return - */ + /** Possibilities are UNKNOWN, BITMAP, DRAWING, AUDIO, VIDEO, MULTIMEDIA, + * OFFICE, TEXT, EXECUTABLE, and ARCHIVE. + * @return + */ def mediaType = column[String]("img_media_type") - /** - * Possibilities are unknown, application, audio, chemical, image, message, model, multipart, text, and video. - * @return - */ + /** Possibilities are unknown, application, audio, chemical, image, message, + * model, multipart, text, and video. + * @return + */ def majorMime = column[String]("img_major_mime") - /** - * E.g. jpeg, gif, png, etc. - * @return - */ + /** E.g. jpeg, gif, png, etc. + * @return + */ def minorMime = column[String]("img_minor_mime") - /** - * Description field given during upload. It's not the description page (associated File: wiki page), - * but the "summary" provided by the user in case of reupload. - * @return - */ + /** Description field given during upload. It's not the description page + * (associated File: wiki page), but the "summary" provided by the user in + * case of reupload. + * @return + */ def description = column[String]("img_description") - /** - * User ID of who uploaded the file. - * @return - */ + /** User ID of who uploaded the file. + * @return + */ def userId = column[Long]("img_user") - /** - * User name of who uploaded the file. - * @return - */ + /** User name of who uploaded the file. + * @return + */ def userText = column[String]("img_user_text") - /** - * Timestamp of when upload took place. Not necessarily the same timestamp as logging.log_timestamp. (?) - * @return - */ + /** Timestamp of when upload took place. Not necessarily the same timestamp as + * logging.log_timestamp. (?) + * @return + */ def timestamp = column[String]("img_timestamp") - /** - * The SHA-1 hash of the file contents in base 36 format. - * @return - */ + /** The SHA-1 hash of the file contents in base 36 format. + * @return + */ def sha1 = column[String]("img_sha1") def url = column[Option[String]]("img_url") @@ -104,42 +94,90 @@ class Images(tag: Tag, tableName: String) extends Table[Image](tag, tableName) { def pageId = column[Option[Long]]("img_page_id") - def * = (name, size, width, height, url, userId, userText, author, pageId) <>(fromDb, toDb) - - def fromDb(t: (String, Long, Int, Int, Option[String], Long, String, Option[String], Option[Long])) = + def * = ( + name, + size, + width, + height, + url, + userId, + userText, + author, + pageId + ) <> (fromDb, toDb) + + def fromDb( + t: ( + String, + Long, + Int, + Int, + Option[String], + Long, + String, + Option[String], + Option[Long] + ) + ) = Images.fromDb(t) - def toDb(i: Image) = Some(( - i.title, - i.size.getOrElse(0L), - i.width.getOrElse(0), - i.height.getOrElse(0), - i.url, - i.uploader.flatMap(_.id).getOrElse(0L), - i.uploader.flatMap(_.name).getOrElse(""), - i.author, - i.pageId - )) + def toDb(i: Image) = Some( + ( + i.title, + i.size.getOrElse(0L), + i.width.getOrElse(0), + i.height.getOrElse(0), + i.url, + i.uploader.flatMap(_.id).getOrElse(0L), + i.uploader.flatMap(_.name).getOrElse(""), + i.author, + i.pageId + ) + ) } - object Images { - def fromDb(t: (String, Long, Int, Int, Option[String], Long, String, Option[String], Option[Long])) = + def fromDb( + t: ( + String, + Long, + Int, + Int, + Option[String], + Long, + String, + Option[String], + Option[Long] + ) + ) = new Image( title = t._1, size = Some(t._2), width = Some(t._3), height = Some(t._4), url = t._5, - uploader = Contributor(Option(t._6), Option(t._7)).collect { case u: User => u }, + uploader = Contributor(Option(t._6), Option(t._7)).collect { + case u: User => u + }, author = t._8, pageId = t._9 ) - def fromDbJoin(t: (Option[String], Option[Long], Option[Int], Option[Int], Option[String], Option[Long], Option[String], - Option[String], Option[Long])): Option[Image] = + def fromDbJoin( + t: ( + Option[String], + Option[Long], + Option[Int], + Option[Int], + Option[String], + Option[Long], + Option[String], + Option[String], + Option[Long] + ) + ): Option[Image] = t._1.map { title => new Image( title = title, diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/MwDatabase.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/MwDatabase.scala index af8e6ca7..afca60b7 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/MwDatabase.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/MwDatabase.scala @@ -10,7 +10,10 @@ import spray.util.pimpFuture import scala.concurrent.ExecutionContext.Implicits.global -class MwDatabase(val dc: DatabaseConfig[JdbcProfile], val dbName: Option[String] = None) { +class MwDatabase( + val dc: DatabaseConfig[JdbcProfile], + val dbName: Option[String] = None +) { val db = dc.db @@ -20,23 +23,24 @@ class MwDatabase(val dc: DatabaseConfig[JdbcProfile], val dbName: Option[String] def prefixed(tableName: String) = dbName.fold("")(_ + "_") + tableName - val categories = TableQuery[Categories]( - (tag: Tag) => new Categories(tag, prefixed("category"))) + val categories = TableQuery[Categories]((tag: Tag) => + new Categories(tag, prefixed("category")) + ) - val images = TableQuery[Images]( - (tag: Tag) => new Images(tag, prefixed("image"))) + val images = + TableQuery[Images]((tag: Tag) => new Images(tag, prefixed("image"))) - val pages = TableQuery[Pages]( - (tag: Tag) => new Pages(tag, prefixed("page"), dbName)) + val pages = + TableQuery[Pages]((tag: Tag) => new Pages(tag, prefixed("page"), dbName)) - val revisions = TableQuery[Revisions]( - (tag: Tag) => new Revisions(tag, prefixed("revision"), dbName)) + val revisions = TableQuery[Revisions]((tag: Tag) => + new Revisions(tag, prefixed("revision"), dbName) + ) - val texts = TableQuery[Texts]( - (tag: Tag) => new Texts(tag, prefixed("text"))) + val texts = TableQuery[Texts]((tag: Tag) => new Texts(tag, prefixed("text"))) - val users = TableQuery[Users]( - (tag: Tag) => new Users(tag, prefixed("user"), dbName)) + val users = + TableQuery[Users]((tag: Tag) => new Users(tag, prefixed("user"), dbName)) val imageDao = new ImageDao(this, images, driver) val textDao = new TextDao(this, texts, driver) @@ -59,18 +63,17 @@ class MwDatabase(val dc: DatabaseConfig[JdbcProfile], val dbName: Option[String] } def existingTables(existing: Boolean) = { - db.run(MTable.getTables).map { - dbTables => - val dbTableNames = dbTables.map(_.name.name).toSet - tables.filter { t => - val tableName = t.baseTableRow.tableName - val contains = dbTableNames.contains(tableName) - - if (existing) - contains - else - !contains - } + db.run(MTable.getTables).map { dbTables => + val dbTableNames = dbTables.map(_.name.name).toSet + tables.filter { t => + val tableName = t.baseTableRow.tableName + val contains = dbTableNames.contains(tableName) + + if (existing) + contains + else + !contains + } } } @@ -101,8 +104,8 @@ object MwDatabase { def dbName(host: String): String = { host.split("\\.").toList match { case "commons" :: "wikimedia" :: xs => "commonswiki_p" - case x :: "wikipedia" :: xs => x + "wiki_p" - case x1 :: x2 :: xs => x1 + x2 + case x :: "wikipedia" :: xs => x + "wiki_p" + case x1 :: x2 :: xs => x1 + x2 } } } diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Pages.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Pages.scala index 48ff0cd2..f2c67203 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Pages.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Pages.scala @@ -4,89 +4,98 @@ import org.scalawiki.dto.{Revision, Page} import slick.driver.H2Driver.api._ -/** - * https://www.mediawiki.org/wiki/Manual:Page_table - * Each page in a MediaWiki installation has an entry here which identifies it by title and contains some essential metadata. - * It was first introduced in r6710, in MediaWiki 1.5. The text of the page itself is stored in the text table. - * To retrieve the text of an article, MediaWiki first searches for page_title in the page table. - * Then, page_latest is used to search the revision table for rev_id, and rev_text_id is obtained in the process. - * The value obtained for rev_text_id is used to search for old_id in the text table to retrieve the text. - * When a page is deleted, the revisions are moved to the archive table - * @param tag - */ -class Pages(tag: Tag, tableName: String, val dbPrefix: Option[String]) extends Table[Page](tag, tableName) { +/** https://www.mediawiki.org/wiki/Manual:Page_table Each page in a MediaWiki + * installation has an entry here which identifies it by title and contains + * some essential metadata. It was first introduced in r6710, in MediaWiki 1.5. + * The text of the page itself is stored in the text table. To retrieve the + * text of an article, MediaWiki first searches for page_title in the page + * table. Then, page_latest is used to search the revision table for rev_id, + * and rev_text_id is obtained in the process. The value obtained for + * rev_text_id is used to search for old_id in the text table to retrieve the + * text. When a page is deleted, the revisions are moved to the archive table + * @param tag + */ +class Pages(tag: Tag, tableName: String, val dbPrefix: Option[String]) + extends Table[Page](tag, tableName) { def withPrefix(name: String) = dbPrefix.fold("")(_ + "_") + name - /** - * Uniquely identifying primary key. This value is preserved across edits and renames. - * There is an analogous field in the archive table to preserve this value in MediaWiki 1.11 and later; however, - * it is [[https://phabricator.wikimedia.org/T28123 not used]] in Special:Undelete, the interface for undeleting pages used by project administrators. - * @return - */ + /** Uniquely identifying primary key. This value is preserved across edits and + * renames. There is an analogous field in the archive table to preserve this + * value in MediaWiki 1.11 and later; however, it is + * [[https://phabricator.wikimedia.org/T28123 not used]] in Special:Undelete, + * the interface for undeleting pages used by project administrators. + * @return + */ def id = column[Option[Long]]("page_id", O.PrimaryKey, O.AutoInc) - /** - * A page name is broken into a [[https://www.mediawiki.org/wiki/Manual:Namespace namespace]] and a title. - * The namespace keys are UI-language-independent constants, defined in includes/Defines.php. - * This field contains the number of the page's namespace. The values range from 0 to 15 for the standard namespaces, - * and from 100 to 2147483647 for [[https://www.mediawiki.org/wiki/Manual:Using_custom_namespaces custom namespaces]]. - * @return - */ + /** A page name is broken into a + * [[https://www.mediawiki.org/wiki/Manual:Namespace namespace]] and a title. + * The namespace keys are UI-language-independent constants, defined in + * includes/Defines.php. This field contains the number of the page's + * namespace. The values range from 0 to 15 for the standard namespaces, and + * from 100 to 2147483647 for + * [[https://www.mediawiki.org/wiki/Manual:Using_custom_namespaces custom namespaces]]. + * @return + */ def namespace = column[Option[Int]]("page_namespace") - /** - * The sanitized [[https://www.mediawiki.org/wiki/Manual:Page_title page title]], without the title of its namespace - * with a maximum of 255 characters (binary), e.g. "[255 chars]" or "Talk:[255 chars]" or "Category discussion:[255 chars here]". - * It is stored as text, with spaces replaced by underscores. The real title shown in articles is just this title - * with underscores (_) converted to spaces ( ). - * @return - */ + /** The sanitized + * [[https://www.mediawiki.org/wiki/Manual:Page_title page title]], without + * the title of its namespace with a maximum of 255 characters (binary), e.g. + * "[255 chars]" or "Talk:[255 chars]" or "Category discussion:[255 chars + * here]". It is stored as text, with spaces replaced by underscores. The + * real title shown in articles is just this title with underscores (_) + * converted to spaces ( ). + * @return + */ def title = column[String]("page_title") - /** - * A value of 1 here indicates the article is a redirect; it is 0 in all other cases. - * @return - */ + /** A value of 1 here indicates the article is a redirect; it is 0 in all + * other cases. + * @return + */ def isRedirect = column[Boolean]("page_is_redirect") def isNew = column[Boolean]("page_is_new") def random = column[Double]("page_random") - /** - * This timestamp is updated whenever the page changes in a way requiring it to be re-rendered, invalidating caches. - * Aside from editing this includes permission changes, creation or deletion of linked pages, and alteration of contained templates. - * @return - */ + /** This timestamp is updated whenever the page changes in a way requiring it + * to be re-rendered, invalidating caches. Aside from editing this includes + * permission changes, creation or deletion of linked pages, and alteration + * of contained templates. + * @return + */ def touched = column[String]("page_touched") - /** - * This timestamp is updated whenever a page is re-parsed and it has all the link tracking tables updated for it. - * This is useful for de-duplicating expensive backlink update jobs. Set to the default value of NULL when the page is created by - * @return - */ + /** This timestamp is updated whenever a page is re-parsed and it has all the + * link tracking tables updated for it. This is useful for de-duplicating + * expensive backlink update jobs. Set to the default value of NULL when the + * page is created by + * @return + */ def linksUpdated = column[String]("page_links_updated") - /** - * This is a foreign key to rev_id for the current revision. It may be 0 during page creation. - * It needs to link to a revision with a valid revision.rev_page, - * or there will be the "The revision #0 of the page named 'Foo' does not exist" error when one tries to view the page - * @return - */ + /** This is a foreign key to rev_id for the current revision. It may be 0 + * during page creation. It needs to link to a revision with a valid + * revision.rev_page, or there will be the "The revision #0 of the page named + * 'Foo' does not exist" error when one tries to view the page + * @return + */ def pageLatest = column[Long]("page_latest", O.Default(0)) - /** - * Uncompressed length in bytes of the page's current source text. - * @return - */ + /** Uncompressed length in bytes of the page's current source text. + * @return + */ def pageLen = column[Int]("page_len") def contentModel = column[String]("page_content_model") def lang = column[String]("page_lang") - def nameTitle = index(withPrefix("name_title"), (namespace, title), unique = true) + def nameTitle = + index(withPrefix("name_title"), (namespace, title), unique = true) // def revision = foreignKey("revisionFK", pageLatest, MediaWiki.revisions)(_.id) @@ -95,9 +104,10 @@ class Pages(tag: Tag, tableName: String, val dbPrefix: Option[String]) extends T def fromDb(t: (Option[Long], Option[Int], String, Long)) = { val pageId = t._1 val pageLatest = t._4 - val revisions = if (pageLatest != 0) - Seq(new Revision(revId = Some(pageLatest), pageId = pageId)) - else Seq.empty + val revisions = + if (pageLatest != 0) + Seq(new Revision(revId = Some(pageLatest), pageId = pageId)) + else Seq.empty new Page( id = pageId, @@ -107,10 +117,12 @@ class Pages(tag: Tag, tableName: String, val dbPrefix: Option[String]) extends T ) } - def toDb(p: Page) = Some(( - p.id, - p.ns, - p.title, - p.revisions.head.id.getOrElse(0L) - )) -} \ No newline at end of file + def toDb(p: Page) = Some( + ( + p.id, + p.ns, + p.title, + p.revisions.head.id.getOrElse(0L) + ) + ) +} diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Revisions.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Revisions.scala index b78a26d1..6d9759dc 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Revisions.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Revisions.scala @@ -4,101 +4,102 @@ import org.scalawiki.dto.{Contributor, Revision} import slick.driver.H2Driver.api._ -/** - * https://www.mediawiki.org/wiki/Manual:Revision_table - * The revision table holds metadata for every edit done to a page within the wiki. - * Every edit of a page creates a revision row, which holds information such as the user who made the edit, - * the time at which the edit was made, and a reference to the new wikitext in the text table. - * @param tag - */ -class Revisions(tag: Tag, tableName: String, val dbPrefix: Option[String]) extends Table[Revision](tag, tableName) { +/** https://www.mediawiki.org/wiki/Manual:Revision_table The revision table + * holds metadata for every edit done to a page within the wiki. Every edit of + * a page creates a revision row, which holds information such as the user who + * made the edit, the time at which the edit was made, and a reference to the + * new wikitext in the text table. + * @param tag + */ +class Revisions(tag: Tag, tableName: String, val dbPrefix: Option[String]) + extends Table[Revision](tag, tableName) { def withPrefix(name: String) = dbPrefix.fold("")(_ + "_") + name - /** - * This field holds the primary key for each revision. page_latest is a foreign key to this field. - * @return - */ + + /** This field holds the primary key for each revision. page_latest is a + * foreign key to this field. + * @return + */ def id = column[Option[Long]]("rev_id", O.PrimaryKey, O.AutoInc) - /** - * This field holds a reference to the page to which this revision pertains. - * The number in this field is equal to the page_id field of said page. - * @return - */ + /** This field holds a reference to the page to which this revision pertains. + * The number in this field is equal to the page_id field of said page. + * @return + */ def pageId = column[Long]("rev_page") - /** - * This is a foreign key to old_id in the text table. (The text table is where the actual bulk text is stored.) - * It's possible for multiple revisions to use the same text—for instance, revisions where only metadata is altered, - * or where a rollback is done to a previous version. - * @return - */ + /** This is a foreign key to old_id in the text table. (The text table is + * where the actual bulk text is stored.) It's possible for multiple + * revisions to use the same text—for instance, revisions where only metadata + * is altered, or where a rollback is done to a previous version. + * @return + */ def textId = column[Long]("rev_text_id") - /** - * This field holds an editor's edit summary (editor's comment on revision). This text is shown in the history and contributions. - * (The recentchanges table contains a copy used for recent changes, related changes, watchlists, and, in the case of page creation, - * for the list of new pages.) It is rendered in a sanitized subset of wiki markup. - * @return - */ + /** This field holds an editor's edit summary (editor's comment on revision). + * This text is shown in the history and contributions. (The recentchanges + * table contains a copy used for recent changes, related changes, + * watchlists, and, in the case of page creation, for the list of new pages.) + * It is rendered in a sanitized subset of wiki markup. + * @return + */ def comment = column[String]("rev_comment") - /** - * This is equal to the user_id of the user who made this edit. The value for this field is 0 for anonymous edits, - * initializations scripts, and for some mass imports. - * @return - */ + /** This is equal to the user_id of the user who made this edit. The value for + * this field is 0 for anonymous edits, initializations scripts, and for some + * mass imports. + * @return + */ def userId = column[Long]("rev_user", O.Default(0)) - /** - * This field holds the text of the editor's username, or the IP address of the editor if the revision was done by an unregistered user. - * In anonymous revisions imported from UseModWiki or early incarnations of the Phase II software, - * this field may contain an IP address with the final octet obscured (i.e. \d{1,3}\.\d{1,3}\.\d{1,3}\.xxx such as 24.150.61.xxx; - * see bug 3631). Some edits imported from UseModWiki may contain a Reverse DNS lookup hostname like ppfree165-153-bz.aknet.it or office.bomis.com. - * @return - */ + /** This field holds the text of the editor's username, or the IP address of + * the editor if the revision was done by an unregistered user. In anonymous + * revisions imported from UseModWiki or early incarnations of the Phase II + * software, this field may contain an IP address with the final octet + * obscured (i.e. \d{1,3}\.\d{1,3}\.\d{1,3}\.xxx such as 24.150.61.xxx; see + * bug 3631). Some edits imported from UseModWiki may contain a Reverse DNS + * lookup hostname like ppfree165-153-bz.aknet.it or office.bomis.com. + * @return + */ def userText = column[String]("rev_user_text", O.Default("")) - /** - * Holds the timestamp of the edit. - * @return - */ + /** Holds the timestamp of the edit. + * @return + */ def timestamp = column[String]("rev_timestamp", O.Default("")) - /** - * Records whether the user marked the 'minor edit' checkbox. - * If the value for this field is 1, then the edit was declared as 'minor'; - * it is 0 otherwise. Many automated edits are marked as minor. - * @return - */ + /** Records whether the user marked the 'minor edit' checkbox. If the value + * for this field is 1, then the edit was declared as 'minor'; it is 0 + * otherwise. Many automated edits are marked as minor. + * @return + */ def minorEdit = column[Boolean]("rev_minor_edit", O.Default(false)) - /** - * This field is reserved for the [[https://www.mediawiki.org/wiki/Manual:RevisionDelete RevisionDelete system]]. - * It's a bitfield in which the values are DELETED_TEXT = 1; DELETED_COMMENT = 2; DELETED_USER = 4; and DELETED_RESTRICTED = 8. - * So, for example, if nothing has been deleted from that revision, then the value is 0; - * if both the comment and user have been deleted, then the value is 6. - * @return - */ + /** This field is reserved for the + * [[https://www.mediawiki.org/wiki/Manual:RevisionDelete RevisionDelete system]]. + * It's a bitfield in which the values are DELETED_TEXT = 1; DELETED_COMMENT + * \= 2; DELETED_USER = 4; and DELETED_RESTRICTED = 8. So, for example, if + * nothing has been deleted from that revision, then the value is 0; if both + * the comment and user have been deleted, then the value is 6. + * @return + */ def deleted = column[Int]("rev_deleted", O.Default(0)) - /** - * This field contains the length of the article after the revision, in bytes. - * @return - */ + /** This field contains the length of the article after the revision, in + * bytes. + * @return + */ def len = column[Long]("rev_len") - /** - * The rev_id of the previous revision to the page. Corresponds to rc_last_oldid. - * For edits which are new page creations, rev_parent_id = 0. - * @return - */ + /** The rev_id of the previous revision to the page. Corresponds to + * rc_last_oldid. For edits which are new page creations, rev_parent_id = 0. + * @return + */ def parentId = column[Long]("rev_parent_id") - /** - * This field is used to add the SHA-1 text content hash in base-36 - * @return - */ + /** This field is used to add the SHA-1 text content hash in base-36 + * @return + */ def sha1 = column[String]("rev_sha1") def contentModel = column[String]("rev_content_model") @@ -112,7 +113,7 @@ class Revisions(tag: Tag, tableName: String, val dbPrefix: Option[String]) exten // def userTimestamp = index("user_timestamp", (userId, timestamp)) // def userTextTimestamp = index("usertext_timestamp", (userText, timestamp)) - //def pageUserTimestamp = index("page_user_timestamp", (pageId, userId, timestamp)) + // def pageUserTimestamp = index("page_user_timestamp", (pageId, userId, timestamp)) // def page = foreignKey("pageFK", pageId, MediaWiki.pages)(_.id) // @@ -120,7 +121,7 @@ class Revisions(tag: Tag, tableName: String, val dbPrefix: Option[String]) exten // // def user = foreignKey("userFK", userId, MediaWiki.users)(_.id) - def * = (id, pageId, parentId, textId, userId, userText) <>(fromDb, toDb) + def * = (id, pageId, parentId, textId, userId, userText) <> (fromDb, toDb) def fromDb(t: (Option[Long], Long, Long, Long, Long, String)) = Revision( @@ -131,18 +132,14 @@ class Revisions(tag: Tag, tableName: String, val dbPrefix: Option[String]) exten user = Contributor(Option(t._5), Option(t._6)) ) - def toDb(r: Revision) = Some(( - r.id, - r.pageId.get, - r.parentId.getOrElse(0L), - r.textId.get, - r.user.flatMap(_.id).getOrElse(0L), - r.user.flatMap(_.name).getOrElse("") - )) + def toDb(r: Revision) = Some( + ( + r.id, + r.pageId.get, + r.parentId.getOrElse(0L), + r.textId.get, + r.user.flatMap(_.id).getOrElse(0L), + r.user.flatMap(_.name).getOrElse("") + ) + ) } - - - - - - diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Texts.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Texts.scala index 6229a84b..01ce9b47 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Texts.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Texts.scala @@ -2,59 +2,47 @@ package org.scalawiki.sql import slick.driver.H2Driver.api._ -/** - * https://www.mediawiki.org/wiki/Manual:Text_table - * The text table holds the wikitext of individual page revisions. - * If using Postgres or Oracle, this table is named pagecontent. - * Field names are a holdover from the 'old' revisions table in MediaWiki 1.4 and earlier. - * @param tag - */ +/** https://www.mediawiki.org/wiki/Manual:Text_table The text table holds the + * wikitext of individual page revisions. If using Postgres or Oracle, this + * table is named pagecontent. Field names are a holdover from the 'old' + * revisions table in MediaWiki 1.4 and earlier. + * @param tag + */ class Texts(tag: Tag, tableName: String) extends Table[Text](tag, tableName) { - /** - * revision.rev_text_id in revision table is a key to this column. - * (In MediaWiki 1.5+, archive.ar_text_id is also a key to this column.) - * @return - */ + + /** revision.rev_text_id in revision table is a key to this column. (In + * MediaWiki 1.5+, archive.ar_text_id is also a key to this column.) + * @return + */ def id = column[Option[Long]]("old_id", O.PrimaryKey, O.AutoInc) - /** - * The wikitext of the page - * @return - */ + /** The wikitext of the page + * @return + */ def text = column[String]("old_text") - /** - * Comma-separated list of flags. Contains the following possible values: - *
    - *
    gzip
    - *
    Text is compressed with PHP's gzdeflate() function. - * Note: If the $wgCompressRevisions option is on, new rows (=current revisions) will be gzipped transparently at save time. - * Previous revisions can also be compressed by using the script compressOld.php - *
    - *
    utf-8
    - *
    Text was stored as UTF-8. - * Note: If the $wgLegacyEncoding option is on, rows *without* this flag will be converted to UTF-8 transparently at load time. - *
    - *
    object
    - *
    Text field contained a serialized PHP object. - * Note: The object either contains multiple versions compressed together to achieve a better compression ratio, - * or it refers to another row where the text can be found. - *
    - *
    external
    - *
    Text was stored in an external location specified by old_text. - * Note: Any additional flags apply to the data stored at that URL, not the URL itself. - * The 'object' flag is not set for URLs of the form 'DB://cluster/id/itemid', - * because the external storage system itself decompresses these. - *
    - *
    - * @return - */ + /** Comma-separated list of flags. Contains the following possible values: + *
    gzip
    Text is compressed with PHP's gzdeflate() + * function. Note: If the $wgCompressRevisions option is on, new rows + * (=current revisions) will be gzipped transparently at save time. Previous + * revisions can also be compressed by using the script compressOld.php
    + *
    utf-8
    Text was stored as UTF-8. Note: If the + * $wgLegacyEncoding option is on, rows *without* this flag will be converted + * to UTF-8 transparently at load time.
    object
    Text + * field contained a serialized PHP object. Note: The object either contains + * multiple versions compressed together to achieve a better compression + * ratio, or it refers to another row where the text can be found.
    + * external
    Text was stored in an external location specified by + * old_text. Note: Any additional flags apply to the data stored at that URL, + * not the URL itself. The 'object' flag is not set for URLs of the form + * 'DB://cluster/id/itemid', because the external storage system itself + * decompresses these.
    + * @return + */ def flags = column[String]("old_flags") - def * = (id, text, flags) <>(Text.tupled, Text.unapply) + def * = (id, text, flags) <> (Text.tupled, Text.unapply) } case class Text(id: Option[Long], text: String, flags: String = "utf-8") - - diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Users.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Users.scala index 06cd9546..36196891 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/Users.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/Users.scala @@ -4,130 +4,115 @@ import org.scalawiki.dto.User import slick.driver.H2Driver.api._ -/** - * https://www.mediawiki.org/wiki/Manual:User_table - * The user table is where MediaWiki stores information about users. If using Postgres, this table is named mwuser. - * @param tag - */ -class Users(tag: Tag, tableName: String, val dbPrefix: Option[String]) extends Table[User](tag, tableName) { +/** https://www.mediawiki.org/wiki/Manual:User_table The user table is where + * MediaWiki stores information about users. If using Postgres, this table is + * named mwuser. + * @param tag + */ +class Users(tag: Tag, tableName: String, val dbPrefix: Option[String]) + extends Table[User](tag, tableName) { def withPrefix(name: String) = dbPrefix.fold("")(_ + "_") + name def id = column[Option[Long]]("user_id", O.PrimaryKey, O.AutoInc) - /** - * Usernames must be unique, and must not be in the form of an IP address. - * Shouldn't allow slashes or case conflicts. Spaces are allowed, and are not converted to underscores like titles. - * @return - */ + /** Usernames must be unique, and must not be in the form of an IP address. + * Shouldn't allow slashes or case conflicts. Spaces are allowed, and are not + * converted to underscores like titles. + * @return + */ def name = column[String]("user_name") - /** - * stores the user's real name (optional) as provided by the user in their "Preferences" section. - * @return - */ + /** stores the user's real name (optional) as provided by the user in their + * "Preferences" section. + * @return + */ def realName = column[String]("user_real_name") - /** - * user_password is one of three formats, depending on the setting of $wgPasswordSalt and $wgPasswordDefault: - * -Since MediaWiki 1.24, $wgPasswordDefault defaults to pbkdf2. In this case you will get a concatenation of: - * The string ":pbkdf2:". - * The hashing algorithm used inside the pbkdf2 layer, by default "sha256". - * The colon character (":"). - * The cost for this algorithm, by default "10000". - * The colon character (":"). - * The length (OF WHAT?), by default "128". - * The colon character (":"). - * Another string, e.g. "kkdejKlBYFV7+LP2m2thYA==" - * And finally another key. - * -Since MediaWiki 1.24, if the maintenance script wrapOldPasswords.php has been used, passwords may also start with ":pbkdf2-legacyA:" or ":pbkdf2-legacyB:" like ":pbkdf2-legacyB:!sha256:10000:128!...". - * -In MediaWiki 1.23 and older, if $wgPasswordSalt is true (default) it is a concatenation of: - * The string ":B:", - * A pseudo-random hexadecimal 31-bit salt between 0x0 and 0x7fff ffff (inclusive), - * The colon character (":"), and - * The MD5 hash of a concatenation of the salt, a dash ("-"), and the MD5 hash of the password. - * -In MediaWiki 1.23 and older, if $wgPasswordSalt is false, it is a concatenation of: - * The string ":A:" and - * The MD5 hash of the password. - * @return - */ + /** user_password is one of three formats, depending on the setting of + * $wgPasswordSalt and $wgPasswordDefault: -Since MediaWiki 1.24, + * $wgPasswordDefault defaults to pbkdf2. In this case you will get a + * concatenation of: The string ":pbkdf2:". The hashing algorithm used inside + * the pbkdf2 layer, by default "sha256". The colon character (":"). The cost + * for this algorithm, by default "10000". The colon character (":"). The + * length (OF WHAT?), by default "128". The colon character (":"). Another + * string, e.g. "kkdejKlBYFV7+LP2m2thYA==" And finally another key. -Since + * MediaWiki 1.24, if the maintenance script wrapOldPasswords.php has been + * used, passwords may also start with ":pbkdf2-legacyA:" or + * ":pbkdf2-legacyB:" like ":pbkdf2-legacyB:!sha256:10000:128!...". -In + * MediaWiki 1.23 and older, if $wgPasswordSalt is true (default) it is a + * concatenation of: The string ":B:", A pseudo-random hexadecimal 31-bit + * salt between 0x0 and 0x7fff ffff (inclusive), The colon character (":"), + * and The MD5 hash of a concatenation of the salt, a dash ("-"), and the MD5 + * hash of the password. -In MediaWiki 1.23 and older, if $wgPasswordSalt is + * false, it is a concatenation of: The string ":A:" and The MD5 hash of the + * password. + * @return + */ def password = column[Int]("user_password") - /** - * newPassword is generated for the mail-a-new-password feature - * @return - */ + /** newPassword is generated for the mail-a-new-password feature + * @return + */ def newPassword = column[String]("user_newpassword") - /** - * newPassTime is set to the current timestamp (wfTimestampNow()) when a new password is set. Like the other timestamps, - * it is in in MediaWiki's timestamp format (yyyymmddhhmmss, e.g. 20130824025644). - * @return - */ + /** newPassTime is set to the current timestamp (wfTimestampNow()) when a new + * password is set. Like the other timestamps, it is in in MediaWiki's + * timestamp format (yyyymmddhhmmss, e.g. 20130824025644). + * @return + */ def newPassTime = column[Int]("user_newpass_time") - /** - * - * @return - */ + /** @return + */ def email = column[String]("user_email") - /** - * touched is the last time a user made a change on the site, including logins, changes to pages (any namespace), watchlistings, and preference changes. - * note: the user_touched time resets when a user is left a talkpage message. - * @return - */ + /** touched is the last time a user made a change on the site, including + * logins, changes to pages (any namespace), watchlistings, and preference + * changes. note: the user_touched time resets when a user is left a talkpage + * message. + * @return + */ def touched = column[String]("user_touched") - /** - * token is a pseudorandomly generated value. When a user checks "Remember my login on this browser" the value is - * stored in a persistent browser cookie ${wgCookiePrefix}Token that authenticates the user while being resistant to spoofing. - * @return - */ + /** token is a pseudorandomly generated value. When a user checks "Remember my + * login on this browser" the value is stored in a persistent browser cookie + * ${wgCookiePrefix}Token that authenticates the user while being resistant + * to spoofing. + * @return + */ def token = column[Boolean]("user_token") - /** - * - * @return - */ + /** @return + */ def emailAuthenticated = column[Boolean]("user_email_authenticated") - /** - * - * @return - */ + /** @return + */ def emailToken = column[Int]("user_email_token") - /** - * - * @return - */ + /** @return + */ def emailTokenExpires = column[Int]("user_email_token_expires") - /** - * - * @return - */ + /** @return + */ def userRegistration = column[String]("user_registration") - /** - * - * @return - */ + /** @return + */ def userEditcount = column[String]("user_editcount") - /** - * - * @return - */ + /** @return + */ def userPasswordExpires = column[String]("user_password_expires") def nameIndex = index(withPrefix("user_name"), name, unique = true) - def * = (id, name) <>(fromDb, toDb) + def * = (id, name) <> (fromDb, toDb) def fromDb(t: (Option[Long], String)) = User( @@ -135,9 +120,10 @@ class Users(tag: Tag, tableName: String, val dbPrefix: Option[String]) extends T login = Some(t._2) ) - def toDb(u: User) = Some(( - u.id, - u.login.orNull - )) + def toDb(u: User) = Some( + ( + u.id, + u.login.orNull + ) + ) } - diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/ImageDao.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/ImageDao.scala index 23de526d..fff4aa29 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/ImageDao.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/ImageDao.scala @@ -8,7 +8,11 @@ import spray.util.pimpFuture import scala.language.higherKinds -class ImageDao(val mwDb: MwDatabase, val query: TableQuery[Images], val driver: JdbcProfile) { +class ImageDao( + val mwDb: MwDatabase, + val query: TableQuery[Images], + val driver: JdbcProfile +) { import driver.api._ @@ -28,4 +32,3 @@ class ImageDao(val mwDb: MwDatabase, val query: TableQuery[Images], val driver: db.run(query.filter(_.name === name).result.headOption).await } - diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/PageDao.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/PageDao.scala index 1d03a56f..121f3b39 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/PageDao.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/PageDao.scala @@ -30,14 +30,16 @@ class PageDao(val mwDb: MwDatabase, val driver: JdbcProfile) { def insertAll(pageSeq: Iterable[Page]): Iterable[Option[Long]] = { val revisionSeq = pageSeq.flatMap(_.revisions.headOption) - require(revisionSeq.size == pageSeq.size, "Pages should have revisions") // Fail on any absent for now + require( + revisionSeq.size == pageSeq.size, + "Pages should have revisions" + ) // Fail on any absent for now val hasIds = pageSeq.head.id.isDefined val pageIds = if (hasIds) { db.run(pages.forceInsertAll(pageSeq)).await pageSeq.map(_.id) - } - else { + } else { db.run(autoInc.forceInsertAll(pageSeq)).await } @@ -49,7 +51,9 @@ class PageDao(val mwDb: MwDatabase, val driver: JdbcProfile) { if (revisionSeq.exists(_.revId.isEmpty)) { updateLastRevision(pageIds, revIds) } - val images = pageSeq.zip(pageIds).flatMap { case (p, id) => p.images.map(_.copy(pageId = id)) } + val images = pageSeq.zip(pageIds).flatMap { case (p, id) => + p.images.map(_.copy(pageId = id)) + } if (images.nonEmpty) { imageDao.insertAll(images) } @@ -58,18 +62,25 @@ class PageDao(val mwDb: MwDatabase, val driver: JdbcProfile) { } // TODO batchUpdate or case/when/then - def updateLastRevision(pageIds: Iterable[Option[Long]], revIds: Iterable[Option[Long]]): Iterable[Int] = { - Future.traverse(pageIds.zip(revIds))({ - case (pageId, revId) => - db.run(pages.filter(_.id === pageId) - .map(p => p.pageLatest) - .update(revId.get)) - }).await + def updateLastRevision( + pageIds: Iterable[Option[Long]], + revIds: Iterable[Option[Long]] + ): Iterable[Int] = { + Future + .traverse(pageIds.zip(revIds))({ case (pageId, revId) => + db.run( + pages + .filter(_.id === pageId) + .map(p => p.pageLatest) + .update(revId.get) + ) + }) + .await } def insert(page: Page): Long = { require(page.revisions.nonEmpty, "page has no revisions") - val newRevs = page.revisions //.filter(_.revId.isEmpty) + val newRevs = page.revisions // .filter(_.revId.isEmpty) val pageIdF = ( if (page.id.isDefined) { @@ -77,14 +88,12 @@ class PageDao(val mwDb: MwDatabase, val driver: JdbcProfile) { db.run(pages.forceInsert(page)).map(_ => page.id) else Future.successful(page.id) - } - else { + } else { db.run(autoInc += page) } - ).map(_.get) + ).map(_.get) pageIdF.map { pageId => - addRevisions(pageId, newRevs) addImages(pageId, page.images) @@ -98,9 +107,12 @@ class PageDao(val mwDb: MwDatabase, val driver: JdbcProfile) { revisionDao.insert(withPage) } - db.run(pages.filter(_.id === pageId) - .map(p => p.pageLatest) - .update(revIds.last)).await + db.run( + pages + .filter(_.id === pageId) + .map(p => p.pageLatest) + .update(revIds.last) + ).await } def addImages(pageId: Long, images: Seq[Image]) = { @@ -130,10 +142,11 @@ class PageDao(val mwDb: MwDatabase, val driver: JdbcProfile) { (pages join revisions on (_.pageLatest === _.id) join texts on (_._2.textId === _.id) - sortBy { case ((p, r), t) => p.id } - ).result + sortBy { case ((p, r), t) => p.id }).result ).map { pages => - pages.map { case ((p, r), t) => p.copy(revisions = Seq(r.copy(content = Some(t.text)))) } + pages.map { case ((p, r), t) => + p.copy(revisions = Seq(r.copy(content = Some(t.text)))) + } }.await def findWithText(ids: Iterable[Long]): Seq[Page] = @@ -141,34 +154,51 @@ class PageDao(val mwDb: MwDatabase, val driver: JdbcProfile) { (pages.filter(_.id inSet ids) join revisions on (_.pageLatest === _.id) join texts on (_._2.textId === _.id) - sortBy { case ((p, r), t) => p.id } - ).result + sortBy { case ((p, r), t) => p.id }).result ).map { pages => - pages.map { case ((p, r), t) => p.copy(revisions = Seq(r.copy(content = Some(t.text)))) } + pages.map { case ((p, r), t) => + p.copy(revisions = Seq(r.copy(content = Some(t.text)))) + } }.await - def findByRevIds(pageIds: Iterable[Long], revIds: Iterable[Long]): Seq[Page] = { + def findByRevIds( + pageIds: Iterable[Long], + revIds: Iterable[Long] + ): Seq[Page] = { db.run( (pages.filter(_.id inSet pageIds) join revisions.filter(_.id inSet revIds) on (_.id === _.pageId) join texts on (_._2.textId === _.id) - joinLeft images on (_._1._1.id === _.pageId) - ).sortBy { case (((p, r), t), i) => p.id }.result).map { pages => - pages.map { - case (((p, r), t), i) => p.copy(revisions = Seq(r.copy(content = Some(t.text))), images = i.toSeq) + joinLeft images on (_._1._1.id === _.pageId)).sortBy { + case (((p, r), t), i) => p.id + }.result + ).map { pages => + pages.map { case (((p, r), t), i) => + p.copy( + revisions = Seq(r.copy(content = Some(t.text))), + images = i.toSeq + ) } }.await } - def findByPageAndRevIdsOpt(pageIds: Iterable[Long], revIds: Iterable[Long]): Seq[Page] = { + def findByPageAndRevIdsOpt( + pageIds: Iterable[Long], + revIds: Iterable[Long] + ): Seq[Page] = { db.run( (pages.filter(_.id inSet pageIds) joinLeft revisions.filter(_.id inSet revIds) on (_.id === _.pageId) joinLeft texts on (_._2.map(_.textId) === _.id) - joinLeft images on (_._1._1.id === _.pageId) - ).sortBy { case (((p, r), t), i) => p.id }.result).map { pages => - pages.map { - case (((p, r), t), i) => p.copy(revisions = r.toSeq.map(_.copy(content = t.map(_.text))), images = i.toSeq) + joinLeft images on (_._1._1.id === _.pageId)).sortBy { + case (((p, r), t), i) => p.id + }.result + ).map { pages => + pages.map { case (((p, r), t), i) => + p.copy( + revisions = r.toSeq.map(_.copy(content = t.map(_.text))), + images = i.toSeq + ) } }.await } @@ -177,24 +207,25 @@ class PageDao(val mwDb: MwDatabase, val driver: JdbcProfile) { db.run( (pages.filter(_.id === id) join revisions on (_.pageLatest === _.id) - join texts on (_._2.textId === _.id) - ).result).map { pages => - pages.map { - case ((p, r), t) => p.copy(revisions = Seq(r.copy(content = Some(t.text)))) + join texts on (_._2.textId === _.id)).result + ).map { pages => + pages.map { case ((p, r), t) => + p.copy(revisions = Seq(r.copy(content = Some(t.text)))) }.head }.await def withRevisions(id: Long): Page = { - db.run((( - for { - p <- pages if p.id === id - r <- revisions if r.pageId === p.id - t <- texts if r.textId === t.id - } yield (p, r, t) - ) sortBy { case (p, r, t) => r.id.desc } - ).result).map { pages => - val rows = pages.map { - case (p, r, t) => (p, r.copy(content = Some(t.text))) + db.run( + (( + for { + p <- pages if p.id === id + r <- revisions if r.pageId === p.id + t <- texts if r.textId === t.id + } yield (p, r, t) + ) sortBy { case (p, r, t) => r.id.desc }).result + ).map { pages => + val rows = pages.map { case (p, r, t) => + (p, r.copy(content = Some(t.text))) } val revs = rows.map { case (p, r) => r } rows.headOption.map { case (p, r) => p.copy(revisions = revs) }.get diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/RevisionDao.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/RevisionDao.scala index 8e0e499b..92171e53 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/RevisionDao.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/RevisionDao.scala @@ -35,8 +35,7 @@ class RevisionDao(val mwDb: MwDatabase, val driver: JdbcProfile) { val revIds = if (hasIds) { db.run(revisions.forceInsertAll(withTextIds)).await revisionSeq.map(_.id) - } - else { + } else { db.run(autoInc.forceInsertAll(withTextIds)).await } revIds @@ -49,10 +48,10 @@ class RevisionDao(val mwDb: MwDatabase, val driver: JdbcProfile) { if (e) Future.successful(revision.id) else - db.run(revisions.forceInsert(addUser(addText(revision)))).map(_ => revision.id) + db.run(revisions.forceInsert(addUser(addText(revision)))) + .map(_ => revision.id) } - } - else { + } else { db.run(autoInc += addUser(addText(revision))) } revId.map(_.get).await @@ -78,7 +77,9 @@ class RevisionDao(val mwDb: MwDatabase, val driver: JdbcProfile) { if (dbUser.isDefined) { revision.copy(user = dbUser) } else { - throw new IllegalArgumentException(s"No user with id $userId exists") + throw new IllegalArgumentException( + s"No user with id $userId exists" + ) } } else if (user.name.isDefined) { val username = user.name.get @@ -86,7 +87,9 @@ class RevisionDao(val mwDb: MwDatabase, val driver: JdbcProfile) { if (dbUser.isDefined) { revision.copy(user = dbUser) } else { - throw new IllegalArgumentException(s"No user with name $username exists") + throw new IllegalArgumentException( + s"No user with name $username exists" + ) } } else { revision @@ -106,7 +109,6 @@ class RevisionDao(val mwDb: MwDatabase, val driver: JdbcProfile) { revisionSeq } - def list = db.run(revisions.result).await def count = db.run(revisions.length.result).await @@ -120,10 +122,12 @@ class RevisionDao(val mwDb: MwDatabase, val driver: JdbcProfile) { // get(id).recover { case _ => false }.map(_ => true) def withText(id: Long): Future[Revision] = - db.run((revisions.filter(_.id === id) - join texts on (_.textId === _.id)).result).map { rows => - rows.map { - case (r, t) => r.copy(content = Some(t.text)) + db.run( + (revisions.filter(_.id === id) + join texts on (_.textId === _.id)).result + ).map { rows => + rows.map { case (r, t) => + r.copy(content = Some(t.text)) }.head } } diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/TextDao.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/TextDao.scala index d7698673..9f588f71 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/TextDao.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/TextDao.scala @@ -7,7 +7,11 @@ import slick.driver.JdbcProfile import slick.lifted.TableQuery import spray.util.pimpFuture -class TextDao(val mwDb: MwDatabase, val query: TableQuery[Texts], val driver: JdbcProfile) { +class TextDao( + val mwDb: MwDatabase, + val query: TableQuery[Texts], + val driver: JdbcProfile +) { import driver.api._ diff --git a/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/UserDao.scala b/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/UserDao.scala index 16a8ba43..2531a442 100644 --- a/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/UserDao.scala +++ b/scalawiki-sql/src/main/scala/org/scalawiki/sql/dao/UserDao.scala @@ -8,7 +8,11 @@ import spray.util.pimpFuture import scala.language.higherKinds -class UserDao(val mwDb: MwDatabase, val query: TableQuery[Users], val driver: JdbcProfile) { +class UserDao( + val mwDb: MwDatabase, + val query: TableQuery[Users], + val driver: JdbcProfile +) { import driver.api._ @@ -24,8 +28,7 @@ class UserDao(val mwDb: MwDatabase, val query: TableQuery[Users], val driver: Jd if (user.id.isDefined) { db.run(query.forceInsert(user)).await user.id - } - else { + } else { db.run(autoInc += user).await } } diff --git a/scalawiki-sql/src/test/scala/org/scalawiki/sql/DslQueryDbCacheBlackBoxSpec.scala b/scalawiki-sql/src/test/scala/org/scalawiki/sql/DslQueryDbCacheBlackBoxSpec.scala index fbe70f5c..14b27ac9 100644 --- a/scalawiki-sql/src/test/scala/org/scalawiki/sql/DslQueryDbCacheBlackBoxSpec.scala +++ b/scalawiki-sql/src/test/scala/org/scalawiki/sql/DslQueryDbCacheBlackBoxSpec.scala @@ -12,7 +12,10 @@ import spray.util.pimpFuture import scala.concurrent.duration._ -class DslQueryDbCacheBlackBoxSpec extends Specification with MockBotSpec with BeforeAfter { +class DslQueryDbCacheBlackBoxSpec + extends Specification + with MockBotSpec + with BeforeAfter { sequential @@ -33,7 +36,7 @@ class DslQueryDbCacheBlackBoxSpec extends Specification with MockBotSpec with Be val dummyAction = Action(DummyActionArg) override def getBot(commands: HttpStub*) = { - val apiBot = super.getBot(commands:_*) + val apiBot = super.getBot(commands: _*) mwDb.dropTables() mwDb.createTables() @@ -47,7 +50,7 @@ class DslQueryDbCacheBlackBoxSpec extends Specification with MockBotSpec with Be } override def after = { - // dc.db.close() + // dc.db.close() } val pageText1 = "some vandalism" @@ -56,20 +59,40 @@ class DslQueryDbCacheBlackBoxSpec extends Specification with MockBotSpec with Be val someUser1 = Some(User(7L, "Formator")) val someUser2 = Some(User(9L, "OtherUser")) - def revisionContent(content: Option[String]) = content.fold("") { - text => s""", "*": "$text" """ + def revisionContent(content: Option[String]) = content.fold("") { text => + s""", "*": "$text" """ } - def page(pageId: Long, title: String, revId: Long, content: Option[String] = None, ns: Int = 0, user: String = "Formator", userId: Long = 7) = + def page( + pageId: Long, + title: String, + revId: Long, + content: Option[String] = None, + ns: Int = 0, + user: String = "Formator", + userId: Long = 7 + ) = s""" "$pageId": { "pageid": $pageId, "ns": $ns, "title": "$title", - "revisions": [{"revid": $revId, "user": "$user", "userid": $userId ${revisionContent(content)} }] + "revisions": [{"revid": $revId, "user": "$user", "userid": $userId ${revisionContent( + content + )} }] }""" - def page1(content: Option[String] = None, revId: Long = 11, user: String = "OtherUser", userId: Long = 9) = + def page1( + content: Option[String] = None, + revId: Long = 11, + user: String = "OtherUser", + userId: Long = 9 + ) = page(569559, "Talk:Welfare reform", revId, content, 1, user, userId) - def page2(content: Option[String] = None, revId: Long = 12, user: String = "Formator", userId: Long = 7) = + def page2( + content: Option[String] = None, + revId: Long = 12, + user: String = "Formator", + userId: Long = 7 + ) = page(4571809, "User:Formator", revId, content, 2, user, userId) def pagesJson(pages: Seq[String]) = @@ -78,72 +101,175 @@ class DslQueryDbCacheBlackBoxSpec extends Specification with MockBotSpec with Be def responseWithRevIds(pages: Seq[String] = Seq(page2(), page1())) = pagesJson(pages) - def responseWithContent(pages: Seq[String] = Seq(page2(Some(pageText2)), page1(Some(pageText1)))) = + def responseWithContent( + pages: Seq[String] = Seq(page2(Some(pageText2)), page1(Some(pageText1))) + ) = pagesJson(pages) "get revisions text with generator and caching" should { "first run should use generator" in { - val expectedCommands = Seq(HttpStub(Map( - "action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|user|userid", // TODO remove user? - "continue" -> ""), responseWithRevIds()), - - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|userid", - "continue" -> ""), responseWithContent()) + val expectedCommands = Seq( + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|user|userid", // TODO remove user? + "continue" -> "" + ), + responseWithRevIds() + ), + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|userid", + "continue" -> "" + ), + responseWithContent() + ) ) bot = getBot(expectedCommands: _*) - val future = bot.page("Category:SomeCategory") - .revisionsByGenerator("categorymembers", "cm", Set.empty, Set("ids", "content", "user", "userid")) + val future = bot + .page("Category:SomeCategory") + .revisionsByGenerator( + "categorymembers", + "cm", + Set.empty, + Set("ids", "content", "user", "userid") + ) val result = future.await(timeout = 1.minutes).toSeq result must have size 2 - result.head === Page(Some(4571809L), Some(2), "User:Formator", - Seq(Revision(Some(12L), Some(4571809L), None, someUser1, None, None, Some(pageText2))) + result.head === Page( + Some(4571809L), + Some(2), + "User:Formator", + Seq( + Revision( + Some(12L), + Some(4571809L), + None, + someUser1, + None, + None, + Some(pageText2) + ) + ) ) - result(1) === Page(Some(569559L), Some(1), "Talk:Welfare reform", - Seq(Revision(Some(11L), Some(569559L), None, someUser2, None, None, Some(pageText1))) + result(1) === Page( + Some(569559L), + Some(1), + "Talk:Welfare reform", + Seq( + Revision( + Some(11L), + Some(569559L), + None, + someUser2, + None, + None, + Some(pageText1) + ) + ) ) } "from cache in" in { val expectedCommands = Seq( - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|user|userid", - "continue" -> ""), responseWithRevIds()), - - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|userid", - "continue" -> ""), responseWithContent()), - - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|user|userid", - "continue" -> ""), responseWithRevIds()) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|user|userid", + "continue" -> "" + ), + responseWithRevIds() + ), + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|userid", + "continue" -> "" + ), + responseWithContent() + ), + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|user|userid", + "continue" -> "" + ), + responseWithRevIds() + ) ) bot = getBot(expectedCommands: _*) - val future = bot.page("Category:SomeCategory") - .revisionsByGenerator("categorymembers", "cm", Set.empty, Set("ids", "content", "user", "userid")) + val future = bot + .page("Category:SomeCategory") + .revisionsByGenerator( + "categorymembers", + "cm", + Set.empty, + Set("ids", "content", "user", "userid") + ) val result = future.await.toSeq result must have size 2 - result(0) === Page(Some(4571809L), Some(2), "User:Formator", - Seq(Revision(Some(12L), Some(4571809L), None, someUser1, None, None, Some(pageText2))) + result(0) === Page( + Some(4571809L), + Some(2), + "User:Formator", + Seq( + Revision( + Some(12L), + Some(4571809L), + None, + someUser1, + None, + None, + Some(pageText2) + ) + ) ) - result(1) === Page(Some(569559L), Some(1), "Talk:Welfare reform", - Seq(Revision(Some(11L), Some(569559L), None, someUser2, None, None, Some(pageText1))) + result(1) === Page( + Some(569559L), + Some(1), + "Talk:Welfare reform", + Seq( + Revision( + Some(11L), + Some(569559L), + None, + someUser2, + None, + None, + Some(pageText1) + ) + ) ) pageDao.list.size must be_==(2).eventually @@ -151,71 +277,158 @@ class DslQueryDbCacheBlackBoxSpec extends Specification with MockBotSpec with Be byRevs.size === 2 byRevs.map(_.title) === Seq("Talk:Welfare reform", "User:Formator") byRevs.map(_.revisions.head.id.get) === Seq(11L, 12L) - byRevs.map(_.revisions.head.content.get) === Seq("some vandalism", "more vandalism") + byRevs.map(_.revisions.head.content.get) === Seq( + "some vandalism", + "more vandalism" + ) - val futureDb = bot.page("Category:SomeCategory") - .revisionsByGenerator("categorymembers", "cm", Set.empty, Set("ids", "content", "user", "userid")) + val futureDb = bot + .page("Category:SomeCategory") + .revisionsByGenerator( + "categorymembers", + "cm", + Set.empty, + Set("ids", "content", "user", "userid") + ) val resultDb = futureDb.await.toSeq resultDb must have size 2 - resultDb.head === Page(Some(569559L), Some(1), "Talk:Welfare reform", - Seq(Revision(Some(11L), Some(569559L), Some(0), someUser2, None, None, Some(pageText1), - textId = resultDb.head.revisions.head.textId)) + resultDb.head === Page( + Some(569559L), + Some(1), + "Talk:Welfare reform", + Seq( + Revision( + Some(11L), + Some(569559L), + Some(0), + someUser2, + None, + None, + Some(pageText1), + textId = resultDb.head.revisions.head.textId + ) + ) ) - resultDb(1) === Page(Some(4571809L), Some(2), "User:Formator", - Seq(Revision(Some(12L), Some(4571809L), Some(0), someUser1, None, None, Some(pageText2), - textId = resultDb(1).revisions.head.textId)) + resultDb(1) === Page( + Some(4571809L), + Some(2), + "User:Formator", + Seq( + Revision( + Some(12L), + Some(4571809L), + Some(0), + someUser1, + None, + None, + Some(pageText2), + textId = resultDb(1).revisions.head.textId + ) + ) ) } "add page to cache" in { val expectedCommands = Seq( // query for page 1 ids in category - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|user|userid", - "continue" -> ""), pagesJson(Seq(page1())) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|user|userid", + "continue" -> "" + ), + pagesJson(Seq(page1())) ), // query for page 1 content - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|userid", - "continue" -> ""), pagesJson(Seq(page1(Some(pageText1)))) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|userid", + "continue" -> "" + ), + pagesJson(Seq(page1(Some(pageText1)))) ), // query for page 1 and page2 ids in category - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|user|userid", - "continue" -> ""), pagesJson(Seq(page1(), page2())) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|user|userid", + "continue" -> "" + ), + pagesJson(Seq(page1(), page2())) ), // fetch for page2 content for cache - HttpStub(Map("action" -> "query", - "pageids" -> "4571809", - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|userid", "continue" -> ""), + HttpStub( + Map( + "action" -> "query", + "pageids" -> "4571809", + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|userid", + "continue" -> "" + ), pagesJson(Seq(page2(Some(pageText2)))) ), // query for page 1 and page2 ids in category. content should be in cache by now - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|user|userid", - "continue" -> ""), pagesJson(Seq(page1(), page2())) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|user|userid", + "continue" -> "" + ), + pagesJson(Seq(page1(), page2())) ) ) bot = getBot(expectedCommands: _*) - val future = bot.page("Category:SomeCategory") - .revisionsByGenerator("categorymembers", "cm", Set.empty, Set("ids", "content", "user", "userid")) + val future = bot + .page("Category:SomeCategory") + .revisionsByGenerator( + "categorymembers", + "cm", + Set.empty, + Set("ids", "content", "user", "userid") + ) val result = future.await.toSeq result must have size 1 - result(0) === Page(Some(569559L), Some(1), "Talk:Welfare reform", - Seq(Revision(Some(11L), Some(569559L), None, someUser2, None, None, Some(pageText1))) + result(0) === Page( + Some(569559L), + Some(1), + "Talk:Welfare reform", + Seq( + Revision( + Some(11L), + Some(569559L), + None, + someUser2, + None, + None, + Some(pageText1) + ) + ) ) pageDao.list.size must be_==(1).eventually @@ -225,37 +438,105 @@ class DslQueryDbCacheBlackBoxSpec extends Specification with MockBotSpec with Be byRevs.map(_.revisions.head.id.get) === Seq(11L) byRevs.map(_.revisions.head.content.get) === Seq("some vandalism") - val plus1Future = bot.page("Category:SomeCategory") - .revisionsByGenerator("categorymembers", "cm", Set.empty, Set("ids", "content", "user", "userid")) + val plus1Future = bot + .page("Category:SomeCategory") + .revisionsByGenerator( + "categorymembers", + "cm", + Set.empty, + Set("ids", "content", "user", "userid") + ) val plus1 = plus1Future.await.toSeq plus1 must have size 2 - plus1(0) === Page(Some(569559L), Some(1), "Talk:Welfare reform", - Seq(Revision(Some(11L), Some(569559L), Some(0), someUser2, None, None, Some(pageText1), - textId = plus1(0).revisions.head.textId)) + plus1(0) === Page( + Some(569559L), + Some(1), + "Talk:Welfare reform", + Seq( + Revision( + Some(11L), + Some(569559L), + Some(0), + someUser2, + None, + None, + Some(pageText1), + textId = plus1(0).revisions.head.textId + ) + ) ) - plus1(1) === Page(Some(4571809L), Some(2), "User:Formator", - Seq(Revision(Some(12L), Some(4571809L), None, someUser1, None, None, Some(pageText2))) + plus1(1) === Page( + Some(4571809L), + Some(2), + "User:Formator", + Seq( + Revision( + Some(12L), + Some(4571809L), + None, + someUser1, + None, + None, + Some(pageText2) + ) + ) ) pageDao.list.size must be_==(2).eventually - val byRevsFinal = pageDao.findByRevIds(Seq(4571809L, 569559L), Seq(12, 11)) + val byRevsFinal = + pageDao.findByRevIds(Seq(4571809L, 569559L), Seq(12, 11)) byRevsFinal.size === 2 byRevsFinal.map(_.title) === Seq("Talk:Welfare reform", "User:Formator") - byRevsFinal.map(_.revisions.head.content.get) === Seq("some vandalism", "more vandalism") + byRevsFinal.map(_.revisions.head.content.get) === Seq( + "some vandalism", + "more vandalism" + ) - val futureFinal = bot.page("Category:SomeCategory") - .revisionsByGenerator("categorymembers", "cm", Set.empty, Set("ids", "content", "user", "userid")) + val futureFinal = bot + .page("Category:SomeCategory") + .revisionsByGenerator( + "categorymembers", + "cm", + Set.empty, + Set("ids", "content", "user", "userid") + ) val resultFinal = futureFinal.await.toSeq resultFinal must have size 2 - resultFinal(0) === Page(Some(569559L), Some(1), "Talk:Welfare reform", - Seq(Revision(Some(11L), Some(569559L), Some(0), someUser2, None, None, Some(pageText1), - textId = resultFinal(0).revisions.head.textId)) + resultFinal(0) === Page( + Some(569559L), + Some(1), + "Talk:Welfare reform", + Seq( + Revision( + Some(11L), + Some(569559L), + Some(0), + someUser2, + None, + None, + Some(pageText1), + textId = resultFinal(0).revisions.head.textId + ) + ) ) - resultFinal(1) === Page(Some(4571809L), Some(2), "User:Formator", - Seq(Revision(Some(12L), Some(4571809L), Some(0), someUser1, None, None, Some(pageText2), - textId = resultFinal(1).revisions.head.textId)) + resultFinal(1) === Page( + Some(4571809L), + Some(2), + "User:Formator", + Seq( + Revision( + Some(12L), + Some(4571809L), + Some(0), + someUser1, + None, + None, + Some(pageText2), + textId = resultFinal(1).revisions.head.textId + ) + ) ) } @@ -263,47 +544,97 @@ class DslQueryDbCacheBlackBoxSpec extends Specification with MockBotSpec with Be // TODO more pages? val expectedCommands = Seq( - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|user|userid", - "continue" -> ""), pagesJson(Seq(page1(revId = 11))) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|user|userid", + "continue" -> "" + ), + pagesJson(Seq(page1(revId = 11))) ), - - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|userid", - "continue" -> ""), pagesJson(Seq(page1(Some(pageText1), revId = 11))) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|userid", + "continue" -> "" + ), + pagesJson(Seq(page1(Some(pageText1), revId = 11))) ), - - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|user|userid", - "continue" -> ""), pagesJson(Seq(page1(revId = 12))) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|user|userid", + "continue" -> "" + ), + pagesJson(Seq(page1(revId = 12))) ), - - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|userid", - "continue" -> ""), pagesJson(Seq(page1(Some(pageText2), revId = 12))) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|userid", + "continue" -> "" + ), + pagesJson(Seq(page1(Some(pageText2), revId = 12))) ), - - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|user|userid", - "continue" -> ""), pagesJson(Seq(page1(revId = 12))) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|user|userid", + "continue" -> "" + ), + pagesJson(Seq(page1(revId = 12))) ) ) bot = getBot(expectedCommands: _*) - val future = bot.page("Category:SomeCategory") - .revisionsByGenerator("categorymembers", "cm", Set.empty, Set("ids", "content", "user", "userid")) + val future = bot + .page("Category:SomeCategory") + .revisionsByGenerator( + "categorymembers", + "cm", + Set.empty, + Set("ids", "content", "user", "userid") + ) val result = future.await.toSeq result must have size 1 - result.head === Page(Some(569559L), Some(1), "Talk:Welfare reform", - Seq(Revision(Some(11L), Some(569559L), None, someUser2, None, None, Some(pageText1))) + result.head === Page( + Some(569559L), + Some(1), + "Talk:Welfare reform", + Seq( + Revision( + Some(11L), + Some(569559L), + None, + someUser2, + None, + None, + Some(pageText1) + ) + ) ) pageDao.list.size must be_==(1).eventually @@ -312,13 +643,32 @@ class DslQueryDbCacheBlackBoxSpec extends Specification with MockBotSpec with Be byRevs.map(_.title) === Seq("Talk:Welfare reform") byRevs.map(_.revisions.head.content.get) === Seq("some vandalism") - val plus1Future = bot.page("Category:SomeCategory") - .revisionsByGenerator("categorymembers", "cm", Set.empty, Set("ids", "content", "user", "userid")) + val plus1Future = bot + .page("Category:SomeCategory") + .revisionsByGenerator( + "categorymembers", + "cm", + Set.empty, + Set("ids", "content", "user", "userid") + ) val plus1 = plus1Future.await.toSeq plus1 must have size 1 - plus1.head === Page(Some(569559L), Some(1), "Talk:Welfare reform", - Seq(Revision(Some(12L), Some(569559L), None, someUser2, None, None, Some(pageText2))) + plus1.head === Page( + Some(569559L), + Some(1), + "Talk:Welfare reform", + Seq( + Revision( + Some(12L), + Some(569559L), + None, + someUser2, + None, + None, + Some(pageText2) + ) + ) ) revisionDao.list.size must be_==(2).eventually @@ -327,14 +677,33 @@ class DslQueryDbCacheBlackBoxSpec extends Specification with MockBotSpec with Be byRevsFinal.map(_.title) === Seq("Talk:Welfare reform") byRevsFinal.map(_.revisions.head.content.get) === Seq("more vandalism") - val futureFinal = bot.page("Category:SomeCategory") - .revisionsByGenerator("categorymembers", "cm", Set.empty, Set("ids", "content", "user", "userid")) + val futureFinal = bot + .page("Category:SomeCategory") + .revisionsByGenerator( + "categorymembers", + "cm", + Set.empty, + Set("ids", "content", "user", "userid") + ) val resultFinal = futureFinal.await.toSeq resultFinal must have size 1 - resultFinal.head === Page(Some(569559L), Some(1), "Talk:Welfare reform", - Seq(Revision(Some(12L), Some(569559L), Some(0), someUser2, None, None, Some(pageText2), - textId = resultFinal.head.revisions.head.textId)) + resultFinal.head === Page( + Some(569559L), + Some(1), + "Talk:Welfare reform", + Seq( + Revision( + Some(12L), + Some(569559L), + Some(0), + someUser2, + None, + None, + Some(pageText2), + textId = resultFinal.head.revisions.head.textId + ) + ) ) } } diff --git a/scalawiki-sql/src/test/scala/org/scalawiki/sql/DslQueryDbCacheModularSpec.scala b/scalawiki-sql/src/test/scala/org/scalawiki/sql/DslQueryDbCacheModularSpec.scala index c16e79f9..5a59105c 100644 --- a/scalawiki-sql/src/test/scala/org/scalawiki/sql/DslQueryDbCacheModularSpec.scala +++ b/scalawiki-sql/src/test/scala/org/scalawiki/sql/DslQueryDbCacheModularSpec.scala @@ -14,7 +14,10 @@ import slick.backend.DatabaseConfig import slick.driver.JdbcProfile import spray.util.pimpFuture -class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with BeforeAfter { +class DslQueryDbCacheModularSpec + extends Specification + with MockBotSpec + with BeforeAfter { sequential @@ -35,7 +38,7 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef val dummyAction = Action(DummyActionArg) override def getBot(commands: HttpStub*) = { - val apiBot = super.getBot(commands:_*) + val apiBot = super.getBot(commands: _*) mwDb.dropTables() mwDb.createTables() @@ -49,7 +52,7 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef } override def after = { - //dc.db.close() + // dc.db.close() } val pageText1 = "some vandalism" @@ -58,20 +61,40 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef val someUser1 = Some(User(7L, "Formator")) val someUser2 = Some(User(9L, "OtherUser")) - def revisionContent(content: Option[String]) = content.fold("") { - text => s""", "*": "$text" """ + def revisionContent(content: Option[String]) = content.fold("") { text => + s""", "*": "$text" """ } - def page(pageId: Long, title: String, revId: Long, content: Option[String] = None, ns: Int = 0, user: String = "Formator", userId: Long = 7) = + def page( + pageId: Long, + title: String, + revId: Long, + content: Option[String] = None, + ns: Int = 0, + user: String = "Formator", + userId: Long = 7 + ) = s""" "$pageId": { "pageid": $pageId, "ns": $ns, "title": "$title", - "revisions": [{"revid": $revId, "user": "$user", "userid": $userId ${revisionContent(content)} }] + "revisions": [{"revid": $revId, "user": "$user", "userid": $userId ${revisionContent( + content + )} }] }""" - def page1(content: Option[String] = None, revId: Long = 11, user: String = "OtherUser", userId: Long = 9) = + def page1( + content: Option[String] = None, + revId: Long = 11, + user: String = "OtherUser", + userId: Long = 9 + ) = page(569559, "Talk:Welfare reform", revId, content, 1, user, userId) - def page2(content: Option[String] = None, revId: Long = 12, user: String = "Formator", userId: Long = 7) = + def page2( + content: Option[String] = None, + revId: Long = 12, + user: String = "Formator", + userId: Long = 7 + ) = page(4571809, "User:Formator", revId, content, 2, user, userId) def pagesJson(pages: Seq[String]) = @@ -80,11 +103,11 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef def responseWithRevIds(pages: Seq[String] = Seq(page2(), page1())) = pagesJson(pages) - def responseWithContent(pages: Seq[String] = Seq(page2(Some(pageText2)), page1(Some(pageText1)))) = + def responseWithContent( + pages: Seq[String] = Seq(page2(Some(pageText2)), page1(Some(pageText1))) + ) = pagesJson(pages) - - "toDb" should { "insert new pages" in { @@ -97,11 +120,15 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef (1 to 10) map ("text" + _), 51L to 60L, 21L to 30L - ) + ) val pages = titles.zip(texts).zip(pageIds).zip(revIds) map { case (((title, text), pageId), revId) => - val revision: Revision = new Revision(revId = Some(revId), pageId = Some(pageId), content = Some(text)) + val revision: Revision = new Revision( + revId = Some(revId), + pageId = Some(pageId), + content = Some(text) + ) Page(Some(pageId), Some(0), title, Seq(revision)) } @@ -114,7 +141,9 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef revs.map(_.textId) aka "textIds in revisions" must_== dbTexts.map(_.id) val dbPages = pageDao.list - dbPages.map(_.revisions.headOption.flatMap(_.revId)) aka "revIds in pages" must_== revs.map(_.id) + dbPages.map( + _.revisions.headOption.flatMap(_.revId) + ) aka "revIds in pages" must_== revs.map(_.id) val fromDb = pageDao.listWithText @@ -140,27 +169,35 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef (1 to 10) map ("text" + _), 51L to 60L, 21L to 30L - ) + ) val pages = titles.zip(texts).zip(pageIds).zip(revIds) map { case (((title, text), pageId), revId) => - val revision: Revision = new Revision(revId = Some(revId), pageId = Some(pageId), content = Some(text)) + val revision: Revision = new Revision( + revId = Some(revId), + pageId = Some(pageId), + content = Some(text) + ) Page(Some(pageId), Some(0), title, Seq(revision)) } pageDao.insertAll(pages) - val newPages = pages.map { - p => - val rev = p.revisions.head - val newRev = rev.copy(revId = rev.id.map(_ + 100), content = rev.content.map("new_" + _)) - p.copy(revisions = Seq(newRev)) + val newPages = pages.map { p => + val rev = p.revisions.head + val newRev = rev.copy( + revId = rev.id.map(_ + 100), + content = rev.content.map("new_" + _) + ) + p.copy(revisions = Seq(newRev)) } cache.toDb(newPages, pages.flatMap(_.id).toSet) val dbPages = pageDao.list - dbPages.flatMap(_.revisions.headOption.flatMap(_.revId)) aka "revIds in pages" must_== revIds.map(_ + 100) + dbPages.flatMap( + _.revisions.headOption.flatMap(_.revId) + ) aka "revIds in pages" must_== revIds.map(_ + 100) val fromDb = pageDao.listWithText @@ -189,11 +226,15 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef (1 to 10) map ("text" + _), 51L to 60L, 21L to 30L - ) + ) val dbPages = titles.zip(texts).zip(pageIds).zip(revIds) map { case (((title, text), pageId), revId) => - val revision: Revision = new Revision(revId = Some(revId), pageId = Some(pageId), content = Some(text)) + val revision: Revision = new Revision( + revId = Some(revId), + pageId = Some(pageId), + content = Some(text) + ) Page(Some(pageId), Some(0), title, Seq(revision)) } @@ -209,7 +250,7 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef (1 to 10) map ("text" + _), 51L to 60L, 21L to 30L - ) + ) val pageJsons = titles.zip(texts).zip(pageIds).zip(revIds) map { case (((title, text), pageId), revId) => @@ -218,10 +259,16 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef val commands = Seq( // fetch for page2 content for cache - HttpStub(Map("action" -> "query", - "pageids" -> pageIds.drop(5).mkString("|"), - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|userid", "continue" -> ""), - pagesJson(pageJsons.drop(5))) + HttpStub( + Map( + "action" -> "query", + "pageids" -> pageIds.drop(5).mkString("|"), + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|userid", + "continue" -> "" + ), + pagesJson(pageJsons.drop(5)) + ) ) bot = getBot(commands: _*) @@ -229,12 +276,27 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef val query = Query( Prop( Info(), - Revisions(RvProp(RvPropArgs.byNames(Seq("ids", "content", "user", "userid")): _*)) + Revisions( + RvProp( + RvPropArgs.byNames(Seq("ids", "content", "user", "userid")): _* + ) + ) ), - Generator(ListArgs.toDsl("categorymembers", Some("Category:SomeCategory"), None, Set.empty, Some("max")).get) + Generator( + ListArgs + .toDsl( + "categorymembers", + Some("Category:SomeCategory"), + None, + Set.empty, + Some("max") + ) + .get + ) ) - val cache = new DslQueryDbCache(new DslQuery(Action(query), bot), database) + val cache = + new DslQueryDbCache(new DslQuery(Action(query), bot), database) val dbPages = titles.zip(texts).zip(pageIds).zip(revIds) map { case (((title, text), pageId), revId) => @@ -242,11 +304,13 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef revId = Some(revId), pageId = Some(pageId), content = Some(text), - user = someUser1) + user = someUser1 + ) Page(Some(pageId), Some(0), title, Seq(revision)) } - val slice = dbPages.take(5) ++ dbPages.drop(5).map(_.copy(revisions = Seq.empty)) + val slice = + dbPages.take(5) ++ dbPages.drop(5).map(_.copy(revisions = Seq.empty)) val notInDb = cache.notInDb(query, pageIds.toSet, slice).await notInDb === dbPages.drop(5) @@ -258,7 +322,7 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef (1 to 100) map ("text" + _), 100L to 199L, 200L to 299L - ) + ) val pageJsons = titles.zip(texts).zip(pageIds).zip(revIds) map { case (((title, text), pageId), revId) => @@ -266,10 +330,18 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef } val commands = Seq( - HttpStub(Map("action" -> "query", - "generator" -> "categorymembers", "gcmtitle" -> "Category:SomeCategory", "gcmlimit" -> "max", - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|userid", - "continue" -> ""), pagesJson(pageJsons)) + HttpStub( + Map( + "action" -> "query", + "generator" -> "categorymembers", + "gcmtitle" -> "Category:SomeCategory", + "gcmlimit" -> "max", + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|userid", + "continue" -> "" + ), + pagesJson(pageJsons) + ) ) bot = getBot(commands: _*) @@ -277,12 +349,27 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef val query = Query( Prop( Info(), - Revisions(RvProp(RvPropArgs.byNames(Seq("ids", "content", "user", "userid")): _*)) + Revisions( + RvProp( + RvPropArgs.byNames(Seq("ids", "content", "user", "userid")): _* + ) + ) ), - Generator(ListArgs.toDsl("categorymembers", Some("Category:SomeCategory"), None, Set.empty, Some("max")).get) + Generator( + ListArgs + .toDsl( + "categorymembers", + Some("Category:SomeCategory"), + None, + Set.empty, + Some("max") + ) + .get + ) ) - val cache = new DslQueryDbCache(new DslQuery(Action(query), bot), database) + val cache = + new DslQueryDbCache(new DslQuery(Action(query), bot), database) val dbPages = titles.zip(texts).zip(pageIds).zip(revIds) map { case (((title, text), pageId), revId) => @@ -290,7 +377,8 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef revId = Some(revId), pageId = Some(pageId), content = Some(text), - user = someUser1) + user = someUser1 + ) Page(Some(pageId), Some(0), title, Seq(revision)) } @@ -299,14 +387,13 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef notInDb === dbPages } - "return in batches" in { val (titles, texts, pageIds, revIds) = ( (1 to 100) map ("title" + _), (1 to 100) map ("text" + _), 100L to 199L, 200L to 299L - ) + ) val pageJsons = titles.zip(texts).zip(pageIds).zip(revIds) map { case (((title, text), pageId), revId) => @@ -315,15 +402,26 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef val commands = Seq( // fetch for page2 content for cache - HttpStub(Map("action" -> "query", - "pageids" -> pageIds.slice(5, 55).mkString("|"), - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|userid", "continue" -> ""), - pagesJson(pageJsons.slice(5, 55))), - - HttpStub(Map("action" -> "query", - "pageids" -> pageIds.slice(55, 95).mkString("|"), - "prop" -> "info|revisions", "rvprop" -> "ids|content|user|userid", "continue" -> ""), - pagesJson(pageJsons.slice(55, 95))) + HttpStub( + Map( + "action" -> "query", + "pageids" -> pageIds.slice(5, 55).mkString("|"), + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|userid", + "continue" -> "" + ), + pagesJson(pageJsons.slice(5, 55)) + ), + HttpStub( + Map( + "action" -> "query", + "pageids" -> pageIds.slice(55, 95).mkString("|"), + "prop" -> "info|revisions", + "rvprop" -> "ids|content|user|userid", + "continue" -> "" + ), + pagesJson(pageJsons.slice(55, 95)) + ) ) bot = getBot(commands: _*) @@ -331,12 +429,27 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef val query = Query( Prop( Info(), - Revisions(RvProp(RvPropArgs.byNames(Seq("ids", "content", "user", "userid")): _*)) + Revisions( + RvProp( + RvPropArgs.byNames(Seq("ids", "content", "user", "userid")): _* + ) + ) ), - Generator(ListArgs.toDsl("categorymembers", Some("Category:SomeCategory"), None, Set.empty, Some("max")).get) + Generator( + ListArgs + .toDsl( + "categorymembers", + Some("Category:SomeCategory"), + None, + Set.empty, + Some("max") + ) + .get + ) ) - val cache = new DslQueryDbCache(new DslQuery(Action(query), bot), database) + val cache = + new DslQueryDbCache(new DslQuery(Action(query), bot), database) val dbPages = titles.zip(texts).zip(pageIds).zip(revIds) map { case (((title, text), pageId), revId) => @@ -344,7 +457,8 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef revId = Some(revId), pageId = Some(pageId), content = Some(text), - user = someUser1) + user = someUser1 + ) Page(Some(pageId), Some(0), title, Seq(revision)) } @@ -375,18 +489,21 @@ class DslQueryDbCacheModularSpec extends Specification with MockBotSpec with Bef ) val notInDbIds = Seq(1L, 2L, 3L) - val notInDbQuery = new DslQueryDbCache(new DslQuery(Action(query), super.getBot()), database).notInDBQuery(query, notInDbIds) - - notInDbQuery === Seq(Query( - Prop( - Info(), - Revisions(RvProp(Ids, Content)) - ), - PageIdsParam(notInDbIds) - )) + val notInDbQuery = new DslQueryDbCache( + new DslQuery(Action(query), super.getBot()), + database + ).notInDBQuery(query, notInDbIds) + + notInDbQuery === Seq( + Query( + Prop( + Info(), + Revisions(RvProp(Ids, Content)) + ), + PageIdsParam(notInDbIds) + ) + ) } } } - - diff --git a/scalawiki-sql/src/test/scala/org/scalawiki/sql/ImageDaoSpec.scala b/scalawiki-sql/src/test/scala/org/scalawiki/sql/ImageDaoSpec.scala index c926e99f..cb170ed4 100644 --- a/scalawiki-sql/src/test/scala/org/scalawiki/sql/ImageDaoSpec.scala +++ b/scalawiki-sql/src/test/scala/org/scalawiki/sql/ImageDaoSpec.scala @@ -27,7 +27,7 @@ class ImageDaoSpec extends Specification with BeforeAfter { } override def after = { - //mwDb.db.close() + // mwDb.db.close() } "image" should { @@ -38,8 +38,11 @@ class ImageDaoSpec extends Specification with BeforeAfter { val title = "Image.jpg" val image = new Image( title, - Some("http://Image.jpg"), None, - Some(1000 * 1000), Some(800), Some(600) + Some("http://Image.jpg"), + None, + Some(1000 * 1000), + Some(800), + Some(600) ) imageDao.insert(image) @@ -55,9 +58,13 @@ class ImageDaoSpec extends Specification with BeforeAfter { val title = "Image.jpg" val image = new Image( title, - Some("http://Image.jpg"), None, - Some(1000 * 1000), Some(800), Some(600), - user.name, Some(user) + Some("http://Image.jpg"), + None, + Some(1000 * 1000), + Some(800), + Some(600), + user.name, + Some(user) ) imageDao.insert(image) @@ -72,8 +79,11 @@ class ImageDaoSpec extends Specification with BeforeAfter { val title = "Image.jpg" val image = new Image( title, - Some("http://Image.jpg"), None, - Some(1000 * 1000), Some(800), Some(600) + Some("http://Image.jpg"), + None, + Some(1000 * 1000), + Some(800), + Some(600) ) val image2 = image.copy() diff --git a/scalawiki-sql/src/test/scala/org/scalawiki/sql/MwDatabaseSpec.scala b/scalawiki-sql/src/test/scala/org/scalawiki/sql/MwDatabaseSpec.scala index 5d46fd67..6cd8ccfc 100644 --- a/scalawiki-sql/src/test/scala/org/scalawiki/sql/MwDatabaseSpec.scala +++ b/scalawiki-sql/src/test/scala/org/scalawiki/sql/MwDatabaseSpec.scala @@ -1,6 +1,5 @@ package org.scalawiki.sql - import slick.driver.JdbcProfile import slick.backend.DatabaseConfig import org.specs2.mutable.{BeforeAfter, Specification} @@ -23,15 +22,10 @@ class MwDatabaseSpec extends Specification with BeforeAfter { } override def after = { - //mwDb.db.close() + // mwDb.db.close() } - val tableNames = Set("category", - "image", - "page", - "revision", - "text", - "user") + val tableNames = Set("category", "image", "page", "revision", "text", "user") "ddls" should { "create schema" in { @@ -68,7 +62,8 @@ class MwDatabaseSpec extends Specification with BeforeAfter { dbs.foreach(_.dropTables()) - val expectedNames = prefixes.flatMap(prefix => tableNames.map(prefix + "_" + _)).toSet + val expectedNames = + prefixes.flatMap(prefix => tableNames.map(prefix + "_" + _)).toSet names === expectedNames getTableNames.isEmpty === true diff --git a/scalawiki-sql/src/test/scala/org/scalawiki/sql/PageDaoBulkSpec.scala b/scalawiki-sql/src/test/scala/org/scalawiki/sql/PageDaoBulkSpec.scala index 1e298792..09307a6b 100644 --- a/scalawiki-sql/src/test/scala/org/scalawiki/sql/PageDaoBulkSpec.scala +++ b/scalawiki-sql/src/test/scala/org/scalawiki/sql/PageDaoBulkSpec.scala @@ -33,7 +33,7 @@ class PageDaoBulkSpec extends Specification with BeforeAfter { } override def after = { - //mwDb.db.close() + // mwDb.db.close() } "page" should { @@ -51,11 +51,10 @@ class PageDaoBulkSpec extends Specification with BeforeAfter { val (titles, texts) = ( (1 to 10) map ("title" + _), (1 to 10) map ("text" + _) - ) + ) - val pages = titles.zip(texts) map { - case (title, text) => - Page(None, Some(0), title, Seq(Revision.one(text))) + val pages = titles.zip(texts) map { case (title, text) => + Page(None, Some(0), title, Seq(Revision.one(text))) } pageDao.insertAll(pages) @@ -71,7 +70,9 @@ class PageDaoBulkSpec extends Specification with BeforeAfter { revs.map(_.textId) aka "textIds in revisions" must_== dbTexts.map(_.id) val dbPages = pageDao.list - dbPages.map(_.revisions.headOption.flatMap(_.revId)) aka "revIds in pages" must_== revs.map(_.id) + dbPages.map( + _.revisions.headOption.flatMap(_.revId) + ) aka "revIds in pages" must_== revs.map(_.id) val fromDb = pageDao.listWithText @@ -92,7 +93,7 @@ class PageDaoBulkSpec extends Specification with BeforeAfter { (1 to 10) map ("title" + _), (1 to 10) map ("text" + _), 51L to 60L - ) + ) val pages = titles.zip(texts).zip(pageIds) map { case ((title, text), pageId) => @@ -112,7 +113,9 @@ class PageDaoBulkSpec extends Specification with BeforeAfter { revs.map(_.textId) aka "textIds in revisions" must_== dbTexts.map(_.id) val dbPages = pageDao.list - dbPages.map(_.revisions.headOption.flatMap(_.revId)) aka "revIds in pages" must_== revs.map(_.id) + dbPages.map( + _.revisions.headOption.flatMap(_.revId) + ) aka "revIds in pages" must_== revs.map(_.id) val fromDb = pageDao.listWithText @@ -135,11 +138,15 @@ class PageDaoBulkSpec extends Specification with BeforeAfter { (1 to 10) map ("text" + _), 51L to 60L, 21L to 30L - ) + ) val pages = titles.zip(texts).zip(pageIds).zip(revIds) map { case (((title, text), pageId), revId) => - val revision: Revision = new Revision(revId = Some(revId), pageId = Some(pageId), content = Some(text)) + val revision: Revision = new Revision( + revId = Some(revId), + pageId = Some(pageId), + content = Some(text) + ) Page(Some(pageId), Some(0), title, Seq(revision)) } @@ -156,7 +163,9 @@ class PageDaoBulkSpec extends Specification with BeforeAfter { revs.map(_.textId) aka "textIds in revisions" must_== dbTexts.map(_.id) val dbPages = pageDao.list - dbPages.map(_.revisions.headOption.flatMap(_.revId)) aka "revIds in pages" must_== revs.map(_.id) + dbPages.map( + _.revisions.headOption.flatMap(_.revId) + ) aka "revIds in pages" must_== revs.map(_.id) val fromDb = pageDao.listWithText @@ -182,13 +191,16 @@ class PageDaoBulkSpec extends Specification with BeforeAfter { 21L to 30L, 31L to 40L, (1 to 10) map ("user" + _) - ) + ) - val pages = (0 to 9) map { - i => - val revision: Revision = new Revision(revId = Some(revIds(i)), pageId = Some(pageIds(i)), content = Some(texts(i)), - user = Some(User(userIds(i), userNames(i)))) - Page(Some(pageIds(i)), Some(0), titles(i), Seq(revision)) + val pages = (0 to 9) map { i => + val revision: Revision = new Revision( + revId = Some(revIds(i)), + pageId = Some(pageIds(i)), + content = Some(texts(i)), + user = Some(User(userIds(i), userNames(i))) + ) + Page(Some(pageIds(i)), Some(0), titles(i), Seq(revision)) } pageDao.insertAll(pages) @@ -204,7 +216,9 @@ class PageDaoBulkSpec extends Specification with BeforeAfter { revs.map(_.textId) aka "textIds in revisions" must_== dbTexts.map(_.id) val dbPages = pageDao.list - dbPages.map(_.revisions.headOption.flatMap(_.revId)) aka "revIds in pages" must_== revs.map(_.id) + dbPages.map( + _.revisions.headOption.flatMap(_.revId) + ) aka "revIds in pages" must_== revs.map(_.id) val fromDb = pageDao.listWithText @@ -230,11 +244,16 @@ class PageDaoBulkSpec extends Specification with BeforeAfter { val title = "Image.jpg" val image = new Image( title, - Some("http://Image.jpg"), None, - Some(1000 * 1000), Some(800), Some(600), - user.name, Some(user) + Some("http://Image.jpg"), + None, + Some(1000 * 1000), + Some(800), + Some(600), + user.name, + Some(user) ) - val revision: Revision = new Revision(user = Some(user), content = Some("revision text")) + val revision: Revision = + new Revision(user = Some(user), content = Some("revision text")) val page = Page(None, Some(0), title, Seq(revision), Seq(image)) diff --git a/scalawiki-sql/src/test/scala/org/scalawiki/sql/PageDaoSpec.scala b/scalawiki-sql/src/test/scala/org/scalawiki/sql/PageDaoSpec.scala index c36a9bee..3bb1c9b4 100644 --- a/scalawiki-sql/src/test/scala/org/scalawiki/sql/PageDaoSpec.scala +++ b/scalawiki-sql/src/test/scala/org/scalawiki/sql/PageDaoSpec.scala @@ -35,7 +35,7 @@ class PageDaoSpec extends Specification with BeforeAfter { } override def after = { - //mwDb.db.close() + // mwDb.db.close() } "page" should { @@ -88,7 +88,8 @@ class PageDaoSpec extends Specification with BeforeAfter { createSchema() val text = "revision text" - val revision: Revision = new Revision(revId = Some(5L), pageId = Some(3L), content = Some(text)) + val revision: Revision = + new Revision(revId = Some(5L), pageId = Some(3L), content = Some(text)) val page: Page = new Page(Some(3L), Some(0), "title", Seq(revision)) @@ -109,7 +110,8 @@ class PageDaoSpec extends Specification with BeforeAfter { val username = Some("username") val user = User(5, username.get) - val revision: Revision = new Revision(user = Some(user), content = Some("revision text")) + val revision: Revision = + new Revision(user = Some(user), content = Some("revision text")) val page = Page(None, Some(0), "title", Seq(revision)) @@ -132,7 +134,10 @@ class PageDaoSpec extends Specification with BeforeAfter { val user = User(5, username.get) userDao.insert(user) - val revision: Revision = new Revision(user = Some(user.copy(login = None)), content = Some("revision text")) + val revision: Revision = new Revision( + user = Some(user.copy(login = None)), + content = Some("revision text") + ) val page = Page(None, Some(0), "title", Seq(revision)) @@ -155,7 +160,10 @@ class PageDaoSpec extends Specification with BeforeAfter { val user = User(5, username.get) userDao.insert(user) - val revision: Revision = new Revision(user = Some(user.copy(id = None)), content = Some("revision text")) + val revision: Revision = new Revision( + user = Some(user.copy(id = None)), + content = Some("revision text") + ) val page = Page(None, Some(0), "title", Seq(revision)) @@ -176,7 +184,8 @@ class PageDaoSpec extends Specification with BeforeAfter { val username = Some("username") val user = new User(None, username) - val revision: Revision = new Revision(user = Some(user), content = Some("revision text")) + val revision: Revision = + new Revision(user = Some(user), content = Some("revision text")) val page = Page(None, Some(0), "title", Seq(revision)) @@ -195,11 +204,16 @@ class PageDaoSpec extends Specification with BeforeAfter { val title = "Image.jpg" val image = new Image( title, - Some("http://Image.jpg"), None, - Some(1000 * 1000), Some(800), Some(600), - user.name, Some(user) + Some("http://Image.jpg"), + None, + Some(1000 * 1000), + Some(800), + Some(600), + user.name, + Some(user) ) - val revision: Revision = new Revision(user = Some(user), content = Some("revision text")) + val revision: Revision = + new Revision(user = Some(user), content = Some("revision text")) val page = Page(None, Some(0), title, Seq(revision), Seq(image)) @@ -259,8 +273,10 @@ class PageDaoSpec extends Specification with BeforeAfter { val title = "title" val (text1, text2) = ("text1", "text2") - val page1 = Page(None, Some(Namespace.MAIN), title, Seq(Revision.one(text1))) - val page2 = Page(None, Some(Namespace.FILE), title, Seq(Revision.one(text2))) + val page1 = + Page(None, Some(Namespace.MAIN), title, Seq(Revision.one(text1))) + val page2 = + Page(None, Some(Namespace.FILE), title, Seq(Revision.one(text2))) val pageId1 = pageDao.insert(page1) val pageId2 = pageDao.insert(page2) @@ -305,8 +321,8 @@ class PageDaoSpec extends Specification with BeforeAfter { val names = Seq("page1", "page2", "page3") val texts = Seq("page1Text", "page2Text", "page3Text") // val revisions = texts.map(Revision.one) - val pages = names.zip(texts).map { - case (n, t) => Page(None, Some(0), n, Seq(Revision.one(t))) + val pages = names.zip(texts).map { case (n, t) => + Page(None, Some(0), n, Seq(Revision.one(t))) } val pageIds = pages.flatMap(p => pageDao.insert(p).toOption) @@ -323,8 +339,8 @@ class PageDaoSpec extends Specification with BeforeAfter { val names = Seq("page1", "page2", "page3") val texts = Seq("page1Text", "page2Text", "page3Text") // val revisions = texts.map(Revision.one) - val pages = names.zip(texts).map { - case (n, t) => Page(None, Some(0), n, Seq(Revision.one(t))) + val pages = names.zip(texts).map { case (n, t) => + Page(None, Some(0), n, Seq(Revision.one(t))) } val pageIds = pages.flatMap(p => pageDao.insert(p).toOption) @@ -342,8 +358,8 @@ class PageDaoSpec extends Specification with BeforeAfter { val names = Seq("page1", "page2", "page3") val texts = Seq("page1Text", "page2Text", "page3Text") // val revisions = texts.map(Revision.one) - val pages = names.zip(texts).map { - case (n, t) => Page(None, Some(0), n, Seq(Revision.one(t))) + val pages = names.zip(texts).map { case (n, t) => + Page(None, Some(0), n, Seq(Revision.one(t))) } val pageIds = pages.flatMap(p => pageDao.insert(p).toOption) @@ -365,8 +381,8 @@ class PageDaoSpec extends Specification with BeforeAfter { Seq("page3Text2", "page3Text1") ) // val revisions = texts.map(Revision.one) - val pages = names.zip(texts).map { - case (n, t) => Page(None, Some(0), n, t.map(Revision.one)) + val pages = names.zip(texts).map { case (n, t) => + Page(None, Some(0), n, t.map(Revision.one)) } val pageIds = pages.flatMap(p => pageDao.insert(p).toOption.toSeq) @@ -378,7 +394,10 @@ class PageDaoSpec extends Specification with BeforeAfter { val revIds = withRevisions.flatMap(_.revisions.last.id) - val dbPages = pageDao.findByRevIds(Set(pageIds.head, pageIds.last), Set(revIds.head, revIds.last)) + val dbPages = pageDao.findByRevIds( + Set(pageIds.head, pageIds.last), + Set(revIds.head, revIds.last) + ) dbPages.size === 2 dbPages.map(_.title) === Seq("page1", "page3") dbPages.flatMap(_.text.toSeq) === Seq("page1Text1", "page3Text1") @@ -394,8 +413,8 @@ class PageDaoSpec extends Specification with BeforeAfter { Seq("page3Text2", "page3Text1") ) // val revisions = texts.map(Revision.one) - val pages = names.zip(texts).map { - case (n, t) => Page(None, Some(0), n, t.map(Revision.one)) + val pages = names.zip(texts).map { case (n, t) => + Page(None, Some(0), n, t.map(Revision.one)) } val pageIds = pages.flatMap(p => pageDao.insert(p).toOption.toSeq) @@ -407,12 +426,18 @@ class PageDaoSpec extends Specification with BeforeAfter { val revIds = withRevisions.flatMap(_.revisions.last.id) - val dbPages = pageDao.findByPageAndRevIdsOpt(Set(pageIds.head, pageIds.last), Set(revIds.head, revIds.last)) + val dbPages = pageDao.findByPageAndRevIdsOpt( + Set(pageIds.head, pageIds.last), + Set(revIds.head, revIds.last) + ) dbPages.size === 2 dbPages.map(_.title) === Seq("page1", "page3") dbPages.flatMap(_.text.toSeq) === Seq("page1Text1", "page3Text1") - val dbPagesOpt = pageDao.findByPageAndRevIdsOpt(Set(pageIds.head, pageIds.last), Set(revIds.head, 123L)) + val dbPagesOpt = pageDao.findByPageAndRevIdsOpt( + Set(pageIds.head, pageIds.last), + Set(revIds.head, 123L) + ) dbPagesOpt.size === 2 dbPagesOpt.map(_.title) === Seq("page1", "page3") dbPagesOpt.flatMap(_.text.toSeq) === Seq("page1Text1") diff --git a/scalawiki-sql/src/test/scala/org/scalawiki/sql/UserDaoSpec.scala b/scalawiki-sql/src/test/scala/org/scalawiki/sql/UserDaoSpec.scala index 68aa3e48..addbdcc9 100644 --- a/scalawiki-sql/src/test/scala/org/scalawiki/sql/UserDaoSpec.scala +++ b/scalawiki-sql/src/test/scala/org/scalawiki/sql/UserDaoSpec.scala @@ -21,12 +21,12 @@ class UserDaoSpec extends Specification with BeforeAfter { } override def before = { - val dc = DatabaseConfig.forConfig[JdbcProfile]("h2mem") - mwDb = new MwDatabase(dc) + val dc = DatabaseConfig.forConfig[JdbcProfile]("h2mem") + mwDb = new MwDatabase(dc) } override def after = { - //mwDb.db.close() + // mwDb.db.close() } "user" should { @@ -83,7 +83,9 @@ class UserDaoSpec extends Specification with BeforeAfter { dbUser.id.isDefined === true dbUser.id === userId - userDao.insert(dbUser.copy(login = Some("other name"))) must throwA[SQLException] + userDao.insert(dbUser.copy(login = Some("other name"))) must throwA[ + SQLException + ] userDao.list.size === 1 } @@ -108,8 +110,8 @@ class UserDaoSpec extends Specification with BeforeAfter { createSchema() val names = Seq("user1", "user2", "user3") - val users = names.map { - name => User(None, Some(name)) + val users = names.map { name => + User(None, Some(name)) } val userIds = users.flatMap(u => userDao.insert(u)) diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CampaignList.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CampaignList.scala index 11c5f3b7..97c2a807 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CampaignList.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CampaignList.scala @@ -15,20 +15,33 @@ class CampaignList { self: HasBot => def categoryMembers(parent: String): Future[Iterable[Page]] = { - bot.run(Action(Query(ListParam(CategoryMembers( - CmTitle(parent), CmNamespace(Seq(Namespace.CATEGORY)), CmLimit("max")) - )))) + bot.run( + Action( + Query( + ListParam( + CategoryMembers( + CmTitle(parent), + CmNamespace(Seq(Namespace.CATEGORY)), + CmLimit("max") + ) + ) + ) + ) + ) } - def titles(pages: Seq[Iterable[Page]]): Seq[String] = pages.flatten.map(_.title) + def titles(pages: Seq[Iterable[Page]]): Seq[String] = + pages.flatten.map(_.title) def getContests(hasImages: HasImagesCategory): Future[Iterable[Contest]] = contestsFromCategory(hasImages.imagesCategory) def contestsFromCategory(parent: String): Future[Iterable[Contest]] = { - for (cats <- categoryMembers(parent)) yield - for (cat <- cats; - contest <- CountryParser.fromCategoryName(cat.title)) + for (cats <- categoryMembers(parent)) + yield for ( + cat <- cats; + contest <- CountryParser.fromCategoryName(cat.title) + ) yield contest } @@ -41,16 +54,21 @@ object CampaignList extends CampaignList with WithBot { override def host = MwBot.commons - def yearsContests(types: Seq[ContestType] = ContestType.all): Future[Seq[Contest]] = { + def yearsContests( + types: Seq[ContestType] = ContestType.all + ): Future[Seq[Contest]] = { for (yearsCats <- categoriesMembers(types.map(_.imagesCategory))) - yield for (contest <- titles(yearsCats).flatMap(CountryParser.fromCategoryName)) + yield for ( + contest <- titles(yearsCats).flatMap(CountryParser.fromCategoryName) + ) yield contest } - def main(args: Array[String]): Unit = { for (yearContests <- yearsContests()) { - for (campaignCats <- categoriesMembers(yearContests.map(_.imagesCategory))) { + for ( + campaignCats <- categoriesMembers(yearContests.map(_.imagesCategory)) + ) { titles(campaignCats).foreach(println) } } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/Checker.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/Checker.scala index 14b199ae..0835fe1a 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/Checker.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/Checker.scala @@ -14,9 +14,9 @@ object Checker { val underscored = line.replace(' ', '_') val underscoredNoFile = underscored.replace("File:", "") winners.exists(_.startsWith(spaced)) || - winners.exists(_.startsWith(underscored)) || - winners.exists(_.startsWith(spacedNoFile)) || - winners.exists(_.startsWith(underscoredNoFile)) + winners.exists(_.startsWith(underscored)) || + winners.exists(_.startsWith(spacedNoFile)) || + winners.exists(_.startsWith(underscoredNoFile)) } println(s"Missing size: ${missing.size}") @@ -31,7 +31,9 @@ object Checker { val missingWinners = winnerFiles.filterNot { line => val spaced = line.replace('_', ' ') val underscored = line.replace(' ', '_') - jury.contains(spaced) || jury.contains(underscored) || line == "File:INSERT FILE NAME" + jury.contains(spaced) || jury.contains( + underscored + ) || line == "File:INSERT FILE NAME" } println(s"missingWinners size: ${missingWinners.size}") missingWinners.foreach(println) diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CountryParser.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CountryParser.scala index 6ff08215..f573438d 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CountryParser.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/CountryParser.scala @@ -8,8 +8,7 @@ import scala.util.Try import scala.util.matching.Regex import scala.util.matching.Regex.Match -/** - * Parses information about contest country +/** Parses information about contest country */ object CountryParser { @@ -18,36 +17,41 @@ object CountryParser { val contestCategory = "Category\\:Images from ([a-zA-Z ]+) (\\d+)" val contestRegex = (contestCategory + "$").r - val contestWithCountryRegex = (contestCategory + " in ([a-zA-Z\\&\\- ]+)($|\\|)").r + val contestWithCountryRegex = + (contestCategory + " in ([a-zA-Z\\&\\- ]+)($|\\|)").r - val contestLinkRegex = "\\[\\[Commons\\:([a-zA-Z ]+) (\\d+) in ([a-zA-Z\\& ]+)\\|".r + val contestLinkRegex = + "\\[\\[Commons\\:([a-zA-Z ]+) (\\d+) in ([a-zA-Z\\& ]+)\\|".r def isContestCategory(s: String) = contestRegex.pattern.matcher(s).matches() val parser = new RegexCampaignParser(contestWithCountryRegex) - /** - * Parses information from wiki-table of participating countries like + /** Parses information from wiki-table of participating countries like * https://commons.wikimedia.org/wiki/Commons:Wiki_Loves_Earth_2016 * - * @param wikiText participating countries table wiki-markup - * @return list of contests from the table + * @param wikiText + * participating countries table wiki-markup + * @return + * list of contests from the table */ def parse(wikiText: String): Seq[Contest] = { (Try { parseTable(wikiText) - } recoverWith { case _ => Try { - parsePageLinks(wikiText) - } + } recoverWith { case _ => + Try { + parsePageLinks(wikiText) + } }).getOrElse(Seq.empty) } - /** - * Parses information from wiki-table of participating countries like + /** Parses information from wiki-table of participating countries like * https://commons.wikimedia.org/wiki/Commons:Wiki_Loves_Earth_2016 * - * @param wikiText participating countries table wiki-markup - * @return list of contests from the table + * @param wikiText + * participating countries table wiki-markup + * @return + * list of contests from the table */ def parseTable(wikiText: String): Seq[Contest] = { val table = TableParser.parse(wikiText) @@ -55,8 +59,8 @@ object CountryParser { val headers = table.headers.toIndexedSeq val uploadsIndex = headers.indexWhere(_.contains("Uploads")) - if (uploadsIndex < 0) throw - new IllegalArgumentException("Could not find the country lists") + if (uploadsIndex < 0) + throw new IllegalArgumentException("Could not find the country lists") def rowToContest(data: Iterable[String]): Option[Contest] = fromCategoryName(data.toIndexedSeq(uploadsIndex)) @@ -72,10 +76,16 @@ object CountryParser { } -class RegexCampaignParser(r: Regex, typeIndex: Int = 1, yearIndex: Int = 2, countryIndex: Int = 3) { +class RegexCampaignParser( + r: Regex, + typeIndex: Int = 1, + yearIndex: Int = 2, + countryIndex: Int = 3 +) { def findWithBackup(cell: String): Option[Contest] = { - r.findFirstMatchIn(cell).flatMap(matchToContestWithCountry) + r.findFirstMatchIn(cell) + .flatMap(matchToContestWithCountry) .orElse { contestRegex.findFirstMatchIn(cell).flatMap(categoryToContest) } @@ -89,10 +99,14 @@ class RegexCampaignParser(r: Regex, typeIndex: Int = 1, yearIndex: Int = 2, coun for (contest <- categoryToContest(m)) yield { val countryStr = m.group(countryIndex) - val country = countries.find(_.name == countryStr).getOrElse(new Country("", countryStr)) + val country = countries + .find(_.name == countryStr) + .getOrElse(new Country("", countryStr)) - val campaignCode = s"${contest.contestType.code}_${country.code}".toLowerCase - val uploadConfigs = Contest.load(campaignCode + ".conf").map(_.uploadConfigs).getOrElse(Nil) + val campaignCode = + s"${contest.contestType.code}_${country.code}".toLowerCase + val uploadConfigs = + Contest.load(campaignCode + ".conf").map(_.uploadConfigs).getOrElse(Nil) contest.copy(country = country, uploadConfigs = uploadConfigs) } @@ -102,7 +116,7 @@ class RegexCampaignParser(r: Regex, typeIndex: Int = 1, yearIndex: Int = 2, coun val typeStr = m.group(typeIndex) val year = m.group(yearIndex).toInt - for (typ <- ContestType.byName(typeStr)) yield - new Contest(typ, NoAdmDivision, year) + for (typ <- ContestType.byName(typeStr)) + yield new Contest(typ, NoAdmDivision, year) } } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala index 3d0a4f90..59e4f70d 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ImageDB.scala @@ -6,10 +6,12 @@ import org.scalawiki.wlx.query.ImageQuery import scala.concurrent.Future -case class ImageDB(contest: Contest, - images: Iterable[Image], - monumentDb: Option[MonumentDB], - minMpx: Option[Float] = None) { +case class ImageDB( + contest: Contest, + images: Iterable[Image], + monumentDb: Option[MonumentDB], + minMpx: Option[Float] = None +) { def this(contest: Contest, images: Seq[Image]) = this(contest, images, None) @@ -25,14 +27,16 @@ case class ImageDB(contest: Contest, lazy val sansIneligible: Iterable[Image] = withCorrectIds .filterNot( - _.categories.contains( - s"Ineligible submissions for WLM ${contest.year} in Ukraine")) + _.categories + .contains(s"Ineligible submissions for WLM ${contest.year} in Ukraine") + ) .filter(_.atLeastMpx(minMpx)) lazy val ineligible: Iterable[Image] = withCorrectIds .filter( - _.categories.contains( - s"Ineligible submissions for WLM ${contest.year} in Ukraine")) + _.categories + .contains(s"Ineligible submissions for WLM ${contest.year} in Ukraine") + ) .filterNot(_.atLeastMpx(minMpx)) lazy val _byId: Grouping[String, Image] = @@ -78,7 +82,8 @@ case class ImageDB(contest: Contest, val parentId = regId.substring(0, 2) val topImages = _byRegion.by(parentId) topImages.filter( - _.monumentId.exists(_.replace("-", "").startsWith(regId))) + _.monumentId.exists(_.replace("-", "").startsWith(regId)) + ) } def idsByRegion(regId: String): Set[String] = { @@ -93,10 +98,12 @@ case class ImageDB(contest: Contest, def idByAuthor(author: String): Set[String] = _byAuthorAndId.by(author).keys - def authorsByRegion(regId: String): Set[String] = _byRegionAndAuthor.by(regId).keys + def authorsByRegion(regId: String): Set[String] = + _byRegionAndAuthor.by(regId).keys def byMegaPixelFilterAuthorMap( - predicate: Int => Boolean): Map[String, Seq[Image]] = { + predicate: Int => Boolean + ): Map[String, Seq[Image]] = { _byMegaPixels.grouped .filterKeys(mpx => mpx >= 0 && predicate(mpx)) .values @@ -112,9 +119,8 @@ case class ImageDB(contest: Contest, def byNumberOfAuthors: Map[Int, Map[String, Iterable[Image]]] = { _byId.grouped - .groupBy { - case (id, photos) => - photos.flatMap(_.author).toSet.size + .groupBy { case (id, photos) => + photos.flatMap(_.author).toSet.size } .mapValues(_.toMap) .toMap @@ -122,15 +128,17 @@ case class ImageDB(contest: Contest, def byNumberOfPhotos: Map[Int, Map[String, Iterable[Image]]] = { _byId.grouped - .groupBy { - case (id, photos) => photos.size + .groupBy { case (id, photos) => + photos.size } .mapValues(_.toMap) .toMap } - def subSet(monuments: Seq[Monument], - withFalseIds: Boolean = false): ImageDB = { + def subSet( + monuments: Seq[Monument], + withFalseIds: Boolean = false + ): ImageDB = { val subSetMonumentDb = new MonumentDB(monumentDb.get.contest, monuments, withFalseIds) val subSetImages = @@ -188,7 +196,8 @@ object ImageGrouping { def byMonument: Image => String = (i: Image) => i.monumentId.getOrElse("") - def byRegion: Image => String = (i: Image) => Monument.getRegionId(i.monumentId) + def byRegion: Image => String = (i: Image) => + Monument.getRegionId(i.monumentId) def byAuthor: Image => String = (i: Image) => i.author.getOrElse("") @@ -198,10 +207,12 @@ object ImageDB { import scala.concurrent.ExecutionContext.Implicits.global - def create(contest: Contest, - imageQuery: ImageQuery, - monumentDb: Option[MonumentDB], - minMpx: Option[Float] = None): Future[ImageDB] = { + def create( + contest: Contest, + imageQuery: ImageQuery, + monumentDb: Option[MonumentDB], + minMpx: Option[Float] = None + ): Future[ImageDB] = { imageQuery.imagesFromCategoryAsync(contest.imagesCategory, contest).map { images => new ImageDB(contest, images, monumentDb, minMpx) diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/KatotthMonumentListCreator.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/KatotthMonumentListCreator.scala index 5af89424..952a8d2c 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/KatotthMonumentListCreator.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/KatotthMonumentListCreator.scala @@ -2,22 +2,44 @@ package org.scalawiki.wlx import org.scalawiki.MwBot import org.scalawiki.dto.markup.Table -import org.scalawiki.wlx.dto.{AdmDivision, Contest, Country, Katotth, Koatuu, Monument, RegionType} +import org.scalawiki.wlx.dto.{ + AdmDivision, + Contest, + Country, + Katotth, + Koatuu, + Monument, + RegionType +} import org.scalawiki.wlx.query.MonumentQuery import org.scalawiki.wlx.stat.reports.Output import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -case class Koatuu2Katotth(koatuu: Option[String], katotth: Option[AdmDivision], monumentIds: Seq[String]/*, candidates: Candidate*/) +case class Koatuu2Katotth( + koatuu: Option[String], + katotth: Option[AdmDivision], + monumentIds: Seq[String] /*, candidates: Candidate*/ +) object KatotthMonumentListCreator { - val UkraineKatotth: Country = new Country("UA", "Ukraine", Seq("uk"), Katotth.regions(() => Some(UkraineKatotth))) + val UkraineKatotth: Country = new Country( + "UA", + "Ukraine", + Seq("uk"), + Katotth.regions(() => Some(UkraineKatotth)) + ) val regionsKatotth = UkraineKatotth.regions val katotthMap = UkraineKatotth.mapByCode - val UkraineKoatuu: Country = new Country("UA", "Ukraine", Seq("uk"), Koatuu.regionsNew(() => Some(UkraineKoatuu))) + val UkraineKoatuu: Country = new Country( + "UA", + "Ukraine", + Seq("uk"), + Koatuu.regionsNew(() => Some(UkraineKoatuu)) + ) val regionsKoatuu = UkraineKoatuu.regions val koatuuMap = UkraineKoatuu.mapByCode @@ -32,10 +54,12 @@ object KatotthMonumentListCreator { val (mapped, unmapped) = sequence.partition(_.katotth.nonEmpty) val grouped = groupByAdm(mapped) - reportUnmapped(monumentDB, unmapped - .filter(_.koatuu.nonEmpty) - .filterNot(_.koatuu.contains("80")) - .filterNot(_.koatuu.contains("85")) + reportUnmapped( + monumentDB, + unmapped + .filter(_.koatuu.nonEmpty) + .filterNot(_.koatuu.contains("80")) + .filterNot(_.koatuu.contains("85")) ) val citiRegions = Set("51100370000040590", "12080090000039979") @@ -43,8 +67,12 @@ object KatotthMonumentListCreator { val village2Regions = Set("12080110000055958", "59080150000013842") Future.sequence(grouped.map { case (adm, k2k) => - if (!Set("Автономна Республіка Крим", "Київ", "Севастополь").contains(adm.namesList.tail.head)) { - val pageNameGuess = s"Вікіпедія:Вікі любить пам'ятки/новий АТУ/${adm.namesList.tail.mkString("/")}" + if ( + !Set("Автономна Республіка Крим", "Київ", "Севастополь") + .contains(adm.namesList.tail.head) + ) { + val pageNameGuess = + s"Вікіпедія:Вікі любить пам'ятки/новий АТУ/${adm.namesList.tail.mkString("/")}" val newPageName = if (citiRegions.contains(adm.code)) { pageNameGuess.replace("громада", "міська громада") @@ -56,22 +84,37 @@ object KatotthMonumentListCreator { pageNameGuess } - val (cities, other) = k2k.partition(_.katotth.exists(_.regionType.exists(_.code == "М"))) + val (cities, other) = + k2k.partition(_.katotth.exists(_.regionType.exists(_.code == "М"))) val cityMonumentIds = cities.flatMap(_.monumentIds).toSet val otherMonumentIds = other.flatMap(_.monumentIds).toSet val header = "{{ВЛП нові списки}}\n" + s"{{Вікіпедія:Вікі любить пам'ятки/новий АТУ/${adm.namesList.tail.head}/navbar}}\n" + "{{WLM-шапка}}\n" - val monuments = monumentDB.monuments.filter(m => cityMonumentIds.contains(m.id)) ++ - monumentDB.monuments.filter(m => otherMonumentIds.contains(m.id)) + val monuments = + monumentDB.monuments.filter(m => cityMonumentIds.contains(m.id)) ++ + monumentDB.monuments.filter(m => otherMonumentIds.contains(m.id)) def monumentPage(monument: Monument) = { val oldPage = monument.page if (oldPage.last == ')') { val suffix = oldPage.substring(oldPage.lastIndexOf("(")) - val koatuu = Katotth.toKoatuu.get(adm.code).map(_.take(5)).getOrElse("") - if (Set("63101", "12101", "51101", "48101", "35101", "46101", "26101", "73101", "53240").contains(koatuu)) { + val koatuu = + Katotth.toKoatuu.get(adm.code).map(_.take(5)).getOrElse("") + if ( + Set( + "63101", + "12101", + "51101", + "48101", + "35101", + "46101", + "26101", + "73101", + "53240" + ).contains(koatuu) + ) { newPageName + s" $suffix" } else { newPageName @@ -82,14 +125,18 @@ object KatotthMonumentListCreator { val subPageFutures = bySubPage.map { case (page, pageMonuments) => val koatuuPages = pageMonuments.map(_.page).distinct.sorted - val pageText = header + pageMonuments.map(_.asWiki(Some("WLM-рядок"))).mkString("") + "\n|}" + - "\n== Взято з ==\n" + koatuuPages.map(page => s"* [[$page]]").mkString("\n") + + val pageText = header + pageMonuments + .map(_.asWiki(Some("WLM-рядок"))) + .mkString("") + "\n|}" + + "\n== Взято з ==\n" + koatuuPages + .map(page => s"* [[$page]]") + .mkString("\n") + "\n== Примітки ==\n{{reflist}}" if (pageText.length > 1000 * 1000) { println(s" $page ${pageText.length}") } - //Future.successful() + // Future.successful() ukWiki.page(page).edit(pageText) } Future.sequence(subPageFutures) @@ -97,12 +144,18 @@ object KatotthMonumentListCreator { }) } - def groupByAdm(sequence: Seq[Koatuu2Katotth]): Map[AdmDivision, Seq[Koatuu2Katotth]] = { + def groupByAdm( + sequence: Seq[Koatuu2Katotth] + ): Map[AdmDivision, Seq[Koatuu2Katotth]] = { sequence.groupBy { k2k => val k = k2k.katotth.get var groupK = k - while (!groupK.regionType.exists(rt => parentCodes.contains(rt.code)) && groupK.parent().nonEmpty) { + while ( + !groupK.regionType.exists(rt => parentCodes.contains(rt.code)) && groupK + .parent() + .nonEmpty + ) { groupK = groupK.parent().get } groupK @@ -111,13 +164,20 @@ object KatotthMonumentListCreator { def reportUnmapped(mDb: MonumentDB, sequence: Seq[Koatuu2Katotth]) = { Output.unknownPlaces(mDb) - val byKoatuu = sequence.map { k2k => - k2k.koatuu.get -> k2k.monumentIds - }.groupBy(_._1).mapValues(_.flatMap(_._2)).toMap - - val table = Table(Seq("koatuu", "monumentIds"), byKoatuu.toSeq.sortBy(_._1).map{ - case (koatuu, ids) => Seq(koatuu, ids.mkString(", ")) - }) + val byKoatuu = sequence + .map { k2k => + k2k.koatuu.get -> k2k.monumentIds + } + .groupBy(_._1) + .mapValues(_.flatMap(_._2)) + .toMap + + val table = Table( + Seq("koatuu", "monumentIds"), + byKoatuu.toSeq.sortBy(_._1).map { case (koatuu, ids) => + Seq(koatuu, ids.mkString(", ")) + } + ) val text = table.asWiki val pageName = s"Вікіпедія:${mDb.contest.contestType.name}/unmappedPlaces" @@ -131,7 +191,8 @@ object KatotthMonumentListCreator { val koatuuOpt = placeByMonumentId.get(m.id) val katotthOpt = koatuuOpt.flatMap { koatuu => val paddedKoatuu = koatuu.padTo(10, "0").mkString - val candidates = Katotth.toKatotth.getOrElse(paddedKoatuu, Nil).flatMap(katotthMap.get) + val candidates = + Katotth.toKatotth.getOrElse(paddedKoatuu, Nil).flatMap(katotthMap.get) if (candidates.nonEmpty) { Some(candidates.maxBy(_.level)) } else { diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListFileNSRemover.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListFileNSRemover.scala index f954687f..813a940a 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListFileNSRemover.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListFileNSRemover.scala @@ -18,7 +18,9 @@ object ListFileNSRemover { } } -class ListFileNSRemoverTask(val host: String, monumentDb: MonumentDB) extends PageUpdateTask with SwebleParser { +class ListFileNSRemoverTask(val host: String, monumentDb: MonumentDB) + extends PageUpdateTask + with SwebleParser { val config: WikiConfig = DefaultConfigEnWp.generate @@ -37,19 +39,26 @@ class ListFileNSRemoverTask(val host: String, monumentDb: MonumentDB) extends Pa if (needsUpdate(monument)) { added += 1 - val value = monument.photo.get.replaceFirst("File:", "").replaceFirst("Файл:", "") + val value = + monument.photo.get.replaceFirst("File:", "").replaceFirst("Файл:", "") val name = wlxParser.image.get swTemplate.setTemplateParam(name, value) } } - val newText = replace(pageText, { case t: WtTemplate if getTemplateName(t) == template => t }, mapper) + val newText = replace( + pageText, + { case t: WtTemplate if getTemplateName(t) == template => t }, + mapper + ) val comment = s"fixing $added image(s)" (newText, comment) } def needsUpdate(m: Monument): Boolean = - m.photo.exists(photo => photo.trim.startsWith("File:") || photo.trim.startsWith("Файл:")) + m.photo.exists(photo => + photo.trim.startsWith("File:") || photo.trim.startsWith("Файл:") + ) def pagesToFix(monumentDb: MonumentDB): Set[String] = { diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListUpdater.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListUpdater.scala index ad9de86d..d2787474 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListUpdater.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/ListUpdater.scala @@ -23,7 +23,12 @@ trait MonumentUpdater { def updatedParams(m: Monument): Map[String, String] } -class ListUpdaterTask(val host: String, monumentDb: MonumentDB, monumentUpdater: MonumentUpdater) extends PageUpdateTask with SwebleParser { +class ListUpdaterTask( + val host: String, + monumentDb: MonumentDB, + monumentUpdater: MonumentUpdater +) extends PageUpdateTask + with SwebleParser { val config: WikiConfig = DefaultConfigEnWp.generate @@ -55,7 +60,11 @@ class ListUpdaterTask(val host: String, monumentDb: MonumentDB, monumentUpdater: .replace("", "") - val newText = replace(noComments, { case t: WtTemplate if getTemplateName(t) == template => t }, mapper) + val newText = replace( + noComments, + { case t: WtTemplate if getTemplateName(t) == template => t }, + mapper + ) val withComments = newText .replace("", "", start + 4) if (end > 0) { removeComments(s.substring(0, start) + s.substring(end + 3, s.length)) - } - else s.substring(0, start) + } else s.substring(0, start) } else { s } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/AdmDivisionFlat.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/AdmDivisionFlat.scala index 9321fee7..2ccbe6d0 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/AdmDivisionFlat.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/AdmDivisionFlat.scala @@ -2,12 +2,25 @@ package org.scalawiki.wlx.dto import org.scalawiki.wlx.dto.Koatuu.{betterName, shortCode, skipGroups} -case class AdmDivisionFlat(codeLevels: Seq[String], name: String, regionType: Option[RegionType]) { +case class AdmDivisionFlat( + codeLevels: Seq[String], + name: String, + regionType: Option[RegionType] +) { def code: String = codeLevels.last - def toHierarchy(regions: Seq[AdmDivision], - parent: () => Option[AdmDivision]): AdmDivision = { - val hier = AdmDivision(code, name, regions, parent, regionType, parent().map(p => Option(p).fold(1)(_.level + 1)).getOrElse(0)) + def toHierarchy( + regions: Seq[AdmDivision], + parent: () => Option[AdmDivision] + ): AdmDivision = { + val hier = AdmDivision( + code, + name, + regions, + parent, + regionType, + parent().map(p => Option(p).fold(1)(_.level + 1)).getOrElse(0) + ) regions.map(_.withParents(() => Some(hier))) hier } @@ -15,21 +28,38 @@ case class AdmDivisionFlat(codeLevels: Seq[String], name: String, regionType: Op } object AdmDivisionFlat { - def apply(l1: String, l2: String, l3: String, l4: String, name: String, regionType: Option[RegionType]): AdmDivisionFlat = { + def apply( + l1: String, + l2: String, + l3: String, + l4: String, + name: String, + regionType: Option[RegionType] + ): AdmDivisionFlat = { val levels = Seq(l1, l2, l3, l4).filterNot(_.isEmpty) - val codes = levels.zipWithIndex.map { case (c, level) => shortCode(c, level + 2) } + val codes = levels.zipWithIndex.map { case (c, level) => + shortCode(c, level + 2) + } AdmDivisionFlat(codes, betterName(name), regionType) } - def makeHierarchy(flat: Seq[AdmDivisionFlat], - parent: () => Option[AdmDivision] = () => None, - level: Int = 0): Seq[AdmDivision] = { - flat.groupBy(_.codeLevels(level)).map { case (code, regions) => - val (topList, subRegions) = regions.partition(_.codeLevels.size == level + 1) - val top = topList.head - val children = skipGroups(makeHierarchy(subRegions, () => None, level + 1)) - top.toHierarchy(children, parent) - }.toSeq.sortBy(_.code) + def makeHierarchy( + flat: Seq[AdmDivisionFlat], + parent: () => Option[AdmDivision] = () => None, + level: Int = 0 + ): Seq[AdmDivision] = { + flat + .groupBy(_.codeLevels(level)) + .map { case (code, regions) => + val (topList, subRegions) = + regions.partition(_.codeLevels.size == level + 1) + val top = topList.head + val children = + skipGroups(makeHierarchy(subRegions, () => None, level + 1)) + top.toHierarchy(children, parent) + } + .toSeq + .sortBy(_.code) } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Campaign.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Campaign.scala index 586425cf..f521f01d 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Campaign.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Campaign.scala @@ -1,11 +1,13 @@ package org.scalawiki.wlx.dto -case class Campaign(enabled: Boolean, - title: String, - headerLabel: String, - thanksLabel: String, - defaultCategories: Seq[String], - fields: Seq[CampaignField]) +case class Campaign( + enabled: Boolean, + title: String, + headerLabel: String, + thanksLabel: String, + defaultCategories: Seq[String], + fields: Seq[CampaignField] +) case class CampaignField(wikitext: String, label: String) @@ -21,7 +23,7 @@ object Campaign { implicit val fieldReads: Reads[CampaignField] = ( (__ \ "wikitext").read[String] and (__ \ "label").read[String] - ) (CampaignField.apply _) + )(CampaignField.apply _) val campaignReads: Reads[Campaign] = ( (__ \ "enabled").read[Boolean] and @@ -30,6 +32,6 @@ object Campaign { (__ \ "display" \ "thanksLabel").read[String] and (__ \ "defaults" \ "categories").read[Seq[String]] and (__ \ "fields").read[Seq[CampaignField]] - ) (Campaign.apply _) + )(Campaign.apply _) -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Contest.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Contest.scala index 8a63340d..f039bc38 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Contest.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Contest.scala @@ -2,13 +2,17 @@ package org.scalawiki.wlx.dto import java.time.ZonedDateTime -import com.typesafe.config.{Config, ConfigFactory, ConfigParseOptions, ConfigResolveOptions} +import com.typesafe.config.{ + Config, + ConfigFactory, + ConfigParseOptions, + ConfigResolveOptions +} import org.scalawiki.wlx.stat.RateConfig import scala.util.Try -/** - * Represents Wiki Loves X contest +/** Represents Wiki Loves X contest * * @param contestType * @param country @@ -19,16 +23,16 @@ import scala.util.Try * @param specialNominations */ case class Contest( - contestType: ContestType, - country: Country, - year: Int, - startDate: String = "", - endDate: String = "", - uploadConfigs: Seq[UploadConfig] = Nil, - specialNominations: Seq[SpecialNomination] = Nil, - rateConfig: RateConfig = RateConfig(), - config: Option[Config] = None - ) extends HasImagesCategory { + contestType: ContestType, + country: Country, + year: Int, + startDate: String = "", + endDate: String = "", + uploadConfigs: Seq[UploadConfig] = Nil, + specialNominations: Seq[SpecialNomination] = Nil, + rateConfig: RateConfig = RateConfig(), + config: Option[Config] = None +) extends HasImagesCategory { def campaign = contestType.code + "-" + country.code @@ -40,20 +44,23 @@ case class Contest( else None - /** - * @return Name of category containing contest images + /** @return + * Name of category containing contest images */ - override def imagesCategory: String = s"Category:Images from $name".replaceAll(" ", "_") + override def imagesCategory: String = + s"Category:Images from $name".replaceAll(" ", "_") - /** - * @return name of template that monument lists consist of + /** @return + * name of template that monument lists consist of */ - def listTemplate: Option[String] = uploadConfigs.headOption.map(_.listTemplate) + def listTemplate: Option[String] = + uploadConfigs.headOption.map(_.listTemplate) - /** - * @return name of template that marks a contest image with monument id + /** @return + * name of template that marks a contest image with monument id */ - def fileTemplate: Option[String] = uploadConfigs.headOption.map(_.fileTemplate) + def fileTemplate: Option[String] = + uploadConfigs.headOption.map(_.fileTemplate) def listsHost: Option[String] = { uploadConfigs.head.listsHost @@ -75,10 +82,10 @@ object Contest { val Campaign = "(\\w+)_(\\w+).conf".r name match { case Campaign(typeCode, countryCode) => - for (contestType <- ContestType.byCode(typeCode); - country <- Country.byCode(countryCode) - ) yield - Contest(contestType, country, ZonedDateTime.now.getYear) + for ( + contestType <- ContestType.byCode(typeCode); + country <- Country.byCode(countryCode) + ) yield Contest(contestType, country, ZonedDateTime.now.getYear) case _ => None } } @@ -92,17 +99,39 @@ object Contest { val (typeStr, countryStr, year) = ( config.getString("type"), config.getString("country"), - config.getInt("year")) + config.getInt("year") + ) val uploadConfig = UploadConfig.fromConfig(config) - for (contestType <- ContestType.byCode(typeStr.toLowerCase); - country <- Country.fromJavaLocales.find(country => country.name == countryStr || country.code == countryStr)) - yield new Contest(contestType, country, year, uploadConfigs = Seq(uploadConfig), config = Some(config)) + for ( + contestType <- ContestType.byCode(typeStr.toLowerCase); + country <- Country.fromJavaLocales.find(country => + country.name == countryStr || country.code == countryStr + ) + ) + yield new Contest( + contestType, + country, + year, + uploadConfigs = Seq(uploadConfig), + config = Some(config) + ) } - def ESPCUkraine(year: Int, startDate: String = "01-09", endDate: String = "30-09") = - new Contest(ContestType.ESPC, Country.Ukraine, year, startDate, endDate, Nil) + def ESPCUkraine( + year: Int, + startDate: String = "01-09", + endDate: String = "30-09" + ) = + new Contest( + ContestType.ESPC, + Country.Ukraine, + year, + startDate, + endDate, + Nil + ) def WLMUkraine(year: Int) = load("wlm_ua.conf").get.copy(year = year) @@ -111,6 +140,3 @@ object Contest { load("wle_ua.conf").get.copy(year = year) } - - - diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/ContestType.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/ContestType.scala index 9fe5d728..bce67324 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/ContestType.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/ContestType.scala @@ -4,7 +4,8 @@ trait HasImagesCategory { def imagesCategory: String } -class ContestType(val code: String, val name: String) extends HasImagesCategory { +class ContestType(val code: String, val name: String) + extends HasImagesCategory { override def imagesCategory: String = "Category:Images from " + name } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Coordinate.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Coordinate.scala index 4930dd2b..5a5a82bf 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Coordinate.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Coordinate.scala @@ -6,6 +6,8 @@ case class Coordinate(lat: String, lon: String) { } object Coordinate { - def apply(latOpt:Option[String], lonOpt:Option[String]) = for (lat<-latOpt; lon <-lonOpt) + def apply(latOpt: Option[String], lonOpt: Option[String]) = for ( + lat <- latOpt; lon <- lonOpt + ) yield new Coordinate(lat, lon) } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Country.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Country.scala index cc4513cf..a63d10c8 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Country.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Country.scala @@ -11,9 +11,11 @@ trait AdmDivision { def name: String - lazy val namesList: Seq[String] = parent().map(_.namesList).getOrElse(Nil) :+ fullName + lazy val namesList: Seq[String] = + parent().map(_.namesList).getOrElse(Nil) :+ fullName - lazy val shortNamesList: Seq[String] = parent().map(_.shortNamesList).getOrElse(Nil) :+ name + lazy val shortNamesList: Seq[String] = + parent().map(_.shortNamesList).getOrElse(Nil) :+ name def regionType: Option[RegionType] @@ -60,33 +62,43 @@ trait AdmDivision { def byName(name: String, level: Int, max: Int): Seq[AdmDivision] = { Seq(this).filter(_.name.toLowerCase == name.toLowerCase) ++ (if (level < max) { - regions.flatMap(_.byName(name, level + 1, max)) - } else Nil) + regions.flatMap(_.byName(name, level + 1, max)) + } else Nil) } - def byRegion(monumentIds: Set[String]): Map[AdmDivision, Set[String]] = { - val entries = monumentIds.flatMap(id => byMonumentId(id).map(adm => id -> adm)) + val entries = + monumentIds.flatMap(id => byMonumentId(id).map(adm => id -> adm)) entries .groupBy { case (id, adm) => adm } - .mapValues(_.toMap.keySet).toMap + .mapValues(_.toMap.keySet) + .toMap } - def byIdAndName(regionId: String, rawName: String, cityType: Option[String] = None): Seq[AdmDivision] = { + def byIdAndName( + regionId: String, + rawName: String, + cityType: Option[String] = None + ): Seq[AdmDivision] = { val cleanName = AdmDivision.cleanName(rawName) - val candidates = byMonumentId(regionId).map { region => - val here = region.byName(cleanName) - if (here.isEmpty) { - region.parent().map(_.byName(cleanName)).getOrElse(Nil) - } else { - here + val candidates = byMonumentId(regionId) + .map { region => + val here = region.byName(cleanName) + if (here.isEmpty) { + region.parent().map(_.byName(cleanName)).getOrElse(Nil) + } else { + here + } } - }.getOrElse(Nil) + .getOrElse(Nil) - val types = cityType.fold(KoatuuTypes.nameToType(rawName).toSet.filterNot(_.code == "Р")) { code => - KoatuuTypes.codeToType.get(code) + val types = cityType.fold( + KoatuuTypes.nameToType(rawName).toSet.filterNot(_.code == "Р") + ) { code => + KoatuuTypes.codeToType + .get(code) .map(Set(_)) .getOrElse(KoatuuTypes.nameToType(code).toSet.filterNot(_.code == "Р")) } @@ -126,13 +138,18 @@ trait AdmDivision { } trait AdmRegion extends AdmDivision { - lazy val regionIds: SortedSet[String] = SortedSet(regions.map(_.shortCode): _*) + lazy val regionIds: SortedSet[String] = SortedSet( + regions.map(_.shortCode): _* + ) - lazy val regionNames: Seq[String] = regions.sortBy(_.shortCode).map(_.fullName) + lazy val regionNames: Seq[String] = + regions.sortBy(_.shortCode).map(_.fullName) - lazy val regionById: Map[String, AdmDivision] = regions.groupBy(_.shortCode).mapValues(_.head).toMap + lazy val regionById: Map[String, AdmDivision] = + regions.groupBy(_.shortCode).mapValues(_.head).toMap - override def regionName(regId: String) = byMonumentId(regId).map(_.fullName).getOrElse("") + override def regionName(regId: String) = + byMonumentId(regId).map(_.fullName).getOrElse("") override def byMonumentId(monumentId: String): Option[AdmDivision] = { regionById.get(regionId(monumentId)).flatMap { region => @@ -141,15 +158,15 @@ trait AdmRegion extends AdmDivision { } } - object NoAdmDivision extends Country("", "") -case class Country(code: String, - name: String, - val languageCodes: Seq[String] = Nil, - var regions: Seq[AdmDivision] = Nil, - val codeToRegion: Map[String, AdmDivision] = Map.empty - ) extends AdmRegion { +case class Country( + code: String, + name: String, + val languageCodes: Seq[String] = Nil, + var regions: Seq[AdmDivision] = Nil, + val codeToRegion: Map[String, AdmDivision] = Map.empty +) extends AdmRegion { val parent: () => Option[AdmDivision] = () => None @@ -159,7 +176,9 @@ case class Country(code: String, monumentId.take(2) } - override def withParents(parent: () => Option[AdmDivision] = () => None): AdmDivision = { + override def withParents( + parent: () => Option[AdmDivision] = () => None + ): AdmDivision = { regions = regionsWithParents() this } @@ -169,14 +188,18 @@ case class Country(code: String, override def level: Int = 0 } -case class Region(code: String, name: String, - var regions: Seq[AdmDivision] = Nil, - var parent: () => Option[AdmDivision] = () => None, - regionType: Option[RegionType] = None, - level: Int = 0) - extends AdmRegion { - - override def withParents(parent: () => Option[AdmDivision] = () => None): AdmDivision = { +case class Region( + code: String, + name: String, + var regions: Seq[AdmDivision] = Nil, + var parent: () => Option[AdmDivision] = () => None, + regionType: Option[RegionType] = None, + level: Int = 0 +) extends AdmRegion { + + override def withParents( + parent: () => Option[AdmDivision] = () => None + ): AdmDivision = { this.parent = parent this.regions = regionsWithParents() this @@ -184,17 +207,23 @@ case class Region(code: String, name: String, override def fullName: String = name + regionType .filterNot(_.noSuffix.contains(name)) - .flatMap(_.nameSuffix).getOrElse("") + .flatMap(_.nameSuffix) + .getOrElse("") } -case class NoRegions(code: String, name: String, - var parent: () => Option[AdmDivision] = () => None, - regionType: Option[RegionType] = None, level: Int = 0) - extends AdmDivision { +case class NoRegions( + code: String, + name: String, + var parent: () => Option[AdmDivision] = () => None, + regionType: Option[RegionType] = None, + level: Int = 0 +) extends AdmDivision { val regions: Seq[AdmDivision] = Nil - override def withParents(parent: () => Option[AdmDivision] = () => None): AdmDivision = { + override def withParents( + parent: () => Option[AdmDivision] = () => None + ): AdmDivision = { this.parent = parent this } @@ -202,10 +231,14 @@ case class NoRegions(code: String, name: String, } object AdmDivision { - def apply(code: String, name: String, - regions: Seq[AdmDivision], - parent: () => Option[AdmDivision], - regionType: Option[RegionType], level: Int = 0): AdmDivision = { + def apply( + code: String, + name: String, + regions: Seq[AdmDivision], + parent: () => Option[AdmDivision], + regionType: Option[RegionType], + level: Int = 0 + ): AdmDivision = { if (regions.isEmpty) { NoRegions(code, name, parent, regionType, level) } else { @@ -238,8 +271,10 @@ object AdmDivision { .replace(",", "") .replace("’", "'") .replace("”", "'") - .split("\\(").head - .split("\\|").head + .split("\\(") + .head + .split("\\|") + .head .trim } @@ -249,22 +284,20 @@ object Country { val Azerbaijan = new Country("AZ", "Azerbaijan", Seq("az")) - val Ukraine: Country = new Country("UA", "Ukraine", Seq("uk"), Koatuu.regions(() => Some(Ukraine))) + val Ukraine: Country = + new Country("UA", "Ukraine", Seq("uk"), Koatuu.regions(() => Some(Ukraine))) val customCountries = Seq(Ukraine) def langMap: Map[String, Seq[String]] = { Locale.getAvailableLocales .groupBy(_.getCountry) - .map { - case (countryCode, locales) => - - val langs = locales.toSeq.flatMap { - locale => - Option(locale.getLanguage) - .filter(_.nonEmpty) - } - countryCode -> langs + .map { case (countryCode, locales) => + val langs = locales.toSeq.flatMap { locale => + Option(locale.getLanguage) + .filter(_.nonEmpty) + } + countryCode -> langs } } @@ -272,27 +305,33 @@ object Country { val countryLangs = langMap - val fromJava = Locale.getISOCountries.map { countryCode => + val fromJava = Locale.getISOCountries + .map { countryCode => + val langCodes = countryLangs.getOrElse(countryCode, Seq.empty) - val langCodes = countryLangs.getOrElse(countryCode, Seq.empty) + val locales = + langCodes.map(langCode => new Locale(langCode, countryCode)) - val locales = langCodes.map(langCode => new Locale(langCode, countryCode)) + val locale = locales.headOption.getOrElse(new Locale("", countryCode)) - val locale = locales.headOption.getOrElse(new Locale("", countryCode)) + val langs = locales.map(_.getDisplayLanguage(Locale.ENGLISH)).distinct - val langs = locales.map(_.getDisplayLanguage(Locale.ENGLISH)).distinct - - new Country(locale.getCountry, - locale.getDisplayCountry(Locale.ENGLISH), - langCodes - ) - }.filterNot(_.code == "ua") + new Country( + locale.getCountry, + locale.getDisplayCountry(Locale.ENGLISH), + langCodes + ) + } + .filterNot(_.code == "ua") Seq(Ukraine) ++ fromJava } lazy val countryMap: Map[String, Country] = - (fromJavaLocales ++ customCountries).groupBy(_.code.toLowerCase).mapValues(_.head).toMap + (fromJavaLocales ++ customCountries) + .groupBy(_.code.toLowerCase) + .mapValues(_.head) + .toMap def byCode(code: String): Option[Country] = countryMap.get(code.toLowerCase) } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Katotth.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Katotth.scala index 13e29b0f..56d950b3 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Katotth.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Katotth.scala @@ -8,7 +8,12 @@ import java.io.InputStreamReader object KatotthTypes extends RegionTypes { override val regionTypes = Seq( - RegionType("O", Seq("область"), Some(" область"), Set("Автономна Республіка Крим")), + RegionType( + "O", + Seq("область"), + Some(" область"), + Set("Автономна Республіка Крим") + ), RegionType("K", Seq("місто")), // що має спеціальний статус; RegionType("P", Seq("район"), Some(" район")), RegionType("H", Seq("громада"), Some(" громада")), @@ -37,24 +42,39 @@ object Katotth { }.toMap } - lazy val toKatotth: Map[String, Seq[String]] = toKoatuu.toSeq.groupBy(_._2).mapValues(_.map(_._1)).toMap + lazy val toKatotth: Map[String, Seq[String]] = + toKoatuu.toSeq.groupBy(_._2).mapValues(_.map(_._1)).toMap def toFlat(row: Seq[String]): AdmDivisionFlat = { - val mainLevels = row.take(4).filterNot(_.isEmpty).map(_.replace("UA", "")).distinct - val codes = mainLevels.zipWithIndex.map { case (c, level) => shortCode(c, level + 2) } - val additional = Seq(row(4)).filterNot(_.isEmpty).map(mainLevels.last + _.replace("UA", "")) - AdmDivisionFlat(codeLevels = codes ++ additional, + val mainLevels = + row.take(4).filterNot(_.isEmpty).map(_.replace("UA", "")).distinct + val codes = mainLevels.zipWithIndex.map { case (c, level) => + shortCode(c, level + 2) + } + val additional = Seq(row(4)) + .filterNot(_.isEmpty) + .map(mainLevels.last + _.replace("UA", "")) + AdmDivisionFlat( + codeLevels = codes ++ additional, name = betterName(row(6)), regionType = KatotthTypes.codeToType.get(row(5)) ) } def readCsv(filename: String): Seq[List[String]] = { - val csv = CSVReader.open(new InputStreamReader(getClass.getResourceAsStream(s"/$filename.csv"), "UTF-8")) + val csv = CSVReader.open( + new InputStreamReader( + getClass.getResourceAsStream(s"/$filename.csv"), + "UTF-8" + ) + ) csv.all().tail } - def regions(parent: () => Option[AdmDivision] = () => None, size: Int = 1): Seq[AdmDivision] = { + def regions( + parent: () => Option[AdmDivision] = () => None, + size: Int = 1 + ): Seq[AdmDivision] = { val flat = readCsv("katotth").map(toFlat) AdmDivisionFlat.makeHierarchy(flat, parent) } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Koatuu.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Koatuu.scala index 9ddb7b08..40985102 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Koatuu.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Koatuu.scala @@ -5,9 +5,12 @@ import play.api.libs.json._ import play.api.libs.json.Reads._ import play.api.libs.functional.syntax._ -case class RegionType(code: String, names: Seq[String], - nameSuffix: Option[String] = None, - noSuffix: Set[String] = Set.empty) +case class RegionType( + code: String, + names: Seq[String], + nameSuffix: Option[String] = None, + noSuffix: Set[String] = Set.empty +) trait RegionTypes { @@ -15,9 +18,11 @@ trait RegionTypes { def codeToType: Map[String, RegionType] - def groupTypes(types: Seq[RegionType]): Map[String, RegionType] = types.groupBy(_.code).mapValues(_.head).toMap + def groupTypes(types: Seq[RegionType]): Map[String, RegionType] = + types.groupBy(_.code).mapValues(_.head).toMap - def nameToType(name: String): Seq[RegionType] = regionTypes.filter(_.names.exists(name.toLowerCase.contains)) + def nameToType(name: String): Seq[RegionType] = + regionTypes.filter(_.names.exists(name.toLowerCase.contains)) } @@ -37,34 +42,53 @@ object KoatuuTypes extends RegionTypes { object Koatuu { - def regions(parent: () => Option[AdmDivision] = () => None) = regionsNew(parent) + def regions(parent: () => Option[AdmDivision] = () => None) = regionsNew( + parent + ) - def regionsNew(parent: () => Option[AdmDivision] = () => None, size: Int = 1): Seq[AdmDivision] = { + def regionsNew( + parent: () => Option[AdmDivision] = () => None, + size: Int = 1 + ): Seq[AdmDivision] = { val stream = getClass.getResourceAsStream("/koatuu_new.json") val json = Json.parse(stream) AdmDivisionFlat.makeHierarchy(parse(json), parent) } - def regionReads(level: Int, parent: () => Option[AdmDivision] = () => None): Reads[AdmDivision] = ( + def regionReads( + level: Int, + parent: () => Option[AdmDivision] = () => None + ): Reads[AdmDivision] = ( (__ \ "code").read[String].map(c => shortCode(c, level)) and (__ \ "name").read[String].map(betterName) and (__ \ ("level" + level)) .lazyReadNullable(Reads.seq[AdmDivision](regionReads(level + 1))) - .map(_.getOrElse(Nil)).map(skipGroups) and + .map(_.getOrElse(Nil)) + .map(skipGroups) and Reads.pure(parent) and - (__ \ "type").readNullable[String].map(_.flatMap(KoatuuTypes.codeToType.get)) - ) (AdmDivision.apply(_, _, _, _, _, level)) - - val groupNames = Seq("Міста обласного підпорядкування", "Міста", "Райони", "Селища міського типу", "Населені пункти", "Сільради") + (__ \ "type") + .readNullable[String] + .map(_.flatMap(KoatuuTypes.codeToType.get)) + )(AdmDivision.apply(_, _, _, _, _, level)) + + val groupNames = Seq( + "Міста обласного підпорядкування", + "Міста", + "Райони", + "Селища міського типу", + "Населені пункти", + "Сільради" + ) .map(_.toUpperCase) - def groupPredicate(r: AdmDivision) = groupNames.exists(r.name.toUpperCase.startsWith) + def groupPredicate(r: AdmDivision) = + groupNames.exists(r.name.toUpperCase.startsWith) def skipGroups(regions: Seq[AdmDivision]): Seq[AdmDivision] = { regions flatMap { _ match { case r if groupPredicate(r) => skipGroups(r.regions) - case r => Seq(r) + case r => Seq(r) } } } @@ -81,9 +105,15 @@ object Koatuu { val s1 = s.split("/").head.toLowerCase.capitalize s1 - .split("-").map(_.capitalize).mkString("-") - .split(" ").map(_.capitalize).mkString(" ") - .split("[Мм]\\.[ ]?").map(_.capitalize).mkString("м. ") + .split("-") + .map(_.capitalize) + .mkString("-") + .split(" ") + .map(_.capitalize) + .mkString(" ") + .split("[Мм]\\.[ ]?") + .map(_.capitalize) + .mkString("м. ") .replaceFirst("^[Мм]\\.[ ]?", "") .replaceAll("Область$", "область") .replaceAll("Район$", "район") @@ -95,4 +125,4 @@ object Koatuu { def main(args: Array[String]): Unit = { println(regions(() => Some(Country.Ukraine))) } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/KoatuuNew.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/KoatuuNew.scala index d904e0fd..1cbee18c 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/KoatuuNew.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/KoatuuNew.scala @@ -7,7 +7,8 @@ import play.api.libs.json._ object KoatuuNew { - val readStringFromLong: Reads[String] = implicitly[Reads[Long]].map(x => x.toString) + val readStringFromLong: Reads[String] = + implicitly[Reads[Long]].map(x => x.toString) def stringOrLong(name: String): Reads[String] = { (__ \ name).read[String] or @@ -20,8 +21,10 @@ object KoatuuNew { stringOrLong("Третій рівень") and stringOrLong("Четвертий рівень") and (__ \ "Назва об'єкта українською мовою").read[String].map(betterName) and - (__ \ "Категорія").readNullable[String].map(_.flatMap(KoatuuTypes.codeToType.get)) - ) (AdmDivisionFlat.apply(_, _, _, _, _, _)) + (__ \ "Категорія") + .readNullable[String] + .map(_.flatMap(KoatuuTypes.codeToType.get)) + )(AdmDivisionFlat.apply(_, _, _, _, _, _)) def parse(json: JsValue): Seq[AdmDivisionFlat] = { json.as[Seq[AdmDivisionFlat]] diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Monument.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Monument.scala index ff645aca..85f0e734 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Monument.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/Monument.scala @@ -9,37 +9,40 @@ import org.scalawiki.wlx.dto.lists.ListConfig import scala.collection.immutable.ListMap import scala.util.Try -case class Monument(page: String = "", - id: String, - name: String, - year: Option[String] = None, - description: Option[String] = None, - article: Option[String] = None, - city: Option[String] = None, - cityType: Option[String] = None, - place: Option[String] = None, - user: Option[String] = None, - area: Option[String] = None, - // coordinate: Option[Coordinate], - lat: Option[String] = None, - lon: Option[String] = None, - typ: Option[String] = None, - subType: Option[String] = None, - photo: Option[String] = None, - gallery: Option[String] = None, - resolution: Option[String] = None, - stateId: Option[String] = None, - contest: Option[Long] = None, - source: Option[String] = None, - otherParams: Map[String, String] = Map.empty, - listConfig: Option[ListConfig] = None - ) extends Ordered[Monument] { +case class Monument( + page: String = "", + id: String, + name: String, + year: Option[String] = None, + description: Option[String] = None, + article: Option[String] = None, + city: Option[String] = None, + cityType: Option[String] = None, + place: Option[String] = None, + user: Option[String] = None, + area: Option[String] = None, + // coordinate: Option[Coordinate], + lat: Option[String] = None, + lon: Option[String] = None, + typ: Option[String] = None, + subType: Option[String] = None, + photo: Option[String] = None, + gallery: Option[String] = None, + resolution: Option[String] = None, + stateId: Option[String] = None, + contest: Option[Long] = None, + source: Option[String] = None, + otherParams: Map[String, String] = Map.empty, + listConfig: Option[ListConfig] = None +) extends Ordered[Monument] { val cityName = AdmDivision.cleanName(city.getOrElse("")) def toUrls = Monument.wikiLinkToUrl(name + " * " + place, "uk.wikipedia.org") - def galleryLink = gallery.fold("") { title => s" [[:Category:$title|$title]]" } + def galleryLink = gallery.fold("") { title => + s" [[:Category:$title|$title]]" + } def regionId = Monument.getRegionId(id) @@ -72,21 +75,29 @@ case class Monument(page: String = "", "subType" -> subType, "photo" -> photo, "gallery" -> gallery, - "resolution" -> resolution) - - val names = listConfig.fold(paramValues.filter(_._2.nonEmpty).keys)(_.namesMap.values.toSeq) - val namesMap = listConfig.map(_.namesMap).getOrElse(names.map(name => name -> name).toMap) + "resolution" -> resolution + ) + + val names = listConfig + .fold(paramValues.filter(_._2.nonEmpty).keys)(_.namesMap.values.toSeq) + val namesMap = listConfig + .map(_.namesMap) + .getOrElse(names.map(name => name -> name).toMap) val longest = names.map(_.length).max - val namesMapPadded = if (pad) namesMap.mapValues(_.padTo(longest, ' ')) else namesMap + val namesMapPadded = + if (pad) namesMap.mapValues(_.padTo(longest, ' ')) else namesMap val params = - namesMapPadded.toSeq.map { - case (englName, mappedName) => mappedName -> paramValues.get(englName).flatten.getOrElse("") + namesMapPadded.toSeq.map { case (englName, mappedName) => + mappedName -> paramValues.get(englName).flatten.getOrElse("") } ++ otherParams.toSeq - val template = Template(templateName.orElse(listConfig.map(_.templateName)).get, ListMap(params: _*)) + val template = Template( + templateName.orElse(listConfig.map(_.templateName)).get, + ListMap(params: _*) + ) template.text + "\n" } @@ -114,25 +125,34 @@ object Monument { wikiText.fold("") { t => wikiLinkToUrl(t, host) } def wikiLinkToUrl(wikiText: String, host: String): String = { - val r1 = "\\[\\[([^|]*?)\\]\\]".r.replaceAllIn(wikiText, { - m => + val r1 = "\\[\\[([^|]*?)\\]\\]".r.replaceAllIn( + wikiText, + { m => val url = URLEncoder.encode(m.group(1).replaceAll(" ", "_"), "UTF-8") val title = m.group(1) s"""$title""" - }) + } + ) - val r2 = "\\[\\[(.*?)\\|(.*?)\\]\\]".r.replaceAllIn(r1, { - m => + val r2 = "\\[\\[(.*?)\\|(.*?)\\]\\]".r.replaceAllIn( + r1, + { m => val url = URLEncoder.encode(m.group(1).replaceAll(" ", "_"), "UTF-8") val title = m.group(2) s"""$title""" - }) + } + ) r2 } - def monumentsFromText(text: String, page: String, template: String, listConfig: ListConfig): Iterable[Monument] = - init(text, page, listConfig) //.toSet + def monumentsFromText( + text: String, + page: String, + template: String, + listConfig: ListConfig + ): Iterable[Monument] = + init(text, page, listConfig) // .toSet def getRegionId(monumentId: String): String = { val parts = monumentId.split("\\-") @@ -143,7 +163,7 @@ object Monument { .getOrElse("") } - def getRegionId(monumentId: Option[String]): String = monumentId.fold("")(getRegionId) + def getRegionId(monumentId: Option[String]): String = + monumentId.fold("")(getRegionId) } - diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/SpecialNomination.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/SpecialNomination.scala index 5cae3168..365a0d8d 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/SpecialNomination.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/SpecialNomination.scala @@ -9,15 +9,24 @@ import org.scalawiki.wlx.stat.reports.DesnaRegionSpecialNomination import scala.util.Try -/** - * Describes monument lists for contest special nominations - * - * @param name Name of the special nomination - * @param listTemplate name of template that monument lists consist of - * @param pages pages that contain lists of monuments, ot templates that contains links to these pages - */ -case class SpecialNomination(name: String, listTemplate: Option[String], pages: Seq[String], years: Seq[Int] = Nil, - cities: Seq[AdmDivision] = Nil, fileTemplate: Option[String] = None) +/** Describes monument lists for contest special nominations + * + * @param name + * Name of the special nomination + * @param listTemplate + * name of template that monument lists consist of + * @param pages + * pages that contain lists of monuments, ot templates that contains links to + * these pages + */ +case class SpecialNomination( + name: String, + listTemplate: Option[String], + pages: Seq[String], + years: Seq[Int] = Nil, + cities: Seq[AdmDivision] = Nil, + fileTemplate: Option[String] = None +) object SpecialNomination { @@ -34,59 +43,74 @@ object SpecialNomination { }.recover { case x: Throwable => println(x) new java.util.ArrayList() - }.toOption.map(_.asScala.toSeq.map { c => - new SpecialNomination( - c.getString("name"), - c.as[Option[String]]("listTemplate"), - c.as[Option[Seq[String]]]("pages").getOrElse(Nil), - c.as[Option[Seq[Int]]]("years").getOrElse(Nil), - if (c.hasPath("cities")) { - c.getConfigList("cities").asScala.toSeq.map { citiConf => - val name = citiConf.getString("name") - val code = citiConf.getString("code") - lookupCity(name, code).head - } - } else Nil, - c.as[Option[String]]("fileTemplate") - ) - }).getOrElse(Seq.empty) + }.toOption + .map(_.asScala.toSeq.map { c => + new SpecialNomination( + c.getString("name"), + c.as[Option[String]]("listTemplate"), + c.as[Option[Seq[String]]]("pages").getOrElse(Nil), + c.as[Option[Seq[Int]]]("years").getOrElse(Nil), + if (c.hasPath("cities")) { + c.getConfigList("cities").asScala.toSeq.map { citiConf => + val name = citiConf.getString("name") + val code = citiConf.getString("code") + lookupCity(name, code).head + } + } else Nil, + c.as[Option[String]]("fileTemplate") + ) + }) + .getOrElse(Seq.empty) } lazy val nominations = load("wlm_ua.conf") - def getMonumentsMap(nominations: Seq[SpecialNomination], stat: ContestStat): Map[SpecialNomination, Seq[Monument]] = { + def getMonumentsMap( + nominations: Seq[SpecialNomination], + stat: ContestStat + ): Map[SpecialNomination, Seq[Monument]] = { val contest = stat.contest - val monumentQuery = MonumentQuery.create(contest, reportDifferentRegionIds = false) - nominations.filter(_.listTemplate.nonEmpty).flatMap { nomination => - nomination.listTemplate.map { listTemplate => - val monuments = if (nomination.pages.nonEmpty && nomination.name != "Пам'ятки Подесення") { - nomination.pages.flatMap { page => - monumentQuery.byPage(page, listTemplate) - } - } else if (nomination.cities.nonEmpty) { - monumentsInCities(nomination.cities, stat.monumentDb.get) - } else if (nomination.name == "Пам'ятки Подесення") { - val desna = DesnaRegionSpecialNomination() - val placeIds = desna.places.flatMap(desna.getPlace).map(_.code) - val k2k = placeIds.flatMap(Katotth.toKoatuu.get) - val allPlaceIds = (placeIds ++ k2k).toSet + val monumentQuery = + MonumentQuery.create(contest, reportDifferentRegionIds = false) + nominations + .filter(_.listTemplate.nonEmpty) + .flatMap { nomination => + nomination.listTemplate.map { listTemplate => + val monuments = if ( + nomination.pages.nonEmpty && nomination.name != "Пам'ятки Подесення" + ) { + nomination.pages.flatMap { page => + monumentQuery.byPage(page, listTemplate) + } + } else if (nomination.cities.nonEmpty) { + monumentsInCities(nomination.cities, stat.monumentDb.get) + } else if (nomination.name == "Пам'ятки Подесення") { + val desna = DesnaRegionSpecialNomination() + val placeIds = desna.places.flatMap(desna.getPlace).map(_.code) + val k2k = placeIds.flatMap(Katotth.toKoatuu.get) + val allPlaceIds = (placeIds ++ k2k).toSet - val monumentDb = stat.monumentDb.get - monumentDb.allMonuments.filter { monument => - monumentDb.placeByMonumentId.get(monument.id).exists(allPlaceIds.contains) - } ++ nomination.pages.flatMap { page => - monumentQuery.byPage(page, listTemplate) + val monumentDb = stat.monumentDb.get + monumentDb.allMonuments.filter { monument => + monumentDb.placeByMonumentId + .get(monument.id) + .exists(allPlaceIds.contains) + } ++ nomination.pages.flatMap { page => + monumentQuery.byPage(page, listTemplate) + } + } else { + Nil } + (nomination, monuments) } - else { - Nil - } - (nomination, monuments) } - }.toMap + .toMap } - def monumentsInCities(cities: Seq[AdmDivision], monumentDb: MonumentDB): Seq[Monument] = { + def monumentsInCities( + cities: Seq[AdmDivision], + monumentDb: MonumentDB + ): Seq[Monument] = { val placeIds = cities.map(_.code).toSet monumentDb.allMonuments.filter { monument => monumentDb.placeByMonumentId.get(monument.id).exists(placeIds.contains) @@ -97,4 +121,4 @@ object SpecialNomination { Country.Ukraine.byIdAndName(code.take(2), name).filter(_.code == code) } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/UploadConfig.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/UploadConfig.scala index 60c177d0..44ca640e 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/UploadConfig.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/UploadConfig.scala @@ -3,14 +3,23 @@ package org.scalawiki.wlx.dto import com.typesafe.config.Config import org.scalawiki.wlx.dto.lists.ListConfig -/** - * @param campaign name of upload campaign (https://commons.wikimedia.org/wiki/Commons:Upload_campaigns) - * @param listTemplate name of template that monument lists consist of - * @param fileTemplate name of template that marks a contest image with monument id - * @param listConfig configuration of monument list fields +/** @param campaign + * name of upload campaign + * (https://commons.wikimedia.org/wiki/Commons:Upload_campaigns) + * @param listTemplate + * name of template that monument lists consist of + * @param fileTemplate + * name of template that marks a contest image with monument id + * @param listConfig + * configuration of monument list fields */ -case class UploadConfig(campaign: String, listTemplate: String, fileTemplate: String, listConfig: ListConfig, listsHost: Option[String] = None) - +case class UploadConfig( + campaign: String, + listTemplate: String, + fileTemplate: String, + listConfig: ListConfig, + listsHost: Option[String] = None +) object UploadConfig { def fromConfig(c: Config): UploadConfig = { @@ -18,11 +27,11 @@ object UploadConfig { val (campaign, listTemplate, fileTemplate) = ( c.getString("campaign"), c.getString("listTemplate"), - c.getString("fileTemplate")) + c.getString("fileTemplate") + ) val listConfig = ListConfig.fromConfig(c) new UploadConfig(campaign, listTemplate, fileTemplate, listConfig) } } - diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/lists/ListConfig.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/lists/ListConfig.scala index 04d72389..70e37ce7 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/lists/ListConfig.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/dto/lists/ListConfig.scala @@ -4,8 +4,8 @@ import com.typesafe.config.{Config, ConfigFactory} import scala.collection.immutable.ListMap -/** - * Maps constant English field names to possibly localized field names in monument lists +/** Maps constant English field names to possibly localized field names in + * monument lists */ trait ListConfig { def templateName: String @@ -14,7 +14,8 @@ trait ListConfig { } -case class ListConfigImpl(templateName: String, namesMap: Map[String, String]) extends ListConfig +case class ListConfigImpl(templateName: String, namesMap: Map[String, String]) + extends ListConfig object ListConfig { @@ -27,10 +28,9 @@ object ListConfig { def fromConfig(c: Config): ListConfig = { val names = c.getConfigList("names") - val kvs = names.asScala.map { - n => - val entry = n.entrySet().iterator().next() - entry.getKey -> entry.getValue.unwrapped().toString + val kvs = names.asScala.map { n => + val entry = n.entrySet().iterator().next() + entry.getKey -> entry.getValue.unwrapped().toString } new ListConfigImpl(c.getString("listTemplate"), ListMap(kvs.toSeq: _*)) @@ -41,7 +41,8 @@ object ListConfig { val WleTh = load("wle_th.conf") } -class OtherTemplateListConfig(val templateName: String, base: ListConfig) extends ListConfig { +class OtherTemplateListConfig(val templateName: String, base: ListConfig) + extends ListConfig { override def namesMap: Map[String, String] = base.namesMap } @@ -50,7 +51,3 @@ object EmptyListConfig extends ListConfig { override def namesMap: Map[String, String] = Map.empty } - - - - diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/query/ImageQuery.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/query/ImageQuery.scala index 677ae9b6..e1de8a7f 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/query/ImageQuery.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/query/ImageQuery.scala @@ -12,30 +12,60 @@ import scala.concurrent.Future trait ImageQuery { - def imagesFromCategoryAsync(category: String, contest: Contest): Future[Iterable[Image]] + def imagesFromCategoryAsync( + category: String, + contest: Contest + ): Future[Iterable[Image]] - def imagesWithTemplateAsync(template: String, contest: Contest): Future[Iterable[Image]] + def imagesWithTemplateAsync( + template: String, + contest: Contest + ): Future[Iterable[Image]] } class ImageQueryApi(bot: ActionBot) extends ImageQuery with QueryLibrary { - override def imagesFromCategoryAsync(category: String, contest: Contest): Future[Iterable[Image]] = { - val generator: Generator = Generator(CategoryMembers(CmTitle(category), CmNamespace(Seq(Namespace.FILE)), CmLimit("400"))) // 5000 / 10 + override def imagesFromCategoryAsync( + category: String, + contest: Contest + ): Future[Iterable[Image]] = { + val generator: Generator = Generator( + CategoryMembers( + CmTitle(category), + CmNamespace(Seq(Namespace.FILE)), + CmLimit("400") + ) + ) // 5000 / 10 imagesByGenerator(contest, generator) } - override def imagesWithTemplateAsync(template: String, contest: Contest): Future[Iterable[Image]] = { - imagesByGenerator(contest, generatorWithTemplate(template, Set(Namespace.FILE))) + override def imagesWithTemplateAsync( + template: String, + contest: Contest + ): Future[Iterable[Image]] = { + imagesByGenerator( + contest, + generatorWithTemplate(template, Set(Namespace.FILE)) + ) } - def imagesByGenerator(contest: Contest, generator: Generator): Future[Iterable[Image]] = { + def imagesByGenerator( + contest: Contest, + generator: Generator + ): Future[Iterable[Image]] = { val specialNominationTemplates = SpecialNomination.nominations - .filter(n => n.years.contains(contest.year)).flatMap(_.fileTemplate) + .filter(n => n.years.contains(contest.year)) + .flatMap(_.fileTemplate) for (pages <- bot.run(imagesByGenerator(generator))) yield { - val optionalImages = for (page <- pages) - yield Image.fromPage(page, contest.fileTemplate, specialNominationTemplates) + val optionalImages = + for (page <- pages) + yield Image.fromPage( + page, + contest.fileTemplate, + specialNominationTemplates + ) optionalImages.flatten } } @@ -43,6 +73,8 @@ class ImageQueryApi(bot: ActionBot) extends ImageQuery with QueryLibrary { object ImageQuery { - def create(db: Boolean = false)(implicit bot: ActionBot = MwBot.fromHost(MwBot.commons)): ImageQuery = new ImageQueryApi(bot) + def create(db: Boolean = false)(implicit + bot: ActionBot = MwBot.fromHost(MwBot.commons) + ): ImageQuery = new ImageQueryApi(bot) -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/query/MonumentQuery.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/query/MonumentQuery.scala index 6dbbb5cf..e8adb277 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/query/MonumentQuery.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/query/MonumentQuery.scala @@ -23,23 +23,38 @@ trait MonumentQuery { def defaultListTemplate: String = contest.uploadConfigs.head.listTemplate - def byMonumentTemplateAsync(generatorTemplate: String = defaultListTemplate, - date: Option[ZonedDateTime] = None, - listTemplate: Option[String] = None): Future[Iterable[Monument]] - - def byPageAsync(page: String, template: String, date: Option[ZonedDateTime] = None): Future[Iterable[Monument]] - - final def byMonumentTemplate(generatorTemplate: String = defaultListTemplate, - date: Option[ZonedDateTime] = None, - listTemplate: Option[String] = None): Iterable[Monument] = - Await.result(byMonumentTemplateAsync(generatorTemplate, date, listTemplate), Timeout) + def byMonumentTemplateAsync( + generatorTemplate: String = defaultListTemplate, + date: Option[ZonedDateTime] = None, + listTemplate: Option[String] = None + ): Future[Iterable[Monument]] + + def byPageAsync( + page: String, + template: String, + date: Option[ZonedDateTime] = None + ): Future[Iterable[Monument]] + + final def byMonumentTemplate( + generatorTemplate: String = defaultListTemplate, + date: Option[ZonedDateTime] = None, + listTemplate: Option[String] = None + ): Iterable[Monument] = + Await.result( + byMonumentTemplateAsync(generatorTemplate, date, listTemplate), + Timeout + ) final def byPage(page: String, template: String): Iterable[Monument] = Await.result(byPageAsync(page, template), Timeout) } -class MonumentQueryApi(val contest: Contest, reportDifferentRegionIds: Boolean = false)(implicit val bot: MwBot) - extends MonumentQuery with QueryLibrary { +class MonumentQueryApi( + val contest: Contest, + reportDifferentRegionIds: Boolean = false +)(implicit val bot: MwBot) + extends MonumentQuery + with QueryLibrary { val host = getHost.get @@ -47,73 +62,134 @@ class MonumentQueryApi(val contest: Contest, reportDifferentRegionIds: Boolean = def getHost: Option[String] = contest.listsHost - override def byMonumentTemplateAsync(generatorTemplate: String, - date: Option[ZonedDateTime] = None, - listTemplate: Option[String] = None): Future[Iterable[Monument]] = { + override def byMonumentTemplateAsync( + generatorTemplate: String, + date: Option[ZonedDateTime] = None, + listTemplate: Option[String] = None + ): Future[Iterable[Monument]] = { val differentRegionIds = new ArrayBuffer[String]() - val title = if (generatorTemplate.startsWith("Template")) generatorTemplate else "Template:" + generatorTemplate - val listConfig = listTemplate.fold(defaultListConfig)(new OtherTemplateListConfig(_, defaultListConfig)) + val title = + if (generatorTemplate.startsWith("Template")) generatorTemplate + else "Template:" + generatorTemplate + val listConfig = listTemplate.fold(defaultListConfig)( + new OtherTemplateListConfig(_, defaultListConfig) + ) if (date.isEmpty) { - bot.page(title).revisionsByGenerator("embeddedin", "ei", - Set(Namespace.PROJECT, Namespace.MAIN), - Set("ids", "content", "timestamp", "user", "userid", "comment"), None, "100") map { pages => + bot + .page(title) + .revisionsByGenerator( + "embeddedin", + "ei", + Set(Namespace.PROJECT, Namespace.MAIN), + Set("ids", "content", "timestamp", "user", "userid", "comment"), + None, + "100" + ) map { pages => val monuments = pages.flatMap { page => if (!page.title.contains("новий АТУ")) { - val monuments = Monument.monumentsFromText(page.text.getOrElse(""), page.title, listTemplate.getOrElse(generatorTemplate), listConfig) - val regionIds = monuments.map(_.id.split("-").init.mkString("-")).toSet + val monuments = Monument.monumentsFromText( + page.text.getOrElse(""), + page.title, + listTemplate.getOrElse(generatorTemplate), + listConfig + ) + val regionIds = + monuments.map(_.id.split("-").init.mkString("-")).toSet if (regionIds.size > 1 && reportDifferentRegionIds) { - differentRegionIds.append(s"* [[${page.title}]]: ${regionIds.toSeq.sorted.mkString(", ")}") + differentRegionIds.append( + s"* [[${page.title}]]: ${regionIds.toSeq.sorted.mkString(", ")}" + ) } monuments } else Nil } if (reportDifferentRegionIds) { - Await.result(bot.page(s"Вікіпедія:${contest.name}/differentRegionIds").edit(differentRegionIds.sorted.mkString("\n")), 10.seconds) + Await.result( + bot + .page(s"Вікіпедія:${contest.name}/differentRegionIds") + .edit(differentRegionIds.sorted.mkString("\n")), + 10.seconds + ) } monuments } } else { - monumentsByDate(title, listTemplate.getOrElse(generatorTemplate), date.get) + monumentsByDate( + title, + listTemplate.getOrElse(generatorTemplate), + date.get + ) } } - override def byPageAsync(page: String, template: String, date: Option[ZonedDateTime] = None): Future[Iterable[Monument]] = { + override def byPageAsync( + page: String, + template: String, + date: Option[ZonedDateTime] = None + ): Future[Iterable[Monument]] = { val config = new OtherTemplateListConfig(template, defaultListConfig) if (!page.startsWith("Template")) { - bot.page(page).revisions(Set.empty, Set("content", "timestamp", "user", "userid", "comment")).map { revs => - revs.headOption.map(page => - Monument.monumentsFromText(page.text.getOrElse(""), page.title, template, config).toSeq).getOrElse(Seq.empty) - } + bot + .page(page) + .revisions( + Set.empty, + Set("content", "timestamp", "user", "userid", "comment") + ) + .map { revs => + revs.headOption + .map(page => + Monument + .monumentsFromText( + page.text.getOrElse(""), + page.title, + template, + config + ) + .toSeq + ) + .getOrElse(Seq.empty) + } } else { byMonumentTemplateAsync(page, date, Some(template)) } } - def monumentsByDate(page: String, template: String, date: ZonedDateTime): Future[Iterable[Monument]] = { - articlesWithTemplate(page).flatMap { - ids => - Future.traverse(ids)(id => pageRevisions(id, date)).map { - pages => - pages.flatten.flatMap(page => Monument.monumentsFromText(page.text.getOrElse(""), page.title, template, defaultListConfig)) - } + def monumentsByDate( + page: String, + template: String, + date: ZonedDateTime + ): Future[Iterable[Monument]] = { + articlesWithTemplate(page).flatMap { ids => + Future.traverse(ids)(id => pageRevisions(id, date)).map { pages => + pages.flatten.flatMap(page => + Monument.monumentsFromText( + page.text.getOrElse(""), + page.title, + template, + defaultListConfig + ) + ) + } } } def pageRevisions(id: Long, date: ZonedDateTime): Future[Option[Page]] = { import org.scalawiki.dto.cmd.query.prop.rvprop._ - val action = Action(Query( - PageIdsParam(Seq(id)), - Prop( - Info(), - Revisions( - RvProp(Content, Ids, Size, User, UserId, Timestamp), - RvLimit("max"), - RvStart(date) + val action = Action( + Query( + PageIdsParam(Seq(id)), + Prop( + Info(), + Revisions( + RvProp(Content, Ids, Size, User, UserId, Timestamp), + RvLimit("max"), + RvStart(date) + ) ) ) - )) + ) bot.run(action).map { pages => pages.headOption @@ -124,8 +200,9 @@ class MonumentQueryApi(val contest: Contest, reportDifferentRegionIds: Boolean = object MonumentQuery { - def create(contest: Contest, reportDifferentRegionIds: Boolean = false)(implicit bot: MwBot = MwBot.fromHost(MwBot.ukWiki)): MonumentQuery = + def create(contest: Contest, reportDifferentRegionIds: Boolean = false)( + implicit bot: MwBot = MwBot.fromHost(MwBot.ukWiki) + ): MonumentQuery = new MonumentQueryApi(contest, reportDifferentRegionIds)(bot) } - diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/RateRanges.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/RateRanges.scala index 2a843f3f..77c49501 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/RateRanges.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/RateRanges.scala @@ -4,20 +4,29 @@ import com.typesafe.config.Config import scala.util.Try -case class RateRanges(rangeMap: Map[(Int, Int), Double], default: Int = 0, sameAuthorZeroBonus: Boolean = false) { +case class RateRanges( + rangeMap: Map[(Int, Int), Double], + default: Int = 0, + sameAuthorZeroBonus: Boolean = false +) { verify() val max = Try(rangeMap.keys.map(_._2).max).toOption.getOrElse(0) private def verify(): Unit = { rangeMap.keys.foreach { case (x1, x2) => - if (x1 > x2) throw new IllegalArgumentException(s"Invalid ends order in range $x1-$x2: $x1 > $x2") + if (x1 > x2) + throw new IllegalArgumentException( + s"Invalid ends order in range $x1-$x2: $x1 > $x2" + ) } rangeMap.keys.toBuffer.sorted.foreach { r1 => rangeMap.keys.toBuffer.sorted.foreach { r2 => - if ((r1 ne r2) && + if ( + (r1 ne r2) && r1._1 <= r2._2 && - r2._1 <= r1._2) { + r2._1 <= r1._2 + ) { throw new IllegalArgumentException(s"Ranges $r1 and $r2 overlap") } } @@ -25,16 +34,20 @@ case class RateRanges(rangeMap: Map[(Int, Int), Double], default: Int = 0, sameA } def rate(param: Int): Double = { - rangeMap.collectFirst { case ((start, end), rate) - if start <= param && param <= end => rate - }.getOrElse(0) + rangeMap + .collectFirst { + case ((start, end), rate) if start <= param && param <= end => rate + } + .getOrElse(0) } def rateWithRange(param: Int): (Double, Int, Option[Int]) = { - rangeMap.collectFirst { case ((start, end), rate) - if start <= param && param <= end => - (rate, start, Some(end)) - }.getOrElse((default, max, None)) + rangeMap + .collectFirst { + case ((start, end), rate) if start <= param && param <= end => + (rate, start, Some(end)) + } + .getOrElse((default, max, None)) } } @@ -44,15 +57,19 @@ object RateRanges { import scala.collection.JavaConverters._ def apply(config: Config): RateRanges = { - val sameAuthorZeroBonus = Try(config.getBoolean("same-author-zero-bonus")).toOption.getOrElse(false) - val map = config.entrySet().asScala + val sameAuthorZeroBonus = + Try(config.getBoolean("same-author-zero-bonus")).toOption.getOrElse(false) + val map = config + .entrySet() + .asScala .filterNot(_.getKey == "same-author-zero-bonus") .map { entry => - val key = entry.getKey - val rangeSeq = key.split("-").map(_.toInt).take(2) - val rate = entry.getValue.unwrapped().asInstanceOf[Number].doubleValue() - ((rangeSeq.head, rangeSeq.last), rate) - }.toMap + val key = entry.getKey + val rangeSeq = key.split("-").map(_.toInt).take(2) + val rate = entry.getValue.unwrapped().asInstanceOf[Number].doubleValue() + ((rangeSeq.head, rangeSeq.last), rate) + } + .toMap new RateRanges(map, sameAuthorZeroBonus = sameAuthorZeroBonus) } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Rater.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Rater.scala index 661e4329..b6c8260c 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Rater.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Rater.scala @@ -8,16 +8,19 @@ import org.scalawiki.wlx.stat.reports.RateInputDistribution import scala.collection.mutable import scala.util.Try -case class RateConfig(newObjectRating: Option[Double] = None, - newAuthorObjectRating: Option[Double] = None, - numberOfAuthorsBonus: Boolean = false, - numberOfImagesBonus: Boolean = false, - baseRate: Double = 1) +case class RateConfig( + newObjectRating: Option[Double] = None, + newAuthorObjectRating: Option[Double] = None, + numberOfAuthorsBonus: Boolean = false, + numberOfImagesBonus: Boolean = false, + baseRate: Double = 1 +) object RateConfig { def apply(conf: StatParams): RateConfig = { - apply(conf.newObjectRating.toOption, + apply( + conf.newObjectRating.toOption, conf.newAuthorObjectRating.toOption, conf.numberOfAuthorsBonus.getOrElse(false), conf.numberOfImagesBonus.getOrElse(false), @@ -43,7 +46,13 @@ trait Rater { } def rateRegion(regionId: String, author: String): Double = { - rateMonumentIds(imageDb._byAuthorAndRegion.by(author, regionId).flatMap(_.monumentId).toSet, author) + rateMonumentIds( + imageDb._byAuthorAndRegion + .by(author, regionId) + .flatMap(_.monumentId) + .toSet, + author + ) } val oldImages = stat.oldImages @@ -61,11 +70,17 @@ object Rater { val config = stat.contest.rateConfig val raters = Seq(new NumberOfMonuments(stat, config.baseRate)) ++ - config.newAuthorObjectRating.map(r => - new NewlyPicturedPerAuthorBonus(stat, config.newObjectRating.getOrElse(1), r) - ).orElse( - config.newObjectRating.map(new NewlyPicturedBonus(stat, _)) - ) + config.newAuthorObjectRating + .map(r => + new NewlyPicturedPerAuthorBonus( + stat, + config.newObjectRating.getOrElse(1), + r + ) + ) + .orElse( + config.newObjectRating.map(new NewlyPicturedBonus(stat, _)) + ) if (raters.tail.isEmpty) { raters.head @@ -77,13 +92,28 @@ object Rater { def fromConfig(stat: ContestStat, config: Config): Rater = { val rateCfg = config.getConfig(s"rates.${stat.contest.year}") - val raters = Seq(new NumberOfMonuments(stat, Try(rateCfg.getDouble("base-rate")).toOption.getOrElse(1))) ++ + val raters = Seq( + new NumberOfMonuments( + stat, + Try(rateCfg.getDouble("base-rate")).toOption.getOrElse(1) + ) + ) ++ (if (rateCfg.hasPath("number-of-authors-bonus")) { - Seq(new NumberOfAuthorsBonus(stat, RateRanges(rateCfg.getConfig("number-of-authors-bonus")))) - } else Nil) ++ + Seq( + new NumberOfAuthorsBonus( + stat, + RateRanges(rateCfg.getConfig("number-of-authors-bonus")) + ) + ) + } else Nil) ++ (if (rateCfg.hasPath("number-of-images-bonus")) { - Seq(new NumberOfImagesInPlaceBonus(stat, RateRanges(rateCfg.getConfig("number-of-images-bonus")))) - } else Nil) + Seq( + new NumberOfImagesInPlaceBonus( + stat, + RateRanges(rateCfg.getConfig("number-of-images-bonus")) + ) + ) + } else Nil) if (raters.tail.isEmpty) { raters.head @@ -102,13 +132,15 @@ class NumberOfMonuments(val stat: ContestStat, baseRate: Double) extends Rater { } override def explain(monumentId: String, author: String): String = { - if (monumentIds.contains(monumentId)) s"Base rate = $baseRate" else "Not a known monument = 0" + if (monumentIds.contains(monumentId)) s"Base rate = $baseRate" + else "Not a known monument = 0" } override def withRating: Boolean = false } -class NewlyPicturedBonus(val stat: ContestStat, newlyPicturedRate: Double) extends Rater { +class NewlyPicturedBonus(val stat: ContestStat, newlyPicturedRate: Double) + extends Rater { override def rate(monumentId: String, author: String): Double = { if (!oldMonumentIds.contains(monumentId)) @@ -125,20 +157,26 @@ class NewlyPicturedBonus(val stat: ContestStat, newlyPicturedRate: Double) exten } } -class NewlyPicturedPerAuthorBonus(val stat: ContestStat, - newlyPicturedRate: Double, - newlyPicturedPerAuthorRate: Double) extends Rater { +class NewlyPicturedPerAuthorBonus( + val stat: ContestStat, + newlyPicturedRate: Double, + newlyPicturedPerAuthorRate: Double +) extends Rater { val oldMonumentIdsByAuthor: Map[String, Set[String]] = oldImages .groupBy(_.author.getOrElse("")) - .mapValues(_.flatMap(_.monumentId).toSet).toMap + .mapValues(_.flatMap(_.monumentId).toSet) + .toMap override def rate(monumentId: String, author: String): Double = { val rate = monumentId match { case id if disqualify(id, author) => 0 case id if !oldMonumentIds.contains(id) => newlyPicturedRate - 1 - case id if !oldMonumentIdsByAuthor.getOrElse(author, Set.empty).contains(id) => + case id + if !oldMonumentIdsByAuthor + .getOrElse(author, Set.empty) + .contains(id) => newlyPicturedPerAuthorRate - 1 case _ => 0 @@ -149,10 +187,13 @@ class NewlyPicturedPerAuthorBonus(val stat: ContestStat, override def explain(monumentId: String, author: String): String = { monumentId match { case id if disqualify(id, author) => - "Disqualified for reuploading similar images = 0" + "Disqualified for reuploading similar images = 0" case id if !oldMonumentIds.contains(id) => s"Newly pictured bonus = ${newlyPicturedRate - 1}" - case id if !oldMonumentIdsByAuthor.getOrElse(author, Set.empty).contains(id) => + case id + if !oldMonumentIdsByAuthor + .getOrElse(author, Set.empty) + .contains(id) => s"Newly pictured per author bonus = ${newlyPicturedPerAuthorRate - 1}" case _ => s"Not newly pictured = 0" @@ -161,29 +202,41 @@ class NewlyPicturedPerAuthorBonus(val stat: ContestStat, override def disqualify(monumentId: String, author: String): Boolean = { Set("Петро Халява", "SnizhokAM").contains(author) && - oldMonumentIdsByAuthor.getOrElse(author, Set.empty).contains(monumentId) + oldMonumentIdsByAuthor.getOrElse(author, Set.empty).contains(monumentId) } } -class NumberOfAuthorsBonus(val stat: ContestStat, val rateRanges: RateRanges) extends Rater { - val authorsByMonument: Map[String, Set[String]] = oldImages.groupBy(_.monumentId.getOrElse("")) +class NumberOfAuthorsBonus(val stat: ContestStat, val rateRanges: RateRanges) + extends Rater { + val authorsByMonument: Map[String, Set[String]] = oldImages + .groupBy(_.monumentId.getOrElse("")) .mapValues { images => images.map(_.author.getOrElse("")).toSet - }.toMap + } + .toMap - val authorsNumberByMonument: Map[String, Int] = authorsByMonument.mapValues(_.size).toMap + val authorsNumberByMonument: Map[String, Int] = + authorsByMonument.mapValues(_.size).toMap - val distribution: Map[Int, Int] = authorsNumberByMonument.values.groupBy(identity).mapValues(_.size).toMap + val distribution: Map[Int, Int] = + authorsNumberByMonument.values.groupBy(identity).mapValues(_.size).toMap if (stat.config.exists(_.rateInputDistribution)) { - new RateInputDistribution(stat, distribution, "Number of authors distribution", + new RateInputDistribution( + stat, + distribution, + "Number of authors distribution", Seq("Number of authors", "Number of monuments") ).updateWiki(MwBot.fromHost(MwBot.commons)) } override def rate(monumentId: String, author: String): Double = { - if (disqualify(monumentId, author)) 0 else - if (rateRanges.sameAuthorZeroBonus && authorsByMonument.getOrElse(monumentId, Set.empty).contains(author)) { + if (disqualify(monumentId, author)) 0 + else if ( + rateRanges.sameAuthorZeroBonus && authorsByMonument + .getOrElse(monumentId, Set.empty) + .contains(author) + ) { 0 } else { rateRanges.rate(authorsNumberByMonument.getOrElse(monumentId, 0)) @@ -194,8 +247,11 @@ class NumberOfAuthorsBonus(val stat: ContestStat, val rateRanges: RateRanges) ex val number = rate(monumentId, author) if (disqualify(monumentId, author)) { "Disqualified for reuploading similar images = 0" - } else - if (rateRanges.sameAuthorZeroBonus && authorsByMonument.getOrElse(monumentId, Set.empty).contains(author)) { + } else if ( + rateRanges.sameAuthorZeroBonus && authorsByMonument + .getOrElse(monumentId, Set.empty) + .contains(author) + ) { s"Pictured by same author before = $number" } else { val picturedBy = authorsNumberByMonument.getOrElse(monumentId, 0) @@ -211,10 +267,15 @@ class NumberOfAuthorsBonus(val stat: ContestStat, val rateRanges: RateRanges) ex } } -case class PerPlaceStat(imagesPerPlace: Map[String, Int], placeByMonument: Map[String, String]) { +case class PerPlaceStat( + imagesPerPlace: Map[String, Int], + placeByMonument: Map[String, String] +) { val distribution = placeByMonument.values .map(place => imagesPerPlace.getOrElse(place, 0)) - .groupBy(identity).mapValues(_.size).toMap + .groupBy(identity) + .mapValues(_.size) + .toMap } object PerPlaceStat { @@ -222,108 +283,180 @@ object PerPlaceStat { val fallbackMapInverse = Map( "5121085201" -> Set("51-210-0002", "51-210-0011", "51-210-0080"), // Усатове "7125785201" -> Set("71-257-0008"), // Матусів - "5322610199" -> Set("53-226-0068", "53-226-0069", "53-226-0070", "53-226-0071", "53-226-0072", "53-226-0073", "53-226-9001"), // урочище Шумейкове + "5322610199" -> Set( + "53-226-0068", + "53-226-0069", + "53-226-0070", + "53-226-0071", + "53-226-0072", + "53-226-0073", + "53-226-9001" + ), // урочище Шумейкове "2611040399" -> Set("26-110-0006"), // Говерла - "0120481999" -> Set("01-204-0065"), // Голубинська сільська рада (Бахчисарайський район) + "0120481999" -> Set( + "01-204-0065" + ), // Голубинська сільська рада (Бахчисарайський район) "3222099719" -> Set("32-220-0071"), // центр колишнього села Красне - "3222000739" -> Set("32-220-0073"), //колишнє село Купувате - "3222000769" -> Set("32-220-0076"), //колишнє село Ладижичі - "3222000799" -> Set("32-220-0079"), //колишнє село Машеве - "3222000839" -> Set("32-220-0083", "32-220-0084"), //колишнє село Опачичі - "3222000859" -> Set("32-220-0085"), //колишнє село Паришів - "3222000889" -> Set("32-220-0088"), //колишнє село Розсоха - "3222000899" -> Set("32-220-0089"), //колишнє село Роз'їждже - "3222000919" -> Set("32-220-0091"), //колишнє село Старі Шепеличі - "3222000929" -> Set("32-220-0092"), //колишнє село Старосілля - "3222000939" -> Set("32-220-0093"), //колишнє село Стечанка - "3222000949" -> Set("32-220-0094"), //колишнє село Стечанка - "3222000969" -> Set("32-220-0096"), //центр колишнього села Товстий Ліс - "3222000989" -> Set("32-220-0098"), //колишнє село Чапаєвка - "3222001009" -> Set("32-220-0100"), //центр колишнього села Чистогалівка - "0111590019" -> Set("01-115-0007", "01-115-0009", "01-115-0010", "01-115-9001"), //Армянська міськрада (Перекопський вал), за 5 км від міста - "0122302009" -> Set("01-223-0041", "01-223-0103", "01-223-0104", "01-223-0200"), //Армянська та Красноперекопська міськради (на межі двох) - "5322690019" -> Set("53-226-0068", "53-226-0069", "53-226-0070", "53-226-0071", "53-226-0072", "53-226-0073", "53-226-9001"), //урочище Шумейкове (15 км від лохвиці) - "1423055701" -> Set("14-133-0017"), //uk:Торське (Лиманська міська громада) - "1420989202" -> Set("14-209-0065"), //uk:Григорівка (Сіверська міська громада) - "1422783201" -> Set("14-227-0018"), //uk:Михайлівка (Покровський район, Михайлівська сільська рада) - "3220490019" -> Set("32-204-9001"), //між селами Городище-Пустоварівське Володарського і Щербаки Білоцерківського районів - "5322610600" -> Set("53-226-0062"), //uk:Заводське - "5324000829" -> Set("53-240-0082"), //ур. Ступки - "4624801019" -> Set("46-248-0101"), //Городиловичі (село більше не існує) ) - "5322681102" -> Set("53-226-0041"), // Забодаква — колишнє село + "3222000739" -> Set("32-220-0073"), // колишнє село Купувате + "3222000769" -> Set("32-220-0076"), // колишнє село Ладижичі + "3222000799" -> Set("32-220-0079"), // колишнє село Машеве + "3222000839" -> Set("32-220-0083", "32-220-0084"), // колишнє село Опачичі + "3222000859" -> Set("32-220-0085"), // колишнє село Паришів + "3222000889" -> Set("32-220-0088"), // колишнє село Розсоха + "3222000899" -> Set("32-220-0089"), // колишнє село Роз'їждже + "3222000919" -> Set("32-220-0091"), // колишнє село Старі Шепеличі + "3222000929" -> Set("32-220-0092"), // колишнє село Старосілля + "3222000939" -> Set("32-220-0093"), // колишнє село Стечанка + "3222000949" -> Set("32-220-0094"), // колишнє село Стечанка + "3222000969" -> Set("32-220-0096"), // центр колишнього села Товстий Ліс + "3222000989" -> Set("32-220-0098"), // колишнє село Чапаєвка + "3222001009" -> Set("32-220-0100"), // центр колишнього села Чистогалівка + "0111590019" -> Set( + "01-115-0007", + "01-115-0009", + "01-115-0010", + "01-115-9001" + ), // Армянська міськрада (Перекопський вал), за 5 км від міста + "0122302009" -> Set( + "01-223-0041", + "01-223-0103", + "01-223-0104", + "01-223-0200" + ), // Армянська та Красноперекопська міськради (на межі двох) + "5322690019" -> Set( + "53-226-0068", + "53-226-0069", + "53-226-0070", + "53-226-0071", + "53-226-0072", + "53-226-0073", + "53-226-9001" + ), // урочище Шумейкове (15 км від лохвиці) + "1423055701" -> Set("14-133-0017"), // uk:Торське (Лиманська міська громада) + "1420989202" -> Set( + "14-209-0065" + ), // uk:Григорівка (Сіверська міська громада) + "1422783201" -> Set( + "14-227-0018" + ), // uk:Михайлівка (Покровський район, Михайлівська сільська рада) + "3220490019" -> Set( + "32-204-9001" + ), // між селами Городище-Пустоварівське Володарського і Щербаки Білоцерківського районів + "5322610600" -> Set("53-226-0062"), // uk:Заводське + "5324000829" -> Set("53-240-0082"), // ур. Ступки + "4624801019" -> Set( + "46-248-0101" + ), // Городиловичі (село більше не існує) ) + "5322681102" -> Set("53-226-0041"), // Забодаква — колишнє село "5322688402" -> Set("53-226-0110"), // - "5322681910" -> Set("53-226-0117"), // Перевалівка + "5322681910" -> Set("53-226-0117"), // Перевалівка "0111948301" -> Set("01-119-0349"), "0111949300" -> Set("01-119-0370"), "0111949702" -> Set("01-119-9002"), - "5121085201" -> Set("51-210-0074", "51-210-0075", "51-210-0082", "51-210-0002", "51-210-0011", "51-210-0080"), + "5121085201" -> Set( + "51-210-0074", + "51-210-0075", + "51-210-0082", + "51-210-0002", + "51-210-0011", + "51-210-0080" + ), "6524710101" -> Set("65-247-1251"), "4810800001" -> Set("48-108-0004"), "3222055103" -> Set("32-220-0060") ) - val fallbackMap = for ((koatuu, ids) <- fallbackMapInverse; id <- ids) yield id -> koatuu + val fallbackMap = + for ((koatuu, ids) <- fallbackMapInverse; id <- ids) yield id -> koatuu def apply(imageDB: ImageDB): PerPlaceStat = { val country = imageDB.contest.country val monumentDb = imageDB.monumentDb.get - val imagesPerPlace = (for (id <- imageDB.ids.toSeq; - place <- monumentDb.placeByMonumentId.get(id)) - yield (place -> imageDB.byId(id).size)).groupBy(_._1).mapValues(_.map(_._2).sum) + val imagesPerPlace = (for ( + id <- imageDB.ids.toSeq; + place <- monumentDb.placeByMonumentId.get(id) + ) + yield (place -> imageDB.byId(id).size)) + .groupBy(_._1) + .mapValues(_.map(_._2).sum) PerPlaceStat(imagesPerPlace.toMap, monumentDb.placeByMonumentId) } } -class NumberOfImagesInPlaceBonus(val stat: ContestStat, val rateRanges: RateRanges) extends Rater { +class NumberOfImagesInPlaceBonus( + val stat: ContestStat, + val rateRanges: RateRanges +) extends Rater { val oldImagesDb = new ImageDB(stat.contest, oldImages, stat.monumentDb) val perPlaceStat = PerPlaceStat(oldImagesDb) val unknownPlaceMonumentsByAuthor = mutable.Map[String, Set[String]]() - val authorsByMonument: Map[String, Set[String]] = oldImages.groupBy(_.monumentId.getOrElse("")) + val authorsByMonument: Map[String, Set[String]] = oldImages + .groupBy(_.monumentId.getOrElse("")) .mapValues { images => images.map(_.author.getOrElse("")).toSet - }.toMap + } + .toMap val distribution: Map[Int, Int] = perPlaceStat.distribution if (stat.config.exists(_.rateInputDistribution)) { - new RateInputDistribution(stat, distribution, "Number of images in place distribution", + new RateInputDistribution( + stat, + distribution, + "Number of images in place distribution", Seq("Number of images in place", "Number of monuments") ).updateWiki(MwBot.fromHost(MwBot.commons)) } override def rate(monumentId: String, author: String): Double = { - if (rateRanges.sameAuthorZeroBonus && authorsByMonument.getOrElse(monumentId, Set.empty).contains(author)) { + if ( + rateRanges.sameAuthorZeroBonus && authorsByMonument + .getOrElse(monumentId, Set.empty) + .contains(author) + ) { 0.0 } else { - perPlaceStat.placeByMonument.get(monumentId).map { place => - rateRanges.rate(perPlaceStat.imagesPerPlace.getOrElse(place, 0)) - }.getOrElse { - val monumentIds = unknownPlaceMonumentsByAuthor.getOrElse(author, Set.empty[String]) - if (!monumentIds.contains(monumentId)) { - unknownPlaceMonumentsByAuthor(author) = monumentIds + monumentId + perPlaceStat.placeByMonument + .get(monumentId) + .map { place => + rateRanges.rate(perPlaceStat.imagesPerPlace.getOrElse(place, 0)) + } + .getOrElse { + val monumentIds = + unknownPlaceMonumentsByAuthor.getOrElse(author, Set.empty[String]) + if (!monumentIds.contains(monumentId)) { + unknownPlaceMonumentsByAuthor(author) = monumentIds + monumentId + } + 0.0 } - 0.0 - } } } override def explain(monumentId: String, author: String): String = { - if (rateRanges.sameAuthorZeroBonus && authorsByMonument.getOrElse(monumentId, Set.empty).contains(author)) { + if ( + rateRanges.sameAuthorZeroBonus && authorsByMonument + .getOrElse(monumentId, Set.empty) + .contains(author) + ) { s"Pictured by same author before = 0" } else { - perPlaceStat.placeByMonument.get(monumentId).map { place => - val perPlace = perPlaceStat.imagesPerPlace.getOrElse(place, 0) - val (rate, start, end) = rateRanges.rateWithRange(perPlace) - s"$perPlace ($start-${end.getOrElse("")}) images were of $place, bonus = $rate" - }.getOrElse { - val monumentIds = unknownPlaceMonumentsByAuthor.getOrElse(author, Set.empty[String]) - if (!monumentIds.contains(monumentId)) { - unknownPlaceMonumentsByAuthor(author) = monumentIds + monumentId + perPlaceStat.placeByMonument + .get(monumentId) + .map { place => + val perPlace = perPlaceStat.imagesPerPlace.getOrElse(place, 0) + val (rate, start, end) = rateRanges.rateWithRange(perPlace) + s"$perPlace ($start-${end.getOrElse("")}) images were of $place, bonus = $rate" + } + .getOrElse { + val monumentIds = + unknownPlaceMonumentsByAuthor.getOrElse(author, Set.empty[String]) + if (!monumentIds.contains(monumentId)) { + unknownPlaceMonumentsByAuthor(author) = monumentIds + monumentId + } + s"unknown place of monument $monumentId, bonus = 0" } - s"unknown place of monument $monumentId, bonus = 0" - } } } } @@ -336,9 +469,13 @@ class RateSum(val stat: ContestStat, val raters: Seq[Rater]) extends Rater { } override def explain(monumentId: String, author: String): String = { - val disqulifiedRater = raters.find(_.disqualify(monumentId, author)) - disqulifiedRater.fold(s"Rating = ${raters.map(_.rate(monumentId, author)).sum}, is a sum of: " + raters.map(_.explain(monumentId, author)).mkString(", ")) { - rater => rater.explain(monumentId, author) + val disqulifiedRater = raters.find(_.disqualify(monumentId, author)) + disqulifiedRater.fold( + s"Rating = ${raters.map(_.rate(monumentId, author)).sum}, is a sum of: " + raters + .map(_.explain(monumentId, author)) + .mkString(", ") + ) { rater => + rater.explain(monumentId, author) } } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala index 6e7a4707..3f108f69 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Statistics.scala @@ -12,130 +12,227 @@ import org.scalawiki.wlx.{ImageDB, MonumentDB} import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -/** - * Holds fetched contest data +/** Holds fetched contest data * - * @param contest contest: contest type (WLM/WLE), country, year, etc. - * @param startYear the first year contest was held, or the first year that we are interested in - * @param monumentDb cultural/natural monuments database for the contest - * @param currentYearImageDb image database for current year's contest - * @param totalImageDb image database that holds images of all monuments from the contest, regardless of when they where uploaded - * @param dbsByYear image databases split by contest year - * @param monumentDbOld monument database at current year's contest start. Used to rate users who submitted newly pictured monuments. + * @param contest + * contest: contest type (WLM/WLE), country, year, etc. + * @param startYear + * the first year contest was held, or the first year that we are interested + * in + * @param monumentDb + * cultural/natural monuments database for the contest + * @param currentYearImageDb + * image database for current year's contest + * @param totalImageDb + * image database that holds images of all monuments from the contest, + * regardless of when they where uploaded + * @param dbsByYear + * image databases split by contest year + * @param monumentDbOld + * monument database at current year's contest start. Used to rate users who + * submitted newly pictured monuments. */ -case class ContestStat(contest: Contest, - startYear: Int, - monumentDb: Option[MonumentDB] = None, - currentYearImageDb: Option[ImageDB] = None, - totalImageDb: Option[ImageDB] = None, - dbsByYear: Seq[ImageDB] = Seq.empty, - monumentDbOld: Option[MonumentDB] = None, - config: Option[StatConfig] = None) { +case class ContestStat( + contest: Contest, + startYear: Int, + monumentDb: Option[MonumentDB] = None, + currentYearImageDb: Option[ImageDB] = None, + totalImageDb: Option[ImageDB] = None, + dbsByYear: Seq[ImageDB] = Seq.empty, + monumentDbOld: Option[MonumentDB] = None, + config: Option[StatConfig] = None +) { val imageDbsByYear = dbsByYear.groupBy(_.contest.year) val yearSeq = imageDbsByYear.keys.toSeq.sorted lazy val oldImages: Iterable[Image] = { - for (total <- totalImageDb; - current <- currentYearImageDb) + for ( + total <- totalImageDb; + current <- currentYearImageDb + ) yield { val currentImageIds = current.images.flatMap(_.pageId).toSet - total.images.filter(image => !currentImageIds.contains(image.pageId.get)) + total.images.filter(image => + !currentImageIds.contains(image.pageId.get) + ) } }.getOrElse(Nil) def imageDbByYear(year: Int) = imageDbsByYear.get(year).map(_.head) def mapYears[T](f: ImageDB => T) = { - for (year <- yearSeq; - imageDb <- imageDbByYear(year)) + for ( + year <- yearSeq; + imageDb <- imageDbByYear(year) + ) yield f(imageDb) } } -/** - * Coordinates fetching contest statistics and creating reports/galleries etc. Needs refactoring. +/** Coordinates fetching contest statistics and creating reports/galleries etc. + * Needs refactoring. * - * @param contest contest: contest type (WLM/WLE), country, year, etc. - * @param startYear the first year contest was held, or the first year that we are interested in - * @param monumentQuery monuments fetcher - * @param imageQuery images fetcher - * @param bot scalawiki bot instance + * @param contest + * contest: contest type (WLM/WLE), country, year, etc. + * @param startYear + * the first year contest was held, or the first year that we are interested + * in + * @param monumentQuery + * monuments fetcher + * @param imageQuery + * images fetcher + * @param bot + * scalawiki bot instance */ -class Statistics(contest: Contest, - startYear: Option[Int], - monumentQuery: MonumentQuery, - imageQuery: ImageQuery, - imageQueryWiki: Option[ImageQuery], - bot: MwBot, - config: StatConfig) { - - def this(contest: Contest, - startYear: Option[Int] = None, - monumentQuery: MonumentQuery, - imageQuery: ImageQuery = ImageQuery.create(), - imageQueryWiki: Option[ImageQuery] = None, - bot: MwBot = MwBot.fromHost(MwBot.commons), - config: Option[StatConfig] = None) = - this(contest, startYear, monumentQuery, imageQuery, imageQueryWiki, bot, config.getOrElse(StatConfig(contest.campaign))) +class Statistics( + contest: Contest, + startYear: Option[Int], + monumentQuery: MonumentQuery, + imageQuery: ImageQuery, + imageQueryWiki: Option[ImageQuery], + bot: MwBot, + config: StatConfig +) { + + def this( + contest: Contest, + startYear: Option[Int] = None, + monumentQuery: MonumentQuery, + imageQuery: ImageQuery = ImageQuery.create(), + imageQueryWiki: Option[ImageQuery] = None, + bot: MwBot = MwBot.fromHost(MwBot.commons), + config: Option[StatConfig] = None + ) = + this( + contest, + startYear, + monumentQuery, + imageQuery, + imageQueryWiki, + bot, + config.getOrElse(StatConfig(contest.campaign)) + ) private val currentYear = contest.year - private val contests = (startYear.getOrElse(currentYear) to currentYear).map(y => contest.copy(year = y)) + private val contests = + (startYear.getOrElse(currentYear) to currentYear).map(y => + contest.copy(year = y) + ) - /** - * Fetches contest data + /** Fetches contest data * - * @param total whether to fetch image database that holds images of all monuments from the contest, regardless of when they where uploaded - * @return asynchronously returned contest data + * @param total + * whether to fetch image database that holds images of all monuments from + * the contest, regardless of when they where uploaded + * @return + * asynchronously returned contest data */ def gatherData(total: Boolean): Future[ContestStat] = { val (monumentDb, monumentDbOld) = ( Some(MonumentDB.getMonumentDb(contest, monumentQuery)), contest.rateConfig.newObjectRating.map { _ => - MonumentDB.getMonumentDb(contest, monumentQuery, date = Some(ZonedDateTime.of(2017, 4, 30, 23, 59, 0, 0, ZoneOffset.UTC))) + MonumentDB.getMonumentDb( + contest, + monumentQuery, + date = + Some(ZonedDateTime.of(2017, 4, 30, 23, 59, 0, 0, ZoneOffset.UTC)) + ) } ) - for (byYear <- Future.sequence(contests.map(contestImages(monumentDb))); - totalImages <- if (total) imagesByTemplate(monumentDb) - else Future.successful(Some(new ImageDB(contest, byYear.find(_.contest.year == currentYear).map(_.images).getOrElse(Nil), monumentDb))) + for ( + byYear <- Future.sequence(contests.map(contestImages(monumentDb))); + totalImages <- + if (total) imagesByTemplate(monumentDb) + else + Future.successful( + Some( + new ImageDB( + contest, + byYear + .find(_.contest.year == currentYear) + .map(_.images) + .getOrElse(Nil), + monumentDb + ) + ) + ) ) yield { val currentYearImages = byYear.find(_.contest.year == currentYear) - val mDbOld: Option[MonumentDB] = currentYearImages.flatMap(getOldImagesMonumentDb(monumentDb, monumentDbOld, totalImages, _)) + val mDbOld: Option[MonumentDB] = currentYearImages.flatMap( + getOldImagesMonumentDb(monumentDb, monumentDbOld, totalImages, _) + ) - ContestStat(contest, startYear.getOrElse(contest.year), monumentDb, currentYearImages, totalImages, byYear, mDbOld, Some(config)) + ContestStat( + contest, + startYear.getOrElse(contest.year), + monumentDb, + currentYearImages, + totalImages, + byYear, + mDbOld, + Some(config) + ) } } private def contestImages(monumentDb: Some[MonumentDB])(contest: Contest) = ImageDB.create(contest, imageQuery, monumentDb, config.minMpx) - private def imagesByTemplate(monumentDb: Some[MonumentDB], imageQuery: ImageQuery = imageQuery) = - for (commons <- imageQuery.imagesWithTemplateAsync(contest.uploadConfigs.head.fileTemplate, contest); - wiki <- imageQueryWiki.map(_.imagesWithTemplateAsync(contest.uploadConfigs.head.fileTemplate, contest)) - .getOrElse(Future.successful(Nil))) yield { + private def imagesByTemplate( + monumentDb: Some[MonumentDB], + imageQuery: ImageQuery = imageQuery + ) = + for ( + commons <- imageQuery.imagesWithTemplateAsync( + contest.uploadConfigs.head.fileTemplate, + contest + ); + wiki <- imageQueryWiki + .map( + _.imagesWithTemplateAsync( + contest.uploadConfigs.head.fileTemplate, + contest + ) + ) + .getOrElse(Future.successful(Nil)) + ) yield { Some(new ImageDB(contest, commons ++ wiki, monumentDb)) } - def getOldImagesMonumentDb(monumentDb: Option[MonumentDB], monumentDbOld: Option[MonumentDB], - totalImages: Option[ImageDB], imageDB: ImageDB): Option[MonumentDB] = { - for (mDb <- monumentDb; - mdbOld <- monumentDbOld; - total <- totalImages.orElse(Some(new ImageDB(contest, Seq.empty)))) yield { + def getOldImagesMonumentDb( + monumentDb: Option[MonumentDB], + monumentDbOld: Option[MonumentDB], + totalImages: Option[ImageDB], + imageDB: ImageDB + ): Option[MonumentDB] = { + for ( + mDb <- monumentDb; + mdbOld <- monumentDbOld; + total <- totalImages.orElse(Some(new ImageDB(contest, Seq.empty))) + ) yield { val oldIds = mdbOld.monuments.filter(_.photo.isDefined).map(_.id).toSet ++ - total.images.filterNot(ti => imageDB.images.exists(i => i.pageId.exists(ti.pageId.contains))).flatMap(_.monumentId) + total.images + .filterNot(ti => + imageDB.images.exists(i => i.pageId.exists(ti.pageId.contains)) + ) + .flatMap(_.monumentId) new MonumentDB(contest, mDb.monuments.filter(m => oldIds.contains(m.id))) } } def init(total: Boolean): Unit = { - gatherData(total = total).map { stat => - new ReporterRegistry(stat, config).output() - }.failed.map(println) + gatherData(total = total) + .map { stat => + new ReporterRegistry(stat, config).output() + } + .failed + .map(println) } def articleStatistics(monumentDb: MonumentDB): Unit = { @@ -169,13 +266,17 @@ object Statistics { val contest = getContest(cfg) val cacheName = s"${cfg.campaign}-${contest.year}" - val imageQuery = ImageQuery.create()(new CachedBot(Site.commons, cacheName, true)) - val imageQueryWiki = ImageQuery.create()(new CachedBot(Site.ukWiki, cacheName + "-wiki", true, entries = 100)) + val imageQuery = + ImageQuery.create()(new CachedBot(Site.commons, cacheName, true)) + val imageQueryWiki = ImageQuery.create()( + new CachedBot(Site.ukWiki, cacheName + "-wiki", true, entries = 100) + ) val stat = new Statistics( contest, startYear = Some(cfg.years.head), - monumentQuery = MonumentQuery.create(contest, reportDifferentRegionIds = true), + monumentQuery = + MonumentQuery.create(contest, reportDifferentRegionIds = true), config = Some(cfg), imageQuery = imageQuery, imageQueryWiki = Some(imageQueryWiki) diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Stats.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Stats.scala index fd061d43..2e327ab3 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Stats.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/Stats.scala @@ -14,7 +14,8 @@ object Stats { columnAggregations = Seq( All, MonumentsWithArticles, - new PercentageAggregation("%", MonumentsWithArticles)), + new PercentageAggregation("%", MonumentsWithArticles) + ), rowOrdering = new StringColumnOrdering(0), rowKeyMapping = Some(new RegionNameById(monumentDb.contest.country)) ) @@ -22,12 +23,16 @@ object Stats { } -object MonumentsByRegion extends Grouping[Monument, String]( - "Region KOATUU code", _.regionId) +object MonumentsByRegion + extends Grouping[Monument, String]("Region KOATUU code", _.regionId) -object MonumentsWithArticles extends Aggregation[Monument, Int]( - "With Articles", _.count(_.name.contains("[["))) +object MonumentsWithArticles + extends Aggregation[Monument, Int]( + "With Articles", + _.count(_.name.contains("[[")) + ) object All extends Aggregation[Any, Int]("All", _.size) -class RegionNameById(country: AdmDivision) extends Mapping[String, String]("Region name", country.regionName) +class RegionNameById(country: AdmDivision) + extends Mapping[String, String]("Region name", country.regionName) diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/generic/Records.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/generic/Records.scala index d6ddccd8..dcd3fc88 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/generic/Records.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/generic/Records.scala @@ -2,27 +2,34 @@ package org.scalawiki.wlx.stat.generic import org.scalawiki.dto.markup.Table -class Aggregation[-T, +V](val name: String, aggFunc: Iterable[T] => V) extends Function[Iterable[T], V] { +class Aggregation[-T, +V](val name: String, aggFunc: Iterable[T] => V) + extends Function[Iterable[T], V] { override def apply(t: Iterable[T]): V = aggFunc(t) } -class Grouping[T, V](val name: String, groupFunc: T => V) extends Function[Iterable[T], Map[V, Iterable[T]]] { +class Grouping[T, V](val name: String, groupFunc: T => V) + extends Function[Iterable[T], Map[V, Iterable[T]]] { override def apply(t: Iterable[T]): Map[V, Iterable[T]] = t.groupBy(groupFunc) } -class Mapping[-T, +V](val name: String, mapFunc: T => V) extends Function[T, V] { +class Mapping[-T, +V](val name: String, mapFunc: T => V) + extends Function[T, V] { override def apply(t: T): V = mapFunc(t) } -class PercentageAggregation[T](name: String, valueAggregation: Aggregation[T, Int]) extends Aggregation[T, Long]( - name, - t => (valueAggregation(t) * 100.0 / t.size).round -) +class PercentageAggregation[T]( + name: String, + valueAggregation: Aggregation[T, Int] +) extends Aggregation[T, Long]( + name, + t => (valueAggregation(t) * 100.0 / t.size).round + ) -class StringColumnOrdering(column: Int, reverse: Boolean = false) extends Ordering[Seq[Any]] { +class StringColumnOrdering(column: Int, reverse: Boolean = false) + extends Ordering[Seq[Any]] { override def compare(x: Seq[Any], y: Seq[Any]): Int = x(column).toString .compareTo( @@ -30,7 +37,8 @@ class StringColumnOrdering(column: Int, reverse: Boolean = false) extends Orderi ) * (if (reverse) -1 else 1) } -class LongColumnOrdering(column: Int, reverse: Boolean = false) extends Ordering[Seq[Any]] { +class LongColumnOrdering(column: Int, reverse: Boolean = false) + extends Ordering[Seq[Any]] { override def compare(x: Seq[Any], y: Seq[Any]): Int = x(column).toString.toLong @@ -39,22 +47,22 @@ class LongColumnOrdering(column: Int, reverse: Boolean = false) extends Ordering ) * (if (reverse) -1 else 1) } - class Records[T, RK]( - data: Iterable[T], - rowGrouping: Grouping[T, RK], - columnAggregations: Seq[Aggregation[T, Any]], - rowOrdering: Ordering[Seq[Any]], - rowKeyMapping: Option[Mapping[RK, Any]] = None - ) { + data: Iterable[T], + rowGrouping: Grouping[T, RK], + columnAggregations: Seq[Aggregation[T, Any]], + rowOrdering: Ordering[Seq[Any]], + rowKeyMapping: Option[Mapping[RK, Any]] = None +) { val byRowKey: Map[RK, Iterable[T]] = rowGrouping(data) - val rows = byRowKey.map { - case (rk, tSeq) => + val rows = byRowKey + .map { case (rk, tSeq) => Seq(rk) ++ rowKeyMapping.map(_(rk)).toSeq ++ columnAggregations.map(_(tSeq)) - }.toSeq + } + .toSeq .sorted(rowOrdering) val total: Seq[Any] = Seq("Total") ++ rowKeyMapping.map(_ => "Total").toSeq ++ diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/AuthorMonuments.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/AuthorMonuments.scala index 5e0b18b8..71203d7f 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/AuthorMonuments.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/AuthorMonuments.scala @@ -5,9 +5,11 @@ import org.scalawiki.dto.markup.Table import org.scalawiki.wlx.ImageDB import org.scalawiki.wlx.stat._ -class AuthorMonuments(val stat: ContestStat, - gallery: Boolean = false, - commons: Option[MwBot] = None) extends Reporter { +class AuthorMonuments( + val stat: ContestStat, + gallery: Boolean = false, + commons: Option[MwBot] = None +) extends Reporter { override def contest = stat.contest @@ -20,7 +22,9 @@ class AuthorMonuments(val stat: ContestStat, val totalImageDb = stat.totalImageDb.get - val oldImages = totalImageDb.images.filter(image => !currentImageIds.contains(image.pageId.get)) + val oldImages = totalImageDb.images.filter(image => + !currentImageIds.contains(image.pageId.get) + ) val oldImageDb = new ImageDB(contest, oldImages, stat.monumentDb) @@ -32,7 +36,11 @@ class AuthorMonuments(val stat: ContestStat, def withRating: Boolean = rater.withRating - def rowData(ids: Set[String], images: Int, userOpt: Option[String] = None): Seq[String] = { + def rowData( + ids: Set[String], + images: Int, + userOpt: Option[String] = None + ): Seq[String] = { val objects = optionalUserGalleryLink(ids.size, userOpt) @@ -42,17 +50,25 @@ class AuthorMonuments(val stat: ContestStat, (ids intersect oldIds intersect oldAuthorIds).size, // existing (ids intersect oldIds -- oldAuthorIds).size, // new for author (ids -- oldIds).size, // new - userOpt.map { user => - rater.rateMonumentIds(ids, user).toString - }.getOrElse(ids.size) + userOpt + .map { user => + rater.rateMonumentIds(ids, user).toString + } + .getOrElse(ids.size) ) } else Nil val byRegion = country.regionIds.toSeq.map { regionId => val regionIds = monumentDb.byRegion(regionId).map(_.id).toSet val currentIds = regionIds intersect ids - val rating: Double = userOpt.map(user => rater.rateMonumentIds(currentIds, user)).getOrElse(currentIds.size) - optionalUserGalleryLink(rating, userOpt, country.regionById.get(regionId).map(_.name)) + val rating: Double = userOpt + .map(user => rater.rateMonumentIds(currentIds, user)) + .getOrElse(currentIds.size) + optionalUserGalleryLink( + rating, + userOpt, + country.regionById.get(regionId).map(_.name) + ) } ((objects +: ratingColumns :+ images) ++ byRegion).map(_.toString) @@ -61,7 +77,8 @@ class AuthorMonuments(val stat: ContestStat, override def table: Table = { val columns = Seq("User", "Objects pictured") ++ - (if (withRating) Seq("Existing", "New for author", "New", "Rating") else Seq.empty) ++ + (if (withRating) Seq("Existing", "New for author", "New", "Rating") + else Seq.empty) ++ Seq("Photos uploaded") ++ country.regionNames @@ -76,11 +93,16 @@ class AuthorMonuments(val stat: ContestStat, } val authorsData = authors.map { user => - val noTemplateUser = user.replaceAll("\\{\\{", "").replaceAll("\\}\\}", "") + val noTemplateUser = + user.replaceAll("\\{\\{", "").replaceAll("\\}\\}", "") val userLink = s"[[User:$noTemplateUser|$noTemplateUser]]" userLink +: - rowData(imageDb._byAuthorAndId.by(user).keys, imageDb._byAuthor.by(user).size, Some(user)) + rowData( + imageDb._byAuthorAndId.by(user).keys, + imageDb._byAuthor.by(user).size, + Some(user) + ) } reportUnknownPlaces() @@ -90,13 +112,17 @@ class AuthorMonuments(val stat: ContestStat, def reportUnknownPlaces() = { (rater match { - case sum: RateSum => sum.raters.collectFirst { case r: NumberOfImagesInPlaceBonus => r } + case sum: RateSum => + sum.raters.collectFirst { case r: NumberOfImagesInPlaceBonus => r } case r: NumberOfImagesInPlaceBonus => Some(r) - case _ => None + case _ => None }).map { reportRater => - reportRater.unknownPlaceMonumentsByAuthor.toSeq.sortBy(-_._2.size).map{ case (author, monuments) => + reportRater.unknownPlaceMonumentsByAuthor.toSeq + .sortBy(-_._2.size) + .map { case (author, monuments) => s"# $author, ${monuments.size} unknown place ids: ${monuments.toSeq.sorted.mkString(", ")}" - }.mkString("\n") + } + .mkString("\n") }.map { text => for (bot <- commons) { bot.page(page + " unknown places").edit(text) @@ -104,7 +130,11 @@ class AuthorMonuments(val stat: ContestStat, } } - private def optionalUserGalleryLink(number: Double, userOpt: Option[String], regionOpt: Option[String] = None) = { + private def optionalUserGalleryLink( + number: Double, + userOpt: Option[String], + regionOpt: Option[String] = None + ) = { if (gallery && userOpt.isDefined && number > 0) { userGalleryLink(number, userOpt, regionOpt) } else { @@ -112,13 +142,29 @@ class AuthorMonuments(val stat: ContestStat, } } - private def userGalleryLink(number: Double, userOpt: Option[String], regionOpt: Option[String] = None) = { - val noTemplateUser = userOpt.get.split("\\|").last.replaceAll("\\{\\{", "").replaceAll("\\}\\}", "") - - val galleryPage = "Commons:" + contest.name + "/" + noTemplateUser + regionOpt.fold("") { region => - "#" + region.replaceAll(" ", "_") - } - val galleryText = Output.galleryByRegionAndId(imageDb.monumentDb.get, imageDb.subSet(_.author == userOpt), oldImageDb, rater, stat.config.exists(_.previousYearsGallery)) + private def userGalleryLink( + number: Double, + userOpt: Option[String], + regionOpt: Option[String] = None + ) = { + val noTemplateUser = userOpt.get + .split("\\|") + .last + .replaceAll("\\{\\{", "") + .replaceAll("\\}\\}", "") + + val galleryPage = + "Commons:" + contest.name + "/" + noTemplateUser + regionOpt.fold("") { + region => + "#" + region.replaceAll(" ", "_") + } + val galleryText = Output.galleryByRegionAndId( + imageDb.monumentDb.get, + imageDb.subSet(_.author == userOpt), + oldImageDb, + rater, + stat.config.exists(_.previousYearsGallery) + ) for (bot <- commons if regionOpt.isEmpty) { bot.page(galleryPage).edit(galleryText) @@ -126,4 +172,4 @@ class AuthorMonuments(val stat: ContestStat, "[[" + galleryPage + "|" + number + "]]" } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Charts.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Charts.scala index 868f4629..0ce7ee8e 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Charts.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Charts.scala @@ -29,13 +29,14 @@ class Charts extends WithBot { ChartUtilities.saveChartAsPNG(new File(filename), chart, width, height) } - /** - * Creates a chart. - * - * @param dataset the dataset. - * - * @return A chart. - */ + /** Creates a chart. + * + * @param dataset + * the dataset. + * + * @return + * A chart. + */ def createPieChart(dataset: PieDataset, title: String) = { val chart = ChartFactory.createPieChart( @@ -46,7 +47,6 @@ class Charts extends WithBot { false ) - val plot = chart.getPlot.asInstanceOf[PiePlot] plot.setLabelFont(new Font("SansSerif", Font.BOLD, 14)) plot.setNoDataMessage("No data available") @@ -62,9 +62,13 @@ class Charts extends WithBot { plot.setSectionPaint("2014 & 2015", blend(color2013, color2014)) plot.setSectionPaint("2015", color2014) plot.setSectionPaint("2013 & 2015", blend(color2012, color2014)) - plot.setSectionPaint("2013 & 2014 & 2015", new Color(0x99CC00)) + plot.setSectionPaint("2013 & 2014 & 2015", new Color(0x99cc00)) - val gen = new StandardPieSectionLabelGenerator("{0}:\n{1} ({2})", new DecimalFormat("0"), new DecimalFormat("0%")) + val gen = new StandardPieSectionLabelGenerator( + "{0}:\n{1} ({2})", + new DecimalFormat("0"), + new DecimalFormat("0%") + ) plot.setLabelGenerator(gen) chart } @@ -77,29 +81,31 @@ class Charts extends WithBot { new Color(r.toInt, g.toInt, b.toInt) } - /** - * Returns a sample dataset. - * - * @return The dataset. - */ + /** Returns a sample dataset. + * + * @return + * The dataset. + */ def createTotalDataset(years: Seq[Int], values: Seq[Int]) = { val category1 = "Всього" val dataset = new DefaultCategoryDataset() - years.zip(values).foreach { case (year, value) => dataset.addValue(value, year, category1) } + years.zip(values).foreach { case (year, value) => + dataset.addValue(value, year, category1) + } dataset } - - /** - * Creates a sample chart. - * - * @param dataset the dataset. - * - * @return The chart. - */ + /** Creates a sample chart. + * + * @param dataset + * the dataset. + * + * @return + * The chart. + */ def createChart(dataset: CategoryDataset, rangeAxisLabel: String) = { - //ChartFactory.setChartTheme(StandardChartTheme.createDarknessTheme()); + // ChartFactory.setChartTheme(StandardChartTheme.createDarknessTheme()); // create the chart... val chart = ChartFactory.createBarChart( @@ -121,7 +127,9 @@ class Charts extends WithBot { plot.setBackgroundPaint(Color.white) // plot.setDomainGridlinePaint(Color.white) // plot.setRangeGridlinePaint(Color.white) - plot.getRenderer.asInstanceOf[BarRenderer].setBarPainter(new StandardBarPainter()) + plot.getRenderer + .asInstanceOf[BarRenderer] + .setBarPainter(new StandardBarPainter()) plot.setDomainGridlinePaint(Color.lightGray) plot.setRangeGridlinePaint(Color.lightGray) @@ -147,15 +155,24 @@ class Charts extends WithBot { } // up to 3 years - def intersectionDiagram(title: String, filename: String, years: Seq[Int], idsSeq: Seq[Set[String]], width: Int, height: Int) { + def intersectionDiagram( + title: String, + filename: String, + years: Seq[Int], + idsSeq: Seq[Set[String]], + width: Int, + height: Int + ) { val pieDataset = intersectionDataSet(years, idsSeq) val pieChart = createPieChart(pieDataset, title) saveCharts(pieChart, filename, width, height) } - - def intersectionDataSet(years: Seq[Int], idsSeq: Seq[Set[String]]): DefaultPieDataset = { + def intersectionDataSet( + years: Seq[Int], + idsSeq: Seq[Set[String]] + ): DefaultPieDataset = { require(years.nonEmpty, "years should not be empty") require(idsSeq.nonEmpty, "idsSeq should not be empty") require(years.size == idsSeq.size, "years and idsSeq should have same size") @@ -163,41 +180,41 @@ class Charts extends WithBot { val intersectAll = idsSeq.reduce(_ intersect _) val unionAll = idsSeq.reduce(_ ++ _) - val yearsCounts = unionAll.map { id => id -> idsSeq.count(_.contains(id)) }.toMap + val yearsCounts = unionAll.map { id => + id -> idsSeq.count(_.contains(id)) + }.toMap - val only = idsSeq.zipWithIndex.map { - case (ids, i) => ids -- removeByIndex(idsSeq, i).foldLeft(Set.empty[String])(_ ++ _) + val only = idsSeq.zipWithIndex.map { case (ids, i) => + ids -- removeByIndex(idsSeq, i).foldLeft(Set.empty[String])(_ ++ _) } - val idsMap = unionAll.map { - id => - id -> years.zip(idsSeq).flatMap { - case (year, set) => - if (set.contains(id)) Some(year) else None - } + val idsMap = unionAll.map { id => + id -> years.zip(idsSeq).flatMap { case (year, set) => + if (set.contains(id)) Some(year) else None + } }.toMap - val yearsMap = idsMap.mapValues(_.mkString(" & ")).groupBy(_._2).mapValues(_.map(_._1)) + val yearsMap = + idsMap.mapValues(_.mkString(" & ")).groupBy(_._2).mapValues(_.map(_._1)) val pieDataset = new DefaultPieDataset() - years.zipWithIndex.foreach { - case (year, i) => - - pieDataset.setValue(year.toString, only(i).size) + years.zipWithIndex.foreach { case (year, i) => + pieDataset.setValue(year.toString, only(i).size) - (i + 1).until(years.size).foreach { j => - val intersection = (idsSeq(i) intersect idsSeq(j)).filter { id => yearsCounts(id) == 2 } - pieDataset.setValue(s"$year & ${years(j)}", intersection.size) + (i + 1).until(years.size).foreach { j => + val intersection = (idsSeq(i) intersect idsSeq(j)).filter { id => + yearsCounts(id) == 2 } + pieDataset.setValue(s"$year & ${years(j)}", intersection.size) + } } if (years.size > 3) { - years.zipWithIndex.foreach { - case (year, i) => - val withoutOne = removeByIndex(years, i) - val key = withoutOne.mkString(" & ") - pieDataset.setValue(key, yearsMap(key).size) + years.zipWithIndex.foreach { case (year, i) => + val withoutOne = removeByIndex(years, i) + val key = withoutOne.mkString(" & ") + pieDataset.setValue(key, yearsMap(key).size) } } @@ -208,13 +225,13 @@ class Charts extends WithBot { pieDataset } - def removeByIndex[T](seq: Seq[T], i: Int): Seq[T] = seq.take(i) ++ seq.drop(i + 1) + def removeByIndex[T](seq: Seq[T], i: Int): Seq[T] = + seq.take(i) ++ seq.drop(i + 1) def saveCharts(chart: JFreeChart, name: String, width: Int, height: Int) { - //charts.saveAsJPEG(chart, name + ".jpg", width, height) + // charts.saveAsJPEG(chart, name + ".jpg", width, height) saveAsPNG(chart, name + ".png", width, height) - //charts.saveAsSVG(chart, name + ".svg", width, height) + // charts.saveAsSVG(chart, name + ".svg", width, height) } - } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/DesnaRegionSpecialNomination.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/DesnaRegionSpecialNomination.scala index c83ed322..f93779a7 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/DesnaRegionSpecialNomination.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/DesnaRegionSpecialNomination.scala @@ -7,27 +7,60 @@ import org.scalawiki.wlx.dto.Country.Ukraine import java.io.InputStreamReader class DesnaRegionSpecialNomination(data: Seq[List[String]]) { - val UkraineKatotth: Country = new Country("UA", "Ukraine", Seq("uk"), - Katotth.regions(() => Some(UkraineKatotth))) - - val places: Seq[List[String]] = data.map(_.filter(_.trim.nonEmpty)).filter(_.size == 4) - - def getOblast(row: List[String], country: Country, suffix: String): Option[AdmDivision] = { + val UkraineKatotth: Country = new Country( + "UA", + "Ukraine", + Seq("uk"), + Katotth.regions(() => Some(UkraineKatotth)) + ) + + val places: Seq[List[String]] = + data.map(_.filter(_.trim.nonEmpty)).filter(_.size == 4) + + def getOblast( + row: List[String], + country: Country, + suffix: String + ): Option[AdmDivision] = { getOblast(row.head, country, suffix) } - def getOblast(name: String, country: Country, suffix: String): Option[AdmDivision] = { + def getOblast( + name: String, + country: Country, + suffix: String + ): Option[AdmDivision] = { country.byName(name.trim + suffix, 0, 1).headOption } - def getRaion(row: List[String], country: Country, suffixes: Seq[String]): Seq[AdmDivision] = { - (for (oblast <- getOblast(row, country: Country, suffixes.headOption.getOrElse(""))) - yield getRaion(oblast, row(1) + suffixes.lastOption.getOrElse(""), country: Country)).getOrElse(Nil) + def getRaion( + row: List[String], + country: Country, + suffixes: Seq[String] + ): Seq[AdmDivision] = { + (for ( + oblast <- getOblast( + row, + country: Country, + suffixes.headOption.getOrElse("") + ) + ) + yield getRaion( + oblast, + row(1) + suffixes.lastOption.getOrElse(""), + country: Country + )).getOrElse(Nil) } - def getRaion(oblast: AdmDivision, name: String, country: Country): Seq[AdmDivision] = { + def getRaion( + oblast: AdmDivision, + name: String, + country: Country + ): Seq[AdmDivision] = { val candidates = country.byIdAndName(oblast.code.take(2), name) - candidates.filter(_.regionType.exists(_.code == "P") || name.endsWith(" район")) + candidates.filter( + _.regionType.exists(_.code == "P") || name.endsWith(" район") + ) } def getRaion(row: List[String]): Seq[AdmDivision] = { @@ -55,14 +88,26 @@ class DesnaRegionSpecialNomination(data: Seq[List[String]]) { twoTries } - def getPlace(row: List[String], country: Country, suffixes: Seq[String]): Seq[AdmDivision] = { + def getPlace( + row: List[String], + country: Country, + suffixes: Seq[String] + ): Seq[AdmDivision] = { (for { oblast <- getOblast(row, country, suffixes.headOption.getOrElse("")); - raion <- getRaion(oblast, row(1) + suffixes.lastOption.getOrElse(""), country).headOption + raion <- getRaion( + oblast, + row(1) + suffixes.lastOption.getOrElse(""), + country + ).headOption } yield getPlace(raion, row(3), country)).getOrElse(Nil) } - def getPlace(raion: AdmDivision, name: String, country: Country): Seq[AdmDivision] = { + def getPlace( + raion: AdmDivision, + name: String, + country: Country + ): Seq[AdmDivision] = { val code = raion.code.take(2) + "-" + raion.code.drop(2) country.byIdAndName(code, name) } @@ -74,7 +119,10 @@ object DesnaRegionSpecialNomination { } private def readCsv(filename: String): Seq[List[String]] = { - val reader = new InputStreamReader(getClass.getResourceAsStream(s"/$filename.csv"), "UTF-8") + val reader = new InputStreamReader( + getClass.getResourceAsStream(s"/$filename.csv"), + "UTF-8" + ) val csv = CSVReader.open(reader) csv.all().tail } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Gallery.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Gallery.scala index c67fc419..ad606190 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Gallery.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Gallery.scala @@ -14,7 +14,8 @@ object Gallery { def gallery(imageDb: ImageDB, mDb: MonumentDB): Unit = { - val monuments = monumentsPictured(imageDb, (size: Int) => 4 <= size && size <= 9) + val monuments = + monumentsPictured(imageDb, (size: Int) => 4 <= size && size <= 9) val byReg1 = monuments.groupBy(_.split("-").head) val reg1Links = for (reg1 <- byReg1.keySet.toSeq.sorted) yield { val reg1Ids = byReg1(reg1) @@ -23,28 +24,49 @@ object Gallery { val title = s"Commons:WLM-UA/$reg1" commons.page(title).edit(reg2Text) - val regionName = Ukraine.regionById.get(reg1).map(_.fullName).getOrElse("") + val regionName = + Ukraine.regionById.get(reg1).map(_.fullName).getOrElse("") s"* [[$title]] $regionName" } } } - def monumentsPictured(imageDb: ImageDB, sizePredicate: Int => Boolean): Set[String] = { - val picturesPerMonument = imageDb.images.flatMap(_.monumentIds).groupBy(identity).mapValues(_.size).toMap + def monumentsPictured( + imageDb: ImageDB, + sizePredicate: Int => Boolean + ): Set[String] = { + val picturesPerMonument = imageDb.images + .flatMap(_.monumentIds) + .groupBy(identity) + .mapValues(_.size) + .toMap picturesPerMonument.filter { case (_, size) => sizePredicate(size) }.keySet } - def pageTitle(reg1: String, reg2: String, index: Int) = s"Commons:WLM-UA/$reg1/$reg2/$index" + def pageTitle(reg1: String, reg2: String, index: Int) = + s"Commons:WLM-UA/$reg1/$reg2/$index" - def getReg2Links(imageDb: ImageDB, mDb: MonumentDB, reg1: String, reg1Ids: Set[String]): Seq[String] = { - val byReg2 = reg1Ids.filter(_.split("-").length >= 2).groupBy(_.split("-")(1)) + def getReg2Links( + imageDb: ImageDB, + mDb: MonumentDB, + reg1: String, + reg1Ids: Set[String] + ): Seq[String] = { + val byReg2 = + reg1Ids.filter(_.split("-").length >= 2).groupBy(_.split("-")(1)) for (reg2 <- byReg2.keySet.toSeq.sorted) yield { - val reg2IdsSliding = byReg2(reg2).toSeq.sorted.sliding(monumentsPerPage, monumentsPerPage).zipWithIndex.toSeq + val reg2IdsSliding = byReg2(reg2).toSeq.sorted + .sliding(monumentsPerPage, monumentsPerPage) + .zipWithIndex + .toSeq for ((reg2Ids, index) <- reg2IdsSliding) yield { val texts = for (monumentId <- reg2Ids) yield { Try { monumentInfo(mDb.byId(monumentId).get) + - imageDb.byId(monumentId).map(_.title).sorted + imageDb + .byId(monumentId) + .map(_.title) + .sorted .mkString("\n\n", "\n", "\n\n") }.getOrElse("") } @@ -52,8 +74,11 @@ object Gallery { val title = pageTitle(reg1, reg2, index + 1) commons.page(title).edit(text) } - val regionName = Ukraine.byMonumentId(s"$reg1-$reg2").map(_.fullName).getOrElse("") - val titles = (1 to reg2IdsSliding.size).map(index => s"[[${pageTitle(reg1, reg2, index)}|$index]]").mkString(", ") + val regionName = + Ukraine.byMonumentId(s"$reg1-$reg2").map(_.fullName).getOrElse("") + val titles = (1 to reg2IdsSliding.size) + .map(index => s"[[${pageTitle(reg1, reg2, index)}|$index]]") + .mkString(", ") s"* $titles $regionName" } } @@ -64,6 +89,8 @@ object Gallery { |* Адреса: ${m.place.getOrElse("")} |* Населений пункт: ${m.cityName} |${m.gallery.map(g => s"* Категорія: [[:Category:$g|$g]]").getOrElse("")} - |${m.article.map(a => s"* Стаття: [[:uk:$a|$a]]").getOrElse("")}""".stripMargin + |${m.article + .map(a => s"* Стаття: [[:uk:$a|$a]]") + .getOrElse("")}""".stripMargin } } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MonumentDbStat.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MonumentDbStat.scala index c2f6a464..6b48b7e8 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MonumentDbStat.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MonumentDbStat.scala @@ -22,7 +22,8 @@ class MonumentDbStat { ) def getStat(monumentDbs: Seq[MonumentDB]) = { - val title = monumentDbs.head.contest.contestType.name + " database statistics" + val title = + monumentDbs.head.contest.contestType.name + " database statistics" val data = monumentDbs.map { db => val country = db.contest.country.code diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MonumentsPicturedByRegion.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MonumentsPicturedByRegion.scala index cd7d805a..49a364b8 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MonumentsPicturedByRegion.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MonumentsPicturedByRegion.scala @@ -7,15 +7,18 @@ import org.scalawiki.wlx.dto.AdmDivision import org.scalawiki.wlx.stat.ContestStat import org.scalawiki.wlx.{ImageDB, MonumentDB} -class MonumentsPicturedByRegion(val stat: ContestStat, - uploadImages: Boolean = false, - regionParam: Option[AdmDivision] = None, - gallery: Boolean = false) - extends Reporter { - - def this(imageDbs: Seq[ImageDB], - totalImageDb: Option[ImageDB], - monumentDb: MonumentDB) = { +class MonumentsPicturedByRegion( + val stat: ContestStat, + uploadImages: Boolean = false, + regionParam: Option[AdmDivision] = None, + gallery: Boolean = false +) extends Reporter { + + def this( + imageDbs: Seq[ImageDB], + totalImageDb: Option[ImageDB], + monumentDb: MonumentDB + ) = { this( ContestStat( monumentDb.contest, @@ -26,7 +29,8 @@ class MonumentsPicturedByRegion(val stat: ContestStat, imageDbs.lastOption.orElse(totalImageDb), totalImageDb, imageDbs - )) + ) + ) } val yearSeq: Seq[Int] = stat.yearSeq.reverse @@ -64,19 +68,23 @@ class MonumentsPicturedByRegion(val stat: ContestStat, } (if (parentRegion == contest.country) { - Seq("Region", - "Objects in lists", - s"$numYears years total", - s"$numYears years percentage") + Seq( + "Region", + "Objects in lists", + s"$numYears years total", + s"$numYears years percentage" + ) } else { Seq("Region (KOATUU)", "Objects in lists", "Total", s"Total percentage") }) ++ yearsColumns } - def regionData(regionId: String, - regionalDetails: Boolean = regionalDetails, - totalImageDb: Option[ImageDB], - monumentDb: MonumentDB): Seq[String] = { + def regionData( + regionId: String, + regionalDetails: Boolean = regionalDetails, + totalImageDb: Option[ImageDB], + monumentDb: MonumentDB + ): Seq[String] = { val picturedMonumentsInRegionSet = totalImageDb .map(_.idsByRegion(regionId)) .getOrElse(Set.empty) ++ @@ -113,7 +121,8 @@ class MonumentsPicturedByRegion(val stat: ContestStat, .flatten val newImagesDb = stat.currentYearImageDb.get.subSet(i => - i.monumentIds.exists(newIds.contains)) + i.monumentIds.exists(newIds.contains) + ) if (gallery) { bot @@ -135,15 +144,19 @@ class MonumentsPicturedByRegion(val stat: ContestStat, val child = new MonumentsPicturedByRegion( stat, false, - parentRegion.byMonumentId(regionId), gallery) + parentRegion.byMonumentId(regionId), + gallery + ) child.updateWiki(bot) } columnData } - def monumentsPicturedTable(totalImageDb: Option[ImageDB], - monumentDb: MonumentDB): Table = { + def monumentsPicturedTable( + totalImageDb: Option[ImageDB], + monumentDb: MonumentDB + ): Table = { val regionIds = parentRegion.regions.map(_.code) val rows = regionIds .map(regionData(_, regionalDetails, totalImageDb, monumentDb)) @@ -157,26 +170,30 @@ class MonumentsPicturedByRegion(val stat: ContestStat, val ids = stat.mapYears(_.ids).reverse val photoSize = stat.mapYears(_.images.size).reverse - val totalByYear = ids.zip(photoSize).zipWithIndex.flatMap { - case ((ids, photos), index) => + val totalByYear = + ids.zip(photoSize).zipWithIndex.flatMap { case ((ids, photos), index) => Seq(ids.size, photos) ++ (if (index == 0) Seq( (ids -- oldImagesMonumentIds).size ) else Nil) - } + } val totalData = if (parentRegion == contest.country) { Seq( - Seq("Total", - allMonuments.toString, - picturedMonuments.toString, - (if (allMonuments != 0) 100 * picturedMonuments / allMonuments - else 0).toString) ++ totalByYear.map(_.toString)) + Seq( + "Total", + allMonuments.toString, + picturedMonuments.toString, + (if (allMonuments != 0) 100 * picturedMonuments / allMonuments + else 0).toString + ) ++ totalByYear.map(_.toString) + ) } else { Seq( Seq("Sum") ++ rows.head.indices.tail.map(i => - rows.map(_(i).toInt).sum.toString), + rows.map(_(i).toInt).sum.toString + ), regionData(parentRegion.code, false, totalImageDb, monumentDb) ) } @@ -208,8 +225,8 @@ class MonumentsPicturedByRegion(val stat: ContestStat, val shortRegionName = regionName .replaceAll("область", "") .replaceAll("Автономна Республіка", "АР") - picturedIds.zipWithIndex.foreach { - case (n, i) => dataset.addValue(n, yearSeq(i), shortRegionName) + picturedIds.zipWithIndex.foreach { case (n, i) => + dataset.addValue(n, yearSeq(i), shortRegionName) } } @@ -220,23 +237,28 @@ class MonumentsPicturedByRegion(val stat: ContestStat, val filenamePrefix = contest.name.replace("_", "") Some( - regionalStatImages(filenamePrefix, - categoryName, - yearSeq, - dataset, - ids, - idsSize, - uploadImages)) + regionalStatImages( + filenamePrefix, + categoryName, + yearSeq, + dataset, + ids, + idsSize, + uploadImages + ) + ) } else None } - def regionalStatImages(filenamePrefix: String, - categoryName: String, - yearSeq: Seq[Int], - dataset: DefaultCategoryDataset, - ids: Seq[Set[String]], - idsSize: Seq[Int], - uploadImages: Boolean = false): String = { + def regionalStatImages( + filenamePrefix: String, + categoryName: String, + yearSeq: Seq[Int], + dataset: DefaultCategoryDataset, + ids: Seq[Set[String]], + idsSize: Seq[Int], + uploadImages: Boolean = false + ): String = { val images = s"\n[[File:${filenamePrefix}PicturedByYearTotal.png|$categoryName, monuments pictured by year overall|left]]" + s"\n[[File:${filenamePrefix}PicturedByYearPie.png|$categoryName, monuments pictured by year pie chart|left]]" + @@ -259,12 +281,14 @@ class MonumentsPicturedByRegion(val stat: ContestStat, bot.page(chartTotalFile).upload(chartTotalFile) val intersectionFile = filenamePrefix + "PicturedByYearPie" - charts.intersectionDiagram("Унікальність фотографій пам'яток за роками", - intersectionFile, - yearSeq, - ids, - 900, - 800) + charts.intersectionDiagram( + "Унікальність фотографій пам'яток за роками", + intersectionFile, + yearSeq, + ids, + 900, + 800 + ) bot.page(intersectionFile + ".png").upload(intersectionFile + ".png") } images diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MostPopularMonuments.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MostPopularMonuments.scala index b1bb8d67..638ae87d 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MostPopularMonuments.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/MostPopularMonuments.scala @@ -6,9 +6,11 @@ import org.scalawiki.wlx.{ImageDB, MonumentDB} class MostPopularMonuments(val stat: ContestStat) extends Reporter { - def this(imageDbs: Seq[ImageDB], - totalImageDb: Option[ImageDB], - monumentDb: MonumentDB) = { + def this( + imageDbs: Seq[ImageDB], + totalImageDb: Option[ImageDB], + monumentDb: MonumentDB + ) = { this( ContestStat( monumentDb.contest, @@ -18,8 +20,9 @@ class MostPopularMonuments(val stat: ContestStat) extends Reporter { Some(monumentDb), imageDbs.lastOption.orElse(totalImageDb), totalImageDb, - imageDbs //.headOption.map(_ => imageDbs.init).getOrElse(Seq.empty) - )) + imageDbs // .headOption.map(_ => imageDbs.init).getOrElse(Seq.empty) + ) + ) } val name = "Most photographed objects" @@ -28,13 +31,17 @@ class MostPopularMonuments(val stat: ContestStat) extends Reporter { contest.contestType.name + " in " + contest.country.name def table: Table = - mostPopularMonumentsTable(stat.dbsByYear, - stat.totalImageDb, - stat.monumentDb.get) - - def mostPopularMonumentsTable(imageDbs: Seq[ImageDB], - totalImageDb: Option[ImageDB], - monumentDb: MonumentDB): Table = { + mostPopularMonumentsTable( + stat.dbsByYear, + stat.totalImageDb, + stat.monumentDb.get + ) + + def mostPopularMonumentsTable( + imageDbs: Seq[ImageDB], + totalImageDb: Option[ImageDB], + monumentDb: MonumentDB + ): Table = { val imageDbsByYear = imageDbs.groupBy(_.contest.year) val yearSeq = imageDbsByYear.keys.toSeq.sorted @@ -46,28 +53,32 @@ class MostPopularMonuments(val stat: ContestStat) extends Reporter { val topAuthors = topN(100, authorsCountTotal) - val rows = topAuthors.zipWithIndex.map { - case (id, index) => - val monument = monumentDb.byId(id).get - Seq( - (index + 1).toString, - s"{{nobr|$id}}", - monument.name.replaceAll("\\[\\[", "[[:uk:"), - monument.galleryLink - ) ++ Seq(authorsCountTotal.getOrElse(id, 0).toString, - photosCountTotal.getOrElse(id, 0).toString) + val rows = topAuthors.zipWithIndex.map { case (id, index) => + val monument = monumentDb.byId(id).get + Seq( + (index + 1).toString, + s"{{nobr|$id}}", + monument.name.replaceAll("\\[\\[", "[[:uk:"), + monument.galleryLink + ) ++ Seq( + authorsCountTotal.getOrElse(id, 0).toString, + photosCountTotal.getOrElse(id, 0).toString + ) } val yearsStr = Seq(yearSeq.headOption, yearSeq.lastOption).flatten .mkString("(", " - ", ")") - Table(authorsColumns, - rows, - name + s" in ${stat.imageDbsByYear.size} year(s) $yearsStr") + Table( + authorsColumns, + rows, + name + s" in ${stat.imageDbsByYear.size} year(s) $yearsStr" + ) } def mostPopularMonumentsInRegions( totalImageDb: ImageDB, - monumentDb: MonumentDB): Map[String, Table] = { + monumentDb: MonumentDB + ): Map[String, Table] = { val authorsColumns = Seq("N", "Id", "Name", "Category") ++ Seq("authors", "photos") @@ -79,25 +90,28 @@ class MostPopularMonuments(val stat: ContestStat) extends Reporter { .replaceAll("область", "") .replaceAll("Автономна Республіка", "АР") - val imageDb = ImageDB(totalImageDb.contest, - totalImageDb.imagesByRegion(regionId), - Some(monumentDb)) + val imageDb = ImageDB( + totalImageDb.contest, + totalImageDb.imagesByRegion(regionId), + Some(monumentDb) + ) val photosCountTotal = imageDb.imageCountById val authorsCountTotal = imageDb.authorsCountById val topAuthors = topN(5, authorsCountTotal) - val rows = topAuthors.zipWithIndex.map { - case (id, index) => - val monument = monumentDb.byId(id).get - Seq( - (index + 1).toString, - s"{{nobr|$id}}", - monument.name.replaceAll("\\[\\[", "[[:uk:"), - monument.galleryLink - ) ++ Seq(authorsCountTotal.getOrElse(id, 0).toString, - photosCountTotal.getOrElse(id, 0).toString) + val rows = topAuthors.zipWithIndex.map { case (id, index) => + val monument = monumentDb.byId(id).get + Seq( + (index + 1).toString, + s"{{nobr|$id}}", + monument.name.replaceAll("\\[\\[", "[[:uk:"), + monument.galleryLink + ) ++ Seq( + authorsCountTotal.getOrElse(id, 0).toString, + photosCountTotal.getOrElse(id, 0).toString + ) } shortRegionName -> Table(authorsColumns, rows) @@ -109,12 +123,13 @@ class MostPopularMonuments(val stat: ContestStat) extends Reporter { val categoryText = s"\n[[Category:$category]]" - val byRegions = mostPopularMonumentsInRegions(stat.totalImageDb.get, - stat.monumentDb.get).toSeq + val byRegions = mostPopularMonumentsInRegions( + stat.totalImageDb.get, + stat.monumentDb.get + ).toSeq .sortBy(_._1) - .map { - case (name, table) => - s"\n==$name==\n" + table.asWiki + .map { case (name, table) => + s"\n==$name==\n" + table.asWiki } .mkString diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/NewMonuments.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/NewMonuments.scala index b593f1e0..78584277 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/NewMonuments.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/NewMonuments.scala @@ -10,9 +10,10 @@ class NewMonuments(val stat: ContestStat) extends Reporter { val ids: Seq[Set[String]] = sorted.map(_.ids) - val oldsIds: Seq[Set[String]] = Seq(Set.empty[String]) ++ (1 until ids.size).map { i => - (0 until i).flatMap(ids).toSet - } + val oldsIds: Seq[Set[String]] = + Seq(Set.empty[String]) ++ (1 until ids.size).map { i => + (0 until i).flatMap(ids).toSet + } val newIds: Seq[Set[String]] = ids.indices.map { i => ids(i) -- oldsIds(i) } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Output.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Output.scala index e0f705de..d20c11e0 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Output.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/Output.scala @@ -10,103 +10,164 @@ import scala.concurrent.ExecutionContext object Output { - def monumentsByType(/*imageDbs: Seq[ImageDB], totalImageDb: ImageDB,*/ monumentDb: MonumentDB) = { + def monumentsByType( + /*imageDbs: Seq[ImageDB], totalImageDb: ImageDB,*/ monumentDb: MonumentDB + ) = { val regions = monumentDb.contest.country.regionById - for ((typ, size) <- monumentDb._byType.mapValues(_.size).toSeq.sortBy(-_._2)) { + for ( + (typ, size) <- monumentDb._byType.mapValues(_.size).toSeq.sortBy(-_._2) + ) { val byRegion = monumentDb._byTypeAndRegion(typ) - val regionStat = byRegion.toSeq.sortBy(-_._2.size).map { - case (regionId, monuments) => + val regionStat = byRegion.toSeq + .sortBy(-_._2.size) + .map { case (regionId, monuments) => val byReg1 = s"${regions(regionId)}: ${monuments.size}" val byReg2 = if (byRegion.size == 1) { val byReg2Stat = monuments.groupBy(m => m.id.substring(0, 6)) - byReg2Stat.toSeq.sortBy(-_._2.size).map { - case (regionId2, monuments2) => + byReg2Stat.toSeq + .sortBy(-_._2.size) + .map { case (regionId2, monuments2) => s"$regionId2: ${monuments2.size}" - }.mkString("(", ", ", ")") + } + .mkString("(", ", ", ")") } else "" byReg1 + byReg2 - }.mkString(", ") + } + .mkString(", ") println(s"$typ: ${monumentDb._byType(typ).size}, $regionStat") } } - def galleryByRegionAndId(monumentDb: MonumentDB, authorImageDb: ImageDB, oldImageDb: ImageDB, rater: Rater, previousImageGallery: Boolean): String = { + def galleryByRegionAndId( + monumentDb: MonumentDB, + authorImageDb: ImageDB, + oldImageDb: ImageDB, + rater: Rater, + previousImageGallery: Boolean + ): String = { val contest = monumentDb.contest val country = contest.country - val regionIds = country.regionIds.filter(id => authorImageDb.idsByRegion(id).nonEmpty) + val regionIds = + country.regionIds.filter(id => authorImageDb.idsByRegion(id).nonEmpty) val author = authorImageDb.authors.head - val sansIneligible = authorImageDb.copy(images = authorImageDb.sansIneligible) + val sansIneligible = + authorImageDb.copy(images = authorImageDb.sansIneligible) val ineligible = authorImageDb.copy(images = authorImageDb.ineligible) - val previousImageDb = if (previousImageGallery) Some(oldImageDb.subSet(_.author.contains(author))) else None + val previousImageDb = + if (previousImageGallery) + Some(oldImageDb.subSet(_.author.contains(author))) + else None val rateConfig = contest.rateConfig val tableHeader = "{| class=\"wikitable\"\n! rate !! base " + - (if (rateConfig.numberOfAuthorsBonus || rater.withRating) "!! authors
    bonus " else "") + + (if (rateConfig.numberOfAuthorsBonus || rater.withRating) + "!! authors
    bonus " + else "") + (if (rateConfig.numberOfImagesBonus) "!! images
    bonus " else "") + "!! objects !! ids \n|-\n" val tableTotal = rater match { case rateSum: RateSum => - val groupedTotal = sansIneligible.ids.map { id => - val rateParts = rateSum.raters.map(_.rate(id, author)) - val rateId = s"${rater.rate(id, author)} || " + rateParts.mkString(" || ") - (id, rater.rate(id, author), rateId) - }.groupBy { case (_, rate, rateId) => - (rate, rateId) - }.mapValues(_.map(_._1)).toSeq.sortBy(-_._1._1) + val groupedTotal = sansIneligible.ids + .map { id => + val rateParts = rateSum.raters.map(_.rate(id, author)) + val rateId = + s"${rater.rate(id, author)} || " + rateParts.mkString(" || ") + (id, rater.rate(id, author), rateId) + } + .groupBy { case (_, rate, rateId) => + (rate, rateId) + } + .mapValues(_.map(_._1)) + .toSeq + .sortBy(-_._1._1) s"\n== Summary ==\n$tableHeader" + - groupedTotal.map { - case ((rate, rateId), ids) => + groupedTotal + .map { case ((rate, rateId), ids) => s"| $rateId || ${ids.size} || ${ids.toSeq.sorted.mkString(", ")}" - }.mkString("\n|-\n") + "\n|}\n" + } + .mkString("\n|-\n") + "\n|}\n" case _ => "" } - tableTotal + regionIds.map { - regionId => + tableTotal + regionIds + .map { regionId => val regionName = country.regionById(regionId).name - val regionHeader = s"== [[:uk:Вікіпедія:Вікі любить пам'ятки/$regionName|$regionName]] ==" + val regionHeader = + s"== [[:uk:Вікіпедія:Вікі любить пам'ятки/$regionName|$regionName]] ==" val ids = sansIneligible.idsByRegion(regionId) - val grouped = ids.map { id => - val rateParts = rater.asInstanceOf[RateSum].raters.map(_.rate(id, author)) - val rateId = s"${rater.rate(id, author)} || " + rateParts.mkString(" || ") - (id, rater.rate(id, author), rateId) - }.groupBy { case (_, rate, rateId) => - (rate, rateId) - }.mapValues(_.map(_._1)).toSeq.sortBy(-_._1._1) + val grouped = ids + .map { id => + val rateParts = + rater.asInstanceOf[RateSum].raters.map(_.rate(id, author)) + val rateId = + s"${rater.rate(id, author)} || " + rateParts.mkString(" || ") + (id, rater.rate(id, author), rateId) + } + .groupBy { case (_, rate, rateId) => + (rate, rateId) + } + .mapValues(_.map(_._1)) + .toSeq + .sortBy(-_._1._1) val table1 = s"\n$tableHeader" + - grouped.map { - case ((rate, rateId), ids) => + grouped + .map { case ((rate, rateId), ids) => s"| $rateId || ${ids.size} || ${ids.toSeq.sorted.mkString(", ")}" - }.mkString("\n|-\n") + "\n|}\n" + } + .mkString("\n|-\n") + "\n|}\n" val rating = rater.rateMonumentIds(ids, author) val ratingStr = s"\nRating: '''$rating''' \n " regionHeader + ratingStr + table1 + - gallery("", ids, sansIneligible, monumentDb, Some(rater), author = Some(author), previousImageDb) + + gallery( + "", + ids, + sansIneligible, + monumentDb, + Some(rater), + author = Some(author), + previousImageDb + ) + (if (ineligible.idsByRegion(regionId).nonEmpty) { - gallery(s"$regionName ineligible", ineligible.idsByRegion(regionId), ineligible, monumentDb, None, - author = Some(author), None, Some("ineligible")) - } else { - "" - }) - }.mkString("\n") + gallery( + s"$regionName ineligible", + ineligible.idsByRegion(regionId), + ineligible, + monumentDb, + None, + author = Some(author), + None, + Some("ineligible") + ) + } else { + "" + }) + } + .mkString("\n") } - private def gallery(header: String, ids: Set[String], imageDb: ImageDB, monumentDb: MonumentDB, - rater: Option[Rater] = None, author: Option[String] = None, prevImages: Option[ImageDB] = None, - subHeader: Option[String] = None) = { + private def gallery( + header: String, + ids: Set[String], + imageDb: ImageDB, + monumentDb: MonumentDB, + rater: Option[Rater] = None, + author: Option[String] = None, + prevImages: Option[ImageDB] = None, + subHeader: Option[String] = None + ) = { def sizes(images: Seq[Image]): Seq[String] = { images.map { img => (for (w <- img.width; h <- img.height) yield s"$w x $h").getOrElse("") @@ -115,10 +176,11 @@ object Output { if (ids.nonEmpty) { (if (header.nonEmpty) s"\n=== $header: ${ids.size} ===\n" else "") + - ids.map { - id => + ids + .map { id => val images = imageDb.byId(id).sortBy(_.title) - val rating = rater.map(_.explain(id, author.getOrElse(""))).getOrElse("") + val rating = + rater.map(_.explain(id, author.getOrElse(""))).getOrElse("") s"\n==== ${subHeader.getOrElse("")} $id ====\n" + s"${monumentDb.byId(id).get.name.replace("[[", "[[:uk:")}\n\n" + s"$rating \n" + @@ -126,22 +188,27 @@ object Output { prevImages.fold("") { prevImagesDb => val prevImagesById = prevImagesDb.byId(id).sortBy(_.title) if (prevImagesById.nonEmpty) { - s"\n===== $id previous =====\n" + Image.gallery(prevImagesById.map(_.title), sizes(prevImagesById)) + s"\n===== $id previous =====\n" + Image.gallery( + prevImagesById.map(_.title), + sizes(prevImagesById) + ) } else "" } - }.mkString("\n") + } + .mkString("\n") } else "" } def galleryByMonumentId(imageDb: ImageDB, monumentDb: MonumentDB): String = { val ids = imageDb.ids.toSeq.sorted - ids.map { - id => + ids + .map { id => val images = imageDb.byId(id).map(_.title).sorted s"\n== $id ==\n" + s"${monumentDb.byId(id).get.name.replace("[[", "[[:uk:")}\n\n" + Image.gallery(images) - }.mkString("\n") + } + .mkString("\n") } def photoWithoutArticle(imageDb: ImageDB): String = { @@ -151,25 +218,30 @@ object Output { val all = monumentDb.monuments.filter(m => m.photo.isDefined && m.article.isEmpty && !m.name.contains("[[") && - m.types.map(_.toLowerCase).exists(_.contains("нац") - && imageDb.authorsCountById.getOrElse(m.id, 0) > 1 - && imageDb.byId(m.id).size > 2) + m.types + .map(_.toLowerCase) + .exists( + _.contains("нац") + && imageDb.authorsCountById.getOrElse(m.id, 0) > 1 + && imageDb.byId(m.id).size > 2 + ) ) val byRegion = all.groupBy(_.regionId) - val perRegion = monumentDb.contest.country.regions.sortBy(_.name).map { - region => + val perRegion = + monumentDb.contest.country.regions.sortBy(_.name).map { region => val regionHeader = s"== ${region.name} ==\n" val monuments = byRegion.getOrElse(region.code, Seq.empty) val images = monuments.flatMap(_.photo) - val descriptions = monuments.map(m => s"[[${m.name}]], ${m.city.getOrElse("")}") + val descriptions = + monuments.map(m => s"[[${m.name}]], ${m.city.getOrElse("")}") val gallery = Image.gallery(images, descriptions) regionHeader + gallery - } + } perRegion.mkString("\n") } @@ -181,7 +253,8 @@ object Output { // "Калинівка", "Козятин", "Ладижин", "Липовець", "Могилів-Подільський", "Немирів", // "Погребище", "Тульчин", "Хмільник", "Шаргород", "Ямпіль") - val cities = Seq("Баштанка", + val cities = Seq( + "Баштанка", "Вознесенськ", "Нова Одеса", "Новий Буг", @@ -193,26 +266,29 @@ object Output { val monumentDb = imageDb.monumentDb.get val all = monumentDb.monuments.filter { m => - val city = m.city.getOrElse("").replaceAll("\\[", " ").replaceAll("\\]", " ") - m.photo.isDefined && cities.map(_ + " ").exists(city.contains) && !city.contains("район") && Set("48").contains(m.regionId) + val city = + m.city.getOrElse("").replaceAll("\\[", " ").replaceAll("\\]", " ") + m.photo.isDefined && cities.map(_ + " ").exists(city.contains) && !city + .contains("район") && Set("48").contains(m.regionId) } - def cityShort(city: String) = cities.find(city.contains).getOrElse("").split(" ")(0) + def cityShort(city: String) = + cities.find(city.contains).getOrElse("").split(" ")(0) def page(city: String) = "User:Ilya/Миколаївська область/" + city all.groupBy(m => cityShort(m.city.getOrElse(""))).foreach { case (city, monuments) => - - val galleries = monuments.map { - m => - val images = imageDb.byId(m.id) - val gallery = Image.gallery(images.map(_.title)) - - s"""== ${m.name.replaceAll("\\[\\[", "[[:uk:")} == - |'''Рік:''' ${m.year.getOrElse("")}, '''Адреса:''' ${m.place.getOrElse("")}, '''Тип:''' ${m.typ.getOrElse("")}, - |'''Охоронний номер:''' ${m.stateId.getOrElse("")}\n""".stripMargin + - gallery + val galleries = monuments.map { m => + val images = imageDb.byId(m.id) + val gallery = Image.gallery(images.map(_.title)) + + s"""== ${m.name.replaceAll("\\[\\[", "[[:uk:")} == + |'''Рік:''' ${m.year.getOrElse("")}, '''Адреса:''' ${m.place + .getOrElse("")}, '''Тип:''' ${m.typ.getOrElse("")}, + |'''Охоронний номер:''' ${m.stateId + .getOrElse("")}\n""".stripMargin + + gallery } ukWiki.page(page(city)).edit(galleries.mkString("\n")) } @@ -226,8 +302,8 @@ object Output { for ((regId, byId) <- byRegionAndId.grouped) { val monuments = monumentDb.byRegion(regId).filter { m => m.types.exists(t => t == "комплекс" || t.contains("нац")) && - m.photo.nonEmpty && - m.article.isEmpty + m.photo.nonEmpty && + m.article.isEmpty } val toWrite = monuments.filter { m => val id = m.id @@ -235,14 +311,16 @@ object Output { val authors = images.flatMap(_.author).toSet authors.size > 1 && images.size > 2 } - val gallery = toWrite.map { - m => + val gallery = toWrite + .map { m => m.photo.get + "| [[" + m.name + "]]" + m.city.fold("")(", " + _) - }.mkString("", "\n", "") + } + .mkString("", "\n", "") val region = toWrite.head.page.split("/")(1) - val page = "Вікіпедія:Пам'ятки національного значення із фото і без статей/" + region + val page = + "Вікіпедія:Пам'ятки національного значення із фото і без статей/" + region MwBot.fromHost(MwBot.ukWiki).page(page).edit(gallery) } @@ -259,39 +337,47 @@ object Output { ) val byRegion = all.groupBy(_.regionId) - monumentDb.contest.country.regions.sortBy(_.name).foreach { - region => - val regionHeader = s"== ${region.name} ==\n" - - val monumentSlices = byRegion.getOrElse(region.code, Seq.empty).sliding(100, 100).zipWithIndex + monumentDb.contest.country.regions.sortBy(_.name).foreach { region => + val regionHeader = s"== ${region.name} ==\n" - for ((monuments, index) <- monumentSlices) { + val monumentSlices = byRegion + .getOrElse(region.code, Seq.empty) + .sliding(100, 100) + .zipWithIndex - val images = monuments.map(_.photo.get) - val descriptions = monuments.map(m => s"[[${m.name}]], ${m.city.getOrElse("")}") + for ((monuments, index) <- monumentSlices) { - val gallery = Image.gallery(images, descriptions) + val images = monuments.map(_.photo.get) + val descriptions = + monuments.map(m => s"[[${m.name}]], ${m.city.getOrElse("")}") - val text = regionHeader + gallery + val gallery = Image.gallery(images, descriptions) - val galleries = monuments.map { - m => - val images = imageDb.byId(m.id) - val gallery = Image.gallery(images.map(_.title)) + val text = regionHeader + gallery - s"""== ${m.name.replaceAll("\\[\\[", "[[:uk:")} == - |'''Рік:''' ${m.year.getOrElse("")}, '''Адреса:''' ${m.place.getOrElse("")}, '''Тип:''' ${m.typ.getOrElse("")}, - |'''Охоронний номер:''' ${m.stateId.getOrElse("")}\n""".stripMargin + gallery - } + val galleries = monuments.map { m => + val images = imageDb.byId(m.id) + val gallery = Image.gallery(images.map(_.title)) - val contestTitle = - imageDb.contest.contestType.name - bot.page(s"$contestTitle - ${region.name} - ${index + 1}").edit(galleries.mkString("\n")) + s"""== ${m.name.replaceAll("\\[\\[", "[[:uk:")} == + |'''Рік:''' ${m.year.getOrElse("")}, '''Адреса:''' ${m.place + .getOrElse("")}, '''Тип:''' ${m.typ.getOrElse("")}, + |'''Охоронний номер:''' ${m.stateId + .getOrElse("")}\n""".stripMargin + gallery } + + val contestTitle = + imageDb.contest.contestType.name + bot + .page(s"$contestTitle - ${region.name} - ${index + 1}") + .edit(galleries.mkString("\n")) + } } } - def byRegion(monumentDb: MonumentDB, imageDb: ImageDB)(implicit ec: ExecutionContext) = { + def byRegion(monumentDb: MonumentDB, imageDb: ImageDB)(implicit + ec: ExecutionContext + ) = { val bot = MwBot.fromHost(MwBot.ukWiki) val country = monumentDb.contest.country @@ -302,19 +388,24 @@ object Output { val regionName = region.parent().get.name + "/" + region.name val text = gallery(regionName, ids, imageDb, monumentDb) - val pageName = s"Вікіпедія:${imageDb.contest.contestType.name}/$regionName" + val pageName = + s"Вікіпедія:${imageDb.contest.contestType.name}/$regionName" bot.page(pageName).edit(text).failed.map(println) } - val byParent = byRegion.groupBy { case (region, ids) => - region.parent().get.name - }.mapValues(_.keySet.map(_.name)) + val byParent = byRegion + .groupBy { case (region, ids) => + region.parent().get.name + } + .mapValues(_.keySet.map(_.name)) byParent.map { case (parent, regions) => val pageName = s"Вікіпедія:${imageDb.contest.contestType.name}/$parent" - val text = regions.toSeq.sorted.map { regionName => - s"*[[Вікіпедія:${imageDb.contest.contestType.name}/$parent/$regionName|$regionName]]" - }.mkString("\n") + val text = regions.toSeq.sorted + .map { regionName => + s"*[[Вікіпедія:${imageDb.contest.contestType.name}/$parent/$regionName|$regionName]]" + } + .mkString("\n") bot.page(pageName).edit(text).failed.map(println) } @@ -323,17 +414,22 @@ object Output { def lessThan2MpGallery(contest: Contest, imageDb: ImageDB) = { val bot = MwBot.fromHost(MwBot.commons) val lessThan2Mp = imageDb.byMegaPixelFilterAuthorMap(_ < 2) - val gallery = new AuthorsStat().authorsImages(lessThan2Mp, imageDb.monumentDb) + val gallery = + new AuthorsStat().authorsImages(lessThan2Mp, imageDb.monumentDb) val contestPage = contest.name - bot.page(s"Commons:$contestPage/Less than 2Mp").edit(gallery, Some("updating")) + bot + .page(s"Commons:$contestPage/Less than 2Mp") + .edit(gallery, Some("updating")) } def unknownPlaces(monumentDb: MonumentDB, imageDb: ImageDB): Unit = { val picturedIds = imageDb.ids - val picturedMonuments = monumentDb.allMonuments.filter(m => picturedIds.contains(m.id)) - val picturedMonumentDb = new MonumentDB(monumentDb.contest, picturedMonuments) + val picturedMonuments = + monumentDb.allMonuments.filter(m => picturedIds.contains(m.id)) + val picturedMonumentDb = + new MonumentDB(monumentDb.contest, picturedMonuments) val unknownPlaces = picturedMonumentDb.unknownPlaces() println(s"notFound size: ${unknownPlaces.size}") @@ -343,28 +439,39 @@ object Output { .mkString("\n") val ukWiki = MwBot.fromHost(MwBot.ukWiki) - ukWiki.page(s"Вікіпедія:Вікі любить пам'ятки/notFoundPlaces-${monumentDb.contest.year}").edit(text, Some("updating")) + ukWiki + .page( + s"Вікіпедія:Вікі любить пам'ятки/notFoundPlaces-${monumentDb.contest.year}" + ) + .edit(text, Some("updating")) } def wrongIds(imageDb: ImageDB, monumentDb: MonumentDB): Unit = { val bot = MwBot.fromHost(MwBot.commons) val wrongIdImages = imageDb.images - .filterNot(image => image.monumentId.fold(false)(id => monumentDb.ids.contains(id) - || id.startsWith("88") - || id.startsWith("95") - || id.startsWith("97") - || id.startsWith("98") - || id.startsWith("99") - )) - - val notObvious = wrongIdImages.filterNot(_.categories.exists(_.startsWith("Obviously ineligible"))) + .filterNot(image => + image.monumentId.fold(false)(id => + monumentDb.ids.contains(id) + || id.startsWith("88") + || id.startsWith("95") + || id.startsWith("97") + || id.startsWith("98") + || id.startsWith("99") + ) + ) + + val notObvious = wrongIdImages.filterNot( + _.categories.exists(_.startsWith("Obviously ineligible")) + ) val contest = imageDb.contest val contestPage = contest.name val text = notObvious.map(_.title).mkString("", "\n", "") - bot.page(s"Commons:$contestPage/Images with bad ids").edit(text, Some("updating")) + bot + .page(s"Commons:$contestPage/Images with bad ids") + .edit(text, Some("updating")) } def missingIds(imageDb: ImageDB, monumentDb: MonumentDB): Unit = { @@ -372,13 +479,17 @@ object Output { val images = imageDb.images.filter(_.monumentIds.isEmpty) - val notObvious = images.filterNot(_.categories.exists(_.startsWith("Obviously ineligible"))) + val notObvious = images.filterNot( + _.categories.exists(_.startsWith("Obviously ineligible")) + ) val contest = imageDb.contest val contestPage = contest.name val text = notObvious.map(_.title).mkString("", "\n", "") - bot.page(s"Commons:$contestPage/Images with missing ids").edit(text, Some("updating")) + bot + .page(s"Commons:$contestPage/Images with missing ids") + .edit(text, Some("updating")) } def multipleIds(imageDb: ImageDB, monumentDb: MonumentDB): Unit = { @@ -389,7 +500,9 @@ object Output { val contestPage = imageDb.contest.name val text = images.map(_.title).mkString("", "\n", "") - bot.page(s"Commons:$contestPage/Images with multiple ids").edit(text, Some("updating")) + bot + .page(s"Commons:$contestPage/Images with multiple ids") + .edit(text, Some("updating")) } def regionalStat(stat: ContestStat): Unit = { @@ -401,15 +514,29 @@ object Output { val authorsStat = new AuthorsStat() - val idsStat = monumentDb.map(_ => new MonumentsPicturedByRegion(stat, uploadImages = false, gallery = true).asText).getOrElse("") - - val authorsContributed = authorsStat.authorsContributed(stat.dbsByYear, stat.totalImageDb, monumentDb) + val idsStat = monumentDb + .map(_ => + new MonumentsPicturedByRegion( + stat, + uploadImages = false, + gallery = true + ).asText + ) + .getOrElse("") + + val authorsContributed = authorsStat.authorsContributed( + stat.dbsByYear, + stat.totalImageDb, + monumentDb + ) val toc = "__TOC__" val category = s"\n[[Category:$categoryName]]" val regionalStat = toc + idsStat + authorsContributed + category - bot.page(s"Commons:$categoryName/Regional statistics").edit(regionalStat, Some("updating")) + bot + .page(s"Commons:$categoryName/Regional statistics") + .edit(regionalStat, Some("updating")) authorsStat.authorsContributedPerRegion(stat.totalImageDb.get, bot) } @@ -418,15 +545,22 @@ object Output { } def missingGallery(monumentDB: MonumentDB) = { - val allMissing = monumentDB.allMonuments.toSeq.filter(m => m.gallery.isEmpty && m.photo.nonEmpty) + val allMissing = monumentDB.allMonuments.toSeq.filter(m => + m.gallery.isEmpty && m.photo.nonEmpty + ) val grouped = allMissing.groupBy(_.page).toSeq.sortBy(_._1) - val text = s"Overall missing: ${allMissing.size}\n" + grouped.map { case (page, monuments) => - s"=== [[$page]] - ${monuments.size} ===\n" + monuments.sortBy(_.id).map { m => - s"#[[$page#${m.id}|${m.id}]] ${m.name}\n" - }.mkString + val text = s"Overall missing: ${allMissing.size}\n" + grouped.map { + case (page, monuments) => + s"=== [[$page]] - ${monuments.size} ===\n" + monuments + .sortBy(_.id) + .map { m => + s"#[[$page#${m.id}|${m.id}]] ${m.name}\n" + } + .mkString }.mkString - val pageName = s"Вікіпедія:${monumentDB.contest.contestType.name}/missingGalleriesWithImages" + val pageName = + s"Вікіпедія:${monumentDB.contest.contestType.name}/missingGalleriesWithImages" MwBot.fromHost(MwBot.ukWiki).page(pageName).edit(text, Some("updating")) } @@ -434,12 +568,15 @@ object Output { def unknownPlaces(monumentDB: MonumentDB) = { val places = monumentDB.unknownPlaces() val tables = monumentDB.unknownPlacesTables() - val text = s"Overall unknown places: ${places.size}, monuments: ${places.map(_.monuments.size).sum}" + tables.map { table => - s"\n=== [[${table.title}]] - ${table.data.size} ===\n" + - table.asWiki - }.mkString + val text = + s"Overall unknown places: ${places.size}, monuments: ${places.map(_.monuments.size).sum}" + tables.map { + table => + s"\n=== [[${table.title}]] - ${table.data.size} ===\n" + + table.asWiki + }.mkString - val pageName = s"Вікіпедія:${monumentDB.contest.contestType.name}/unknownPlaces" + val pageName = + s"Вікіпедія:${monumentDB.contest.contestType.name}/unknownPlaces" MwBot.fromHost(MwBot.ukWiki).page(pageName).edit(text, Some("updating")) } diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/RateInputDistribution.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/RateInputDistribution.scala index 4ae538cf..958c595e 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/RateInputDistribution.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/RateInputDistribution.scala @@ -3,15 +3,20 @@ package org.scalawiki.wlx.stat.reports import org.scalawiki.dto.markup.Table import org.scalawiki.wlx.stat.ContestStat -class RateInputDistribution(val stat: ContestStat, - val distribution: Map[Int, Int], - val name: String, - val headers: Seq[String]) extends Reporter { +class RateInputDistribution( + val stat: ContestStat, + val distribution: Map[Int, Int], + val name: String, + val headers: Seq[String] +) extends Reporter { override def table: Table = { - Table(headers, distribution.toSeq.sortBy(_._1).map{ - case (x1, x2) => Seq(x1, x2).map(_.toString) - }) + Table( + headers, + distribution.toSeq.sortBy(_._1).map { case (x1, x2) => + Seq(x1, x2).map(_.toString) + } + ) } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/ReporterRegistry.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/ReporterRegistry.scala index 522f5d91..b227c439 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/ReporterRegistry.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/ReporterRegistry.scala @@ -8,7 +8,9 @@ import org.scalawiki.wlx.{ImageDB, ImageFiller, MonumentDB} import scala.concurrent.ExecutionContext import scala.util.Try -class ReporterRegistry(stat: ContestStat, cfg: StatConfig)(implicit ec: ExecutionContext) { +class ReporterRegistry(stat: ContestStat, cfg: StatConfig)(implicit + ec: ExecutionContext +) { import org.scalawiki.wlx.stat.reports.{ReporterRegistry => RR} @@ -23,22 +25,23 @@ class ReporterRegistry(stat: ContestStat, cfg: StatConfig)(implicit ec: Executio // def authorsMonuments: String = // RR.authorsMonuments(stat.currentYearImageDb.get) - def authorsImages: String = RR.authorsImages(currentYearImageDb.get, monumentDb) + def authorsImages: String = + RR.authorsImages(currentYearImageDb.get, monumentDb) - def authorsContributed: String = RR.authorsContributed(stat.dbsByYear, totalImageDb, monumentDb) + def authorsContributed: String = + RR.authorsContributed(stat.dbsByYear, totalImageDb, monumentDb) def specialNominations(): String = RR.specialNominations(stat) def mostPopularMonuments: String = new MostPopularMonuments(stat).asText - def monumentsPictured: String = new MonumentsPicturedByRegion(stat, gallery = true).asText + def monumentsPictured: String = + new MonumentsPicturedByRegion(stat, gallery = true).asText def withArticles: Option[String] = RR.withArticles(monumentDb) - /** - * Outputs current year reports. - * - */ + /** Outputs current year reports. + */ def currentYear(): Unit = { for (imageDb <- currentYearImageDb) { @@ -68,7 +71,7 @@ class ReporterRegistry(stat: ContestStat, cfg: StatConfig)(implicit ec: Executio } if (cfg.fillLists && cfg.years.size == 1) { - ImageFiller.fillLists(mDb, imageDb) + ImageFiller.fillLists(mDb, imageDb) } if (cfg.missingGallery) { @@ -81,7 +84,9 @@ class ReporterRegistry(stat: ContestStat, cfg: StatConfig)(implicit ec: Executio } if (cfg.mostPopularMonuments) { - new MostPopularMonuments(stat).updateWiki(MwBot.fromHost(MwBot.commons)) + new MostPopularMonuments(stat).updateWiki( + MwBot.fromHost(MwBot.commons) + ) } } } @@ -127,15 +132,24 @@ class ReporterRegistry(stat: ContestStat, cfg: StatConfig)(implicit ec: Executio } -class NumberOfMonumentsByNumberOfPictures(val stat: ContestStat, val imageDb: ImageDB) extends Reporter { - val picturesPerMonument = imageDb.images.flatMap(_.monumentIds).groupBy(identity).values.map(_.size) - val numberOfMonumentsByNumberOfPictures = picturesPerMonument.groupBy(identity).mapValues(_.size).toSeq +class NumberOfMonumentsByNumberOfPictures( + val stat: ContestStat, + val imageDb: ImageDB +) extends Reporter { + val picturesPerMonument = + imageDb.images.flatMap(_.monumentIds).groupBy(identity).values.map(_.size) + val numberOfMonumentsByNumberOfPictures = picturesPerMonument + .groupBy(identity) + .mapValues(_.size) + .toSeq .sortBy { case (pictures, monuments) => -pictures } override val table = Table( Seq("pictures", "monuments"), - numberOfMonumentsByNumberOfPictures.map { case (pictures, monuments) => Seq(pictures.toString, monuments.toString) } + numberOfMonumentsByNumberOfPictures.map { case (pictures, monuments) => + Seq(pictures.toString, monuments.toString) + } ) override def name: String = "Number Of monuments by number of pictures" @@ -152,15 +166,17 @@ object ReporterRegistry { def authorsImages(imageDb: ImageDB, monumentDb: Option[MonumentDB]): String = new AuthorsStat().authorsImages(imageDb._byAuthor.grouped, monumentDb) - def authorsContributed(imageDbs: Seq[ImageDB], totalImageDb: Option[ImageDB], monumentDb: Option[MonumentDB]): String = + def authorsContributed( + imageDbs: Seq[ImageDB], + totalImageDb: Option[ImageDB], + monumentDb: Option[MonumentDB] + ): String = new AuthorsStat().authorsContributed(imageDbs, totalImageDb, monumentDb) def specialNominations(stat: ContestStat): String = - new SpecialNominations(stat, stat.currentYearImageDb.get).specialNomination() + new SpecialNominations(stat, stat.currentYearImageDb.get) + .specialNomination() def withArticles(monumentDb: Option[MonumentDB]): Option[String] = monumentDb.map(db => Stats.withArticles(db).asWiki("").asWiki) } - - - diff --git a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/SpecialNominations.scala b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/SpecialNominations.scala index ee296de4..ac82bfde 100644 --- a/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/SpecialNominations.scala +++ b/scalawiki-wlx/src/main/scala/org/scalawiki/wlx/stat/reports/SpecialNominations.scala @@ -19,34 +19,46 @@ class SpecialNominations(stat: ContestStat, imageDb: ImageDB) { } def nominations: Seq[SpecialNomination] = { - SpecialNomination.nominations.filter(_.years.contains(contest.year)).sortBy(_.name) + SpecialNomination.nominations + .filter(_.years.contains(contest.year)) + .sortBy(_.name) } def specialNomination(): String = { val monumentsMap = SpecialNomination.getMonumentsMap(nominations, stat) val imageDbs = nominations.map { nomination => - val specialNominationImageDb = if (monumentsMap.get(nomination).exists(_.nonEmpty)) { - imageDb.subSet(monumentsMap(nomination), withFalseIds = true) - } else { - imageDb.subSet { i => - nomination.fileTemplate.exists(i.specialNominations.contains) + val specialNominationImageDb = + if (monumentsMap.get(nomination).exists(_.nonEmpty)) { + imageDb.subSet(monumentsMap(nomination), withFalseIds = true) + } else { + imageDb.subSet { i => + nomination.fileTemplate.exists(i.specialNominations.contains) + } } - } nomination -> specialNominationImageDb }.toMap - val headers = Seq("Special nomination", "authors", - "all monuments", "special nomination monuments", "photographed monuments", "photographed special monuments", - "newly pictured monuments", "photos") + val headers = Seq( + "Special nomination", + "authors", + "all monuments", + "special nomination monuments", + "photographed monuments", + "photographed special monuments", + "newly pictured monuments", + "photos" + ) val newImageNames = imageDb.images.map(_.title).toSet val oldMonumentIds = stat.totalImageDb.get.images .filterNot(image => newImageNames.contains(image.title)) - .flatMap(_.monumentIds).toSet + .flatMap(_.monumentIds) + .toSet val rows = for (nomination <- nominations) yield { - val imagesPage = s"Commons:Images from ${contest.name} special nomination ${nomination.name}" + val imagesPage = + s"Commons:Images from ${contest.name} special nomination ${nomination.name}" val imageDb = imageDbs(nomination) galleryByRegion(imagesPage + " by region", imageDb) @@ -56,7 +68,10 @@ class SpecialNominations(stat: ContestStat, imageDb: ImageDB) { nomination.name, imageDb.authors.size.toString, monumentsMap.get(nomination).map(_.size.toString).getOrElse(""), - monumentsMap.get(nomination).map(_.map(_.id).count(isSpecialNominationMonument).toString).getOrElse(""), + monumentsMap + .get(nomination) + .map(_.map(_.id).count(isSpecialNominationMonument).toString) + .getOrElse(""), imageDb.ids.size.toString, imageDb.ids.count(isSpecialNominationMonument).toString, imageDb.ids.diff(oldMonumentIds).size.toString, @@ -83,22 +98,36 @@ class SpecialNominations(stat: ContestStat, imageDb: ImageDB) { if (images.nonEmpty) { val monumentIds = images.flatMap(_.monumentIds) - val byPlace = monumentIds.groupBy { id => - monumentDb.placeByMonumentId.getOrElse(id, "Unknown") - }.mapValues(_.toSet).toMap + val byPlace = monumentIds + .groupBy { id => + monumentDb.placeByMonumentId.getOrElse(id, "Unknown") + } + .mapValues(_.toSet) + .toMap imagesText += s"\n== ${region.name} ${images.size} images ==\n" - imagesText += byPlace.map { case (code, monumentIds) => - val place = Country.Ukraine.byMonumentId(monumentIds.head).map(_.name).getOrElse("Unknown") - val placeImages = images.filter(_.monumentIds.toSet.intersect(monumentIds).nonEmpty) - s"\n=== $place ${placeImages.size} images ===\n" ++ - placeImages.map(_.title).mkString("\n", "\n", "") - }.mkString("\n") + imagesText += byPlace + .map { case (code, monumentIds) => + val place = Country.Ukraine + .byMonumentId(monumentIds.head) + .map(_.name) + .getOrElse("Unknown") + val placeImages = + images.filter(_.monumentIds.toSet.intersect(monumentIds).nonEmpty) + s"\n=== $place ${placeImages.size} images ===\n" ++ + placeImages + .map(_.title) + .mkString("\n", "\n", "") + } + .mkString("\n") } } - MwBot.fromHost(MwBot.commons).page(imagesPage).edit(imagesText, Some("updating")) + MwBot + .fromHost(MwBot.commons) + .page(imagesPage) + .edit(imagesText, Some("updating")) } def galleryByAuthor(imagesPage: String, imageDb: ImageDB): Unit = { @@ -109,10 +138,15 @@ class SpecialNominations(stat: ContestStat, imageDb: ImageDB) { val images = imageDb._byAuthor.by(author) if (images.nonEmpty) { imagesText += s"\n== $author, ${byId.keys.size} monuments ==\n" - imagesText += images.map(_.title).mkString("\n", "\n", "") + imagesText += images + .map(_.title) + .mkString("\n", "\n", "") } } - MwBot.fromHost(MwBot.commons).page(imagesPage).edit(imagesText, Some("updating")) + MwBot + .fromHost(MwBot.commons) + .page(imagesPage) + .edit(imagesText, Some("updating")) } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/CampaignListSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/CampaignListSpec.scala index c11647ae..6c982238 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/CampaignListSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/CampaignListSpec.scala @@ -17,9 +17,10 @@ import scala.concurrent.ExecutionContext.Implicits.global class CampaignListSpec extends Specification with MockBotSpec with Mockito { - implicit def titleToHasImages(title: String): HasImagesCategory = new HasImagesCategory { - override def imagesCategory = title - } + implicit def titleToHasImages(title: String): HasImagesCategory = + new HasImagesCategory { + override def imagesCategory = title + } def campaignList(testBot: MwBot) = new CampaignList() with HasBot { val bot = testBot @@ -27,20 +28,34 @@ class CampaignListSpec extends Specification with MockBotSpec with Mockito { def categoryQueryBot(title: String, responseFile: String) = { val response = resourceAsString(responseFile) - val cmd = HttpStub(Map("action" -> "query", "list" -> "categorymembers", - "cmtitle" -> title, - "cmnamespace" -> Namespace.CATEGORY.toString, - "continue" -> "", "cmlimit" -> "max"), - response) + val cmd = HttpStub( + Map( + "action" -> "query", + "list" -> "categorymembers", + "cmtitle" -> title, + "cmnamespace" -> Namespace.CATEGORY.toString, + "continue" -> "", + "cmlimit" -> "max" + ), + response + ) getBot(cmd) } def mockedBot(title: String, categories: Seq[String]) = { val bot = mock[MwBot] - val action = Action(Query(ListParam(CategoryMembers( - CmTitle(title), CmNamespace(Seq(Namespace.CATEGORY)), CmLimit("max")) - ))) + val action = Action( + Query( + ListParam( + CategoryMembers( + CmTitle(title), + CmNamespace(Seq(Namespace.CATEGORY)), + CmLimit("max") + ) + ) + ) + ) bot.run(action) returns Future { categories.map(cat => Page(title = cat)) @@ -51,9 +66,13 @@ class CampaignListSpec extends Specification with MockBotSpec with Mockito { "CampaignList" should { "return WLM years" in { - val bot = categoryQueryBot("Category:Images from Wiki Loves Monuments", "/org/scalawiki/wlx/WLM_years.json") + val bot = categoryQueryBot( + "Category:Images from Wiki Loves Monuments", + "/org/scalawiki/wlx/WLM_years.json" + ) - val result = campaignList(bot).getContests(ContestType.WLM).map(_.toSeq).await + val result = + campaignList(bot).getContests(ContestType.WLM).map(_.toSeq).await result must have size 8 result.map(_.year) === (2010 to 2017) @@ -61,9 +80,13 @@ class CampaignListSpec extends Specification with MockBotSpec with Mockito { } "return WLE years" in { - val bot = categoryQueryBot("Category:Images from Wiki Loves Earth", "/org/scalawiki/wlx/WLE_years.json") + val bot = categoryQueryBot( + "Category:Images from Wiki Loves Earth", + "/org/scalawiki/wlx/WLE_years.json" + ) - val result = campaignList(bot).getContests(ContestType.WLE).map(_.toSeq).await + val result = + campaignList(bot).getContests(ContestType.WLE).map(_.toSeq).await val byYear = result.sortBy(_.year) byYear must have size 5 @@ -94,7 +117,12 @@ class CampaignListSpec extends Specification with MockBotSpec with Mockito { result.map(_.year).distinct === Seq(2011) result.map(_.contestType).distinct === Seq(ContestType.WLM) - result.map(_.country.name) === Seq("Andorra", "Hungary", "Hungary - international", "the Netherlands") + result.map(_.country.name) === Seq( + "Andorra", + "Hungary", + "Hungary - international", + "the Netherlands" + ) } "return WLM 2012" in { @@ -121,7 +149,14 @@ class CampaignListSpec extends Specification with MockBotSpec with Mockito { result.map(_.year).distinct === Seq(2012) result.map(_.contestType).distinct === Seq(ContestType.WLM) - result.map(_.country.name) === Seq("the Czech Republic", "the Philippines", "South Africa", "Ukraine", "the United States", "an unknown country") + result.map(_.country.name) === Seq( + "the Czech Republic", + "the Philippines", + "South Africa", + "Ukraine", + "the United States", + "an unknown country" + ) } "return WLE 2014" in { @@ -142,7 +177,12 @@ class CampaignListSpec extends Specification with MockBotSpec with Mockito { result.map(_.year).distinct === Seq(2014) result.map(_.contestType).distinct === Seq(ContestType.WLE) - result.map(_.country.name) === Seq("Algeria", "Andorra & Catalan areas", "Armenia & Nagorno-Karabakh", "the Netherlands") + result.map(_.country.name) === Seq( + "Algeria", + "Andorra & Catalan areas", + "Armenia & Nagorno-Karabakh", + "the Netherlands" + ) } "return WLE 2016" in { @@ -161,6 +201,8 @@ class CampaignListSpec extends Specification with MockBotSpec with Mockito { result.map(_.year).distinct === Seq(2016) result.map(_.contestType).distinct === Seq(ContestType.WLE) - result.map(_.country.name) === Seq("Albania"/*, "Biosphere Reserves 2016"*/) + result.map(_.country.name) === Seq( + "Albania" /*, "Biosphere Reserves 2016"*/ + ) } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/CountryListParserSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/CountryListParserSpec.scala index e561785f..f6579cff 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/CountryListParserSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/CountryListParserSpec.scala @@ -11,54 +11,78 @@ class CountryListParserSpec extends Specification { "category name parser" should { "parse WLE" in { - CountryParser.fromCategoryName("Category:Images from Wiki Loves Earth 2015") === Some( + CountryParser.fromCategoryName( + "Category:Images from Wiki Loves Earth 2015" + ) === Some( Contest(ContestType.WLE, NoAdmDivision, 2015) ) } "parse WLM" in { - CountryParser.fromCategoryName("Category:Images from Wiki Loves Monuments 2015") === Some( + CountryParser.fromCategoryName( + "Category:Images from Wiki Loves Monuments 2015" + ) === Some( Contest(ContestType.WLM, NoAdmDivision, 2015) ) } "parse WLE country" in { - CountryParser.fromCategoryName("Category:Images from Wiki Loves Earth 2015 in Algeria") + CountryParser + .fromCategoryName( + "Category:Images from Wiki Loves Earth 2015 in Algeria" + ) .map(c => c.copy(country = c.country.withoutLangCodes)) === Some( Contest(ContestType.WLE, Country("DZ", "Algeria"), 2015) ) } "parse WLM country" in { - CountryParser.fromCategoryName("Category:Images from Wiki Loves Monuments 2015 in Algeria") + CountryParser + .fromCategoryName( + "Category:Images from Wiki Loves Monuments 2015 in Algeria" + ) .map(c => c.copy(country = c.country.withoutLangCodes)) === Some( Contest(ContestType.WLM, Country("DZ", "Algeria"), 2015) ) } "parse WLE UA" in { - CountryParser.fromCategoryName("Category:Images from Wiki Loves Earth 2015 in Ukraine") === Some( - Contest(ContestType.WLE, Country.Ukraine, 2015, + CountryParser.fromCategoryName( + "Category:Images from Wiki Loves Earth 2015 in Ukraine" + ) === Some( + Contest( + ContestType.WLE, + Country.Ukraine, + 2015, uploadConfigs = Seq( UploadConfig( campaign = "wle-ua", listTemplate = "ВЛЗ-рядок", fileTemplate = "UkrainianNaturalHeritageSite", listConfig = ListConfig.WleUa - ))) + ) + ) + ) ) } "parse WLM UA" in { - CountryParser.fromCategoryName("Category:Images from Wiki Loves Monuments 2015 in Ukraine") === Some( - Contest(ContestType.WLM, Country.Ukraine, 2015, + CountryParser.fromCategoryName( + "Category:Images from Wiki Loves Monuments 2015 in Ukraine" + ) === Some( + Contest( + ContestType.WLM, + Country.Ukraine, + 2015, uploadConfigs = Seq( UploadConfig( campaign = "wlm-ua", listTemplate = "ВЛП-рядок", fileTemplate = "Monument Ukraine", listConfig = ListConfig.WlmUa - ))) + ) + ) + ) ) } } @@ -75,7 +99,8 @@ class CountryListParserSpec extends Specification { } "parse wle 2016" in { - val wiki = resourceAsString("/org/scalawiki/wlx/wle_2016_participating.wiki") + val wiki = + resourceAsString("/org/scalawiki/wlx/wle_2016_participating.wiki") val contests = CountryParser.parse(wiki) @@ -110,7 +135,8 @@ class CountryListParserSpec extends Specification { } "parse wlm 2016" in { - val wiki = resourceAsString("/org/scalawiki/wlx/wlm_2016_participating.wiki") + val wiki = + resourceAsString("/org/scalawiki/wlx/wlm_2016_participating.wiki") val contests = CountryParser.parse(wiki) diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/DesnaRegionSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/DesnaRegionSpec.scala index 3175d1f6..952988d8 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/DesnaRegionSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/DesnaRegionSpec.scala @@ -34,28 +34,29 @@ class DesnaRegionSpec extends Specification { "Остер" in { val places = desna.getPlace( - List("Чернігівська", "Чернігівський", "Остерська", "Остер")) + List("Чернігівська", "Чернігівський", "Остерська", "Остер") + ) places.size === 1 } "Новгород-Сіверський" in { val places = desna.getRaion( - List("Чернігівська", - "Новгород-Сіверський", - "Семенівська", - "Лісківщина")) + List("Чернігівська", "Новгород-Сіверський", "Семенівська", "Лісківщина") + ) places.size === 1 } "Козелецький район" in { val places = desna.getRaion( - List("Чернігівська", "Козелецький", "Козелецька", "Тарасів")) + List("Чернігівська", "Козелецький", "Козелецька", "Тарасів") + ) places.size === 1 } "Бірки" in { val places = desna.getPlace( - List("Чернігівська", "Чернігівський", "Остерська", "Бірки")) + List("Чернігівська", "Чернігівський", "Остерська", "Бірки") + ) places.size === 1 } } diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ImageDbSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ImageDbSpec.scala index 13aa2680..4ff5746e 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ImageDbSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ImageDbSpec.scala @@ -8,38 +8,45 @@ import org.specs2.mutable.Specification class ImageDbSpec extends Specification { private val Ukraine = Country.Ukraine - val monuments = Ukraine.regionIds.flatMap { - regionId => - (1 to regionId.toInt).map { i => - Monument( - page = "", - id = regionId + "-001-" + f"$i%04d", - name = "Monument in " + Ukraine.regionName(regionId), - listConfig = Some(ListConfig.WlmUa) - ) - } + val monuments = Ukraine.regionIds.flatMap { regionId => + (1 to regionId.toInt).map { i => + Monument( + page = "", + id = regionId + "-001-" + f"$i%04d", + name = "Monument in " + Ukraine.regionName(regionId), + listConfig = Some(ListConfig.WlmUa) + ) + } } def images(year: Int): Set[Image] = { var imageCount = 0 - Ukraine.regionIds.flatMap { - regionId => - (1 to regionId.toInt).flatMap { i => - val id = regionId + "-001-" + f"$i%04d" - - if (i % 10 == year % 10 || i % 10 - 1 == year % 10) { - imageCount += 1 - Some(Image(s"image of $id taken on $year number $imageCount", None, None, Some(10), Some(10), Some(10), Some(id), - Some(User(None, Some("author"))))) - } else None - } + Ukraine.regionIds.flatMap { regionId => + (1 to regionId.toInt).flatMap { i => + val id = regionId + "-001-" + f"$i%04d" + + if (i % 10 == year % 10 || i % 10 - 1 == year % 10) { + imageCount += 1 + Some( + Image( + s"image of $id taken on $year number $imageCount", + None, + None, + Some(10), + Some(10), + Some(10), + Some(id), + Some(User(None, Some("author"))) + ) + ) + } else None + } } } val allImages = images(2012) ++ images(2013) ++ images(2014) val contest = Contest.WLMUkraine(2014) - // "group images by regions" in { // // val src = images(2014).toSeq @@ -61,17 +68,29 @@ class ImageDbSpec extends Specification { "show image list" in { val noRes = new Image("imageNoRes", width = None, height = None) - val halfMinus = new Image("image_800_600", width = Some(800), height = Some(600)) - val halfPlus = new Image("image_1024_768", width = Some(1024), height = Some(768)) - val one = new Image("image_1000_1000", width = Some(1000), height = Some(1000)) - val onePlus = new Image("image_1280_1024", width = Some(1280), height = Some(1024)) - val twoPlus = new Image("image_1920_1200", width = Some(1900), height = Some(1200)) - val mp12 = new Image("image_12Mp", width = Some(4000), height = Some(3000)) - val mp24 = new Image("image_24Mp", width = Some(6000), height = Some(4000)) - - val allImages = Seq(noRes, halfMinus, halfPlus, one, onePlus, twoPlus, mp12, mp24) - - val imageDb = new ImageDB(contest, allImages, Some(new MonumentDB(contest, Seq.empty))) + val halfMinus = + new Image("image_800_600", width = Some(800), height = Some(600)) + val halfPlus = + new Image("image_1024_768", width = Some(1024), height = Some(768)) + val one = + new Image("image_1000_1000", width = Some(1000), height = Some(1000)) + val onePlus = + new Image("image_1280_1024", width = Some(1280), height = Some(1024)) + val twoPlus = + new Image("image_1920_1200", width = Some(1900), height = Some(1200)) + val mp12 = + new Image("image_12Mp", width = Some(4000), height = Some(3000)) + val mp24 = + new Image("image_24Mp", width = Some(6000), height = Some(4000)) + + val allImages = + Seq(noRes, halfMinus, halfPlus, one, onePlus, twoPlus, mp12, mp24) + + val imageDb = new ImageDB( + contest, + allImages, + Some(new MonumentDB(contest, Seq.empty)) + ) // imageDb.byMegaPixels(None) === Seq(noRes) imageDb.byMegaPixels(0) === Seq(halfMinus, halfPlus) @@ -86,25 +105,86 @@ class ImageDbSpec extends Specification { val user1 = User(1, "user1") val user2 = User(2, "user2") - val noRes = new Image("imageNoRes", width = None, height = None, uploader = Some(user1)) - - val halfMinus = new Image("image_800_600", width = Some(800), height = Some(600), author = Some("user1"), uploader = Some(user1)) - val halfPlus = new Image("image_1024_768", width = Some(1024), height = Some(768), author = Some("user2"), uploader = Some(user2)) - val one = new Image("image_1000_1000", width = Some(1000), height = Some(1000), author = Some("user1"), uploader = Some(user1)) - val onePlus = new Image("image_1280_1024", width = Some(1280), height = Some(1024), author = Some("user2"), uploader = Some(user2)) - val twoPlus = new Image("image_1920_1200", width = Some(1900), height = Some(1200), author = Some("user1"), uploader = Some(user1)) - val mp12 = new Image("image_12Mp", width = Some(4000), height = Some(3000), author = Some("user1"), uploader = Some(user1)) - val mp24 = new Image("image_24Mp", width = Some(6000), height = Some(4000), author = Some("user1"), uploader = Some(user1)) - - val allImages = Seq(noRes, halfMinus, halfPlus, one, onePlus, twoPlus, mp12, mp24) - - val imageDb = new ImageDB(contest, allImages, Some(new MonumentDB(contest, Seq.empty))) - - imageDb._byMegaPixelsAndAuthor(0) === Map("user1" -> Seq(halfMinus), "user2" -> Seq(halfPlus)) - imageDb._byMegaPixelsAndAuthor(1) === Map("user1" -> Seq(one), "user2" -> Seq(onePlus)) + val noRes = new Image( + "imageNoRes", + width = None, + height = None, + uploader = Some(user1) + ) + + val halfMinus = new Image( + "image_800_600", + width = Some(800), + height = Some(600), + author = Some("user1"), + uploader = Some(user1) + ) + val halfPlus = new Image( + "image_1024_768", + width = Some(1024), + height = Some(768), + author = Some("user2"), + uploader = Some(user2) + ) + val one = new Image( + "image_1000_1000", + width = Some(1000), + height = Some(1000), + author = Some("user1"), + uploader = Some(user1) + ) + val onePlus = new Image( + "image_1280_1024", + width = Some(1280), + height = Some(1024), + author = Some("user2"), + uploader = Some(user2) + ) + val twoPlus = new Image( + "image_1920_1200", + width = Some(1900), + height = Some(1200), + author = Some("user1"), + uploader = Some(user1) + ) + val mp12 = new Image( + "image_12Mp", + width = Some(4000), + height = Some(3000), + author = Some("user1"), + uploader = Some(user1) + ) + val mp24 = new Image( + "image_24Mp", + width = Some(6000), + height = Some(4000), + author = Some("user1"), + uploader = Some(user1) + ) + + val allImages = + Seq(noRes, halfMinus, halfPlus, one, onePlus, twoPlus, mp12, mp24) + + val imageDb = new ImageDB( + contest, + allImages, + Some(new MonumentDB(contest, Seq.empty)) + ) + + imageDb._byMegaPixelsAndAuthor(0) === Map( + "user1" -> Seq(halfMinus), + "user2" -> Seq(halfPlus) + ) + imageDb._byMegaPixelsAndAuthor(1) === Map( + "user1" -> Seq(one), + "user2" -> Seq(onePlus) + ) imageDb._byMegaPixelsAndAuthor(2) === Map("user1" -> Seq(twoPlus)) - imageDb.byMegaPixelFilterAuthorMap(_ < 2) === Map("user1" -> Seq(halfMinus, one), "user2" -> Seq(halfPlus, onePlus)) + imageDb.byMegaPixelFilterAuthorMap(_ < 2) === Map( + "user1" -> Seq(halfMinus, one), + "user2" -> Seq(halfPlus, onePlus) + ) } } diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ImageFillerSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ImageFillerSpec.scala index 27fad528..694b0830 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ImageFillerSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ImageFillerSpec.scala @@ -15,9 +15,24 @@ class ImageFillerSpec extends Specification { "ListFiller" should { "select bestImage" in { val images = Seq( - Image("10mp", size = Some(2 * 10 ^ 9), width = Some(5000), height = Some(2000)), - Image("16mp", size = Some(3 * 10 ^ 9), width = Some(4000), height = Some(4000)), - Image("16mpBigger", size = Some(4 * 10 ^ 9), width = Some(4000), height = Some(4000)) + Image( + "10mp", + size = Some(2 * 10 ^ 9), + width = Some(5000), + height = Some(2000) + ), + Image( + "16mp", + size = Some(3 * 10 ^ 9), + width = Some(4000), + height = Some(4000) + ), + Image( + "16mpBigger", + size = Some(4 * 10 ^ 9), + width = Some(4000), + height = Some(4000) + ) ) val best = ImageFiller.bestImage(images) @@ -28,7 +43,8 @@ class ImageFillerSpec extends Specification { val monumentDb = new MonumentDB(contest, Seq.empty) val imageDb = new ImageDB(contest, Seq.empty, Some(monumentDb)) - val task = new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) + val task = + new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) val (newText, comment) = task.updatePage("page", "") newText === "" @@ -41,11 +57,13 @@ class ImageFillerSpec extends Specification { Monument(id = "id2", name = "name2", listConfig = Some(listConfig)), Monument(id = "id3", name = "name3", listConfig = Some(listConfig)) ) - val text = "header\n" + monuments.map(_.asWiki()).mkString("{|\n|}\n") + "\nfooter" + val text = + "header\n" + monuments.map(_.asWiki()).mkString("{|\n|}\n") + "\nfooter" val monumentDb = new MonumentDB(contest, monuments) val imageDb = new ImageDB(contest, Seq.empty, monumentDb) - val task = new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) + val task = + new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) val (newText, comment) = task.updatePage("page", text) newText === text @@ -53,20 +71,46 @@ class ImageFillerSpec extends Specification { } "addPhotosToPageText add 1 image" in { - val monument1 = Monument(id = "id1", name = "name1", photo = Some("Img1.jpg"), listConfig = Some(listConfig)) - val monument2 = Monument(id = "id2", name = "name2", listConfig = Some(listConfig)) - val monument3 = Monument(id = "id3", name = "name3", listConfig = Some(listConfig)) + val monument1 = Monument( + id = "id1", + name = "name1", + photo = Some("Img1.jpg"), + listConfig = Some(listConfig) + ) + val monument2 = + Monument(id = "id2", name = "name2", listConfig = Some(listConfig)) + val monument3 = + Monument(id = "id3", name = "name3", listConfig = Some(listConfig)) val monuments = Seq(monument1, monument2, monument3) val text = "header\n" + monuments.map(_.asWiki()).mkString + "\nfooter" val images = Seq( - Image("File:Img1.jpg", size = Some(10 ^ 6), width = Some(2048), height = Some(1024), monumentIds = List("id1")), - Image("File:Img2.jpg", size = Some(10 ^ 6), width = Some(1280), height = Some(1024), monumentIds = List("id2")), - Image("File:Img2sm.jpg", size = Some(10 ^ 6), width = Some(1024), height = Some(768), monumentIds = List("id2")) + Image( + "File:Img1.jpg", + size = Some(10 ^ 6), + width = Some(2048), + height = Some(1024), + monumentIds = List("id1") + ), + Image( + "File:Img2.jpg", + size = Some(10 ^ 6), + width = Some(1280), + height = Some(1024), + monumentIds = List("id2") + ), + Image( + "File:Img2sm.jpg", + size = Some(10 ^ 6), + width = Some(1024), + height = Some(768), + monumentIds = List("id2") + ) ) val monumentDb = new MonumentDB(contest, monuments) val imageDb = new ImageDB(contest, images, monumentDb) - val task = new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) + val task = + new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) val (newText, comment) = task.updatePage("page", text) val updatedMonuments = Seq( @@ -74,33 +118,62 @@ class ImageFillerSpec extends Specification { monument2.copy(photo = Some("Img2.jpg")), monument3 ) - val expected = "header\n" + updatedMonuments.map(_.asWiki()).mkString + "\nfooter" + val expected = + "header\n" + updatedMonuments.map(_.asWiki()).mkString + "\nfooter" newText === expected comment === "updated 1 monument(s)" } "addPhotosToPageText add 1 image preserve comments" in { - val monument1 = Monument(id = "id1", name = "name1", photo = Some("Img1.jpg"), listConfig = Some(listConfig)) - val monument2 = Monument(id = "id2", name = "name2", listConfig = Some(listConfig)) - val monument3 = Monument(id = "id3", name = "name3", listConfig = Some(listConfig)) + val monument1 = Monument( + id = "id1", + name = "name1", + photo = Some("Img1.jpg"), + listConfig = Some(listConfig) + ) + val monument2 = + Monument(id = "id2", name = "name2", listConfig = Some(listConfig)) + val monument3 = + Monument(id = "id3", name = "name3", listConfig = Some(listConfig)) val monuments = Seq(monument1, monument2, monument3) - val text = "header\n" + Seq(" ", + val text = "header\n" + Seq( + " ", monument2.asWiki(), monument3.asWiki() ).mkString + "\nfooter" val images = Seq( - Image("File:Img1.jpg", size = Some(10 ^ 6), width = Some(2048), height = Some(1024), monumentIds = List("id1")), - Image("File:Img2.jpg", size = Some(10 ^ 6), width = Some(1280), height = Some(1024), monumentIds = List("id2")), - Image("File:Img2sm.jpg", size = Some(10 ^ 6), width = Some(1024), height = Some(768), monumentIds = List("id2")) + Image( + "File:Img1.jpg", + size = Some(10 ^ 6), + width = Some(2048), + height = Some(1024), + monumentIds = List("id1") + ), + Image( + "File:Img2.jpg", + size = Some(10 ^ 6), + width = Some(1280), + height = Some(1024), + monumentIds = List("id2") + ), + Image( + "File:Img2sm.jpg", + size = Some(10 ^ 6), + width = Some(1024), + height = Some(768), + monumentIds = List("id2") + ) ) val monumentDb = new MonumentDB(contest, monuments) val imageDb = new ImageDB(contest, images, monumentDb) - val task = new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) + val task = + new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) val (newText, comment) = task.updatePage("page", text) - val expected = "header\n" + Seq(" ", + val expected = "header\n" + Seq( + " ", monument2.copy(photo = Some("Img2.jpg")).asWiki(), monument3.asWiki() ).mkString + "\nfooter" @@ -109,21 +182,44 @@ class ImageFillerSpec extends Specification { } "addPhotosToPageText should preserve surrounding markup" in { - val monument1 = Monument(id = "id1", name = "name1", listConfig = Some(listConfig)) - val monument2 = Monument(id = "id2", name = "name2", listConfig = Some(listConfig)) - val monument3 = Monument(id = "id3", name = "name3", listConfig = Some(listConfig)) + val monument1 = + Monument(id = "id1", name = "name1", listConfig = Some(listConfig)) + val monument2 = + Monument(id = "id2", name = "name2", listConfig = Some(listConfig)) + val monument3 = + Monument(id = "id3", name = "name3", listConfig = Some(listConfig)) val monuments = Seq(monument1, monument2, monument3) - val text = "header\n" + monuments.map(_.asWiki()).mkString("{|\n|}\n") + "\nfooter" + val text = + "header\n" + monuments.map(_.asWiki()).mkString("{|\n|}\n") + "\nfooter" val images = Seq( - Image("File:Img1.jpg", size = Some(10 ^ 6), width = Some(2048), height = Some(1024), monumentIds = List("id1")), - Image("File:Img2.jpg", size = Some(10 ^ 6), width = Some(1280), height = Some(1024), monumentIds = List("id2")), - Image("File:Img3.jpg", size = Some(10 ^ 6), width = Some(1024), height = Some(768), monumentIds = List("id3")) + Image( + "File:Img1.jpg", + size = Some(10 ^ 6), + width = Some(2048), + height = Some(1024), + monumentIds = List("id1") + ), + Image( + "File:Img2.jpg", + size = Some(10 ^ 6), + width = Some(1280), + height = Some(1024), + monumentIds = List("id2") + ), + Image( + "File:Img3.jpg", + size = Some(10 ^ 6), + width = Some(1024), + height = Some(768), + monumentIds = List("id3") + ) ) val monumentDb = new MonumentDB(contest, monuments) val imageDb = new ImageDB(contest, images, monumentDb) - val task = new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) + val task = + new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) val (newText, comment) = task.updatePage("page", text) val updatedMonuments = Seq( @@ -131,7 +227,9 @@ class ImageFillerSpec extends Specification { monument2.copy(photo = Some("Img2.jpg")), monument3.copy(photo = Some("Img3.jpg")) ) - val expected = "header\n" + updatedMonuments.map(_.asWiki()).mkString("{|\n|}\n") + "\nfooter" + val expected = "header\n" + updatedMonuments + .map(_.asWiki()) + .mkString("{|\n|}\n") + "\nfooter" newText === expected comment === "updated 3 monument(s)" } @@ -141,13 +239,15 @@ class ImageFillerSpec extends Specification { Image("Файл:Image1.jpg", monumentIds = List("id1")) ) - val monument1 = Monument(id = "id1", name = "name1", listConfig = Some(listConfig)) + val monument1 = + Monument(id = "id1", name = "name1", listConfig = Some(listConfig)) val monuments = Seq(monument1) val monumentDb = new MonumentDB(contest, monuments) val imageDb = new ImageDB(contest, images, monumentDb) val text = monuments.map(_.asWiki()).mkString - val task = new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) + val task = + new ListUpdaterTask(host, monumentDb, new ImageFillerUpdater(imageDb)) val (newText, comment) = task.updatePage("page", text) val updatedMonuments = Seq( diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ListFileNSRemoverSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ListFileNSRemoverSpec.scala index bcda4521..d354e91d 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ListFileNSRemoverSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/ListFileNSRemoverSpec.scala @@ -29,7 +29,8 @@ class ListFileNSRemoverSpec extends Specification { Monument(id = "id2", name = "name2", listConfig = Some(listConfig)), Monument(id = "id3", name = "name3", listConfig = Some(listConfig)) ) - val text = "header\n" + monuments.map(_.asWiki()).mkString("{|\n|}\n") + "\nfooter" + val text = + "header\n" + monuments.map(_.asWiki()).mkString("{|\n|}\n") + "\nfooter" val monumentDb = new MonumentDB(contest, Seq.empty) val task = new ListFileNSRemoverTask(host, monumentDb) @@ -40,9 +41,16 @@ class ListFileNSRemoverSpec extends Specification { } "fix 1 image" in { - val monument1 = Monument(id = "id1", name = "name1", photo = Some("File:Img1.jpg"), listConfig = Some(listConfig)) - val monument2 = Monument(id = "id2", name = "name2", listConfig = Some(listConfig)) - val monument3 = Monument(id = "id3", name = "name3", listConfig = Some(listConfig)) + val monument1 = Monument( + id = "id1", + name = "name1", + photo = Some("File:Img1.jpg"), + listConfig = Some(listConfig) + ) + val monument2 = + Monument(id = "id2", name = "name2", listConfig = Some(listConfig)) + val monument3 = + Monument(id = "id3", name = "name3", listConfig = Some(listConfig)) val monuments = Seq(monument1, monument2, monument3) val text = "header\n" + monuments.map(_.asWiki()).mkString + "\nfooter" @@ -51,21 +59,38 @@ class ListFileNSRemoverSpec extends Specification { val (newText, comment) = task.updatePage("page", text) val updatedMonuments = Seq( - monument1.copy(photo = Some("Img1.jpg")), - monument2, - monument3 + monument1.copy(photo = Some("Img1.jpg")), + monument2, + monument3 ) - val expected = "header\n" + updatedMonuments.map(_.asWiki()).mkString + "\nfooter" + val expected = + "header\n" + updatedMonuments.map(_.asWiki()).mkString + "\nfooter" newText === expected comment === "fixing 1 image(s)" } "should preserve surrounding markup" in { - val monument1 = Monument(id = "id1", name = "name1", photo = Some("File:Img1.jpg"), listConfig = Some(listConfig)) - val monument2 = Monument(id = "id2", name = "name2", photo = Some("File:Img2.jpg"), listConfig = Some(listConfig)) - val monument3 = Monument(id = "id3", name = "name3", photo = Some("File:Img3.jpg"), listConfig = Some(listConfig)) + val monument1 = Monument( + id = "id1", + name = "name1", + photo = Some("File:Img1.jpg"), + listConfig = Some(listConfig) + ) + val monument2 = Monument( + id = "id2", + name = "name2", + photo = Some("File:Img2.jpg"), + listConfig = Some(listConfig) + ) + val monument3 = Monument( + id = "id3", + name = "name3", + photo = Some("File:Img3.jpg"), + listConfig = Some(listConfig) + ) val monuments = Seq(monument1, monument2, monument3) - val text = "header\n" + monuments.map(_.asWiki()).mkString("{|\n|}\n") + "\nfooter" + val text = + "header\n" + monuments.map(_.asWiki()).mkString("{|\n|}\n") + "\nfooter" val monumentDb = new MonumentDB(contest, monuments) @@ -77,15 +102,21 @@ class ListFileNSRemoverSpec extends Specification { monument2.copy(photo = Some("Img2.jpg")), monument3.copy(photo = Some("Img3.jpg")) ) - val expected = "header\n" + updatedMonuments.map(_.asWiki()).mkString("{|\n|}\n") + "\nfooter" + val expected = "header\n" + updatedMonuments + .map(_.asWiki()) + .mkString("{|\n|}\n") + "\nfooter" newText === expected comment === "fixing 3 image(s)" } - "fix localized File:" in { - val monument1 = Monument(id = "id1", name = "name1", photo = Some("Файл:Img1.jpg"), listConfig = Some(listConfig)) + val monument1 = Monument( + id = "id1", + name = "name1", + photo = Some("Файл:Img1.jpg"), + listConfig = Some(listConfig) + ) val monuments = Seq(monument1) val monumentDb = new MonumentDB(contest, monuments) diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/MonumentDbSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/MonumentDbSpec.scala index 6d19c503..fbe44a7c 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/MonumentDbSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/MonumentDbSpec.scala @@ -10,25 +10,27 @@ class MonumentDbSpec extends Specification { val contest = Contest.WLMUkraine(2014) val monuments = Ukraine.regionIds.flatMap { regionId => - (1 to regionId.toInt).map { i => - Monument( - id = regionId + "-001-" + f"$i%04d", - name = "Monument in " + Ukraine.regionName(regionId), - listConfig = Some(ListConfig.WleUa) - ) - } - } - - def specialNominationMonuments(prefix: String) = Ukraine.regionIds.flatMap { regionId => (1 to regionId.toInt).map { i => Monument( - id = prefix + "-" + regionId + "1-" + f"$i%04d", - name = "Special Nomination Monument in " + Ukraine.regionName(regionId), - listConfig = Some(ListConfig.WlmUa) + id = regionId + "-001-" + f"$i%04d", + name = "Monument in " + Ukraine.regionName(regionId), + listConfig = Some(ListConfig.WleUa) ) } } + def specialNominationMonuments(prefix: String) = Ukraine.regionIds.flatMap { + regionId => + (1 to regionId.toInt).map { i => + Monument( + id = prefix + "-" + regionId + "1-" + f"$i%04d", + name = + "Special Nomination Monument in " + Ukraine.regionName(regionId), + listConfig = Some(ListConfig.WlmUa) + ) + } + } + "monument db" should { "contain monuments ids" in { val db = new MonumentDB(contest, monuments.toSeq) diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/MonumentSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/MonumentSpec.scala index 74b969f6..804fa411 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/MonumentSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/MonumentSpec.scala @@ -115,12 +115,21 @@ class MonumentSpec extends Specification { "asWiki" should { "output default fields" in { - val m = new Monument("page1", "id1", "name1", photo = Some("Image.jpg"), gallery = Some("category1"), listConfig = None) + val m = new Monument( + "page1", + "id1", + "name1", + photo = Some("Image.jpg"), + gallery = Some("category1"), + listConfig = None + ) val longest = WleUa.namesMap.values.map(_.length).max val names = WleUa.namesMap.mapValues(_.padTo(longest, ' ')) - val keys = names.filterKeys(Set("ID", "name", "year", "description", "photo", "gallery").contains) + val keys = names.filterKeys( + Set("ID", "name", "year", "description", "photo", "gallery").contains + ) val text = m.asWiki(Some("WLE-row"), pad = false) val lines = text.split("\\n", -1).toSeq lines === @@ -136,12 +145,21 @@ class MonumentSpec extends Specification { } "output WLE fields" in { - val m = new Monument("page1", "id1", "name1", photo = Some("Image.jpg"), gallery = Some("category1"), listConfig = Some(WleUa)) + val m = new Monument( + "page1", + "id1", + "name1", + photo = Some("Image.jpg"), + gallery = Some("category1"), + listConfig = Some(WleUa) + ) val longest = WleUa.namesMap.values.map(_.length).max val names = WleUa.namesMap.mapValues(_.padTo(longest, ' ')) - val keys = names.filterKeys(Set("ID", "name", "year", "description", "photo", "gallery").contains) + val keys = names.filterKeys( + Set("ID", "name", "year", "description", "photo", "gallery").contains + ) val text = m.asWiki() val lines = text.split("\\n", -1).toSeq lines === @@ -165,7 +183,14 @@ class MonumentSpec extends Specification { } "output WLM fields" in { - val m = new Monument("page1", "id1", "name1", photo = Some("Image.jpg"), gallery = Some("category1"), listConfig = Some(WlmUa)) + val m = new Monument( + "page1", + "id1", + "name1", + photo = Some("Image.jpg"), + gallery = Some("category1"), + listConfig = Some(WlmUa) + ) val longest = WlmUa.namesMap.values.map(_.length).max @@ -195,7 +220,6 @@ class MonumentSpec extends Specification { } - // @Test def testSetResolution1 { // val m: Monument = Monument.init(withResolution1) // m.addResolution @@ -206,14 +230,23 @@ class MonumentSpec extends Specification { "monuments" should { "parse" in { - val list = Monument.monumentsFromText(listText, "page", "ВЛП-рядок", listConfig = WlmUa).toSeq + val list = Monument + .monumentsFromText(listText, "page", "ВЛП-рядок", listConfig = WlmUa) + .toSeq list.size === 2 list(0).id === "05-105-0001" list(1).id === "05-105-0012" } "parse with rubbish" in { - val list = Monument.monumentsFromText("abcd\n" + listText + "\ndef", "page", "ВЛП-рядок", listConfig = WlmUa).toSeq + val list = Monument + .monumentsFromText( + "abcd\n" + listText + "\ndef", + "page", + "ВЛП-рядок", + listConfig = WlmUa + ) + .toSeq list.size === 2 list(0).id === "05-105-0001" list(1).id === "05-105-0012" diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/RegionFixerSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/RegionFixerSpec.scala index 6f092a3b..65bd280c 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/RegionFixerSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/RegionFixerSpec.scala @@ -9,7 +9,6 @@ class RegionFixerSpec extends Specification { val country = contest.country val listConfig = contest.uploadConfigs.head.listConfig - "fixer" should { "get all regions" in { @@ -20,7 +19,10 @@ class RegionFixerSpec extends Specification { "fix region 1" in { val wiki = resourceAsString("/org/scalawiki/wlx/region_to_fix.wiki") - val parser = new WlxTemplateParser(listConfig, "Вікіпедія:Вікі любить пам'ятки/Житомирська область/Новоград-Волинський район") + val parser = new WlxTemplateParser( + listConfig, + "Вікіпедія:Вікі любить пам'ятки/Житомирська область/Новоград-Волинський район" + ) val monuments = parser.parse(wiki).toSeq val updater = new RegionFixerUpdater(new MonumentDB(contest, monuments)) @@ -32,7 +34,10 @@ class RegionFixerSpec extends Specification { "fix region 2" in { val wiki = resourceAsString("/org/scalawiki/wlx/region_to_fix_2.wiki") - val parser = new WlxTemplateParser(listConfig, "Вікіпедія:Вікі любить пам'ятки/Донецька область/Мангушський район") + val parser = new WlxTemplateParser( + listConfig, + "Вікіпедія:Вікі любить пам'ятки/Донецька область/Мангушський район" + ) val monuments = parser.parse(wiki).toSeq val updater = new RegionFixerUpdater(new MonumentDB(contest, monuments)) @@ -44,7 +49,10 @@ class RegionFixerSpec extends Specification { "fix region 3" in { val wiki = resourceAsString("/org/scalawiki/wlx/region_to_fix_3.wiki") - val parser = new WlxTemplateParser(listConfig, "Вікіпедія:Вікі любить пам'ятки/Сумська область/Білопільський район") + val parser = new WlxTemplateParser( + listConfig, + "Вікіпедія:Вікі любить пам'ятки/Сумська область/Білопільський район" + ) val monuments = parser.parse(wiki).toSeq val updater = new RegionFixerUpdater(new MonumentDB(contest, monuments)) diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/TestListConfigs.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/TestListConfigs.scala index a038ed83..ec452662 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/TestListConfigs.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/TestListConfigs.scala @@ -21,4 +21,3 @@ object NameConfig extends ListConfig { override val templateName = "templateName" } - diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/UnknownPlacesSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/UnknownPlacesSpec.scala index 36d59653..27d1c9c4 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/UnknownPlacesSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/UnknownPlacesSpec.scala @@ -29,11 +29,14 @@ class UnknownPlacesSpec extends Specification { val place = UnknownPlace("page1", "regionId1", "place1", Nil, Seq(monument)) emptyDb.unknownPlacesTables(Seq(place)) === Seq( - Table(headers, - Seq( - Seq("place1", "", "regionId1-1 monument1") - ), - "page1")) + Table( + headers, + Seq( + Seq("place1", "", "regionId1-1 monument1") + ), + "page1" + ) + ) } "Сичівка" in { @@ -139,18 +142,22 @@ class UnknownPlacesSpec extends Specification { ) emptyDb.unknownPlacesTables(places) === Seq( - Table(headers, - Seq( - Seq("place1", "", "regionId1-1 monument1"), - Seq("place2", "", "regionId1-2 monument2") - ), - "page1"), - Table(headers, - Seq( - Seq("place3", "", "regionId2-3 monument3"), - Seq("place4", "", "regionId3-4 monument4") - ), - "page2") + Table( + headers, + Seq( + Seq("place1", "", "regionId1-1 monument1"), + Seq("place2", "", "regionId1-2 monument2") + ), + "page1" + ), + Table( + headers, + Seq( + Seq("place3", "", "regionId2-3 monument3"), + Seq("place4", "", "regionId3-4 monument4") + ), + "page2" + ) ) } } diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlmUaListsSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlmUaListsSpec.scala index 4145dcca..a27c68de 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlmUaListsSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlmUaListsSpec.scala @@ -17,7 +17,8 @@ class WlmUaListsSpec extends Specification { val contest = Contest.byCampaign(campaign).get.copy(year = 2019) val country = contest.country - val bot = new CachedBot(Site.ukWiki, cacheName + "-wiki", true, entries = 1000) + val bot = + new CachedBot(Site.ukWiki, cacheName + "-wiki", true, entries = 1000) val monumentQuery = MonumentQuery.create(contest)(bot) val monumentDb = MonumentDB.getMonumentDb(contest, monumentQuery) val all = monumentDb.allMonuments @@ -36,10 +37,14 @@ class WlmUaListsSpec extends Specification { val updater = new RegionFixerUpdater(monumentDb) updater.raions.size === 490 - val highLevel = all.filter(m => updater.raionNames.contains(m.cityName) && m.place.exists(_.trim.nonEmpty)) + val highLevel = all.filter(m => + updater.raionNames.contains(m.cityName) && m.place.exists( + _.trim.nonEmpty + ) + ) val percentage = highLevel.size * 100 / all.size percentage should be <= 5 } } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlxTableParserSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlxTableParserSpec.scala index ba846e18..24e20647 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlxTableParserSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlxTableParserSpec.scala @@ -61,7 +61,9 @@ class WlxTableParserSpec extends Specification { val monuments = parser.parse(table.asWiki) monuments.size === 2 - monuments.map(m => Seq(m.id, m.name, m.otherParams("_f1"), m.otherParams("_f2"))) === data + monuments.map(m => + Seq(m.id, m.name, m.otherParams("_f1"), m.otherParams("_f2")) + ) === data } "parse thailand" in { @@ -74,4 +76,3 @@ class WlxTableParserSpec extends Specification { } } } - diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlxTemplateParserSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlxTemplateParserSpec.scala index 992d4cd2..7e51de95 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlxTemplateParserSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/WlxTemplateParserSpec.scala @@ -6,8 +6,8 @@ import org.specs2.mutable.Specification class WlxTemplateParserSpec extends Specification { def dataToWiki(data: Seq[Seq[(String, String)]]): String = { - val templates = data.map { - params => new Template("templateName", params.toMap) + val templates = data.map { params => + new Template("templateName", params.toMap) } templates.map(_.text).mkString("\n") @@ -46,8 +46,9 @@ class WlxTemplateParserSpec extends Specification { monuments.size === 2 monuments.map(_.page).toSet === Set("page") - monuments.map(m => Seq(m.id, m.name, m.otherParams("_f1"), m.otherParams("_f2"))) === data.map(_.toMap.values.toSeq) + monuments.map(m => + Seq(m.id, m.name, m.otherParams("_f1"), m.otherParams("_f2")) + ) === data.map(_.toMap.values.toSeq) } } } - diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/CampaignSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/CampaignSpec.scala index 8a96c401..0e41c63c 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/CampaignSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/CampaignSpec.scala @@ -16,10 +16,12 @@ class CampaignSpec extends Specification { "{{Wiki Loves Earth is running|at|{{Upload campaign header wle-at}}}}", "{{Upload campaign use Wiki Loves Earth}}", Seq("National parks of Austria"), - Seq(CampaignField( - "{{Nationalpark Österreich|$1}}", - "Name laut Liste" - )) + Seq( + CampaignField( + "{{Nationalpark Österreich|$1}}", + "Name laut Liste" + ) + ) ) } } diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/CountrySpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/CountrySpec.scala index e300cacc..ae88c839 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/CountrySpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/CountrySpec.scala @@ -17,7 +17,8 @@ class CountrySpec extends Specification { Switzerland.code === "CH" Switzerland.name === "Switzerland" - Switzerland.languageCodes.toSet.intersect( Set("fr", "de", "it")) === Set("fr", "de", "it") + Switzerland.languageCodes.toSet + .intersect(Set("fr", "de", "it")) === Set("fr", "de", "it") } } diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KatotthSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KatotthSpec.scala index b5e19ed7..1f58dd9c 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KatotthSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KatotthSpec.scala @@ -6,8 +6,12 @@ class KatotthSpec extends Specification { args(skipAll = true) - val Ukraine: Country = new Country("UA", "Ukraine", Seq("uk"), - Katotth.regions(() => Some(Ukraine))) + val Ukraine: Country = new Country( + "UA", + "Ukraine", + Seq("uk"), + Katotth.regions(() => Some(Ukraine)) + ) val regions = Ukraine.regions "level1" should { @@ -43,7 +47,9 @@ class KatotthSpec extends Specification { ) "contain country parent" in { - regions.flatMap(_.parent().map(_.name)) === List.fill(topRegions.size)(Country.Ukraine.name) + regions.flatMap(_.parent().map(_.name)) === List.fill(topRegions.size)( + Country.Ukraine.name + ) } "have 27 elements" in { @@ -57,7 +63,9 @@ class KatotthSpec extends Specification { "lookup level1 by code" in { val set = topRegions.keySet val byRegion = Ukraine.byRegion(set) - byRegion.map { case (adm, ids) => ids.head -> adm.fullName } === topRegions + byRegion.map { case (adm, ids) => + ids.head -> adm.fullName + } === topRegions Ukraine.byIdAndName("80-361", "Київ").head.name === "Київ" } @@ -75,20 +83,35 @@ class KatotthSpec extends Specification { "level2" should { "have 490 raions" in { - val withoutCities = Country.Ukraine.regions.filter(adm => !Set("Київ", "Севастополь").contains(adm.name)) + val withoutCities = Country.Ukraine.regions.filter(adm => + !Set("Київ", "Севастополь").contains(adm.name) + ) withoutCities.size === 25 - val raions = withoutCities.flatMap(_.regions).filter(_.name.endsWith("район")) + val raions = + withoutCities.flatMap(_.regions).filter(_.name.endsWith("район")) raions.size === 490 } "contain Kyiv raions" in { - val regionNames = Seq("Голосіївський", "Дарницький", "Деснянський", "Дніпровський", - "Оболонський", "Печерський", "Подільський", "Святошинський", "Солом'янський", "Шевченківський") + val regionNames = Seq( + "Голосіївський", + "Дарницький", + "Деснянський", + "Дніпровський", + "Оболонський", + "Печерський", + "Подільський", + "Святошинський", + "Солом'янський", + "Шевченківський" + ) val kyiv = regions.find(_.name == "Київ").get val kyivRegions = kyiv.regions kyivRegions.map(_.name) === regionNames - kyivRegions.flatMap(_.parent().map(_.name)) === List.fill(regionNames.size)("Київ") + kyivRegions.flatMap(_.parent().map(_.name)) === List.fill( + regionNames.size + )("Київ") } // "find Kyiv raions by code" in { @@ -113,8 +136,15 @@ class KatotthSpec extends Specification { "contain Kyiv oblast regions" in { val ko = regions.find(_.name == "Київська").get - val koRegionNames = Seq("Бориспільський", "Броварський", "Бучанський", "Білоцерківський", "Вишгородський", - "Обухівський", "Фастівський") + val koRegionNames = Seq( + "Бориспільський", + "Броварський", + "Бучанський", + "Білоцерківський", + "Вишгородський", + "Обухівський", + "Фастівський" + ) val names = koRegionNames ++ Seq("Прип'ять") val koRegions = ko.regions @@ -124,19 +154,40 @@ class KatotthSpec extends Specification { "contain Crimea regions" in { val crimea = regions.find(_.name == "Автономна Республіка Крим").get - val regionNames = Seq("Бахчисарайський", "Білогірський", "Джанкойський", "Євпаторійський", "Керченський", "Курманський", - "Перекопський", "Сімферопольський", "Феодосійський", "Ялтинський") + val regionNames = Seq( + "Бахчисарайський", + "Білогірський", + "Джанкойський", + "Євпаторійський", + "Керченський", + "Курманський", + "Перекопський", + "Сімферопольський", + "Феодосійський", + "Ялтинський" + ) crimea.regions.map(_.name) === regionNames - crimea.regions.flatMap(_.parent().map(_.name)) === List.fill(regionNames.size)("Автономна Республіка Крим") + crimea.regions.flatMap(_.parent().map(_.name)) === List.fill( + regionNames.size + )("Автономна Республіка Крим") } "contain Vinnytsya oblast regions" in { val vinnytsyaRegion = regions.find(_.name == "Вінницька").get - val regionNames = Seq("Вінницький", "Гайсинський", "Жмеринський", "Могилів-Подільський", "Тульчинський", "Хмільницький") + val regionNames = Seq( + "Вінницький", + "Гайсинський", + "Жмеринський", + "Могилів-Подільський", + "Тульчинський", + "Хмільницький" + ) vinnytsyaRegion.regions.map(_.name) === regionNames - vinnytsyaRegion.regions.flatMap(_.parent().map(_.name)).toSet === Set("Вінницька") + vinnytsyaRegion.regions.flatMap(_.parent().map(_.name)).toSet === Set( + "Вінницька" + ) } "lookup regions by monumentId" in { @@ -183,8 +234,15 @@ class KatotthSpec extends Specification { "contain Simferopol regions" in { val simferopol = Ukraine.byMonumentId("01-101").get simferopol.name === "Сімферополь" - simferopol.regions.map(_.name) === Seq("Залізничний", "Київський", "Центральний", "Аерофлотський", "Гресівський", - "Комсомольське", "Аграрне") + simferopol.regions.map(_.name) === Seq( + "Залізничний", + "Київський", + "Центральний", + "Аерофлотський", + "Гресівський", + "Комсомольське", + "Аграрне" + ) val hresivskyi = simferopol.regions.find(_.name == "Гресівський").get hresivskyi.regions.map(_.name) === Seq("Бітумне") @@ -200,7 +258,10 @@ class KatotthSpec extends Specification { "differentiate types" in { val smt = Ukraine.byIdAndName("14-215", "смт Андріївка") val village1 = Ukraine.byIdAndName("14-215", "село Андріївка") - val village2 = Ukraine.byIdAndName("14-215", "[[Андріївка (Волноваський район, село)|Андріївка (село)]]") + val village2 = Ukraine.byIdAndName( + "14-215", + "[[Андріївка (Волноваський район, село)|Андріївка (село)]]" + ) smt.size === 1 smt.head.regionType === Some(KoatuuTypes.codeToType("Т")) village1.size === 1 @@ -211,7 +272,7 @@ class KatotthSpec extends Specification { "contain lesser regions" in { Ukraine.byIdAndName("18-240", "Новоград-Волинський район").size === 1 - Ukraine.byIdAndName("14-224", "Іванопільська").size === 1 // сільська рада + Ukraine.byIdAndName("14-224", "Іванопільська").size === 1 // сільська рада Ukraine.byIdAndName("18-211", "Хорошівський район").size === 1 Ukraine.byIdAndName("35-236", "Новоархангельський район").size === 1 Ukraine.byIdAndName("18-254", "Пулинський район").size === 1 @@ -219,4 +280,4 @@ class KatotthSpec extends Specification { // Ukraine.byIdAndName("01-119", "Ялтинська").size === 1 // м/р } } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuComparator.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuComparator.scala index d93e48eb..dab569f9 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuComparator.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuComparator.scala @@ -7,8 +7,12 @@ class KoatuuComparator extends Specification { val oldRegions = Ukraine.regions - val UkraineNew: Country = new Country("UA", "Ukraine", Seq("uk"), - Koatuu.regionsNew(() => Some(UkraineNew))) + val UkraineNew: Country = new Country( + "UA", + "Ukraine", + Seq("uk"), + Koatuu.regionsNew(() => Some(UkraineNew)) + ) val newRegions = UkraineNew.regions diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuFlatParserSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuFlatParserSpec.scala index a47974a5..953e2a46 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuFlatParserSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuFlatParserSpec.scala @@ -205,8 +205,19 @@ class KoatuuFlatParserSpec extends Specification { val kyiv = regions.head kyiv.code === "80" kyiv.name === "Київ" - regions.tail.map(_.name) === Seq("Райони м. Київ", "Голосіївський", "Дарницький", "Деснянський", "Дніпровський", - "Оболонський", "Печерський", "Подільський", "Святошинський", "Солом'янський", "Шевченківський") + regions.tail.map(_.name) === Seq( + "Райони м. Київ", + "Голосіївський", + "Дарницький", + "Деснянський", + "Дніпровський", + "Оболонський", + "Печерський", + "Подільський", + "Святошинський", + "Солом'янський", + "Шевченківський" + ) } } @@ -221,7 +232,9 @@ class KoatuuFlatParserSpec extends Specification { } "parse Simferopol" in { - val regions = makeHierarchy(parse(arr(crimeaJson, simferopolJson, simferopolRegionJson))) + val regions = makeHierarchy( + parse(arr(crimeaJson, simferopolJson, simferopolRegionJson)) + ) regions.size === 1 val crimea = regions.head crimea.code === "01" @@ -233,7 +246,9 @@ class KoatuuFlatParserSpec extends Specification { simferopol.code === "01101" simferopol.name === "Сімферополь" - crimea.regions.flatMap(_.parent().map(_.name)) === Seq("Автономна Республіка Крим") + crimea.regions.flatMap(_.parent().map(_.name)) === Seq( + "Автономна Республіка Крим" + ) } "parse Kyiv city" in { @@ -242,8 +257,18 @@ class KoatuuFlatParserSpec extends Specification { val kyiv = regions.head kyiv.code === "80" kyiv.name === "Київ" - kyiv.regions.map(_.name) === Seq("Голосіївський", "Дарницький", "Деснянський", "Дніпровський", - "Оболонський", "Печерський", "Подільський", "Святошинський", "Солом'янський", "Шевченківський") + kyiv.regions.map(_.name) === Seq( + "Голосіївський", + "Дарницький", + "Деснянський", + "Дніпровський", + "Оболонський", + "Печерський", + "Подільський", + "Святошинський", + "Солом'янський", + "Шевченківський" + ) } } diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuNewSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuNewSpec.scala index 5d4ef78e..f2fa4966 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuNewSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuNewSpec.scala @@ -4,8 +4,12 @@ import org.specs2.mutable.Specification class KoatuuNewSpec extends Specification { - val Ukraine: Country = new Country("UA", "Ukraine", Seq("uk"), - Koatuu.regionsNew(() => Some(Ukraine))) + val Ukraine: Country = new Country( + "UA", + "Ukraine", + Seq("uk"), + Koatuu.regionsNew(() => Some(Ukraine)) + ) val regions = Ukraine.regions "level1" should { @@ -41,7 +45,9 @@ class KoatuuNewSpec extends Specification { ) "contain country parent" in { - regions.flatMap(_.parent().map(_.name)) === List.fill(topRegions.size)(Country.Ukraine.name) + regions.flatMap(_.parent().map(_.name)) === List.fill(topRegions.size)( + Country.Ukraine.name + ) } "have 27 elements" in { @@ -73,20 +79,35 @@ class KoatuuNewSpec extends Specification { "level2" should { "have 490 raions" in { - val withoutCities = Country.Ukraine.regions.filter(adm => !Set("Київ", "Севастополь").contains(adm.name)) + val withoutCities = Country.Ukraine.regions.filter(adm => + !Set("Київ", "Севастополь").contains(adm.name) + ) withoutCities.size === 25 - val raions = withoutCities.flatMap(_.regions).filter(_.name.endsWith("район")) + val raions = + withoutCities.flatMap(_.regions).filter(_.name.endsWith("район")) raions.size === 490 } "contain Kyiv raions" in { - val regionNames = Seq("Голосіївський", "Дарницький", "Деснянський", "Дніпровський", - "Оболонський", "Печерський", "Подільський", "Святошинський", "Солом'янський", "Шевченківський") + val regionNames = Seq( + "Голосіївський", + "Дарницький", + "Деснянський", + "Дніпровський", + "Оболонський", + "Печерський", + "Подільський", + "Святошинський", + "Солом'янський", + "Шевченківський" + ) val kyiv = regions.find(_.name == "Київ").get val kyivRegions = kyiv.regions kyivRegions.map(_.name) === regionNames - kyivRegions.flatMap(_.parent().map(_.name)) === List.fill(regionNames.size)("Київ") + kyivRegions.flatMap(_.parent().map(_.name)) === List.fill( + regionNames.size + )("Київ") } "find Kyiv raions by code" in { @@ -103,7 +124,8 @@ class KoatuuNewSpec extends Specification { "80-391" -> "Шевченківський" ) - val regionToIds = Ukraine.byRegion(idToName.keySet).mapValues(_.head).toMap + val regionToIds = + Ukraine.byRegion(idToName.keySet).mapValues(_.head).toMap regionToIds.keySet.flatMap(_.parent().map(_.name)) === Set("Київ") regionToIds.map(_.swap).mapValues(_.name).toMap === idToName @@ -112,47 +134,135 @@ class KoatuuNewSpec extends Specification { "contain Kyiv oblast regions" in { val ko = regions.find(_.name == "Київська область").get val koRegionNames = Seq( - "Баришівський", "Білоцерківський", "Богуславський", "Бориспільський", "Бородянський", "Броварський", - "Васильківський", "Вишгородський", "Володарський", "Згурівський", "Іванківський", "Кагарлицький", "Києво-Святошинський", - "Макарівський", "Миронівський", "Обухівський", "Переяслав-Хмельницький", "Поліський", "Рокитнянський", "Сквирський", - "Ставищенський", "Таращанський", "Тетіївський", "Фастівський", "Яготинський" + "Баришівський", + "Білоцерківський", + "Богуславський", + "Бориспільський", + "Бородянський", + "Броварський", + "Васильківський", + "Вишгородський", + "Володарський", + "Згурівський", + "Іванківський", + "Кагарлицький", + "Києво-Святошинський", + "Макарівський", + "Миронівський", + "Обухівський", + "Переяслав-Хмельницький", + "Поліський", + "Рокитнянський", + "Сквирський", + "Ставищенський", + "Таращанський", + "Тетіївський", + "Фастівський", + "Яготинський" ).map(_ + " район") - val koCities = Seq("Біла Церква", "Березань", "Бориспіль", "Бровари", "Буча", "Васильків", "Ірпінь", "Обухів", "Переяслав", - "Прип'ять", "Ржищів", "Славутич", "Фастів") + val koCities = Seq( + "Біла Церква", + "Березань", + "Бориспіль", + "Бровари", + "Буча", + "Васильків", + "Ірпінь", + "Обухів", + "Переяслав", + "Прип'ять", + "Ржищів", + "Славутич", + "Фастів" + ) val names = Seq() ++ koCities ++ koRegionNames val koRegions = ko.regions koRegions.map(_.name).sorted === names.sorted - koRegions.flatMap(_.parent().map(_.name)) === List.fill(names.size)("Київська область") + koRegions.flatMap(_.parent().map(_.name)) === List.fill(names.size)( + "Київська область" + ) } "contain Crimea regions" in { val crimea = regions.find(_.name == "Автономна Республіка Крим").get val regionNames = Seq( - "Сімферополь", "Алушта", "Джанкой", "Євпаторія", "Керч", - "Красноперекопськ", "Саки", "Армянськ", "Феодосія", "Судак", "Ялта", - "Бахчисарайський район", "Білогірський район", "Джанкойський район", "Кіровський район", "Красногвардійський район", - "Красноперекопський район", "Ленінський район", "Нижньогірський район", "Первомайський район", "Роздольненський район", - "Сакський район", "Сімферопольський район", "Совєтський район", "Чорноморський район") + "Сімферополь", + "Алушта", + "Джанкой", + "Євпаторія", + "Керч", + "Красноперекопськ", + "Саки", + "Армянськ", + "Феодосія", + "Судак", + "Ялта", + "Бахчисарайський район", + "Білогірський район", + "Джанкойський район", + "Кіровський район", + "Красногвардійський район", + "Красноперекопський район", + "Ленінський район", + "Нижньогірський район", + "Первомайський район", + "Роздольненський район", + "Сакський район", + "Сімферопольський район", + "Совєтський район", + "Чорноморський район" + ) crimea.regions.map(_.name) === regionNames - crimea.regions.flatMap(_.parent().map(_.name)) === List.fill(regionNames.size)("Автономна Республіка Крим") + crimea.regions.flatMap(_.parent().map(_.name)) === List.fill( + regionNames.size + )("Автономна Республіка Крим") } "contain Vinnytsya oblast regions" in { val vinnytsyaRegion = regions.find(_.name == "Вінницька область").get val regionNames = Seq( - "Вінниця", "Жмеринка", "Могилів-Подільський", "Козятин", "Ладижин", "Хмільник", - "Барський район", "Бершадський район", "Вінницький район", "Гайсинський район", "Жмеринський район", - "Іллінецький район", "Козятинський район", "Калинівський район", "Крижопільський район", "Липовецький район", - "Літинський район", "Могилів-Подільський район", "Мурованокуриловецький район", "Немирівський район", - "Оратівський район", "Піщанський район", "Погребищенський район", "Теплицький район", "Томашпільський район", - "Тростянецький район", "Тульчинський район", "Тиврівський район", "Хмільницький район", "Чернівецький район", - "Чечельницький район", "Шаргородський район", "Ямпільський район") + "Вінниця", + "Жмеринка", + "Могилів-Подільський", + "Козятин", + "Ладижин", + "Хмільник", + "Барський район", + "Бершадський район", + "Вінницький район", + "Гайсинський район", + "Жмеринський район", + "Іллінецький район", + "Козятинський район", + "Калинівський район", + "Крижопільський район", + "Липовецький район", + "Літинський район", + "Могилів-Подільський район", + "Мурованокуриловецький район", + "Немирівський район", + "Оратівський район", + "Піщанський район", + "Погребищенський район", + "Теплицький район", + "Томашпільський район", + "Тростянецький район", + "Тульчинський район", + "Тиврівський район", + "Хмільницький район", + "Чернівецький район", + "Чечельницький район", + "Шаргородський район", + "Ямпільський район" + ) vinnytsyaRegion.regions.map(_.name) === regionNames - vinnytsyaRegion.regions.flatMap(_.parent().map(_.name)) === List.fill(regionNames.size)("Вінницька область") + vinnytsyaRegion.regions.flatMap(_.parent().map(_.name)) === List.fill( + regionNames.size + )("Вінницька область") } "lookup regions by monumentId" in { @@ -199,8 +309,15 @@ class KoatuuNewSpec extends Specification { "contain Simferopol regions" in { val simferopol = Ukraine.byMonumentId("01-101").get simferopol.name === "Сімферополь" - simferopol.regions.map(_.name) === Seq("Залізничний", "Київський", "Центральний", "Аерофлотський", "Гресівський", - "Комсомольське", "Аграрне") + simferopol.regions.map(_.name) === Seq( + "Залізничний", + "Київський", + "Центральний", + "Аерофлотський", + "Гресівський", + "Комсомольське", + "Аграрне" + ) val hresivskyi = simferopol.regions.find(_.name == "Гресівський").get hresivskyi.regions.map(_.name) === Seq("Бітумне") @@ -216,7 +333,10 @@ class KoatuuNewSpec extends Specification { "differentiate types" in { val smt = Ukraine.byIdAndName("14-215", "смт Андріївка") val village1 = Ukraine.byIdAndName("14-215", "село Андріївка") - val village2 = Ukraine.byIdAndName("14-215", "[[Андріївка (Волноваський район, село)|Андріївка (село)]]") + val village2 = Ukraine.byIdAndName( + "14-215", + "[[Андріївка (Волноваський район, село)|Андріївка (село)]]" + ) smt.size === 1 smt.head.regionType === Some(KoatuuTypes.codeToType("Т")) village1.size === 1 @@ -227,7 +347,7 @@ class KoatuuNewSpec extends Specification { "contain lesser regions" in { Ukraine.byIdAndName("18-240", "Новоград-Волинський район").size === 1 - Ukraine.byIdAndName("14-224", "Іванопільська").size === 1 // сільська рада + Ukraine.byIdAndName("14-224", "Іванопільська").size === 1 // сільська рада Ukraine.byIdAndName("18-211", "Хорошівський район").size === 1 Ukraine.byIdAndName("35-236", "Новоархангельський район").size === 1 Ukraine.byIdAndName("18-254", "Пулинський район").size === 1 @@ -235,4 +355,4 @@ class KoatuuNewSpec extends Specification { // Ukraine.byIdAndName("01-119", "Ялтинська").size === 1 // м/р } } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuSpec.scala index 44cd670b..35ba75ce 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuSpec.scala @@ -40,7 +40,9 @@ class KoatuuSpec extends Specification { ) "contain country parent" in { - regions.flatMap(_.parent().map(_.name)) === List.fill(topRegions.size)(Country.Ukraine.name) + regions.flatMap(_.parent().map(_.name)) === List.fill(topRegions.size)( + Country.Ukraine.name + ) } "have 27 elements" in { @@ -52,7 +54,8 @@ class KoatuuSpec extends Specification { } "lookup level1 by code" in { - Ukraine.byRegion(topRegions.keySet) + Ukraine + .byRegion(topRegions.keySet) .map { case (adm, ids) => ids.head -> adm.name } === topRegions Ukraine.byIdAndName("80-361", "Київ").head.name === "Київ" @@ -71,19 +74,34 @@ class KoatuuSpec extends Specification { "level2" should { "have 490 raions" in { - val withoutCities = Country.Ukraine.regions.filter(adm => !Set("Київ", "Севастополь").contains(adm.name)) + val withoutCities = Country.Ukraine.regions.filter(adm => + !Set("Київ", "Севастополь").contains(adm.name) + ) withoutCities.size === 25 - val raions = withoutCities.flatMap(_.regions).filter(_.name.endsWith("район")) + val raions = + withoutCities.flatMap(_.regions).filter(_.name.endsWith("район")) raions.size === 490 } "contain Kyiv raions" in { - val regionNames = Seq("Голосіївський", "Дарницький", "Деснянський", "Дніпровський", - "Оболонський", "Печерський", "Подільський", "Святошинський", "Солом'янський", "Шевченківський") + val regionNames = Seq( + "Голосіївський", + "Дарницький", + "Деснянський", + "Дніпровський", + "Оболонський", + "Печерський", + "Подільський", + "Святошинський", + "Солом'янський", + "Шевченківський" + ) val kyiv = regions.find(_.name == "Київ").get kyiv.regions.map(_.name) === regionNames - kyiv.regions.flatMap(_.parent().map(_.name)) === List.fill(regionNames.size)("Київ") + kyiv.regions.flatMap(_.parent().map(_.name)) === List.fill( + regionNames.size + )("Київ") } "find Kyiv raions by code" in { @@ -100,7 +118,8 @@ class KoatuuSpec extends Specification { "80-391" -> "Шевченківський" ) - val regionToIds = Ukraine.byRegion(idToName.keySet).mapValues(_.head).toMap + val regionToIds = + Ukraine.byRegion(idToName.keySet).mapValues(_.head).toMap regionToIds.keySet.flatMap(_.parent().map(_.name)) === Set("Київ") regionToIds.map(_.swap).mapValues(_.name).toMap === idToName @@ -109,47 +128,135 @@ class KoatuuSpec extends Specification { "contain Kyiv oblast regions" in { val ko = regions.find(_.name == "Київська область").get val koRegions = Seq( - "Баришівський", "Білоцерківський", "Богуславський", "Бориспільський", "Бородянський", "Броварський", - "Васильківський", "Вишгородський", "Володарський", "Згурівський", "Іванківський", "Кагарлицький", "Києво-Святошинський", - "Макарівський", "Миронівський", "Обухівський", "Переяслав-Хмельницький", "Поліський", "Рокитнянський", "Сквирський", - "Ставищенський", "Таращанський", "Тетіївський", "Фастівський", "Яготинський" + "Баришівський", + "Білоцерківський", + "Богуславський", + "Бориспільський", + "Бородянський", + "Броварський", + "Васильківський", + "Вишгородський", + "Володарський", + "Згурівський", + "Іванківський", + "Кагарлицький", + "Києво-Святошинський", + "Макарівський", + "Миронівський", + "Обухівський", + "Переяслав-Хмельницький", + "Поліський", + "Рокитнянський", + "Сквирський", + "Ставищенський", + "Таращанський", + "Тетіївський", + "Фастівський", + "Яготинський" ).map(_ + " район") - val koCities = Seq("Біла Церква", "Березань", "Бориспіль", "Бровари", "Буча", "Васильків", "Ірпінь", "Обухів", "Переяслав", - "Прип'ять", "Ржищів", "Славутич", "Фастів") + val koCities = Seq( + "Біла Церква", + "Березань", + "Бориспіль", + "Бровари", + "Буча", + "Васильків", + "Ірпінь", + "Обухів", + "Переяслав", + "Прип'ять", + "Ржищів", + "Славутич", + "Фастів" + ) val names = Seq() ++ koCities ++ koRegions ko.regions.map(_.name).sorted === names.sorted - ko.regions.flatMap(_.parent().map(_.name)) === List.fill(names.size)("Київська область") + ko.regions.flatMap(_.parent().map(_.name)) === List.fill(names.size)( + "Київська область" + ) } "contain Crimea regions" in { val crimea = regions.find(_.name == "Автономна Республіка Крим").get val regionNames = Seq( - "Сімферополь", "Алушта", "Джанкой", "Євпаторія", "Керч", - "Красноперекопськ", "Саки", "Армянськ", "Феодосія", "Судак", "Ялта", - "Бахчисарайський район", "Білогірський район", "Джанкойський район", "Кіровський район", "Красногвардійський район", - "Красноперекопський район", "Ленінський район", "Нижньогірський район", "Первомайський район", "Роздольненський район", - "Сакський район", "Сімферопольський район", "Совєтський район", "Чорноморський район") + "Сімферополь", + "Алушта", + "Джанкой", + "Євпаторія", + "Керч", + "Красноперекопськ", + "Саки", + "Армянськ", + "Феодосія", + "Судак", + "Ялта", + "Бахчисарайський район", + "Білогірський район", + "Джанкойський район", + "Кіровський район", + "Красногвардійський район", + "Красноперекопський район", + "Ленінський район", + "Нижньогірський район", + "Первомайський район", + "Роздольненський район", + "Сакський район", + "Сімферопольський район", + "Совєтський район", + "Чорноморський район" + ) crimea.regions.map(_.name) === regionNames - crimea.regions.flatMap(_.parent().map(_.name)) === List.fill(regionNames.size)("Автономна Республіка Крим") + crimea.regions.flatMap(_.parent().map(_.name)) === List.fill( + regionNames.size + )("Автономна Республіка Крим") } "contain Vinnytsya oblast regions" in { val vinnytsyaRegion = regions.find(_.name == "Вінницька область").get val regionNames = Seq( - "Вінниця", "Жмеринка", "Могилів-Подільський", "Козятин", "Ладижин", "Хмільник", - "Барський район", "Бершадський район", "Вінницький район", "Гайсинський район", "Жмеринський район", - "Іллінецький район", "Козятинський район", "Калинівський район", "Крижопільський район", "Липовецький район", - "Літинський район", "Могилів-Подільський район", "Мурованокуриловецький район", "Немирівський район", - "Оратівський район", "Піщанський район", "Погребищенський район", "Теплицький район", "Томашпільський район", - "Тростянецький район", "Тульчинський район", "Тиврівський район", "Хмільницький район", "Чернівецький район", - "Чечельницький район", "Шаргородський район", "Ямпільський район") + "Вінниця", + "Жмеринка", + "Могилів-Подільський", + "Козятин", + "Ладижин", + "Хмільник", + "Барський район", + "Бершадський район", + "Вінницький район", + "Гайсинський район", + "Жмеринський район", + "Іллінецький район", + "Козятинський район", + "Калинівський район", + "Крижопільський район", + "Липовецький район", + "Літинський район", + "Могилів-Подільський район", + "Мурованокуриловецький район", + "Немирівський район", + "Оратівський район", + "Піщанський район", + "Погребищенський район", + "Теплицький район", + "Томашпільський район", + "Тростянецький район", + "Тульчинський район", + "Тиврівський район", + "Хмільницький район", + "Чернівецький район", + "Чечельницький район", + "Шаргородський район", + "Ямпільський район" + ) vinnytsyaRegion.regions.map(_.name) === regionNames - vinnytsyaRegion.regions.flatMap(_.parent().map(_.name)) === List.fill(regionNames.size)("Вінницька область") + vinnytsyaRegion.regions.flatMap(_.parent().map(_.name)) === List.fill( + regionNames.size + )("Вінницька область") } "lookup regions by monumentId" in { @@ -176,12 +283,21 @@ class KoatuuSpec extends Specification { } "lookup by monument id" in { - Ukraine.byMonumentId("18-240").map(_.fullName).getOrElse("") === "Новоград-Волинський район" + Ukraine + .byMonumentId("18-240") + .map(_.fullName) + .getOrElse("") === "Новоград-Волинський район" Ukraine.byMonumentId("07-101").map(_.fullName).getOrElse("") === "Луцьк" - Ukraine.byMonumentId("01-101").map(_.fullName).getOrElse("") === "Сімферополь" + Ukraine + .byMonumentId("01-101") + .map(_.fullName) + .getOrElse("") === "Сімферополь" Ukraine.byMonumentId("01-103").map(_.fullName).getOrElse("") === "Алушта" Ukraine.byMonumentId("01-106").map(_.fullName).getOrElse("") === "Джанкой" - Ukraine.byMonumentId("01-109").map(_.fullName).getOrElse("") === "Євпаторія" + Ukraine + .byMonumentId("01-109") + .map(_.fullName) + .getOrElse("") === "Євпаторія" Ukraine.byMonumentId("05-101").map(_.fullName).getOrElse("") === "Вінниця" } } @@ -206,8 +322,15 @@ class KoatuuSpec extends Specification { "contain Simferopol regions" in { val simferopol = Ukraine.byMonumentId("01-101").get simferopol.name === "Сімферополь" - simferopol.regions.map(_.name) === Seq("Залізничний", "Київський", "Центральний", "Аерофлотський", "Гресівський", - "Комсомольське", "Аграрне") + simferopol.regions.map(_.name) === Seq( + "Залізничний", + "Київський", + "Центральний", + "Аерофлотський", + "Гресівський", + "Комсомольське", + "Аграрне" + ) val hresivskyi = simferopol.regions.find(_.name == "Гресівський").get hresivskyi.regions.map(_.name) === Seq("Бітумне") @@ -223,7 +346,10 @@ class KoatuuSpec extends Specification { "differentiate types" in { val smt = Ukraine.byIdAndName("14-215", "смт Андріївка") val village1 = Ukraine.byIdAndName("14-215", "село Андріївка") - val village2 = Ukraine.byIdAndName("14-215", "[[Андріївка (Волноваський район, село)|Андріївка (село)]]") + val village2 = Ukraine.byIdAndName( + "14-215", + "[[Андріївка (Волноваський район, село)|Андріївка (село)]]" + ) smt.size === 1 smt.head.regionType === Some(KoatuuTypes.codeToType("Т")) village1.size === 1 @@ -234,7 +360,7 @@ class KoatuuSpec extends Specification { "contain lesser regions" in { Ukraine.byIdAndName("18-240", "Новоград-Волинський район").size === 1 - Ukraine.byIdAndName("14-224", "Іванопільська").size === 1 // сільська рада + Ukraine.byIdAndName("14-224", "Іванопільська").size === 1 // сільська рада Ukraine.byIdAndName("18-211", "Хорошівський район").size === 1 Ukraine.byIdAndName("35-236", "Новоархангельський район").size === 1 Ukraine.byIdAndName("18-254", "Пулинський район").size === 1 @@ -242,4 +368,4 @@ class KoatuuSpec extends Specification { // Ukraine.byIdAndName("01-119", "Ялтинська").size === 1 // м/р } } -} \ No newline at end of file +} diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuToKatotthSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuToKatotthSpec.scala index 1c074b91..74f76706 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuToKatotthSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/dto/KoatuuToKatotthSpec.scala @@ -12,7 +12,8 @@ class KoatuuToKatotthSpec extends Specification { "UA", "Ukraine", Seq("uk"), - Katotth.regions(() => Some(UkraineKatotth))) + Katotth.regions(() => Some(UkraineKatotth)) + ) val regionsKatotth = UkraineKatotth.regions val katotthMap = UkraineKatotth.mapByCode @@ -20,7 +21,8 @@ class KoatuuToKatotthSpec extends Specification { "UA", "Ukraine", Seq("uk"), - Koatuu.regionsNew(() => Some(UkraineKoatuu))) + Koatuu.regionsNew(() => Some(UkraineKoatuu)) + ) val regionsKoatuu = UkraineKoatuu.regions val koatuuMap = UkraineKoatuu.mapByCode diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/AuthorsMonumentsSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/AuthorsMonumentsSpec.scala index 25556e31..89fe3e62 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/AuthorsMonumentsSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/AuthorsMonumentsSpec.scala @@ -10,15 +10,19 @@ import org.specs2.mutable.Specification class AuthorsMonumentsSpec extends Specification { - val contest = new Contest(ContestType.WLE, - Country.Ukraine, - 2013, - uploadConfigs = Seq.empty[UploadConfig]) - - def getTable(images: Seq[Image], - monuments: Seq[Monument], - contest: Contest = contest, - gallery: Boolean = false): Table = { + val contest = new Contest( + ContestType.WLE, + Country.Ukraine, + 2013, + uploadConfigs = Seq.empty[UploadConfig] + ) + + def getTable( + images: Seq[Image], + monuments: Seq[Monument], + contest: Contest = contest, + gallery: Boolean = false + ): Table = { val mdb = Some(new MonumentDB(contest, monuments)) val db = new ImageDB(contest, images, mdb) @@ -29,22 +33,29 @@ class AuthorsMonumentsSpec extends Specification { def monument(id: String, name: String) = new Monument(id = id, name = name, listConfig = Some(WlmUa)) - def monuments(n: Int, - regionId: String, - namePrefix: String, - startId: Int = 1): Seq[Monument] = + def monuments( + n: Int, + regionId: String, + namePrefix: String, + startId: Int = 1 + ): Seq[Monument] = (startId until startId + n).map(i => - monument(s"$regionId-xxx-000$i", namePrefix + i)) - - def image(i: Int, - author: Option[String] = None, - monumentId: Option[String] = None): Image = { - Image(s"Image$i.jpg", - author = author, - monumentIds = monumentId.toList, - pageId = Some(i), - width = Some(800), - height = Some(600)) + monument(s"$regionId-xxx-000$i", namePrefix + i) + ) + + def image( + i: Int, + author: Option[String] = None, + monumentId: Option[String] = None + ): Image = { + Image( + s"Image$i.jpg", + author = author, + monumentIds = monumentId.toList, + pageId = Some(i), + width = Some(800), + height = Some(600) + ) } "authorsMonumentsTable" should { @@ -69,7 +80,11 @@ class AuthorsMonumentsSpec extends Specification { val contestStat = ContestStat(contest, 2013, mdb, Some(db), Some(db)) val table = new AuthorMonuments(contestStat).table - table.headers === Seq("User", "Objects pictured", "Photos uploaded") ++ contest.country.regionNames + table.headers === Seq( + "User", + "Objects pictured", + "Photos uploaded" + ) ++ contest.country.regionNames table.data === Seq( Seq("Total", "0.0", "0") ++ Seq.fill(27)("0.0") @@ -81,7 +96,11 @@ class AuthorsMonumentsSpec extends Specification { val monuments = Seq.empty[Monument] val table = getTable(images, monuments) - table.headers === Seq("User", "Objects pictured", "Photos uploaded") ++ contest.country.regionNames + table.headers === Seq( + "User", + "Objects pictured", + "Photos uploaded" + ) ++ contest.country.regionNames table.data === Seq( Seq("Total", "0.0", "0") ++ Seq @@ -94,7 +113,11 @@ class AuthorsMonumentsSpec extends Specification { val monuments = Seq.empty[Monument] val table = getTable(images, monuments) - table.headers === Seq("User", "Objects pictured", "Photos uploaded") ++ contest.country.regionNames + table.headers === Seq( + "User", + "Objects pictured", + "Photos uploaded" + ) ++ contest.country.regionNames table.data === Seq( Seq("Total", "0.0", "1") ++ Seq @@ -108,7 +131,11 @@ class AuthorsMonumentsSpec extends Specification { val monuments = Seq.empty[Monument] val table: Table = getTable(images, monuments) - table.headers === Seq("User", "Objects pictured", "Photos uploaded") ++ contest.country.regionNames + table.headers === Seq( + "User", + "Objects pictured", + "Photos uploaded" + ) ++ contest.country.regionNames table.data === Seq( Seq("Total", "0.0", "1") ++ Seq @@ -123,24 +150,33 @@ class AuthorsMonumentsSpec extends Specification { val table = getTable(images, monuments) - table.headers === Seq("User", "Objects pictured", "Photos uploaded") ++ contest.country.regionNames + table.headers === Seq( + "User", + "Objects pictured", + "Photos uploaded" + ) ++ contest.country.regionNames val data = table.data data.head === Seq("Total", "1.0", "1") ++ Seq.fill( - contest.country.regions.size)("0.0") + contest.country.regions.size + )("0.0") data.size === 2 data.last === Seq("[[User:user|user]]", "1.0", "1") ++ Seq.fill( - contest.country.regions.size)("0.0") + contest.country.regions.size + )("0.0") } "have 1 image with author and monument no regions" in { val noRegions = contest.copy(country = Country.Azerbaijan) val user = "user" val images = Seq( - Image("image1.jpg", - author = Some(user), - monumentIds = List("123"), - pageId = Some(1))) + Image( + "image1.jpg", + author = Some(user), + monumentIds = List("123"), + pageId = Some(1) + ) + ) val monuments = Seq(new Monument(id = "123", name = "123 monument")) val table = getTable(images, monuments, noRegions) @@ -157,10 +193,13 @@ class AuthorsMonumentsSpec extends Specification { val noRegions = contest.copy(country = Country.Azerbaijan) val user = "user" val images = Seq( - Image("image1.jpg", - author = Some(user), - monumentIds = List("123"), - pageId = Some(1))) + Image( + "image1.jpg", + author = Some(user), + monumentIds = List("123"), + pageId = Some(1) + ) + ) val monuments = Seq(new Monument(id = "123", name = "123 monument")) val table = getTable(images, monuments, noRegions, gallery = true) @@ -169,9 +208,11 @@ class AuthorsMonumentsSpec extends Specification { table.data === Seq( Seq("Total", "1.0", "1"), - Seq("[[User:user|user]]", - "[[Commons:Wiki Loves Earth 2013 in Azerbaijan/user|1.0]]", - "1") + Seq( + "[[User:user|user]]", + "[[Commons:Wiki Loves Earth 2013 in Azerbaijan/user|1.0]]", + "1" + ) ) } @@ -179,38 +220,54 @@ class AuthorsMonumentsSpec extends Specification { val noRegions = contest.copy(country = Country.Azerbaijan) val (user1, user2) = ("user1", "user2") val images = Seq( - Image("image11.jpg", - author = Some(user1), - monumentIds = List("11"), - pageId = Some(111)), - Image("image12.jpg", - author = Some(user1), - monumentIds = List("11"), - pageId = Some(112)), - Image("image13.jpg", - author = Some(user1), - monumentIds = List("12"), - pageId = Some(113)), - Image("image21.jpg", - author = Some(user2), - monumentIds = List("21"), - pageId = Some(121)), - Image("image22.jpg", - author = Some(user2), - monumentIds = List("22"), - pageId = Some(122)), - Image("image23.jpg", - author = Some(user2), - monumentIds = List("22"), - pageId = Some(123)), - Image("image24.jpg", - author = Some(user2), - monumentIds = List("23"), - pageId = Some(124)), - Image("image25.jpg", - author = Some(user2), - monumentIds = List("24"), - pageId = Some(125)) + Image( + "image11.jpg", + author = Some(user1), + monumentIds = List("11"), + pageId = Some(111) + ), + Image( + "image12.jpg", + author = Some(user1), + monumentIds = List("11"), + pageId = Some(112) + ), + Image( + "image13.jpg", + author = Some(user1), + monumentIds = List("12"), + pageId = Some(113) + ), + Image( + "image21.jpg", + author = Some(user2), + monumentIds = List("21"), + pageId = Some(121) + ), + Image( + "image22.jpg", + author = Some(user2), + monumentIds = List("22"), + pageId = Some(122) + ), + Image( + "image23.jpg", + author = Some(user2), + monumentIds = List("22"), + pageId = Some(123) + ), + Image( + "image24.jpg", + author = Some(user2), + monumentIds = List("23"), + pageId = Some(124) + ), + Image( + "image25.jpg", + author = Some(user2), + monumentIds = List("24"), + pageId = Some(125) + ) ) val monuments = Seq( @@ -235,32 +292,44 @@ class AuthorsMonumentsSpec extends Specification { "with regions in" in { val images2 = Seq( - Image("File:Img11y2f1.jpg", - monumentIds = List("01-xxx-0001"), - author = Some("FromCrimeaOld"), - pageId = Some(11)), - Image("File:Img12y2f1.jpg", - monumentIds = List("01-xxx-0002"), - author = Some("FromCrimeaNew"), - pageId = Some(12)), - Image("File:Img52y2f1.jpg", - monumentIds = List("05-xxx-0002"), - author = Some("FromPodillyaNew1"), - pageId = Some(5121)), - Image("File:Img52y2f2.jpg", - monumentIds = List("05-xxx-0002"), - author = Some("FromPodillyaNew2"), - pageId = Some(5222)), - Image("File:Img72y2f1.jpg", - monumentIds = List("07-xxx-0002"), - author = Some("FromVolynNew"), - pageId = Some(72)) + Image( + "File:Img11y2f1.jpg", + monumentIds = List("01-xxx-0001"), + author = Some("FromCrimeaOld"), + pageId = Some(11) + ), + Image( + "File:Img12y2f1.jpg", + monumentIds = List("01-xxx-0002"), + author = Some("FromCrimeaNew"), + pageId = Some(12) + ), + Image( + "File:Img52y2f1.jpg", + monumentIds = List("05-xxx-0002"), + author = Some("FromPodillyaNew1"), + pageId = Some(5121) + ), + Image( + "File:Img52y2f2.jpg", + monumentIds = List("05-xxx-0002"), + author = Some("FromPodillyaNew2"), + pageId = Some(5222) + ), + Image( + "File:Img72y2f1.jpg", + monumentIds = List("07-xxx-0002"), + author = Some("FromVolynNew"), + pageId = Some(72) + ) ) - val mDb = new MonumentDB(contest, - monuments(2, "01", "Crimea") ++ - monuments(5, "05", "Podillya") ++ - monuments(7, "07", "Volyn")) + val mDb = new MonumentDB( + contest, + monuments(2, "01", "Crimea") ++ + monuments(5, "05", "Podillya") ++ + monuments(7, "07", "Volyn") + ) val db = new ImageDB(contest, images2, Some(mDb)) val contestStat = @@ -270,86 +339,113 @@ class AuthorsMonumentsSpec extends Specification { data.size === 6 - table.headers.slice(0, 6) === Seq("User", - "Objects pictured", - "Photos uploaded", - "Автономна Республіка Крим", - "Вінницька область", - "Волинська область") + table.headers.slice(0, 6) === Seq( + "User", + "Objects pictured", + "Photos uploaded", + "Автономна Республіка Крим", + "Вінницька область", + "Волинська область" + ) data.head === Seq("Total", "4.0", "5", "2.0", "1.0", "1.0") ++ Seq.fill( - 24)("0.0") + 24 + )("0.0") data.slice(1, 6) === Seq( - Seq("[[User:FromCrimeaNew|FromCrimeaNew]]", - "1.0", - "1", - "1.0", - "0.0", - "0.0") ++ Seq + Seq( + "[[User:FromCrimeaNew|FromCrimeaNew]]", + "1.0", + "1", + "1.0", + "0.0", + "0.0" + ) ++ Seq .fill(24)("0.0"), - Seq("[[User:FromCrimeaOld|FromCrimeaOld]]", - "1.0", - "1", - "1.0", - "0.0", - "0.0") ++ Seq + Seq( + "[[User:FromCrimeaOld|FromCrimeaOld]]", + "1.0", + "1", + "1.0", + "0.0", + "0.0" + ) ++ Seq .fill(24)("0.0"), - Seq("[[User:FromPodillyaNew1|FromPodillyaNew1]]", - "1.0", - "1", - "0.0", - "1.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromPodillyaNew2|FromPodillyaNew2]]", - "1.0", - "1", - "0.0", - "1.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromVolynNew|FromVolynNew]]", - "1.0", - "1", - "0.0", - "0.0", - "1.0") ++ Seq + Seq( + "[[User:FromPodillyaNew1|FromPodillyaNew1]]", + "1.0", + "1", + "0.0", + "1.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromPodillyaNew2|FromPodillyaNew2]]", + "1.0", + "1", + "0.0", + "1.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromVolynNew|FromVolynNew]]", + "1.0", + "1", + "0.0", + "0.0", + "1.0" + ) ++ Seq .fill(24)("0.0") ) } "rate no new images" in { val images2 = Seq( - Image("File:Img11y2f1.jpg", - monumentIds = List("01-xxx-0001"), - author = Some("FromCrimeaOld"), - pageId = Some(11)), - Image("File:Img12y2f1.jpg", - monumentIds = List("01-xxx-0002"), - author = Some("FromCrimeaNew"), - pageId = Some(12)), - Image("File:Img52y2f1.jpg", - monumentIds = List("05-xxx-0002"), - author = Some("FromPodillyaNew1"), - pageId = Some(5221)), - Image("File:Img52y2f2.jpg", - monumentIds = List("05-xxx-0002"), - author = Some("FromPodillyaNew2"), - pageId = Some(5222)), - Image("File:Img72y2f1.jpg", - monumentIds = List("07-xxx-0002"), - author = Some("FromVolynNew"), - pageId = Some(72)) + Image( + "File:Img11y2f1.jpg", + monumentIds = List("01-xxx-0001"), + author = Some("FromCrimeaOld"), + pageId = Some(11) + ), + Image( + "File:Img12y2f1.jpg", + monumentIds = List("01-xxx-0002"), + author = Some("FromCrimeaNew"), + pageId = Some(12) + ), + Image( + "File:Img52y2f1.jpg", + monumentIds = List("05-xxx-0002"), + author = Some("FromPodillyaNew1"), + pageId = Some(5221) + ), + Image( + "File:Img52y2f2.jpg", + monumentIds = List("05-xxx-0002"), + author = Some("FromPodillyaNew2"), + pageId = Some(5222) + ), + Image( + "File:Img72y2f1.jpg", + monumentIds = List("07-xxx-0002"), + author = Some("FromVolynNew"), + pageId = Some(72) + ) ) - val mDb = new MonumentDB(contest, - monuments(2, "01", "Crimea") ++ - monuments(5, "05", "Podillya") ++ - monuments(7, "07", "Volyn")) + val mDb = new MonumentDB( + contest, + monuments(2, "01", "Crimea") ++ + monuments(5, "05", "Podillya") ++ + monuments(7, "07", "Volyn") + ) val images1 = images2.map { i2 => - i2.copy(pageId = i2.pageId.map(_ + 100), - title = i2.title.replace("Img", "Img0")) + i2.copy( + pageId = i2.pageId.map(_ + 100), + title = i2.title.replace("Img", "Img0") + ) } val db = new ImageDB(contest, images2, Some(mDb)) @@ -360,7 +456,8 @@ class AuthorsMonumentsSpec extends Specification { 2013, Some(mDb), Some(db), - Some(totalDb)) + Some(totalDb) + ) val table = new AuthorMonuments(contestStat).table val data = table.data @@ -379,101 +476,125 @@ class AuthorsMonumentsSpec extends Specification { "Волинська область" ) - data.head === Seq("Total", - "4.0", - "0", - "4", - "0", - "4", - "5", - "2.0", - "1.0", - "1.0") ++ Seq + data.head === Seq( + "Total", + "4.0", + "0", + "4", + "0", + "4", + "5", + "2.0", + "1.0", + "1.0" + ) ++ Seq .fill(24)("0.0") data.slice(1, 6) === Seq( - Seq("[[User:FromCrimeaNew|FromCrimeaNew]]", - "1.0", - "1", - "0", - "0", - "1.0", - "1", - "1.0", - "0.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromCrimeaOld|FromCrimeaOld]]", - "1.0", - "1", - "0", - "0", - "1.0", - "1", - "1.0", - "0.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromPodillyaNew1|FromPodillyaNew1]]", - "1.0", - "1", - "0", - "0", - "1.0", - "1", - "0.0", - "1.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromPodillyaNew2|FromPodillyaNew2]]", - "1.0", - "1", - "0", - "0", - "1.0", - "1", - "0.0", - "1.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromVolynNew|FromVolynNew]]", - "1.0", - "1", - "0", - "0", - "1.0", - "1", - "0.0", - "0.0", - "1.0") ++ Seq.fill(24)("0.0") + Seq( + "[[User:FromCrimeaNew|FromCrimeaNew]]", + "1.0", + "1", + "0", + "0", + "1.0", + "1", + "1.0", + "0.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromCrimeaOld|FromCrimeaOld]]", + "1.0", + "1", + "0", + "0", + "1.0", + "1", + "1.0", + "0.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromPodillyaNew1|FromPodillyaNew1]]", + "1.0", + "1", + "0", + "0", + "1.0", + "1", + "0.0", + "1.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromPodillyaNew2|FromPodillyaNew2]]", + "1.0", + "1", + "0", + "0", + "1.0", + "1", + "0.0", + "1.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromVolynNew|FromVolynNew]]", + "1.0", + "1", + "0", + "0", + "1.0", + "1", + "0.0", + "0.0", + "1.0" + ) ++ Seq.fill(24)("0.0") ) } "rate with all new images" in { val images2 = Seq( - Image("File:Img11y2f1.jpg", - monumentIds = List("01-xxx-0001"), - author = Some("FromCrimeaOld"), - pageId = Some(11)), - Image("File:Img12y2f1.jpg", - monumentIds = List("01-xxx-0002"), - author = Some("FromCrimeaNew"), - pageId = Some(12)), - Image("File:Img52y2f1.jpg", - monumentIds = List("05-xxx-0002"), - author = Some("FromPodillyaNew1"), - pageId = Some(5221)), - Image("File:Img52y2f2.jpg", - monumentIds = List("05-xxx-0002"), - author = Some("FromPodillyaNew2"), - pageId = Some(5222)), - Image("File:Img72y2f1.jpg", - monumentIds = List("07-xxx-0002"), - author = Some("FromVolynNew"), - pageId = Some(72)) + Image( + "File:Img11y2f1.jpg", + monumentIds = List("01-xxx-0001"), + author = Some("FromCrimeaOld"), + pageId = Some(11) + ), + Image( + "File:Img12y2f1.jpg", + monumentIds = List("01-xxx-0002"), + author = Some("FromCrimeaNew"), + pageId = Some(12) + ), + Image( + "File:Img52y2f1.jpg", + monumentIds = List("05-xxx-0002"), + author = Some("FromPodillyaNew1"), + pageId = Some(5221) + ), + Image( + "File:Img52y2f2.jpg", + monumentIds = List("05-xxx-0002"), + author = Some("FromPodillyaNew2"), + pageId = Some(5222) + ), + Image( + "File:Img72y2f1.jpg", + monumentIds = List("07-xxx-0002"), + author = Some("FromVolynNew"), + pageId = Some(72) + ) ) - val mDb = new MonumentDB(contest, - monuments(2, "01", "Crimea") ++ - monuments(5, "05", "Podillya") ++ - monuments(7, "07", "Volyn")) + val mDb = new MonumentDB( + contest, + monuments(2, "01", "Crimea") ++ + monuments(5, "05", "Podillya") ++ + monuments(7, "07", "Volyn") + ) val db = new ImageDB(contest, images2, Some(mDb)) @@ -482,7 +603,8 @@ class AuthorsMonumentsSpec extends Specification { 2013, Some(mDb), Some(db), - Some(db)) + Some(db) + ) val table = new AuthorMonuments(contestStat).table val data = table.data @@ -502,116 +624,146 @@ class AuthorsMonumentsSpec extends Specification { "Волинська область" ) - data.head === Seq("Total", - "4.0", - "0", - "0", - "4", - "4", - "5", - "2.0", - "1.0", - "1.0") ++ Seq + data.head === Seq( + "Total", + "4.0", + "0", + "0", + "4", + "4", + "5", + "2.0", + "1.0", + "1.0" + ) ++ Seq .fill(24)("0.0") data.slice(1, 6) === Seq( - Seq("[[User:FromCrimeaNew|FromCrimeaNew]]", - "1.0", - "0", - "0", - "1", - "3.0", - "1", - "3.0", - "0.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromCrimeaOld|FromCrimeaOld]]", - "1.0", - "0", - "0", - "1", - "3.0", - "1", - "3.0", - "0.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromPodillyaNew1|FromPodillyaNew1]]", - "1.0", - "0", - "0", - "1", - "3.0", - "1", - "0.0", - "3.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromPodillyaNew2|FromPodillyaNew2]]", - "1.0", - "0", - "0", - "1", - "3.0", - "1", - "0.0", - "3.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromVolynNew|FromVolynNew]]", - "1.0", - "0", - "0", - "1", - "3.0", - "1", - "0.0", - "0.0", - "3.0") ++ Seq.fill(24)("0.0") + Seq( + "[[User:FromCrimeaNew|FromCrimeaNew]]", + "1.0", + "0", + "0", + "1", + "3.0", + "1", + "3.0", + "0.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromCrimeaOld|FromCrimeaOld]]", + "1.0", + "0", + "0", + "1", + "3.0", + "1", + "3.0", + "0.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromPodillyaNew1|FromPodillyaNew1]]", + "1.0", + "0", + "0", + "1", + "3.0", + "1", + "0.0", + "3.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromPodillyaNew2|FromPodillyaNew2]]", + "1.0", + "0", + "0", + "1", + "3.0", + "1", + "0.0", + "3.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromVolynNew|FromVolynNew]]", + "1.0", + "0", + "0", + "1", + "3.0", + "1", + "0.0", + "0.0", + "3.0" + ) ++ Seq.fill(24)("0.0") ) } "order by rate in" in { val images1 = Seq( - Image("File:Img11y1f1.jpg", - monumentIds = List("01-xxx-0001"), - author = Some("FromCrimea"), - pageId = Some(1111)), - Image("File:Img51y1f1.jpg", - monumentIds = List("05-xxx-0001"), - author = Some("FromPodillya"), - pageId = Some(51)), - Image("File:Img71y1f1.jpg", - monumentIds = List("07-xxx-0001"), - author = Some("FromVolyn"), - pageId = Some(71)) + Image( + "File:Img11y1f1.jpg", + monumentIds = List("01-xxx-0001"), + author = Some("FromCrimea"), + pageId = Some(1111) + ), + Image( + "File:Img51y1f1.jpg", + monumentIds = List("05-xxx-0001"), + author = Some("FromPodillya"), + pageId = Some(51) + ), + Image( + "File:Img71y1f1.jpg", + monumentIds = List("07-xxx-0001"), + author = Some("FromVolyn"), + pageId = Some(71) + ) ) val images2 = Seq( - Image("File:Img11y2f1.jpg", - monumentIds = List("01-xxx-0001"), - author = Some("FromCrimeaOld"), - pageId = Some(1121)), - Image("File:Img12y2f1.jpg", - monumentIds = List("01-xxx-0002"), - author = Some("FromCrimeaNew"), - pageId = Some(1221)), - Image("File:Img52y2f1.jpg", - monumentIds = List("05-xxx-0002"), - author = Some("FromPodillyaNew1"), - pageId = Some(5221)), - Image("File:Img52y2f2.jpg", - monumentIds = List("05-xxx-0002"), - author = Some("FromPodillyaNew2"), - pageId = Some(5222)), - Image("File:Img72y2f1.jpg", - monumentIds = List("07-xxx-0002"), - author = Some("FromVolynNew"), - pageId = Some(72)) + Image( + "File:Img11y2f1.jpg", + monumentIds = List("01-xxx-0001"), + author = Some("FromCrimeaOld"), + pageId = Some(1121) + ), + Image( + "File:Img12y2f1.jpg", + monumentIds = List("01-xxx-0002"), + author = Some("FromCrimeaNew"), + pageId = Some(1221) + ), + Image( + "File:Img52y2f1.jpg", + monumentIds = List("05-xxx-0002"), + author = Some("FromPodillyaNew1"), + pageId = Some(5221) + ), + Image( + "File:Img52y2f2.jpg", + monumentIds = List("05-xxx-0002"), + author = Some("FromPodillyaNew2"), + pageId = Some(5222) + ), + Image( + "File:Img72y2f1.jpg", + monumentIds = List("07-xxx-0002"), + author = Some("FromVolynNew"), + pageId = Some(72) + ) ) - val mDb = new MonumentDB(contest, - monuments(2, "01", "Crimea") ++ - monuments(5, "05", "Podillya") ++ - monuments(7, "07", "Volyn")) + val mDb = new MonumentDB( + contest, + monuments(2, "01", "Crimea") ++ + monuments(5, "05", "Podillya") ++ + monuments(7, "07", "Volyn") + ) val oldIds = images1.flatMap(_.monumentId).toSet val db = new ImageDB(contest, images2, Some(mDb)) @@ -622,7 +774,8 @@ class AuthorsMonumentsSpec extends Specification { 2013, Some(mDb), Some(db), - Some(totalDb)) + Some(totalDb) + ) val table = new AuthorMonuments(contestStat).table val data = table.data @@ -642,70 +795,82 @@ class AuthorsMonumentsSpec extends Specification { "Волинська область" ) - data.head === Seq("Total", - "4.0", - "0", - "1", - "3", - "4", - "5", - "2.0", - "1.0", - "1.0") ++ Seq + data.head === Seq( + "Total", + "4.0", + "0", + "1", + "3", + "4", + "5", + "2.0", + "1.0", + "1.0" + ) ++ Seq .fill(24)("0.0") data.slice(1, 9) === Seq( - Seq("[[User:FromCrimeaNew|FromCrimeaNew]]", - "1.0", - "0", - "0", - "1", - "3.0", - "1", - "3.0", - "0.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromPodillyaNew1|FromPodillyaNew1]]", - "1.0", - "0", - "0", - "1", - "3.0", - "1", - "0.0", - "3.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromPodillyaNew2|FromPodillyaNew2]]", - "1.0", - "0", - "0", - "1", - "3.0", - "1", - "0.0", - "3.0", - "0.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromVolynNew|FromVolynNew]]", - "1.0", - "0", - "0", - "1", - "3.0", - "1", - "0.0", - "0.0", - "3.0") ++ Seq.fill(24)("0.0"), - Seq("[[User:FromCrimeaOld|FromCrimeaOld]]", - "1.0", - "0", - "1", - "0", - "1.0", - "1", - "1.0", - "0.0", - "0.0") ++ Seq.fill(24)("0.0") + Seq( + "[[User:FromCrimeaNew|FromCrimeaNew]]", + "1.0", + "0", + "0", + "1", + "3.0", + "1", + "3.0", + "0.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromPodillyaNew1|FromPodillyaNew1]]", + "1.0", + "0", + "0", + "1", + "3.0", + "1", + "0.0", + "3.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromPodillyaNew2|FromPodillyaNew2]]", + "1.0", + "0", + "0", + "1", + "3.0", + "1", + "0.0", + "3.0", + "0.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromVolynNew|FromVolynNew]]", + "1.0", + "0", + "0", + "1", + "3.0", + "1", + "0.0", + "0.0", + "3.0" + ) ++ Seq.fill(24)("0.0"), + Seq( + "[[User:FromCrimeaOld|FromCrimeaOld]]", + "1.0", + "0", + "1", + "0", + "1.0", + "1", + "1.0", + "0.0", + "0.0" + ) ++ Seq.fill(24)("0.0") ) } } diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/ChartsSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/ChartsSpec.scala index 105d68b8..2fd14644 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/ChartsSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/ChartsSpec.scala @@ -78,17 +78,28 @@ class ChartsSpec extends Specification { val ids = Seq( Set("2_only") ++ twoAndThree ++ twoAndFour ++ intersection, - Set("3_only1", "3_only2") ++ twoAndThree ++ threeAndFour ++ intersection, - Set("4_only1", "4_only2", "4_only3") ++ twoAndFour ++ threeAndFour ++ intersection + Set( + "3_only1", + "3_only2" + ) ++ twoAndThree ++ threeAndFour ++ intersection, + Set( + "4_only1", + "4_only2", + "4_only3" + ) ++ twoAndFour ++ threeAndFour ++ intersection ) val dataset = charts.intersectionDataSet(years, ids) dataset.getKeys.asScala === Seq( - "2012", "2012 & 2013", "2012 & 2014", - "2013", "2013 & 2014", + "2012", + "2012 & 2013", + "2012 & 2014", + "2013", + "2013 & 2014", "2014", - "2012 & 2013 & 2014") + "2012 & 2013 & 2014" + ) dataset.getValue("2012") === 1 dataset.getValue("2012 & 2013") === 6 @@ -135,15 +146,22 @@ class ChartsSpec extends Specification { val dataset = charts.intersectionDataSet(years, ids) dataset.getKeys.asScala === Seq( - "2012", "2012 & 2013", "2012 & 2014", "2012 & 2015", - "2013", "2013 & 2014", "2013 & 2015", - "2014", "2014 & 2015", + "2012", + "2012 & 2013", + "2012 & 2014", + "2012 & 2015", + "2013", + "2013 & 2014", + "2013 & 2015", + "2014", + "2014 & 2015", "2015", "2013 & 2014 & 2015", "2012 & 2014 & 2015", "2012 & 2013 & 2015", "2012 & 2013 & 2014", - "2012 & 2013 & 2014 & 2015") + "2012 & 2013 & 2014 & 2015" + ) dataset.getValue("2012") === 2 dataset.getValue("2012 & 2013") === 6 diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/GroupingSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/GroupingSpec.scala index 2e7d62fc..44aa3da1 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/GroupingSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/GroupingSpec.scala @@ -8,7 +8,8 @@ class GroupingSpec extends Specification { "grouping" should { "group by resolution" in { - val images1 = Seq(Image("File:11.jpg", width = Some(1000), height = Some(1000))) + val images1 = + Seq(Image("File:11.jpg", width = Some(1000), height = Some(1000))) val images2 = Seq( Image("File:21.jpg", width = Some(2000), height = Some(1000)), Image("File:22.jpg", width = Some(2000), height = Some(1100)) @@ -29,7 +30,8 @@ class GroupingSpec extends Specification { Image("File:22.jpg", monumentIds = List("234")) ) - val g = new Grouping("monuments", ImageGrouping.byMonument, images1 ++ images2) + val g = + new Grouping("monuments", ImageGrouping.byMonument, images1 ++ images2) g.keys === Set("123", "234") g.by("123") === images1 @@ -44,7 +46,8 @@ class GroupingSpec extends Specification { Image("File:22.jpg", monumentIds = List("05-234")) ) - val g = new Grouping("monuments", ImageGrouping.byRegion, images1 ++ images2) + val g = + new Grouping("monuments", ImageGrouping.byRegion, images1 ++ images2) g.keys === Set("01", "05") g.by("01") === images1 @@ -59,7 +62,8 @@ class GroupingSpec extends Specification { Image("File:22.jpg", author = Some("B2")) ) - val g = new Grouping("monuments", ImageGrouping.byAuthor, images1 ++ images2) + val g = + new Grouping("monuments", ImageGrouping.byAuthor, images1 ++ images2) g.keys === Set("A1", "B2") g.by("A1") === images1 @@ -68,12 +72,16 @@ class GroupingSpec extends Specification { } "group by author and region" in { - val imagesA1 = Seq(Image("File:A1.jpg", author = Some("A"), monumentIds = List("01-123"))) + val imagesA1 = Seq( + Image("File:A1.jpg", author = Some("A"), monumentIds = List("01-123")) + ) val imagesA2 = Seq( Image("File:A21.jpg", author = Some("A"), monumentIds = List("05-345")), Image("File:A22.jpg", author = Some("A"), monumentIds = List("05-345")) ) - val imagesB1 = Seq(Image("File:B1.jpg", author = Some("B"), monumentIds = List("01-123"))) + val imagesB1 = Seq( + Image("File:B1.jpg", author = Some("B"), monumentIds = List("01-123")) + ) val imagesB2 = Seq( Image("File:B21.jpg", author = Some("B"), monumentIds = List("05-345")), Image("File:B22.jpg", author = Some("B"), monumentIds = List("05-345")) diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/MonumentsPicturedByRegionSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/MonumentsPicturedByRegionSpec.scala index e9a64dcf..db9b342b 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/MonumentsPicturedByRegionSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/MonumentsPicturedByRegionSpec.scala @@ -15,21 +15,34 @@ class MonumentsPicturedByRegionSpec extends Specification { def monument(id: String, name: String) = new Monument(id = id, name = name, listConfig = Some(WlmUa)) - def monuments(n: Int, regionId: String, namePrefix: String, startId: Int = 1): Seq[Monument] = - (startId until startId + n).map(i => monument(s"$regionId-xxx-000$i", namePrefix + i)) + def monuments( + n: Int, + regionId: String, + namePrefix: String, + startId: Int = 1 + ): Seq[Monument] = + (startId until startId + n).map(i => + monument(s"$regionId-xxx-000$i", namePrefix + i) + ) - def getStat(monuments: Seq[Monument], images: Seq[Seq[Image]] = Seq(Seq.empty)): ContestStat = { + def getStat( + monuments: Seq[Monument], + images: Seq[Seq[Image]] = Seq(Seq.empty) + ): ContestStat = { val monumentDb = new MonumentDB(contest, monuments) def imageDb(images: Seq[Image]) = new ImageDB(contest, images, monumentDb) val imagesDbs = images.map(imageDb) - ContestStat(contest, startYear, + ContestStat( + contest, + startYear, Some(monumentDb), imagesDbs.lastOption, Some(imageDb(images.flatten)), - imagesDbs.init) + imagesDbs.init + ) } "monuments pictured" should { @@ -37,7 +50,12 @@ class MonumentsPicturedByRegionSpec extends Specification { val stat = getStat(Seq.empty) val table = new MonumentsPicturedByRegion(stat).table - table.headers === Seq("Region", "Objects in lists", "0 years total", "0 years percentage") + table.headers === Seq( + "Region", + "Objects in lists", + "0 years total", + "0 years percentage" + ) table.data === Seq( Seq("Total", "0", "0", "0") ) @@ -53,7 +71,12 @@ class MonumentsPicturedByRegionSpec extends Specification { val table = new MonumentsPicturedByRegion(stat).table - table.headers === Seq("Region", "Objects in lists", "0 years total", "0 years percentage") + table.headers === Seq( + "Region", + "Objects in lists", + "0 years total", + "0 years percentage" + ) table.data === Seq( Seq("Автономна Республіка Крим", "2", "0", "0"), Seq("Вінницька область", "5", "0", "0"), @@ -63,20 +86,34 @@ class MonumentsPicturedByRegionSpec extends Specification { } "monuments pictured" in { - val monumentDb = new MonumentDB(contest, + val monumentDb = new MonumentDB( + contest, monuments(2, "01", "Crimea") ++ monuments(5, "05", "Podillya") ++ monuments(7, "07", "Volyn") ) - val image = Image("File:Img3.jpg", monumentIds = List("01-xxx-0001"), pageId = Some(1L)) + val image = Image( + "File:Img3.jpg", + monumentIds = List("01-xxx-0001"), + pageId = Some(1L) + ) val totalImageDb = new ImageDB(contest, Seq(image), monumentDb) val imageDbs = Seq.empty - val table = new MonumentsPicturedByRegion(imageDbs, Some(totalImageDb), monumentDb).table + val table = new MonumentsPicturedByRegion( + imageDbs, + Some(totalImageDb), + monumentDb + ).table - table.headers === Seq("Region", "Objects in lists", "0 years total", "0 years percentage") + table.headers === Seq( + "Region", + "Objects in lists", + "0 years total", + "0 years percentage" + ) table.data === Seq( Seq("Автономна Республіка Крим", "2", "1", "50"), Seq("Вінницька область", "5", "0", "0"), @@ -86,77 +123,173 @@ class MonumentsPicturedByRegionSpec extends Specification { } "two years monuments pictured" in { - val monumentDb = new MonumentDB(contest, + val monumentDb = new MonumentDB( + contest, monuments(2, "01", "Crimea") ++ monuments(5, "05", "Podillya") ++ monuments(7, "07", "Volyn") ) val images = Seq( - Seq(Image("File:Img1.jpg", monumentIds = List("01-xxx-0001"), pageId = Some(1L))), - Seq(Image("File:Img2.jpg", monumentIds = List("05-xxx-0001"), pageId = Some(2L)), - Image("File:Img3.jpg", monumentIds = List("07-xxx-0001"), pageId = Some(3L))) + Seq( + Image( + "File:Img1.jpg", + monumentIds = List("01-xxx-0001"), + pageId = Some(1L) + ) + ), + Seq( + Image( + "File:Img2.jpg", + monumentIds = List("05-xxx-0001"), + pageId = Some(2L) + ), + Image( + "File:Img3.jpg", + monumentIds = List("07-xxx-0001"), + pageId = Some(3L) + ) + ) ) val totalImageDb = new ImageDB(contest, images.flatten, monumentDb) - val imageDbs = images.zipWithIndex.map { case (img, i) => new ImageDB(Contest.WLMUkraine(2014 + i), img, monumentDb) } + val imageDbs = images.zipWithIndex.map { case (img, i) => + new ImageDB(Contest.WLMUkraine(2014 + i), img, monumentDb) + } - val table = new MonumentsPicturedByRegion(imageDbs, Some(totalImageDb), monumentDb).table + val table = new MonumentsPicturedByRegion( + imageDbs, + Some(totalImageDb), + monumentDb + ).table - table.headers === Seq("Region", "Objects in lists", "2 years total", "2 years percentage", - "2015 Objects", "2015 Pictures", "2015 newly pictured", - "2014 Objects", "2014 Pictures") + table.headers === Seq( + "Region", + "Objects in lists", + "2 years total", + "2 years percentage", + "2015 Objects", + "2015 Pictures", + "2015 newly pictured", + "2014 Objects", + "2014 Pictures" + ) table.data === Seq( - Seq("Автономна Республіка Крим", "2", "1", "50") ++ Seq(Seq("1", "1"), Seq("0", "0", "0")).reverse.flatten, + Seq("Автономна Республіка Крим", "2", "1", "50") ++ Seq( + Seq("1", "1"), + Seq("0", "0", "0") + ).reverse.flatten, Seq("Вінницька область", "5", "1", "20") ++ Seq( Seq("0", "0"), - Seq("1", "1", "[[Commons:Wiki Loves Monuments 2015 in Ukraine/Monuments newly pictured by region in Вінницька область|1]]") + Seq( + "1", + "1", + "[[Commons:Wiki Loves Monuments 2015 in Ukraine/Monuments newly pictured by region in Вінницька область|1]]" + ) ).reverse.flatten, Seq("Волинська область", "7", "1", "14") ++ Seq( Seq("0", "0"), - Seq("1", "1", "[[Commons:Wiki Loves Monuments 2015 in Ukraine/Monuments newly pictured by region in Волинська область|1]]") + Seq( + "1", + "1", + "[[Commons:Wiki Loves Monuments 2015 in Ukraine/Monuments newly pictured by region in Волинська область|1]]" + ) ).reverse.flatten, - Seq("Total", "14", "3", "21") ++ Seq(Seq("1", "1"), Seq("2", "2", "2")).reverse.flatten + Seq("Total", "14", "3", "21") ++ Seq( + Seq("1", "1"), + Seq("2", "2", "2") + ).reverse.flatten ) } "4 years monuments pictured" in { - val monumentDb = new MonumentDB(contest, + val monumentDb = new MonumentDB( + contest, monuments(2, "01", "Crimea") ++ monuments(5, "05", "Podillya") ++ monuments(7, "07", "Volyn") ) val images = Seq( - Image("File:Img1.jpg", monumentIds = List("01-xxx-0001"), pageId = Some(1L)), - Image("File:Img2.jpg", monumentIds = List("05-xxx-0001"), pageId = Some(2L)), - Image("File:Img3.jpg", monumentIds = List("07-xxx-0001"), pageId = Some(3L)), - Image("File:Img4.jpg", monumentIds = List("01-xxx-0002"), pageId = Some(4L)) + Image( + "File:Img1.jpg", + monumentIds = List("01-xxx-0001"), + pageId = Some(1L) + ), + Image( + "File:Img2.jpg", + monumentIds = List("05-xxx-0001"), + pageId = Some(2L) + ), + Image( + "File:Img3.jpg", + monumentIds = List("07-xxx-0001"), + pageId = Some(3L) + ), + Image( + "File:Img4.jpg", + monumentIds = List("01-xxx-0002"), + pageId = Some(4L) + ) ) val totalImageDb = new ImageDB(contest, images, monumentDb) - val imageDbs = images.zipWithIndex.map { case (img, i) => new ImageDB(Contest.WLMUkraine(2012 + i), Seq(img), monumentDb) } + val imageDbs = images.zipWithIndex.map { case (img, i) => + new ImageDB(Contest.WLMUkraine(2012 + i), Seq(img), monumentDb) + } - val table = new MonumentsPicturedByRegion(imageDbs, Some(totalImageDb), monumentDb).table + val table = new MonumentsPicturedByRegion( + imageDbs, + Some(totalImageDb), + monumentDb + ).table - table.headers === Seq("Region", "Objects in lists", "4 years total", "4 years percentage", - "2015 Objects", "2015 Pictures", "2015 newly pictured", - "2014 Objects", "2014 Pictures", - "2013 Objects", "2013 Pictures", - "2012 Objects", "2012 Pictures" + table.headers === Seq( + "Region", + "Objects in lists", + "4 years total", + "4 years percentage", + "2015 Objects", + "2015 Pictures", + "2015 newly pictured", + "2014 Objects", + "2014 Pictures", + "2013 Objects", + "2013 Pictures", + "2012 Objects", + "2012 Pictures" ) val data = table.data.toSeq data.size === 4 data(0) === Seq("Автономна Республіка Крим", "2", "2", "100") ++ Seq( - Seq("1", "1"), Seq("0", "0"), Seq("0", "0"), Seq("1", "1", "[[Commons:Wiki Loves Monuments 2015 in Ukraine/Monuments newly pictured by region in Автономна Республіка Крим|1]]") + Seq("1", "1"), + Seq("0", "0"), + Seq("0", "0"), + Seq( + "1", + "1", + "[[Commons:Wiki Loves Monuments 2015 in Ukraine/Monuments newly pictured by region in Автономна Республіка Крим|1]]" + ) ).reverse.flatten data(1) === Seq("Вінницька область", "5", "1", "20") ++ Seq( - Seq("0", "0"), Seq("1", "1"), Seq("0", "0"), Seq("0", "0", "0")).reverse.flatten + Seq("0", "0"), + Seq("1", "1"), + Seq("0", "0"), + Seq("0", "0", "0") + ).reverse.flatten data(2) === Seq("Волинська область", "7", "1", "14") ++ Seq( - Seq("0", "0"), Seq("0", "0"), Seq("1", "1"), Seq("0", "0", "0")).reverse.flatten + Seq("0", "0"), + Seq("0", "0"), + Seq("1", "1"), + Seq("0", "0", "0") + ).reverse.flatten data(3) === Seq("Total", "14", "4", "28") ++ Seq( - Seq("1", "1"), Seq("1", "1"), Seq("1", "1"), Seq("1", "1", "1")).reverse.flatten + Seq("1", "1"), + Seq("1", "1"), + Seq("1", "1"), + Seq("1", "1", "1") + ).reverse.flatten } } } diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/MostPopularMonumentsSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/MostPopularMonumentsSpec.scala index 65e1e1c4..aaa3699c 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/MostPopularMonumentsSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/MostPopularMonumentsSpec.scala @@ -15,35 +15,44 @@ class MostPopularMonumentsSpec extends Specification { def monument(id: String, name: String) = new Monument(id = id, name = name, listConfig = Some(WlmUa)) - def monuments(n: Int, - regionId: String, - namePrefix: String, - startId: Int = 1): Seq[Monument] = + def monuments( + n: Int, + regionId: String, + namePrefix: String, + startId: Int = 1 + ): Seq[Monument] = (startId until startId + n).map(i => - monument(s"$regionId-xxx-000$i", namePrefix + i)) + monument(s"$regionId-xxx-000$i", namePrefix + i) + ) "mostPopularMonuments" should { "work on no monuments" in { val monumentDb = new MonumentDB(contest, Nil) val table = - new MostPopularMonuments(Nil, - Some(new ImageDB(contest, Nil, monumentDb)), - monumentDb).table + new MostPopularMonuments( + Nil, + Some(new ImageDB(contest, Nil, monumentDb)), + monumentDb + ).table table.headers === Seq("N", "Id", "Name", "Category", "authors", "photos") table.data.isEmpty === true } "work on no images" in { - val monumentDb = new MonumentDB(contest, - monuments(2, "01", "Crimea") ++ - monuments(5, "05", "Podillya") ++ - monuments(7, "07", "Volyn")) + val monumentDb = new MonumentDB( + contest, + monuments(2, "01", "Crimea") ++ + monuments(5, "05", "Podillya") ++ + monuments(7, "07", "Volyn") + ) val table = - new MostPopularMonuments(Nil, - Some(new ImageDB(contest, Nil, monumentDb)), - monumentDb).table + new MostPopularMonuments( + Nil, + Some(new ImageDB(contest, Nil, monumentDb)), + monumentDb + ).table table.headers === Seq("N", "Id", "Name", "Category", "authors", "photos") table.data.isEmpty === true @@ -51,32 +60,46 @@ class MostPopularMonumentsSpec extends Specification { "work with images" in { val images = Seq( - Image("File:Img11.jpg", - monumentIds = List("01-xxx-0001"), - author = Some("FromCrimea")), - Image("File:Img51.jpg", - monumentIds = List("05-xxx-0001"), - author = Some("FromPodillya1")), - Image("File:Img52.jpg", - monumentIds = List("05-xxx-0001"), - author = Some("FromPodillya2")), - Image("File:Img71.jpg", - monumentIds = List("07-xxx-0001"), - author = Some("FromVolyn")), - Image("File:Img12.jpg", - monumentIds = List("01-xxx-0001"), - author = Some("FromCrimea")) + Image( + "File:Img11.jpg", + monumentIds = List("01-xxx-0001"), + author = Some("FromCrimea") + ), + Image( + "File:Img51.jpg", + monumentIds = List("05-xxx-0001"), + author = Some("FromPodillya1") + ), + Image( + "File:Img52.jpg", + monumentIds = List("05-xxx-0001"), + author = Some("FromPodillya2") + ), + Image( + "File:Img71.jpg", + monumentIds = List("07-xxx-0001"), + author = Some("FromVolyn") + ), + Image( + "File:Img12.jpg", + monumentIds = List("01-xxx-0001"), + author = Some("FromCrimea") + ) ) - val monumentDb = new MonumentDB(contest, - monuments(2, "01", "Crimea") ++ - monuments(5, "05", "Podillya") ++ - monuments(7, "07", "Volyn")) + val monumentDb = new MonumentDB( + contest, + monuments(2, "01", "Crimea") ++ + monuments(5, "05", "Podillya") ++ + monuments(7, "07", "Volyn") + ) val table = - new MostPopularMonuments(Nil, - Some(new ImageDB(contest, images, monumentDb)), - monumentDb).table + new MostPopularMonuments( + Nil, + Some(new ImageDB(contest, images, monumentDb)), + monumentDb + ).table table.headers === Seq("N", "Id", "Name", "Category", "authors", "photos") table.data === Seq( @@ -88,50 +111,71 @@ class MostPopularMonumentsSpec extends Specification { "work with images 2 years" in { val images1 = Seq( - Image("File:Img11y1f1.jpg", - monumentIds = List("01-xxx-0001"), - author = Some("FromCrimea")), - Image("File:Img11y1f2.jpg", - monumentIds = List("01-xxx-0001"), - author = Some("FromCrimea")), - Image("File:Img51y1f1.jpg", - monumentIds = List("05-xxx-0001"), - author = Some("FromPodillya1")), - Image("File:Img51y1f2.jpg", - monumentIds = List("05-xxx-0001"), - author = Some("FromPodillya2")), - Image("File:Img71y1f1.jpg", - monumentIds = List("07-xxx-0001"), - author = Some("FromVolyn")) + Image( + "File:Img11y1f1.jpg", + monumentIds = List("01-xxx-0001"), + author = Some("FromCrimea") + ), + Image( + "File:Img11y1f2.jpg", + monumentIds = List("01-xxx-0001"), + author = Some("FromCrimea") + ), + Image( + "File:Img51y1f1.jpg", + monumentIds = List("05-xxx-0001"), + author = Some("FromPodillya1") + ), + Image( + "File:Img51y1f2.jpg", + monumentIds = List("05-xxx-0001"), + author = Some("FromPodillya2") + ), + Image( + "File:Img71y1f1.jpg", + monumentIds = List("07-xxx-0001"), + author = Some("FromVolyn") + ) ) val images2 = Seq( - Image("File:Img11y2f1.jpg", - monumentIds = List("01-xxx-0001"), - author = Some("FromCrimea1")), - Image("File:Img12y2f1.jpg", - monumentIds = List("01-xxx-0002"), - author = Some("FromCrimea2")), - Image("File:Img52y2f1.jpg", - monumentIds = List("05-xxx-0002"), - author = Some("FromPodillya1")), - Image("File:Img52y2f1.jpg", - monumentIds = List("05-xxx-0002"), - author = Some("FromPodillya2")), - Image("File:Img72y2f1.jpg", - monumentIds = List("07-xxx-0002"), - author = Some("FromVolyn")) + Image( + "File:Img11y2f1.jpg", + monumentIds = List("01-xxx-0001"), + author = Some("FromCrimea1") + ), + Image( + "File:Img12y2f1.jpg", + monumentIds = List("01-xxx-0002"), + author = Some("FromCrimea2") + ), + Image( + "File:Img52y2f1.jpg", + monumentIds = List("05-xxx-0002"), + author = Some("FromPodillya1") + ), + Image( + "File:Img52y2f1.jpg", + monumentIds = List("05-xxx-0002"), + author = Some("FromPodillya2") + ), + Image( + "File:Img72y2f1.jpg", + monumentIds = List("07-xxx-0002"), + author = Some("FromVolyn") + ) ) - val monumentDb = new MonumentDB(contest, - monuments(2, "01", "Crimea") ++ - monuments(5, "05", "Podillya") ++ - monuments(7, "07", "Volyn")) + val monumentDb = new MonumentDB( + contest, + monuments(2, "01", "Crimea") ++ + monuments(5, "05", "Podillya") ++ + monuments(7, "07", "Volyn") + ) val table = new MostPopularMonuments( - Seq(images1, images2).zipWithIndex.map { - case (images, i) => - new ImageDB(Contest.WLMUkraine(2014 + i), images, monumentDb) + Seq(images1, images2).zipWithIndex.map { case (images, i) => + new ImageDB(Contest.WLMUkraine(2014 + i), images, monumentDb) }, Some(new ImageDB(contest, images1 ++ images2, monumentDb)), monumentDb diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/NumberOfAuthorsBonusSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/NumberOfAuthorsBonusSpec.scala index 40d0d52e..584a1792 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/NumberOfAuthorsBonusSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/NumberOfAuthorsBonusSpec.scala @@ -11,34 +11,58 @@ class NumberOfAuthorsBonusSpec extends Specification { val contestStatStub = ContestStat(contest = contest, startYear = 2019) val monumentId1 = "01-123-0001" val monument1 = Monument(id = monumentId1, name = "name 1") - val image1 = Image("File:Image1.jpg", pageId = Some(1)).withAuthor("author 1").withMonument(monumentId1) - val image2 = Image("File:Image2.jpg", pageId = Some(2)).withAuthor("author 1").withMonument(monumentId1) + val image1 = Image("File:Image1.jpg", pageId = Some(1)) + .withAuthor("author 1") + .withMonument(monumentId1) + val image2 = Image("File:Image2.jpg", pageId = Some(2)) + .withAuthor("author 1") + .withMonument(monumentId1) val monumentDbStub = Some(new MonumentDB(contest, Seq(monument1))) val imageDbStub = new ImageDB(contest, Nil, monumentDbStub) "rater" should { - val ranges = RateRanges(ConfigFactory.parseString("""{"0-0": 9, "1-3": 3}""")) + val ranges = + RateRanges(ConfigFactory.parseString("""{"0-0": 9, "1-3": 3}""")) val imageDb = imageDbStub.copy(images = Seq(image1)) - val currentYearStat = contestStatStub.copy(currentYearImageDb = Some(imageDb)) + val currentYearStat = + contestStatStub.copy(currentYearImageDb = Some(imageDb)) "rate non-pictured first year" in { - val rater = new NumberOfAuthorsBonus(currentYearStat.copy(totalImageDb = Some(imageDb)), ranges) + val rater = new NumberOfAuthorsBonus( + currentYearStat.copy(totalImageDb = Some(imageDb)), + ranges + ) rater.rate("01-123-0001", "author 1") === 9 - rater.explain("01-123-0001", "author 1") === "Pictured before by 0 (0-0) authors = 9.0" + rater.explain( + "01-123-0001", + "author 1" + ) === "Pictured before by 0 (0-0) authors = 9.0" } "rate pictured twice by same author with sameAuthorZeroBonus" in { val totalImageDb = imageDbStub.copy(images = Seq(image1, image2)) - val rater = new NumberOfAuthorsBonus(currentYearStat.copy(totalImageDb = Some(totalImageDb)), ranges.copy(sameAuthorZeroBonus = true)) + val rater = new NumberOfAuthorsBonus( + currentYearStat.copy(totalImageDb = Some(totalImageDb)), + ranges.copy(sameAuthorZeroBonus = true) + ) rater.rate("01-123-0001", "author 1") === 0 - rater.explain("01-123-0001", "author 1") === "Pictured by same author before = 0.0" + rater.explain( + "01-123-0001", + "author 1" + ) === "Pictured by same author before = 0.0" } "rate pictured twice by same author without sameAuthorZeroBonus" in { val totalImageDb = imageDbStub.copy(images = Seq(image1, image2)) - val rater = new NumberOfAuthorsBonus(currentYearStat.copy(totalImageDb = Some(totalImageDb)), ranges.copy(sameAuthorZeroBonus = false)) + val rater = new NumberOfAuthorsBonus( + currentYearStat.copy(totalImageDb = Some(totalImageDb)), + ranges.copy(sameAuthorZeroBonus = false) + ) rater.rate("01-123-0001", "author 1") === 3 - rater.explain("01-123-0001", "author 1") === "Pictured before by 1 (1-3) authors = 3.0" + rater.explain( + "01-123-0001", + "author 1" + ) === "Pictured before by 1 (1-3) authors = 3.0" } } } diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/RateRangesSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/RateRangesSpec.scala index ccad7d26..dbe4179a 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/RateRangesSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/RateRangesSpec.scala @@ -12,11 +12,15 @@ class RateRangesSpec extends Specification { } "parse one range" in { - RateRanges(ConfigFactory.parseString("""{"0-0":9}""")).rangeMap === Map(((0, 0), 9)) + RateRanges(ConfigFactory.parseString("""{"0-0":9}""")).rangeMap === Map( + ((0, 0), 9) + ) } "parse several ranges" in { - RateRanges(ConfigFactory.parseString("""{"0-0": 9, "1-3": 3, "4-9": 1}""")).rangeMap === Map( + RateRanges( + ConfigFactory.parseString("""{"0-0": 9, "1-3": 3, "4-9": 1}""") + ).rangeMap === Map( ((0, 0), 9), ((1, 3), 3), ((4, 9), 1) @@ -24,13 +28,19 @@ class RateRangesSpec extends Specification { } "do not allow reversed range" in { - val exception = new IllegalArgumentException("Invalid ends order in range 2-1: 2 > 1") - RateRanges(ConfigFactory.parseString("""{"0-0": 9, "2-1": 3}""")) must throwA(exception) + val exception = + new IllegalArgumentException("Invalid ends order in range 2-1: 2 > 1") + RateRanges( + ConfigFactory.parseString("""{"0-0": 9, "2-1": 3}""") + ) must throwA(exception) } "do not allow overlaps" in { - val exception = new IllegalArgumentException("Ranges (0,0) and (0,1) overlap") - RateRanges(ConfigFactory.parseString("""{"0-0": 9, "0-1": 3}""")) must throwA(exception) + val exception = + new IllegalArgumentException("Ranges (0,0) and (0,1) overlap") + RateRanges( + ConfigFactory.parseString("""{"0-0": 9, "0-1": 3}""") + ) must throwA(exception) } } } diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/RaterSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/RaterSpec.scala index 243e8491..68664a6c 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/RaterSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/RaterSpec.scala @@ -11,11 +11,17 @@ class RaterSpec extends Specification { val contest = Contest(ContestType.WLM, Country.Ukraine, 2020) val monumentDb = Some(new MonumentDB(contest, Nil)) val imageDb = new ImageDB(contest, Nil, monumentDb) - val contestStat = ContestStat(contest = contest, startYear = 2019, monumentDb = monumentDb, - currentYearImageDb = Some(imageDb), totalImageDb = Some(imageDb)) + val contestStat = ContestStat( + contest = contest, + startYear = 2019, + monumentDb = monumentDb, + currentYearImageDb = Some(imageDb), + totalImageDb = Some(imageDb) + ) "parse wlm 2020" in { - val rater = Rater.fromConfig(contestStat, ConfigFactory.load("wlm_ua.conf")) + val rater = + Rater.fromConfig(contestStat, ConfigFactory.load("wlm_ua.conf")) rater must beAnInstanceOf[RateSum] val rateSum = rater.asInstanceOf[RateSum] val raters = rateSum.raters @@ -24,15 +30,20 @@ class RaterSpec extends Specification { raters.find(_.isInstanceOf[NumberOfAuthorsBonus]) must beSome raters.find(_.isInstanceOf[NumberOfImagesInPlaceBonus]) must beSome - val numberOfAuthorsBonus = raters.collect { case r: NumberOfAuthorsBonus => r }.head + val numberOfAuthorsBonus = raters.collect { + case r: NumberOfAuthorsBonus => r + }.head numberOfAuthorsBonus.rateRanges.sameAuthorZeroBonus === false - val numberOfImagesInPlaceBonus = raters.collect { case r: NumberOfImagesInPlaceBonus => r }.head + val numberOfImagesInPlaceBonus = raters.collect { + case r: NumberOfImagesInPlaceBonus => r + }.head numberOfImagesInPlaceBonus.rateRanges.sameAuthorZeroBonus === false } "parse wle 2020" in { - val rater = Rater.fromConfig(contestStat, ConfigFactory.load("wle_ua.conf")) + val rater = + Rater.fromConfig(contestStat, ConfigFactory.load("wle_ua.conf")) rater must beAnInstanceOf[RateSum] val rateSum = rater.asInstanceOf[RateSum] val raters = rateSum.raters @@ -48,7 +59,6 @@ class RaterSpec extends Specification { bonusRater.rateRanges.sameAuthorZeroBonus === true } - // "05-101-0380" in { // val monumentId = "05-101-0380" // val rater = Rater.fromConfig(contestStat, ConfigFactory.load("wlm_ua.conf")) diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatParamsSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatParamsSpec.scala index f54b1b60..786ce84b 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatParamsSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatParamsSpec.scala @@ -20,7 +20,8 @@ class StatParamsSpec extends Specification { } "parse campaign with years" in { - val cfg = StatParams.parse(Seq("--campaign", "wle-ua", "--year", "2015", "2016")) + val cfg = + StatParams.parse(Seq("--campaign", "wle-ua", "--year", "2015", "2016")) cfg === StatConfig("wle-ua", Seq(2015, 2016)) } @@ -30,49 +31,119 @@ class StatParamsSpec extends Specification { } "years sorted" in { - val cfg = StatParams.parse(Seq("--campaign", "wle-ua", "--year", "2016", "2014", "2015", "2012")) + val cfg = StatParams.parse( + Seq("--campaign", "wle-ua", "--year", "2016", "2014", "2015", "2012") + ) cfg === StatConfig("wle-ua", 2012 to 2016) } "start year" in { - val cfg = StatParams.parse(Seq("--campaign", "wle-ua", "-y", "2017", "--start-year", "2012")) + val cfg = StatParams.parse( + Seq("--campaign", "wle-ua", "-y", "2017", "--start-year", "2012") + ) cfg === StatConfig("wle-ua", 2012 to 2017) } "parse new object rating" in { - StatParams.parse(Seq("--campaign", "wle-ua", "--new-object-rating", "7")) === - StatConfig("wle-ua", Seq(thisYear), rateConfig = RateConfig(newObjectRating = Some(7))) - - StatParams.parse(Seq("--campaign", "wle-ua", "--new-author-object-rating", "3")) === - StatConfig("wle-ua", Seq(thisYear), rateConfig = RateConfig(newAuthorObjectRating = Some(3))) - - StatParams.parse(Seq("--campaign", "wle-ua", "--new-object-rating", "10", "--new-author-object-rating", "5")) === - StatConfig("wle-ua", Seq(thisYear), rateConfig = RateConfig(newObjectRating = Some(10), newAuthorObjectRating = Some(5))) + StatParams.parse( + Seq("--campaign", "wle-ua", "--new-object-rating", "7") + ) === + StatConfig( + "wle-ua", + Seq(thisYear), + rateConfig = RateConfig(newObjectRating = Some(7)) + ) + + StatParams.parse( + Seq("--campaign", "wle-ua", "--new-author-object-rating", "3") + ) === + StatConfig( + "wle-ua", + Seq(thisYear), + rateConfig = RateConfig(newAuthorObjectRating = Some(3)) + ) + + StatParams.parse( + Seq( + "--campaign", + "wle-ua", + "--new-object-rating", + "10", + "--new-author-object-rating", + "5" + ) + ) === + StatConfig( + "wle-ua", + Seq(thisYear), + rateConfig = RateConfig( + newObjectRating = Some(10), + newAuthorObjectRating = Some(5) + ) + ) } "parse bonus" in { - StatParams.parse(Seq("--campaign", "wle-ua", "--number-of-authors-bonus")) === - StatConfig("wle-ua", Seq(thisYear), rateConfig = RateConfig(numberOfAuthorsBonus = true)) - - StatParams.parse(Seq("--campaign", "wle-ua", "--number-of-images-bonus")) === - StatConfig("wle-ua", Seq(thisYear), rateConfig = RateConfig(numberOfImagesBonus = true)) - - StatParams.parse(Seq("--campaign", "wle-ua", "--number-of-authors-bonus", "--number-of-images-bonus")) === - StatConfig("wle-ua", Seq(thisYear), rateConfig = RateConfig(numberOfAuthorsBonus = true, numberOfImagesBonus = true)) + StatParams.parse( + Seq("--campaign", "wle-ua", "--number-of-authors-bonus") + ) === + StatConfig( + "wle-ua", + Seq(thisYear), + rateConfig = RateConfig(numberOfAuthorsBonus = true) + ) + + StatParams.parse( + Seq("--campaign", "wle-ua", "--number-of-images-bonus") + ) === + StatConfig( + "wle-ua", + Seq(thisYear), + rateConfig = RateConfig(numberOfImagesBonus = true) + ) + + StatParams.parse( + Seq( + "--campaign", + "wle-ua", + "--number-of-authors-bonus", + "--number-of-images-bonus" + ) + ) === + StatConfig( + "wle-ua", + Seq(thisYear), + rateConfig = + RateConfig(numberOfAuthorsBonus = true, numberOfImagesBonus = true) + ) } "parse gallery" in { - StatParams.parse(Seq("--campaign", "wle-ua")) === StatConfig("wle-ua", Seq(thisYear)) - StatParams.parse(Seq("--campaign", "wle-ua", "--gallery")) === StatConfig("wle-ua", Seq(thisYear), gallery = true) + StatParams.parse(Seq("--campaign", "wle-ua")) === StatConfig( + "wle-ua", + Seq(thisYear) + ) + StatParams.parse(Seq("--campaign", "wle-ua", "--gallery")) === StatConfig( + "wle-ua", + Seq(thisYear), + gallery = true + ) } "parse place-detection" in { - StatParams.parse(Seq("--campaign", "wle-ua")) === StatConfig("wle-ua", Seq(thisYear)) - StatParams.parse(Seq("--campaign", "wle-ua", "--place-detection")) === StatConfig("wle-ua", Seq(thisYear), placeDetection = true) + StatParams.parse(Seq("--campaign", "wle-ua")) === StatConfig( + "wle-ua", + Seq(thisYear) + ) + StatParams.parse( + Seq("--campaign", "wle-ua", "--place-detection") + ) === StatConfig("wle-ua", Seq(thisYear), placeDetection = true) } "parse campaign with regions" in { - val cfg = StatParams.parse(Seq("--campaign", "wle-ua", "--year", "2012", "--region", "01", "02")) + val cfg = StatParams.parse( + Seq("--campaign", "wle-ua", "--year", "2012", "--region", "01", "02") + ) cfg === StatConfig("wle-ua", Seq(2012), Seq("01", "02")) } } diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala index 823c5b13..f246d327 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/StatisticsSpec.scala @@ -14,7 +14,10 @@ import spray.util.pimpFuture import scala.concurrent.Future -class StatisticsSpec(implicit ee: ExecutionEnv) extends Specification with Mockito with FutureMatchers { +class StatisticsSpec(implicit ee: ExecutionEnv) + extends Specification + with Mockito + with FutureMatchers { val contest = Contest.WLEUkraine(2016) @@ -23,7 +26,10 @@ class StatisticsSpec(implicit ee: ExecutionEnv) extends Specification with Mocki val monumentQuery = mock[MonumentQuery] val imageQuery = mock[ImageQuery] - imageQuery.imagesFromCategoryAsync(contest.imagesCategory, contest) returns Future.successful(images) + imageQuery.imagesFromCategoryAsync( + contest.imagesCategory, + contest + ) returns Future.successful(images) monumentQuery.byMonumentTemplate(date = None) returns monuments val cfg = StatConfig(campaign = contest.campaign) @@ -46,7 +52,8 @@ class StatisticsSpec(implicit ee: ExecutionEnv) extends Specification with Mocki } "give some stat" in { - val images = Seq(Image("image1.jpg", author = Some("user"), monumentIds = List("123"))) + val images = + Seq(Image("image1.jpg", author = Some("user"), monumentIds = List("123"))) val monuments = Seq(new Monument(id = "123", name = "123 monument")) val stat = mockedStat(monuments, images) @@ -67,10 +74,14 @@ class StatisticsSpec(implicit ee: ExecutionEnv) extends Specification with Mocki val monumentQuery = mock[MonumentQuery] val imageQuery = mock[ImageQuery] - imageQuery.imagesFromCategoryAsync(contest.imagesCategory, contest) returns Future.failed(new RuntimeException("Error 123")) + imageQuery.imagesFromCategoryAsync( + contest.imagesCategory, + contest + ) returns Future.failed(new RuntimeException("Error 123")) monumentQuery.byMonumentTemplate(date = None) returns monuments - val stat = new Statistics(contest, None, monumentQuery, imageQuery, None, bot) + val stat = + new Statistics(contest, None, monumentQuery, imageQuery, None, bot) stat.gatherData(false) must throwA[RuntimeException].await } @@ -78,38 +89,73 @@ class StatisticsSpec(implicit ee: ExecutionEnv) extends Specification with Mocki def monument(id: String, name: String) = new Monument(id = id, name = name, listConfig = Some(WlmUa)) - def monuments(n: Int, regionId: String, namePrefix: String, startId: Int = 1): Seq[Monument] = - (startId until startId + n).map(i => monument(s"$regionId-xxx-000$i", namePrefix + i)) + def monuments( + n: Int, + regionId: String, + namePrefix: String, + startId: Int = 1 + ): Seq[Monument] = + (startId until startId + n).map(i => + monument(s"$regionId-xxx-000$i", namePrefix + i) + ) "getOldImagesMonumentDb" should { val bot = mock[MwBot] val monumentQuery = mock[MonumentQuery] val imageQuery = mock[ImageQuery] - val stat = new Statistics(contest, None, monumentQuery, imageQuery, None, bot) + val stat = + new Statistics(contest, None, monumentQuery, imageQuery, None, bot) "be empty" in { - stat.getOldImagesMonumentDb(None, None, None, new ImageDB(contest, Seq.empty)) === None + stat.getOldImagesMonumentDb( + None, + None, + None, + new ImageDB(contest, Seq.empty) + ) === None val mDb = new MonumentDB(contest, Seq.empty) - stat.getOldImagesMonumentDb(Some(mDb), None, None, new ImageDB(contest, Seq.empty)) === None + stat.getOldImagesMonumentDb( + Some(mDb), + None, + None, + new ImageDB(contest, Seq.empty) + ) === None } "have images from old monument db" in { - val images1 = Seq(Image("File:Img11y1f1.jpg", monumentIds = List("01-xxx-0001"), author = Some("FromCrimea"))) - - val images2 = Seq(Image("File:Img11y2f1.jpg", monumentIds = List("01-xxx-0002"), author = Some("FromCrimeaOld"))) + val images1 = Seq( + Image( + "File:Img11y1f1.jpg", + monumentIds = List("01-xxx-0001"), + author = Some("FromCrimea") + ) + ) + + val images2 = Seq( + Image( + "File:Img11y2f1.jpg", + monumentIds = List("01-xxx-0002"), + author = Some("FromCrimeaOld") + ) + ) val withoutPhotos = monuments(3, "01", "Crimea") - val withPhotos = withoutPhotos.head.copy(photo = Some(images1.head.title)) +: withoutPhotos.tail + val withPhotos = withoutPhotos.head.copy(photo = + Some(images1.head.title) + ) +: withoutPhotos.tail val mDb = new MonumentDB(contest, withPhotos) - val oldMdb = stat.getOldImagesMonumentDb(Some(mDb), Some(mDb), None, new ImageDB(contest, images2)) + val oldMdb = stat.getOldImagesMonumentDb( + Some(mDb), + Some(mDb), + None, + new ImageDB(contest, images2) + ) oldMdb.map(_.monuments) === Some(Seq(withPhotos.head)) } } } - - diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/generic/MonumentStatSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/generic/MonumentStatSpec.scala index 79607036..c80dc65b 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/generic/MonumentStatSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/generic/MonumentStatSpec.scala @@ -13,8 +13,15 @@ class MonumentStatSpec extends Specification { def monument(id: String, name: String) = new Monument(id = id, name = name, listConfig = Some(WlmUa)) - def monuments(n: Int, regionId: String, namePrefix: String, startId: Int = 1): Seq[Monument] = - (startId until startId + n).map(i => monument(s"$regionId-xxx-000$i", namePrefix + i)) + def monuments( + n: Int, + regionId: String, + namePrefix: String, + startId: Int = 1 + ): Seq[Monument] = + (startId until startId + n).map(i => + monument(s"$regionId-xxx-000$i", namePrefix + i) + ) "with articles stat" should { "work with no regions, no articles" in { @@ -22,7 +29,8 @@ class MonumentStatSpec extends Specification { } "work with no articles" in { - val db = new MonumentDB(contest, + val db = new MonumentDB( + contest, monuments(2, "01", "Crimea") ++ monuments(5, "05", "Podillya") ++ monuments(7, "07", "Volyn") @@ -30,7 +38,13 @@ class MonumentStatSpec extends Specification { val table = Stats.withArticles(db) - table.headers === Seq("Region KOATUU code", "Region name", "All", "With Articles", "%") + table.headers === Seq( + "Region KOATUU code", + "Region name", + "All", + "With Articles", + "%" + ) table.total === Seq("Total", "Total", 14, 0, 0) table.rows === List( List("01", "Автономна Республіка Крим", 2, 0, 0), @@ -40,15 +54,27 @@ class MonumentStatSpec extends Specification { } "work with articles" in { - val db = new MonumentDB(contest, + val db = new MonumentDB( + contest, monuments(1, "01", "Crimea") ++ monuments(2, "01", "[[Crimea]]", 2) ++ - monuments(5, "05", "Podillya") ++ monuments(1, "05", "[[Podillya]]", 6) ++ + monuments(5, "05", "Podillya") ++ monuments( + 1, + "05", + "[[Podillya]]", + 6 + ) ++ monuments(7, "07", "Volyn") ++ monuments(7, "07", "[[Volyn]]", 8) ) val table = Stats.withArticles(db) - table.headers === Seq("Region KOATUU code", "Region name", "All", "With Articles", "%") + table.headers === Seq( + "Region KOATUU code", + "Region name", + "All", + "With Articles", + "%" + ) table.total === Seq("Total", "Total", 23, 10, 43) table.rows === List( List("01", "Автономна Республіка Крим", 3, 2, 67), diff --git a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/generic/RecordsSpec.scala b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/generic/RecordsSpec.scala index d999f014..762881ea 100644 --- a/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/generic/RecordsSpec.scala +++ b/scalawiki-wlx/src/test/scala/org/scalawiki/wlx/stat/generic/RecordsSpec.scala @@ -2,7 +2,6 @@ package org.scalawiki.wlx.stat.generic import org.specs2.mutable.Specification - class RecordsSpec extends Specification { "test" should { @@ -44,7 +43,6 @@ class RecordsSpec extends Specification { ) } - } }
    header1