From 530f7e8996e81ee2281bc9616900a90e1c339e45 Mon Sep 17 00:00:00 2001 From: Alexander Frolov Date: Wed, 24 Nov 2021 14:14:30 +0300 Subject: [PATCH] ignoreLines feature for filtering the input file (#307) * feature/ignoreLines ###What's done: * Added ignoreLines option for Fix plugin. * Added tests for ignoreLines and Fix plugin. * Added ignoreLines option for Warn plugin. * Added tests for ignoreLines and Warn plugin. * Implemented copying a file in Warn plugin in order to filter out lines that match any regex from ignoreLines. (#303) --- .../IgnoreLines/Bug1Expected.kt | 21 +++++ .../IgnoreLinesTest/IgnoreLines/Bug1Test.kt | 12 +++ .../IgnoreLinesTest/IgnoreLines/save.toml | 7 ++ .../IgnoreLinesIsEmpty/Bug1Expected.kt | 22 ++++++ .../IgnoreLinesIsEmpty/Bug1Test.kt | 11 +++ .../IgnoreLinesIsEmpty/save.toml | 7 ++ .../NoIgnoreLines/Bug1Expected.kt | 21 +++++ .../IgnoreLinesTest/NoIgnoreLines/Bug1Test.kt | 11 +++ .../IgnoreLinesTest/NoIgnoreLines/save.toml | 6 ++ .../kotlin/org/cqfn/save/chapter1/save.toml | 6 ++ examples/kotlin-diktat/save.toml | 1 - .../IgnoreLinesTest/EnumTestDetection.kt | 16 ++++ .../warn/chapter1/IgnoreLinesTest/save.toml | 7 ++ examples/kotlin-diktat/warn/save.toml | 2 +- .../kotlin/org/cqfn/save/cli/GeneralTest.kt | 2 +- .../org/cqfn/save/core/plugin/Plugin.kt | 2 +- .../org/cqfn/save/core/plugin/PluginConfig.kt | 6 ++ .../kotlin/org/cqfn/save/core/Save.kt | 2 +- .../save/core/integration/ClassicFixTest.kt | 40 +++++++++- .../save/core/integration/ClassicWarnTest.kt | 15 +++- .../cqfn/save/core/integration/FixDirTest.kt | 2 +- .../plugins/fixandwarn/FixAndWarnPlugin.kt | 1 - .../fixandwarn/FixAndWarnPluginConfig.kt | 11 ++- .../org/cqfn/save/plugins/fix/FixPlugin.kt | 31 +++++--- .../cqfn/save/plugins/fix/FixPluginConfig.kt | 20 ++++- .../org/cqfn/save/plugin/warn/WarnPlugin.kt | 79 +++++++++++-------- .../cqfn/save/plugin/warn/WarnPluginConfig.kt | 23 ++++-- 27 files changed, 319 insertions(+), 65 deletions(-) create mode 100644 examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/Bug1Expected.kt create mode 100644 examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/Bug1Test.kt create mode 100644 examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/save.toml create mode 100644 examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/Bug1Expected.kt create mode 100644 examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/Bug1Test.kt create mode 100644 examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/save.toml create mode 100644 examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/Bug1Expected.kt create mode 100644 examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/Bug1Test.kt create mode 100644 examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/save.toml create mode 100644 examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/chapter1/save.toml create mode 100644 examples/kotlin-diktat/warn/chapter1/IgnoreLinesTest/EnumTestDetection.kt create mode 100644 examples/kotlin-diktat/warn/chapter1/IgnoreLinesTest/save.toml diff --git a/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/Bug1Expected.kt b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/Bug1Expected.kt new file mode 100644 index 000000000..f485b861a --- /dev/null +++ b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/Bug1Expected.kt @@ -0,0 +1,21 @@ +package org.cqfn.save.IgnoreLinesTest.IgnoreLines + +class D { + val x = 0 + + /** + * @return + */ + fun bar(): Bar { + val qux = 42 + return Bar(qux) + } +} + +/** + * @param foo + */ +fun readFile(foo: Foo) { + var bar: Bar +} +// ;warn:0: [TEST] JUST_A_TEST diff --git a/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/Bug1Test.kt b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/Bug1Test.kt new file mode 100644 index 000000000..43886418b --- /dev/null +++ b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/Bug1Test.kt @@ -0,0 +1,12 @@ +package test.smoke.src.main.kotlin + +fun readFile(foo: Foo) { + var bar: Bar +} + +class D {val x = 0 +fun bar(): Bar {val qux = 42; return Bar(qux)} +} +// ;warn:0: [TEST] JUST_A_TEST +// IGNORE_ME +# I WILL DISSAPEAR AS WELL diff --git a/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/save.toml b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/save.toml new file mode 100644 index 000000000..67047f4e8 --- /dev/null +++ b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines/save.toml @@ -0,0 +1,7 @@ +[general] + tags = ["FixIgnoreLines"] + description = "Test for ignoreLines option in Fix" + suiteName = "Autofix: Smoke Tests" + +[fix] + ignoreLines = ["^// IGNORE.*", "^#.*"] \ No newline at end of file diff --git a/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/Bug1Expected.kt b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/Bug1Expected.kt new file mode 100644 index 000000000..43a982386 --- /dev/null +++ b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/Bug1Expected.kt @@ -0,0 +1,22 @@ +package org.cqfn.save.IgnoreLinesTest.IgnoreLinesIsEmpty + +class D { + val x = 0 + + /** + * @return + */ + fun bar(): Bar { + val qux = 42 + return Bar(qux) + } +} + +/** + * @param foo + */ +fun readFile(foo: Foo) { + var bar: Bar +} +// ;warn:0: [TEST] JUST_A_TEST +// IGNORE_ME diff --git a/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/Bug1Test.kt b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/Bug1Test.kt new file mode 100644 index 000000000..7a21be2ad --- /dev/null +++ b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/Bug1Test.kt @@ -0,0 +1,11 @@ +package test.smoke.src.main.kotlin + +fun readFile(foo: Foo) { + var bar: Bar +} + +class D {val x = 0 +fun bar(): Bar {val qux = 42; return Bar(qux)} +} +// ;warn:0: [TEST] JUST_A_TEST +// IGNORE_ME diff --git a/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/save.toml b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/save.toml new file mode 100644 index 000000000..12f77b199 --- /dev/null +++ b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty/save.toml @@ -0,0 +1,7 @@ +[general] + tags = ["smoke"] + description = "Smoke tests for diktat" + suiteName = "Autofix: Smoke Tests" + +[fix] + ignoreLines = [] \ No newline at end of file diff --git a/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/Bug1Expected.kt b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/Bug1Expected.kt new file mode 100644 index 000000000..50c4220cf --- /dev/null +++ b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/Bug1Expected.kt @@ -0,0 +1,21 @@ +package org.cqfn.save.IgnoreLinesTest.NoIgnoreLines + +class D { + val x = 0 + + /** + * @return + */ + fun bar(): Bar { + val qux = 42 + return Bar(qux) + } +} + +/** + * @param foo + */ +fun readFile(foo: Foo) { + var bar: Bar +} +// IGNORE_ME diff --git a/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/Bug1Test.kt b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/Bug1Test.kt new file mode 100644 index 000000000..7a21be2ad --- /dev/null +++ b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/Bug1Test.kt @@ -0,0 +1,11 @@ +package test.smoke.src.main.kotlin + +fun readFile(foo: Foo) { + var bar: Bar +} + +class D {val x = 0 +fun bar(): Bar {val qux = 42; return Bar(qux)} +} +// ;warn:0: [TEST] JUST_A_TEST +// IGNORE_ME diff --git a/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/save.toml b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/save.toml new file mode 100644 index 000000000..f4d6312e1 --- /dev/null +++ b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines/save.toml @@ -0,0 +1,6 @@ +[general] + tags = ["smoke"] + description = "Smoke tests for diktat" + suiteName = "Autofix: Smoke Tests" + +[fix] \ No newline at end of file diff --git a/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/chapter1/save.toml b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/chapter1/save.toml new file mode 100644 index 000000000..f4d6312e1 --- /dev/null +++ b/examples/kotlin-diktat/fix/smoke/src/main/kotlin/org/cqfn/save/chapter1/save.toml @@ -0,0 +1,6 @@ +[general] + tags = ["smoke"] + description = "Smoke tests for diktat" + suiteName = "Autofix: Smoke Tests" + +[fix] \ No newline at end of file diff --git a/examples/kotlin-diktat/save.toml b/examples/kotlin-diktat/save.toml index 0e83dbc05..634123fd1 100644 --- a/examples/kotlin-diktat/save.toml +++ b/examples/kotlin-diktat/save.toml @@ -5,4 +5,3 @@ expectedWarningsPattern = "// ;warn:?(.*):(\\d*): (.+)" timeOutMillis = 30000 - diff --git a/examples/kotlin-diktat/warn/chapter1/IgnoreLinesTest/EnumTestDetection.kt b/examples/kotlin-diktat/warn/chapter1/IgnoreLinesTest/EnumTestDetection.kt new file mode 100644 index 000000000..1ef8decb8 --- /dev/null +++ b/examples/kotlin-diktat/warn/chapter1/IgnoreLinesTest/EnumTestDetection.kt @@ -0,0 +1,16 @@ +package org.cqfn.diktat.test.resources.test.paragraph1.naming.enum_ + +// ;warn:3:1: [MISSING_KDOC_TOP_LEVEL] all public and internal top-level classes and functions should have Kdoc: EnumTestDetection (cannot be auto-corrected) +// ;warn:30: [WRONG_DECLARATIONS_ORDER] declarations of constants and enum members should be sorted alphabetically: enum entries order is incorrect +// ;warn:10:5: [ENUMS_SEPARATED] enum is incorrectly formatted: enums must end with semicolon +enum class EnumTestDetection { + // ;warn:$line+1:5: [ENUM_VALUE] enum values should be{{ in }}selected UPPER_CASE snake/PascalCase format: paSC_SAl_l + paSC_SAl_l, + + // ;warn:5: [ENUM_VALUE] enum values{{ should }}be in selected{{ UPPER_CASE }}snake/PascalCase format: PascAsl_f + PascAsl_f + // ;warn:$line-2:5: [ENUMS_SEPARATED] enum is incorrectly formatted: last enum entry must end with a comma + + // ;warn:1:9: {{.*}}[PACKAGE_NAME_INCORRECT_PREFIX] package name should start from company's domain: org.cqfn.save{{.*}} +} +// ;warn:0:0: [YOU SHOULD NOT SEE THIS] this warning should not be shown diff --git a/examples/kotlin-diktat/warn/chapter1/IgnoreLinesTest/save.toml b/examples/kotlin-diktat/warn/chapter1/IgnoreLinesTest/save.toml new file mode 100644 index 000000000..bc6f5d8a8 --- /dev/null +++ b/examples/kotlin-diktat/warn/chapter1/IgnoreLinesTest/save.toml @@ -0,0 +1,7 @@ +[general] + tags = ["WarnIgnoreLines"] + description = "Test suite that checks if filtering by regex work." + +[warn] + testNameRegex = "^E.*T.*D.*" + ignoreLines = [".*// ;warn:0:0: \\[YOU SHOULD NOT SEE THIS\\] this warning should not be shown.*"] \ No newline at end of file diff --git a/examples/kotlin-diktat/warn/save.toml b/examples/kotlin-diktat/warn/save.toml index a0da02544..fa65a4016 100644 --- a/examples/kotlin-diktat/warn/save.toml +++ b/examples/kotlin-diktat/warn/save.toml @@ -17,4 +17,4 @@ messageCaptureGroupOut = 4 exactWarningsMatch = false warningTextHasColumn = true - warningTextHasLine = true + warningTextHasLine = true \ No newline at end of file diff --git a/save-cli/src/jvmTest/kotlin/org/cqfn/save/cli/GeneralTest.kt b/save-cli/src/jvmTest/kotlin/org/cqfn/save/cli/GeneralTest.kt index aca7bd591..fddbe94fa 100644 --- a/save-cli/src/jvmTest/kotlin/org/cqfn/save/cli/GeneralTest.kt +++ b/save-cli/src/jvmTest/kotlin/org/cqfn/save/cli/GeneralTest.kt @@ -75,7 +75,7 @@ class GeneralTest { val saveFlags = " . --result-output FILE --report-type JSON" // Execute the script from examples val execCmd = "$runCmd$saveBinName $saveFlags" - val pb = ProcessBuilder(true, fs).exec(execCmd, examplesDir, null, 100_000L) + val pb = ProcessBuilder(true, fs).exec(execCmd, examplesDir, null, 200_000L) println("SAVE execution output:\n${pb.stdout.joinToString("\n")}") if (pb.stderr.isNotEmpty()) { println("Warning and errors during SAVE execution:\n${pb.stderr.joinToString("\n")}") diff --git a/save-common/src/commonMain/kotlin/org/cqfn/save/core/plugin/Plugin.kt b/save-common/src/commonMain/kotlin/org/cqfn/save/core/plugin/Plugin.kt index a818a1335..d8eb5a397 100644 --- a/save-common/src/commonMain/kotlin/org/cqfn/save/core/plugin/Plugin.kt +++ b/save-common/src/commonMain/kotlin/org/cqfn/save/core/plugin/Plugin.kt @@ -56,7 +56,7 @@ abstract class Plugin( if (!excludedTests.isNullOrEmpty()) { logDebug("Excluded tests for [${testConfig.location}] : $excludedTests") } - + clean() return if (testFilesList.isNotEmpty()) { // fixme: remove this logging and convert `testFilesList` back to Sequence // or at least make `logDebug` accept lazy messages diff --git a/save-common/src/commonMain/kotlin/org/cqfn/save/core/plugin/PluginConfig.kt b/save-common/src/commonMain/kotlin/org/cqfn/save/core/plugin/PluginConfig.kt index cb6c0052f..70c77db19 100644 --- a/save-common/src/commonMain/kotlin/org/cqfn/save/core/plugin/PluginConfig.kt +++ b/save-common/src/commonMain/kotlin/org/cqfn/save/core/plugin/PluginConfig.kt @@ -25,6 +25,11 @@ interface PluginConfig { */ val type: TestConfigSections + /** + * list of regexes to be ignored + */ + val ignoreLinesPatterns: MutableList + /** * Location of the toml config */ @@ -76,6 +81,7 @@ data class GeneralConfig( val timeOutMillis: Long? = null, ) : PluginConfig { override val type = TestConfigSections.GENERAL + override val ignoreLinesPatterns: MutableList = mutableListOf() @Transient override var configLocation: Path = "undefined_toml_location".toPath() diff --git a/save-core/src/commonNonJsMain/kotlin/org/cqfn/save/core/Save.kt b/save-core/src/commonNonJsMain/kotlin/org/cqfn/save/core/Save.kt index 26bb462d3..3f6977a90 100644 --- a/save-core/src/commonNonJsMain/kotlin/org/cqfn/save/core/Save.kt +++ b/save-core/src/commonNonJsMain/kotlin/org/cqfn/save/core/Save.kt @@ -118,7 +118,7 @@ class Save( } reporter.afterAll() reporter.out.close() - logInfo("SAVE has finished execution. You can rerun with --log debug or --log trace for additional information.") + logInfo("SAVE has finished execution. You can rerun with --log debug or --log all for additional information.") return reporter } diff --git a/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicFixTest.kt b/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicFixTest.kt index e79c923a1..8b198a071 100644 --- a/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicFixTest.kt +++ b/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicFixTest.kt @@ -19,7 +19,7 @@ class ClassicFixTest { runTestsWithDiktat( listOf( "fix/save.toml" - ), 2 + ), 5 ) } @@ -28,7 +28,43 @@ class ClassicFixTest { runTestsWithDiktat( listOf( "fix/smoke/save.toml" - ), 2 + ), 5 + ) + } + + @Test + fun `execute fix plugin on folder`() { + runTestsWithDiktat( + listOf( + "fix/smoke/src/main/kotlin/org/cqfn/save/" + ), 5 + ) + } + + @Test + fun `check NoIgnoreLines`() { + runTestsWithDiktat( + listOf( + "fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/NoIgnoreLines" + ), 1 + ) + } + + @Test + fun `check IgnoreLinesIsEmpty`() { + runTestsWithDiktat( + listOf( + "fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLinesIsEmpty" + ), 1 + ) + } + + @Test + fun `check IgnoreLines`() { + runTestsWithDiktat( + listOf( + "fix/smoke/src/main/kotlin/org/cqfn/save/IgnoreLinesTest/IgnoreLines" + ), 1 ) } } diff --git a/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicWarnTest.kt b/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicWarnTest.kt index fb8d15468..b71132d83 100644 --- a/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicWarnTest.kt +++ b/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicWarnTest.kt @@ -47,7 +47,7 @@ class ClassicWarnTest { runTestsWithDiktat( listOf( "warn/chapter1" - ), 7 + ), 8 ) } @@ -60,6 +60,15 @@ class ClassicWarnTest { ) } + @Test + fun `lines that match ignoreLines should be ignored`() { + runTestsWithDiktat( + listOf( + "warn/chapter1/IgnoreLinesTest" + ), 1 + ) + } + @Test fun `test output file set`() { runTestsWithDiktat( @@ -86,7 +95,7 @@ class ClassicWarnTest { runTestsWithDiktat( listOf( "warn/save.toml" - ), 9 + ), 10 ) } @@ -95,7 +104,7 @@ class ClassicWarnTest { runTestsWithDiktat( listOf( "warn/chapter1/save.toml" - ), 7 + ), 8 ) } diff --git a/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/FixDirTest.kt b/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/FixDirTest.kt index 54bd9db18..dc5b9b2b8 100644 --- a/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/FixDirTest.kt +++ b/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/FixDirTest.kt @@ -6,7 +6,7 @@ import kotlin.test.Test class FixDirTest { @Test fun `execute fix plugin`() { - runTestsWithDiktat(listOf("fix/smoke/src/main/kotlin/org/cqfn/save"), 2) + runTestsWithDiktat(listOf("fix/smoke/src/main/kotlin/org/cqfn/save"), 5) } @Test diff --git a/save-plugins/fix-and-warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fixandwarn/FixAndWarnPlugin.kt b/save-plugins/fix-and-warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fixandwarn/FixAndWarnPlugin.kt index 578100847..a7b22ad7c 100644 --- a/save-plugins/fix-and-warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fixandwarn/FixAndWarnPlugin.kt +++ b/save-plugins/fix-and-warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fixandwarn/FixAndWarnPlugin.kt @@ -138,7 +138,6 @@ class FixAndWarnPlugin( files.forEach { file -> val fileData = fs.readLines(file) filesAndTheirWarningsMap[file] = mutableListOf() - val fileDataWithoutWarnings = fileData.filterIndexed { index, line -> val isLineWithWarning = (generalConfig.expectedWarningsPattern!!.find(line)?.groups != null) if (isLineWithWarning) { diff --git a/save-plugins/fix-and-warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fixandwarn/FixAndWarnPluginConfig.kt b/save-plugins/fix-and-warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fixandwarn/FixAndWarnPluginConfig.kt index 75fd99e4a..aab8fed37 100644 --- a/save-plugins/fix-and-warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fixandwarn/FixAndWarnPluginConfig.kt +++ b/save-plugins/fix-and-warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fixandwarn/FixAndWarnPluginConfig.kt @@ -26,6 +26,9 @@ data class FixAndWarnPluginConfig( override var configLocation: Path = "undefined_toml_location".toPath() override val resourceNamePatternStr: String = "(${fix.resourceNamePatternStr})|(${warn.resourceNamePatternStr})" + @Transient + override val ignoreLinesPatterns: MutableList = mutableListOf() + override fun mergeWith(otherConfig: PluginConfig): PluginConfig { val other = otherConfig as FixAndWarnPluginConfig val mergedFixPluginConfig = fix.mergeWith(other.fix) @@ -33,7 +36,9 @@ data class FixAndWarnPluginConfig( return FixAndWarnPluginConfig( mergedFixPluginConfig as FixPluginConfig, mergedWarnPluginConfig as WarnPluginConfig - ).also { it.configLocation = this.configLocation } + ).also { + it.configLocation = this.configLocation + } } override fun validateAndSetDefaults(): PluginConfig { @@ -49,6 +54,8 @@ data class FixAndWarnPluginConfig( return FixAndWarnPluginConfig( fix.validateAndSetDefaults(), warn.validateAndSetDefaults() - ).also { it.configLocation = this.configLocation } + ).also { + it.configLocation = this.configLocation + } } } diff --git a/save-plugins/fix-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fix/FixPlugin.kt b/save-plugins/fix-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fix/FixPlugin.kt index 94c982ea2..015860503 100644 --- a/save-plugins/fix-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fix/FixPlugin.kt +++ b/save-plugins/fix-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fix/FixPlugin.kt @@ -84,7 +84,9 @@ class FixPlugin( val extraFlags = extraFlagsList.singleOrNull() ?: ExtraFlags("", "") val pathMap = chunk.map { it.test to it.expected } - val pathCopyMap = pathMap.map { (test, expected) -> createTestFile(test, generalConfig) to expected } + val pathCopyMap = pathMap.map { (test, expected) -> + createTestFile(test, generalConfig, fixPluginConfig) to expected + } val testCopyNames = pathCopyMap.joinToString(separator = fixPluginConfig.batchSeparator!!) { (testCopy, _) -> testCopy.toString() } @@ -146,24 +148,33 @@ class FixPlugin( } } - private fun createTestFile(path: Path, generalConfig: GeneralConfig): Path { - val pathCopy: Path = constructPathForCopyOfTestFile( - "${FixPlugin::class.simpleName!!}-${Random.nextInt()}", - path - ) + private fun createTestFile( + path: Path, + generalConfig: GeneralConfig, + fixPluginConfig: FixPluginConfig): Path { + val pathCopy: Path = constructPathForCopyOfTestFile("${FixPlugin::class.simpleName!!}-${Random.nextInt()}", path) tmpDirectory = pathCopy.parent!! createTempDir(tmpDirectory!!) - val expectedWarningPattern = generalConfig.expectedWarningsPattern + val defaultIgnoreLinesPatterns: MutableList = mutableListOf() + generalConfig.expectedWarningsPattern?.let { defaultIgnoreLinesPatterns.add(it) } + generalConfig.runConfigPattern?.let { defaultIgnoreLinesPatterns.add(it) } fs.write(fs.createFile(pathCopy)) { - fs.readLines(path).forEach { - if (expectedWarningPattern == null || !generalConfig.expectedWarningsPattern!!.matches(it)) { + fs.readLines(path) + .filter { line -> + fixPluginConfig.ignoreLines?.let { + fixPluginConfig.ignoreLinesPatterns.none { it.matches(line) } + } + ?: run { + defaultIgnoreLinesPatterns.none { it.matches(line) } + } + } + .forEach { write( (it + "\n").encodeToByteArray() ) } - } } return pathCopy } diff --git a/save-plugins/fix-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fix/FixPluginConfig.kt b/save-plugins/fix-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fix/FixPluginConfig.kt index 69e86b0f0..bf42494d0 100644 --- a/save-plugins/fix-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fix/FixPluginConfig.kt +++ b/save-plugins/fix-plugin/src/commonMain/kotlin/org/cqfn/save/plugins/fix/FixPluginConfig.kt @@ -23,6 +23,7 @@ import kotlinx.serialization.UseSerializers * @property resourceNameTestSuffix suffix name of the test file. * @property resourceNameExpectedSuffix suffix name of the expected file. * @property batchSeparator + * @property ignoreLines mutable list of patterns that later will be used to filter lines in test file */ @Serializable data class FixPluginConfig( @@ -31,12 +32,16 @@ data class FixPluginConfig( val batchSeparator: String? = null, val resourceNameTestSuffix: String? = null, val resourceNameExpectedSuffix: String? = null, + val ignoreLines: MutableList? = null ) : PluginConfig { override val type = TestConfigSections.FIX @Transient override var configLocation: Path = "undefined_toml_location".toPath() + @Transient + override val ignoreLinesPatterns: MutableList = ignoreLines?.map { it.toRegex() }?.toMutableList() ?: mutableListOf() + /** * @property resourceNameTest */ @@ -61,14 +66,23 @@ data class FixPluginConfig( this.batchSeparator ?: other.batchSeparator, this.resourceNameTestSuffix ?: other.resourceNameTestSuffix, this.resourceNameExpectedSuffix ?: other.resourceNameExpectedSuffix, - ).also { it.configLocation = this.configLocation } + other.ignoreLines?.let { + this.ignoreLines?.let { other.ignoreLines.union(this.ignoreLines) } ?: other.ignoreLines + }?.toMutableList() ?: this.ignoreLines + ).also { + it.configLocation = this.configLocation + } } + // due to probable bug in ktoml, ignoreLines = [] and no ignoreLines is ktoml are parsed to be mutableListOf("null") override fun validateAndSetDefaults() = FixPluginConfig( execFlags ?: "", batchSize ?: 1, batchSeparator ?: ", ", resourceNameTest, - resourceNameExpected - ).also { it.configLocation = this.configLocation } + resourceNameExpected, + ignoreLines + ).also { + it.configLocation = this.configLocation + } } diff --git a/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPlugin.kt b/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPlugin.kt index 570060c7e..8d4ec6dbb 100644 --- a/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPlugin.kt +++ b/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPlugin.kt @@ -1,7 +1,7 @@ package org.cqfn.save.plugin.warn import org.cqfn.save.core.config.TestConfig -import org.cqfn.save.core.files.createRelativePathToTheRoot +import org.cqfn.save.core.files.createFile import org.cqfn.save.core.files.readLines import org.cqfn.save.core.logging.describe import org.cqfn.save.core.logging.logWarn @@ -23,6 +23,7 @@ import org.cqfn.save.plugin.warn.utils.getLineNumber import okio.FileNotFoundException import okio.FileSystem import okio.Path +import kotlin.random.Random private typealias WarningMap = Map> @@ -82,13 +83,31 @@ class WarnPlugin( } } + private fun createTestFiles(paths: List, warnPluginConfig: WarnPluginConfig): List { + val dirName = "${WarnPlugin::class.simpleName!!}-${Random.nextInt()}" + val dirPath = constructPathForCopyOfTestFile(dirName, paths[0]).parent!! + createTempDir(dirPath) + + val ignorePatterns = warnPluginConfig.ignoreLinesPatterns + + return paths.map { path -> + val copyPath = constructPathForCopyOfTestFile(dirName, path) + fs.write(fs.createFile(copyPath)) { + fs.readLines(path) + .filter { line -> ignorePatterns.none { it.matches(line) } } + .map { write((it + "\n").encodeToByteArray()) } + } + copyPath + } + } + @Suppress( "TOO_LONG_FUNCTION", "SAY_NO_TO_VAR", "LongMethod", "ReturnCount", "SwallowedException", - "TOO_MANY_LINES_IN_LAMBDA", + "TOO_MANY_LINES_IN_LAMBDA" ) private fun handleTestFile( paths: List, @@ -96,15 +115,13 @@ class WarnPlugin( generalConfig: GeneralConfig ): Sequence { // extracting all warnings from test resource files - val expectedWarningsMap: WarningMap = paths.associate { + val copyPaths: List = createTestFiles(paths, warnPluginConfig) + val expectedWarningsMap: WarningMap = copyPaths.associate { val warningsForCurrentPath = it.collectWarningsWithLineNumbers(warnPluginConfig, generalConfig) it.name to warningsForCurrentPath } - val extraFlagsList = paths.mapNotNull { path -> - extraFlagsExtractor.extractExtraFlagsFrom(path) - } - .distinct() + val extraFlagsList = copyPaths.mapNotNull { extraFlagsExtractor.extractExtraFlagsFrom(it) }.distinct() require(extraFlagsList.size <= 1) { "Extra flags for all files in a batch should be same, but you have batchSize=${warnPluginConfig.batchSize}" + " and there are ${extraFlagsList.size} different sets of flags inside it, namely $extraFlagsList" @@ -124,19 +141,13 @@ class WarnPlugin( // NOTE: SAVE will pass relative paths of Tests (calculated from testRootConfig dir) into the executed tool val fileNamesForExecCmd = warnPluginConfig.wildCardInDirectoryMode?.let { - val directoryPrefix = testConfig - .directory - .createRelativePathToTheRoot(testConfig.getRootConfig().location) // a hack to put only the directory path to the execution command // only in case a directory mode is enabled - "$directoryPrefix$it" - } ?: paths.joinToString(separator = warnPluginConfig.batchSeparator!!) { - it.createRelativePathToTheRoot(testConfig.getRootConfig().location) - } - + "${copyPaths[0].parent!!}$it" + } ?: copyPaths.joinToString(separator = warnPluginConfig.batchSeparator!!) val execFlagsAdjusted = resolvePlaceholdersFrom(warnPluginConfig.execFlags, extraFlags, fileNamesForExecCmd) val execCmd = "${generalConfig.execCmd} $execFlagsAdjusted" - val time = generalConfig.timeOutMillis!!.times(paths.size) + val time = generalConfig.timeOutMillis!!.times(copyPaths.size) val executionResult = try { pb.exec(execCmd, testConfig.getRootConfig().directory.toString(), redirectTo, time) @@ -146,6 +157,7 @@ class WarnPlugin( } catch (ex: ProcessExecutionException) { return failTestResult(paths, ex, execCmd) } + val stdout = warnPluginConfig.testToolResFileOutput?.let { val testToolResFilePath = testConfig.directory / warnPluginConfig.testToolResFileOutput @@ -220,26 +232,27 @@ class WarnPlugin( generalConfig: GeneralConfig ): List { val linesFile = fs.readLines(this) - return linesFile.mapIndexed { index, line -> - val newLine = line.getLineNumber( - generalConfig.expectedWarningsPattern!!, - warnPluginConfig.lineCaptureGroup, - warnPluginConfig.linePlaceholder!!, - index + 1, - this, - linesFile, - ) - with(warnPluginConfig) { - line.extractWarning( + return linesFile + .mapIndexed { index, line -> + val newLine = line.getLineNumber( generalConfig.expectedWarningsPattern!!, - this@collectWarningsWithLineNumbers.name, - newLine, - columnCaptureGroup, - messageCaptureGroup!!, - benchmarkMode!!, + warnPluginConfig.lineCaptureGroup, + warnPluginConfig.linePlaceholder!!, + index + 1, + this, + linesFile ) + with(warnPluginConfig) { + line.extractWarning( + generalConfig.expectedWarningsPattern!!, + this@collectWarningsWithLineNumbers.name, + newLine, + columnCaptureGroup, + messageCaptureGroup!!, + benchmarkMode!! + ) + } } - } .filterNotNull() .sortedBy { warn -> warn.message } } diff --git a/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPluginConfig.kt b/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPluginConfig.kt index b8e4b96f3..74f18b5a5 100644 --- a/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPluginConfig.kt +++ b/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPluginConfig.kt @@ -49,6 +49,7 @@ import kotlinx.serialization.UseSerializers * @property partialWarnTextMatch if true - the regex created from expected warning will be wrapped with '.*': .*warn.*. * That can help a user to write only main information in the warning without any need to add/copy-paste technical info * @property testToolResFileOutput file with actual warnings + * @property ignoreLines mutable list of patterns that later will be ignored in test files * @property benchmarkMode whether to ignore the warning messages */ @Serializable @@ -73,7 +74,8 @@ data class WarnPluginConfig( val patternForRegexInWarning: List? = null, val partialWarnTextMatch: Boolean? = null, val testToolResFileOutput: String? = null, - val benchmarkMode: Boolean? = null, + val ignoreLines: MutableList? = null, + val benchmarkMode: Boolean? = null ) : PluginConfig { @Transient override val type = TestConfigSections.WARN @@ -81,6 +83,9 @@ data class WarnPluginConfig( @Transient override var configLocation: Path = "undefined_toml_location".toPath() + @Transient + override val ignoreLinesPatterns: MutableList = ignoreLines?.map { it.toRegex() }?.toMutableList() ?: mutableListOf() + /** * regex for name of the test file. */ @@ -115,8 +120,13 @@ data class WarnPluginConfig( this.patternForRegexInWarning ?: other.patternForRegexInWarning, this.partialWarnTextMatch ?: other.partialWarnTextMatch, this.testToolResFileOutput ?: other.testToolResFileOutput, - this.benchmarkMode ?: other.benchmarkMode, - ).also { it.configLocation = this.configLocation } + other.ignoreLines?.let { + this.ignoreLines?.let { other.ignoreLines.union(this.ignoreLines) } ?: other.ignoreLines + }?.toMutableList() ?: this.ignoreLines, + this.benchmarkMode ?: other.benchmarkMode + ).also { + it.configLocation = this.configLocation + } } @Suppress( @@ -172,8 +182,11 @@ data class WarnPluginConfig( patternForRegexInWarning ?: defaultPatternForRegexInWarning, partialWarnTextMatch ?: false, testToolResFileOutput, - benchmarkMode ?: false, - ).also { it.configLocation = this.configLocation } + ignoreLines, + benchmarkMode ?: false + ).also { + it.configLocation = this.configLocation + } } private fun requirePositiveIfNotNull(value: Long?) {