From bf4afe3ca36772b8e8b56aadcb6f0c494a5d6eaa Mon Sep 17 00:00:00 2001 From: Nathan Leach Date: Fri, 14 Jul 2023 13:42:34 -0500 Subject: [PATCH] Feature/externalize mongo schema (#224) * dependency updates * workflow update * workflow update * ignore config update * tool created * cleanup * remove reference to wiki * fixed an attribute name * starting to remove schema creation * removing schemas * removed static schemas * update supported platforms * record suppression feature * release notes update * documenation updates * package updates * dependency updates * action version update * action version update * documentdb compat * MongoTool doc update * minor logic update --- .github/workflows/build-ci.yml | 22 ++-- .github/workflows/build-prerelease.yml | 2 +- .github/workflows/publish-build.yml | 4 +- .gitignore | 3 +- .../CxAnalytixCLI/CxAnalytixCLI.csproj | 2 +- .../CxAnalytixDaemon/CxAnalytixDaemon.csproj | 9 +- .../CxAnalytixService.csproj | 5 +- CxAnalytix.sln | 31 ++++- Libs/Configuration/Configuration.csproj | 6 +- Libs/Configuration/Impls/Config.cs | 95 ++++++++------ .../Impls/ConfigElementCollection.cs | 32 +++++ Libs/Configuration/Impls/CxAnalytixService.cs | 37 +++--- .../Configuration/Impls/EnabledTransformer.cs | 5 +- .../Impls/EnabledTransformersCollection.cs | 26 ---- .../Utils/RecordNameConfigAttribute.cs | 10 ++ .../CxAuditTrailsCrawler.csproj | 4 +- Libs/CxAuditTrailsDB/CxAuditTrailsDB.csproj | 5 +- .../CxAuditTrailsDirectDB.csproj | 6 +- Libs/CxRestClient/CxRestClient.csproj | 4 +- Libs/Exceptions/Exceptions.csproj | 2 +- Libs/Executive/Executive.csproj | 2 +- Libs/Extensions/Extensions.csproj | 2 +- Libs/Interfaces/Interfaces.csproj | 2 +- Libs/LogCleaner/LogCleaner.csproj | 2 +- Libs/OutputBootstrapper/Output.cs | 8 +- .../OutputBootstrapper.csproj | 6 +- Libs/OutputBootstrapper/RecordRefWrapper.cs | 32 +++++ .../RecordSuppressionMediator.cs | 32 +++++ Libs/ProjectFilter/ProjectFilter.csproj | 2 +- Libs/Utilities/Utilities.csproj | 4 +- MongoTool/CollectionCreator.cs | 58 +++++++++ MongoTool/Extensions.cs | 24 ++++ MongoTool/MongoTool.csproj | 45 +++++++ MongoTool/Program.cs | 119 ++++++++++++++++++ MongoTool/UserCreator.cs | 68 ++++++++++ Output/AMQPOutput/AMQPOutput.csproj | 6 +- Output/Log4NetOutput/Log4NetOutput.csproj | 5 +- Output/MongoDBOutput/MongoDBOutFactory.cs | 51 +------- Output/MongoDBOutput/MongoDBOutput.csproj | 8 +- .../MongoDBOutput/PolicyViolationsSchema.cs | 45 ------- Output/MongoDBOutput/ProjectInfoSchema.cs | 43 ------- Output/MongoDBOutput/SCADetailSchema.cs | 45 ------- Output/MongoDBOutput/SCASummarySchema.cs | 80 ------------ Output/MongoDBOutput/SastDetailSchema.cs | 74 ----------- Output/MongoDBOutput/SastSummarySchema.cs | 61 --------- Output/MongoDBOutput/ScanStatisticsSchema.cs | 32 ----- QA/RegressionTester/RegressionTester.csproj | 4 +- README.md | 3 + SDK/SDK.Modules/SDK.Modules.csproj | 6 +- .../StringFormatExamples.csproj | 2 +- .../Configuration_Tests.csproj | 6 +- .../CxAuditTrailsCrawler_Tests.csproj | 8 +- .../CxRestClient_Tests.csproj | 6 +- Tests/Extensions_Test/Extensions_Test.csproj | 4 +- .../LogCleaner_Tests/LogCleaner_Tests.csproj | 4 +- .../ProjectFilter_Tests.csproj | 8 +- .../TransformLogic_Tests.csproj | 6 +- Tests/Utilities_Tests/Utilities_Tests.csproj | 8 +- XForm/Common/BaseTransformer.cs | 6 +- XForm/Common/Common.csproj | 2 +- .../CxOneTransformer/CxOneTransformer.csproj | 2 +- XForm/SastTransformer/SastTransformer.csproj | 4 +- XForm/ScaTransformer/ScaTransformer.csproj | 2 +- config_files/dotnet.exe.config | 5 +- manual/configuration-general.tex | 14 +-- manual/deploymentguide.tex | 2 +- manual/installing.tex | 79 +++++++++++- manual/release_notes-content.tex | 18 +++ 68 files changed, 740 insertions(+), 625 deletions(-) create mode 100644 Libs/Configuration/Impls/ConfigElementCollection.cs delete mode 100644 Libs/Configuration/Impls/EnabledTransformersCollection.cs create mode 100644 Libs/Configuration/Utils/RecordNameConfigAttribute.cs create mode 100644 Libs/OutputBootstrapper/RecordRefWrapper.cs create mode 100644 Libs/OutputBootstrapper/RecordSuppressionMediator.cs create mode 100644 MongoTool/CollectionCreator.cs create mode 100644 MongoTool/Extensions.cs create mode 100644 MongoTool/MongoTool.csproj create mode 100644 MongoTool/Program.cs create mode 100644 MongoTool/UserCreator.cs delete mode 100644 Output/MongoDBOutput/PolicyViolationsSchema.cs delete mode 100644 Output/MongoDBOutput/ProjectInfoSchema.cs delete mode 100644 Output/MongoDBOutput/SCADetailSchema.cs delete mode 100644 Output/MongoDBOutput/SCASummarySchema.cs delete mode 100644 Output/MongoDBOutput/SastDetailSchema.cs delete mode 100644 Output/MongoDBOutput/SastSummarySchema.cs delete mode 100644 Output/MongoDBOutput/ScanStatisticsSchema.cs diff --git a/.github/workflows/build-ci.yml b/.github/workflows/build-ci.yml index 1d21d4ed..9cf2bfdb 100644 --- a/.github/workflows/build-ci.yml +++ b/.github/workflows/build-ci.yml @@ -10,14 +10,18 @@ jobs: infer-csharp-pr-comment: runs-on: ubuntu-latest - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' && github.actor != 'dependabot' steps: + - name: Actor + run: | + echo GITHUB_ACTOR + echo ${{ github.actor }} - name: Dotnet Core Install uses: actions/setup-dotnet@v2 with: dotnet-version: 6.* - name: Fetch Code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Platform build win-x64 run: dotnet publish --self-contained -c ReleaseWindows -o artifacts/win-x64 -r win-x64 - name: Platform build linux-x64 @@ -58,7 +62,7 @@ jobs: with: dotnet-version: 6.* - name: Fetch Code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Platform build win-x64 run: dotnet publish --self-contained -c ReleaseWindows -o artifacts -r win-x64 @@ -71,7 +75,7 @@ jobs: with: dotnet-version: 6.* - name: Fetch Code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Platform build linux-x64 run: dotnet publish --self-contained -c ReleaseLinux -o artifacts -r linux-x64 @@ -85,9 +89,9 @@ jobs: dotnet-version: 6.* - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Fetch Code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Platform build linux-x64 run: dotnet publish --self-contained -c ReleaseLinux -o artifacts -r linux-x64 - name: Build Docker Container @@ -102,7 +106,7 @@ jobs: with: dotnet-version: 6.* - name: Fetch Code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Run Tests run: dotnet test --verbosity q @@ -111,7 +115,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Fetch Code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build Manual id: build_manual uses: xu-cheng/latex-action@v2 @@ -126,7 +130,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Fetch Code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Build Release Notes id: build_release_notes uses: xu-cheng/latex-action@v2 diff --git a/.github/workflows/build-prerelease.yml b/.github/workflows/build-prerelease.yml index 3a819dfa..60750719 100644 --- a/.github/workflows/build-prerelease.yml +++ b/.github/workflows/build-prerelease.yml @@ -31,7 +31,7 @@ jobs: steps: - uses: technote-space/workflow-conclusion-action@v2 - name: Fetch Code - uses: actions/checkout@v2 + uses: actions/checkout@v3 if: env.WORKFLOW_CONCLUSION == 'failure' - name: Remove Prerelease Tag run: git push origin ':refs/tags/v${{ needs.create-tag.outputs.tag }}' diff --git a/.github/workflows/publish-build.yml b/.github/workflows/publish-build.yml index 5c601bb9..924a8ac5 100644 --- a/.github/workflows/publish-build.yml +++ b/.github/workflows/publish-build.yml @@ -26,7 +26,7 @@ jobs: steps: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to GitHub Container Registry @@ -43,7 +43,7 @@ jobs: - name: Fetch Code @ tag v${{ inputs.tag }} - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: refs/tags/v${{ inputs.tag }} diff --git a/.gitignore b/.gitignore index 5c16beae..a8e62892 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ artifacts/ .build_tools/ **/TestResults -**/DELETE* \ No newline at end of file +**/DELETE* +**/launchSettings.json diff --git a/Applications/CxAnalytixCLI/CxAnalytixCLI.csproj b/Applications/CxAnalytixCLI/CxAnalytixCLI.csproj index 7bc78879..5b3a4c0c 100644 --- a/Applications/CxAnalytixCLI/CxAnalytixCLI.csproj +++ b/Applications/CxAnalytixCLI/CxAnalytixCLI.csproj @@ -11,7 +11,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix diff --git a/Applications/CxAnalytixDaemon/CxAnalytixDaemon.csproj b/Applications/CxAnalytixDaemon/CxAnalytixDaemon.csproj index 2321d6a5..71bc5dcd 100644 --- a/Applications/CxAnalytixDaemon/CxAnalytixDaemon.csproj +++ b/Applications/CxAnalytixDaemon/CxAnalytixDaemon.csproj @@ -9,7 +9,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -32,10 +32,9 @@ - - - - + + + diff --git a/Applications/CxAnalytixService/CxAnalytixService.csproj b/Applications/CxAnalytixService/CxAnalytixService.csproj index 39832f4a..121fc601 100644 --- a/Applications/CxAnalytixService/CxAnalytixService.csproj +++ b/Applications/CxAnalytixService/CxAnalytixService.csproj @@ -12,7 +12,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -40,8 +40,7 @@ - - + diff --git a/CxAnalytix.sln b/CxAnalytix.sln index 74f572d8..d10f4d27 100644 --- a/CxAnalytix.sln +++ b/CxAnalytix.sln @@ -89,7 +89,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SastTransformer", "XForm\Sa EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Common", "XForm\Common\Common.csproj", "{BE71BE3E-7591-4D62-BFB3-F77D7B57F12A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CxOneTransformer", "XForm\CxOneTransformer\CxOneTransformer.csproj", "{9F56B86B-C692-4D85-BABB-735B969D2D4E}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CxOneTransformer", "XForm\CxOneTransformer\CxOneTransformer.csproj", "{9F56B86B-C692-4D85-BABB-735B969D2D4E}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Install", "Install", "{CDEFBA2F-FB11-44A9-AE7D-8317C42A830C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MongoTool", "MongoTool\MongoTool.csproj", "{8DE4F60A-CD68-40D8-98F0-14E8C06406D2}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -891,6 +895,30 @@ Global {9F56B86B-C692-4D85-BABB-735B969D2D4E}.ReleaseWindows|x64.Build.0 = Release|Any CPU {9F56B86B-C692-4D85-BABB-735B969D2D4E}.ReleaseWindows|x86.ActiveCfg = Release|Any CPU {9F56B86B-C692-4D85-BABB-735B969D2D4E}.ReleaseWindows|x86.Build.0 = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Debug|x64.ActiveCfg = Debug|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Debug|x64.Build.0 = Debug|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Debug|x86.ActiveCfg = Debug|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Debug|x86.Build.0 = Debug|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Release|Any CPU.Build.0 = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Release|x64.ActiveCfg = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Release|x64.Build.0 = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Release|x86.ActiveCfg = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.Release|x86.Build.0 = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseLinux|Any CPU.ActiveCfg = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseLinux|Any CPU.Build.0 = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseLinux|x64.ActiveCfg = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseLinux|x64.Build.0 = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseLinux|x86.ActiveCfg = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseLinux|x86.Build.0 = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseWindows|Any CPU.ActiveCfg = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseWindows|Any CPU.Build.0 = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseWindows|x64.ActiveCfg = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseWindows|x64.Build.0 = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseWindows|x86.ActiveCfg = Release|Any CPU + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2}.ReleaseWindows|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -930,6 +958,7 @@ Global {0788985B-731F-4BA5-A7E3-5C7996074B54} = {203A826C-286F-442C-B7B9-E43F806B8778} {BE71BE3E-7591-4D62-BFB3-F77D7B57F12A} = {203A826C-286F-442C-B7B9-E43F806B8778} {9F56B86B-C692-4D85-BABB-735B969D2D4E} = {203A826C-286F-442C-B7B9-E43F806B8778} + {8DE4F60A-CD68-40D8-98F0-14E8C06406D2} = {CDEFBA2F-FB11-44A9-AE7D-8317C42A830C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {103FD3A6-4F85-4DC0-B65F-7E974EA202FE} diff --git a/Libs/Configuration/Configuration.csproj b/Libs/Configuration/Configuration.csproj index be5bec74..cde3386c 100644 --- a/Libs/Configuration/Configuration.csproj +++ b/Libs/Configuration/Configuration.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -31,9 +31,9 @@ - + - + diff --git a/Libs/Configuration/Impls/Config.cs b/Libs/Configuration/Impls/Config.cs index e43b04ef..7567a4bb 100644 --- a/Libs/Configuration/Impls/Config.cs +++ b/Libs/Configuration/Impls/Config.cs @@ -2,14 +2,18 @@ using CxAnalytix.Configuration.Utils; using CxAnalytix.Exceptions; using log4net; +using log4net.Util; using System; +using System.Collections; +using System.Collections.Generic; using System.Configuration; using System.IO; + namespace CxAnalytix.Configuration.Impls { - public class Config + public static class Config { private static System.Configuration.Configuration _cfgManager; private static ILog _log = LogManager.GetLogger(typeof (Config) ); @@ -32,14 +36,16 @@ static Config() _cfgManager = ConfigurationManager.OpenMappedExeConfiguration(map, ConfigurationUserLevel.None); - if (OperatingSystem.IsWindows()) + + + if (OperatingSystem.IsWindows()) EncryptSensitiveSections(); else _log.Warn("This platform does not support encrypting credentials in the configuration file. Your credentials may be stored in plain text."); var builder = new ContainerBuilder(); - foreach(var sec in _cfgManager.Sections) + foreach(var sec in _cfgManager.Sections.OnlyValid() ) { builder.RegisterInstance(sec).As(sec.GetType()).ExternallyOwned(); } @@ -47,47 +53,60 @@ static Config() } - public static T GetConfig() + private static IEnumerable OnlyValid(this ConfigurationSectionCollection elements) { - return _configDI.Resolve(); + List result = new(); + + for (int i = 0; i < elements.Count; i++) + try + { + result.Add(elements[i]); + } + catch (ConfigurationErrorsException ex) + { + _log.Warn($"Configuration error: {ex.Message}"); + } + + return result; } + public static T GetConfig() => _configDI.Resolve(); - private static void EncryptSensitiveSections() - { - foreach (ConfigurationSection section in _cfgManager.Sections) - { - var attribs = section.GetType().GetCustomAttributes(typeof(SecureConfigSectionAttribute), true); - - if (attribs != null && attribs.Length > 0) - { - bool found = false; - foreach (SecureConfigSectionAttribute attribInst in attribs) - { - if (attribInst.IsPropSet(section.GetType(), section)) - { - found = true; - break; - } - } - - if (!found) - continue; - } - else - continue; - - if (!section.SectionInformation.IsProtected) - { - section.SectionInformation.ProtectSection("DataProtectionConfigurationProvider"); - section.SectionInformation.ForceSave = true; - section.SectionInformation.ForceDeclaration(true); - } - } + private static void EncryptSensitiveSections() + { + foreach (var section in _cfgManager.Sections.OnlyValid()) + { + var attribs = section.GetType().GetCustomAttributes(typeof(SecureConfigSectionAttribute), true); + + if (attribs != null && attribs.Length > 0) + { + bool found = false; + foreach (SecureConfigSectionAttribute attribInst in attribs) + { + if (attribInst.IsPropSet(section.GetType(), section)) + { + found = true; + break; + } + } + + if (!found) + continue; + } + else + continue; + + if (!section.SectionInformation.IsProtected) + { + section.SectionInformation.ProtectSection("DataProtectionConfigurationProvider"); + section.SectionInformation.ForceSave = true; + section.SectionInformation.ForceDeclaration(true); + } + } _cfgManager.Save(ConfigurationSaveMode.Modified); - } + } + - } } diff --git a/Libs/Configuration/Impls/ConfigElementCollection.cs b/Libs/Configuration/Impls/ConfigElementCollection.cs new file mode 100644 index 00000000..e319bede --- /dev/null +++ b/Libs/Configuration/Impls/ConfigElementCollection.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.Configuration; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CxAnalytix.Configuration.Impls +{ + + public interface IKeyProducer + { + String Key { get; } + } + + public class ConfigElementCollection : ConfigurationElementCollection where T : ConfigurationElement, IKeyProducer, new() + { + protected override ConfigurationElement CreateNewElement() + { + return new T(); + } + + protected override object GetElementKey(ConfigurationElement element) + { + if (element == null) + return null; + + return (element as IKeyProducer).Key; + } + + } +} diff --git a/Libs/Configuration/Impls/CxAnalytixService.cs b/Libs/Configuration/Impls/CxAnalytixService.cs index 530057ed..d9936b19 100644 --- a/Libs/Configuration/Impls/CxAnalytixService.cs +++ b/Libs/Configuration/Impls/CxAnalytixService.cs @@ -47,49 +47,56 @@ public String OutputModuleName set { this["OutputModuleName"] = value; } } - [ConfigurationProperty("SASTScanSummaryRecordName", IsRequired = true)] + [RecordNameConfig] + [ConfigurationProperty("SASTScanSummaryRecordName", IsRequired = false, DefaultValue = null)] public String SASTScanSummaryRecordName { get => (String)this["SASTScanSummaryRecordName"]; set { this["SASTScanSummaryRecordName"] = value; } } - [ConfigurationProperty("SASTScanDetailRecordName", IsRequired = true)] + [ConfigurationProperty("SASTScanDetailRecordName", IsRequired = false, DefaultValue = null)] + [RecordNameConfig] public String SASTScanDetailRecordName { get => (String)this["SASTScanDetailRecordName"]; set { this["SASTScanDetailRecordName"] = value; } } - [ConfigurationProperty("SCAScanSummaryRecordName", IsRequired = true)] + [ConfigurationProperty("SCAScanSummaryRecordName", IsRequired = false, DefaultValue = null)] + [RecordNameConfig] public String SCAScanSummaryRecordName { get => (String)this["SCAScanSummaryRecordName"]; set { this["SCAScanSummaryRecordName"] = value; } } - [ConfigurationProperty("SCAScanDetailRecordName", IsRequired = true)] + [ConfigurationProperty("SCAScanDetailRecordName", IsRequired = false, DefaultValue = null)] + [RecordNameConfig] public String SCAScanDetailRecordName { get => (String)this["SCAScanDetailRecordName"]; set { this["SCAScanDetailRecordName"] = value; } } - [ConfigurationProperty("ProjectInfoRecordName", IsRequired = true)] + [ConfigurationProperty("ProjectInfoRecordName", IsRequired = false, DefaultValue = null)] + [RecordNameConfig] public String ProjectInfoRecordName { get => (String)this["ProjectInfoRecordName"]; set { this["ProjectInfoRecordName"] = value; } } - [ConfigurationProperty("PolicyViolationsRecordName", IsRequired = true)] + [ConfigurationProperty("PolicyViolationsRecordName", IsRequired = false, DefaultValue = null)] + [RecordNameConfig] public String PolicyViolationsRecordName { get => (String)this["PolicyViolationsRecordName"]; set { this["PolicyViolationsRecordName"] = value; } } - [ConfigurationProperty("ScanStatisticsRecordName", IsRequired = false)] + [ConfigurationProperty("ScanStatisticsRecordName", IsRequired = false, DefaultValue = null)] + [RecordNameConfig] public String ScanStatisticsRecordName { get => (String)this["ScanStatisticsRecordName"]; @@ -104,20 +111,12 @@ public int ProcessPeriodMinutes } [ConfigurationProperty("EnabledTransformers", IsDefaultCollection = false, IsRequired = true)] - [ConfigurationCollection(typeof(EnabledTransformersCollection), AddItemName = "Transformer")] - public EnabledTransformersCollection Transformers + [ConfigurationCollection(typeof(ConfigElementCollection), AddItemName = "Transformer")] + public ConfigElementCollection Transformers { - get - { - return (EnabledTransformersCollection)this["EnabledTransformers"]; - } - - set - { - this["EnabledTransformers"] = value; - } + get => (ConfigElementCollection )this["EnabledTransformers"]; + set => this["EnabledTransformers"] = value; } - } } diff --git a/Libs/Configuration/Impls/EnabledTransformer.cs b/Libs/Configuration/Impls/EnabledTransformer.cs index f2d27c7b..f56ec340 100644 --- a/Libs/Configuration/Impls/EnabledTransformer.cs +++ b/Libs/Configuration/Impls/EnabledTransformer.cs @@ -7,7 +7,7 @@ namespace CxAnalytix.Configuration.Impls { - public class EnabledTransformer : ConfigurationElement + public class EnabledTransformer : ConfigurationElement, IKeyProducer { [ConfigurationProperty("Name", IsRequired = true)] @@ -17,5 +17,6 @@ public String Name set => base["Name"] = value; } - } + public string Key => Name; + } } diff --git a/Libs/Configuration/Impls/EnabledTransformersCollection.cs b/Libs/Configuration/Impls/EnabledTransformersCollection.cs deleted file mode 100644 index 3749d793..00000000 --- a/Libs/Configuration/Impls/EnabledTransformersCollection.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Configuration; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace CxAnalytix.Configuration.Impls -{ - public class EnabledTransformersCollection : ConfigurationElementCollection - { - protected override ConfigurationElement CreateNewElement() - { - return new EnabledTransformer(); - } - - protected override object GetElementKey(ConfigurationElement element) - { - if (element == null) - return null; - - return (element as EnabledTransformer).Name; - } - - } -} diff --git a/Libs/Configuration/Utils/RecordNameConfigAttribute.cs b/Libs/Configuration/Utils/RecordNameConfigAttribute.cs new file mode 100644 index 00000000..999e8698 --- /dev/null +++ b/Libs/Configuration/Utils/RecordNameConfigAttribute.cs @@ -0,0 +1,10 @@ +using System; + +namespace CxAnalytix.Configuration.Utils +{ + [System.AttributeUsage(System.AttributeTargets.Property, AllowMultiple = false, Inherited = true)] + public class RecordNameConfigAttribute : Attribute + { + public RecordNameConfigAttribute() { } + } +} diff --git a/Libs/CxAuditTrailsCrawler/CxAuditTrailsCrawler.csproj b/Libs/CxAuditTrailsCrawler/CxAuditTrailsCrawler.csproj index 86ff570f..4ea48725 100644 --- a/Libs/CxAuditTrailsCrawler/CxAuditTrailsCrawler.csproj +++ b/Libs/CxAuditTrailsCrawler/CxAuditTrailsCrawler.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -23,7 +23,7 @@ - + diff --git a/Libs/CxAuditTrailsDB/CxAuditTrailsDB.csproj b/Libs/CxAuditTrailsDB/CxAuditTrailsDB.csproj index 4bc6bec6..f674ebe5 100644 --- a/Libs/CxAuditTrailsDB/CxAuditTrailsDB.csproj +++ b/Libs/CxAuditTrailsDB/CxAuditTrailsDB.csproj @@ -9,7 +9,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -38,8 +38,7 @@ - - + diff --git a/Libs/CxAuditTrailsDirectDB/CxAuditTrailsDirectDB.csproj b/Libs/CxAuditTrailsDirectDB/CxAuditTrailsDirectDB.csproj index d76a7747..18894521 100644 --- a/Libs/CxAuditTrailsDirectDB/CxAuditTrailsDirectDB.csproj +++ b/Libs/CxAuditTrailsDirectDB/CxAuditTrailsDirectDB.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -16,8 +16,8 @@ - - + + diff --git a/Libs/CxRestClient/CxRestClient.csproj b/Libs/CxRestClient/CxRestClient.csproj index a2ae8821..e05865dd 100644 --- a/Libs/CxRestClient/CxRestClient.csproj +++ b/Libs/CxRestClient/CxRestClient.csproj @@ -7,7 +7,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -33,7 +33,7 @@ - + diff --git a/Libs/Exceptions/Exceptions.csproj b/Libs/Exceptions/Exceptions.csproj index 7773e0a3..182d721c 100644 --- a/Libs/Exceptions/Exceptions.csproj +++ b/Libs/Exceptions/Exceptions.csproj @@ -9,7 +9,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix diff --git a/Libs/Executive/Executive.csproj b/Libs/Executive/Executive.csproj index 3af89dab..88a6e2b9 100644 --- a/Libs/Executive/Executive.csproj +++ b/Libs/Executive/Executive.csproj @@ -9,7 +9,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix diff --git a/Libs/Extensions/Extensions.csproj b/Libs/Extensions/Extensions.csproj index 6bb64eff..6b52369d 100644 --- a/Libs/Extensions/Extensions.csproj +++ b/Libs/Extensions/Extensions.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix diff --git a/Libs/Interfaces/Interfaces.csproj b/Libs/Interfaces/Interfaces.csproj index 872c04ef..ee0ab5df 100644 --- a/Libs/Interfaces/Interfaces.csproj +++ b/Libs/Interfaces/Interfaces.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix diff --git a/Libs/LogCleaner/LogCleaner.csproj b/Libs/LogCleaner/LogCleaner.csproj index ed458a48..98c93de4 100644 --- a/Libs/LogCleaner/LogCleaner.csproj +++ b/Libs/LogCleaner/LogCleaner.csproj @@ -7,7 +7,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix diff --git a/Libs/OutputBootstrapper/Output.cs b/Libs/OutputBootstrapper/Output.cs index e556dc58..799779b3 100644 --- a/Libs/OutputBootstrapper/Output.cs +++ b/Libs/OutputBootstrapper/Output.cs @@ -44,10 +44,9 @@ static Output() } [MethodImpl(MethodImplOptions.Synchronized)] - public static IOutputTransaction StartTransaction() { - var retVal = _outFactory.StartTransaction(); + var retVal = new RecordSuppressionMediator(_outFactory.StartTransaction()); _log.Trace($"Starting output transaction: {retVal.TransactionId}"); @@ -57,9 +56,12 @@ public static IOutputTransaction StartTransaction() [MethodImpl(MethodImplOptions.Synchronized)] public static IRecordRef RegisterRecord(String name) { + if (String.IsNullOrEmpty(name)) + return new RecordRefWrapper(); + _log.Debug($"Registering record with name {name}"); - return _outFactory.RegisterRecord(name); + return new RecordRefWrapper(_outFactory.RegisterRecord(name)); } } diff --git a/Libs/OutputBootstrapper/OutputBootstrapper.csproj b/Libs/OutputBootstrapper/OutputBootstrapper.csproj index 965aa3d8..1987d3c3 100644 --- a/Libs/OutputBootstrapper/OutputBootstrapper.csproj +++ b/Libs/OutputBootstrapper/OutputBootstrapper.csproj @@ -7,7 +7,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -15,9 +15,9 @@ - + - + diff --git a/Libs/OutputBootstrapper/RecordRefWrapper.cs b/Libs/OutputBootstrapper/RecordRefWrapper.cs new file mode 100644 index 00000000..61952b12 --- /dev/null +++ b/Libs/OutputBootstrapper/RecordRefWrapper.cs @@ -0,0 +1,32 @@ +using CxAnalytix.Interfaces.Outputs; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace OutputBootstrapper +{ + internal class RecordRefWrapper : IRecordRef + { + private IRecordRef _wrapped; + + public RecordRefWrapper() + { + Suppressed = true; + _wrapped = null; + } + + public RecordRefWrapper(IRecordRef original) + { + Suppressed = false; + _wrapped = original; + } + + internal IRecordRef OriginalRef => _wrapped; + + public bool Suppressed { get; private set; } + + public string RecordName => (_wrapped == null) ? ("Suppressed") : (_wrapped.RecordName); + } +} diff --git a/Libs/OutputBootstrapper/RecordSuppressionMediator.cs b/Libs/OutputBootstrapper/RecordSuppressionMediator.cs new file mode 100644 index 00000000..ef775535 --- /dev/null +++ b/Libs/OutputBootstrapper/RecordSuppressionMediator.cs @@ -0,0 +1,32 @@ +using CxAnalytix.Interfaces.Outputs; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace OutputBootstrapper +{ + internal class RecordSuppressionMediator : IOutputTransaction + { + private IOutputTransaction _wrapped; + + public RecordSuppressionMediator(IOutputTransaction wrapped) => _wrapped = wrapped; + + + public string TransactionId => _wrapped.TransactionId; + + public bool Commit() => _wrapped.Commit(); + + public void Dispose() => _wrapped.Dispose(); + + public void write(IRecordRef which, IDictionary record) + { + var wrappedRef = which as RecordRefWrapper; + + if (!wrappedRef.Suppressed) + _wrapped.write(wrappedRef.OriginalRef, record); + + } + } +} diff --git a/Libs/ProjectFilter/ProjectFilter.csproj b/Libs/ProjectFilter/ProjectFilter.csproj index 864326df..1c269eed 100644 --- a/Libs/ProjectFilter/ProjectFilter.csproj +++ b/Libs/ProjectFilter/ProjectFilter.csproj @@ -7,7 +7,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix diff --git a/Libs/Utilities/Utilities.csproj b/Libs/Utilities/Utilities.csproj index 659a4ca6..6c7e4338 100644 --- a/Libs/Utilities/Utilities.csproj +++ b/Libs/Utilities/Utilities.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -17,7 +17,7 @@ - + diff --git a/MongoTool/CollectionCreator.cs b/MongoTool/CollectionCreator.cs new file mode 100644 index 00000000..24dd79d9 --- /dev/null +++ b/MongoTool/CollectionCreator.cs @@ -0,0 +1,58 @@ +using CxAnalytix.Configuration.Impls; +using CxAnalytix.Configuration.Utils; +using CxAnalytix.Exceptions; +using CxAnalytix.Extensions; +using log4net; +using Microsoft.IdentityModel.Tokens; +using MongoDB.Driver; +using System; +using System.Collections.Generic; +using System.Drawing.Printing; +using System.Linq; +using System.Reflection; + +namespace CxAnalytix.MongoTool +{ + internal static class CollectionCreator + { + private static ILog _log = LogManager.GetLogger(typeof(CollectionCreator)); + + public static void CreateCollections(MongoUrl mongoEndpoint) + { + var client = new MongoClient(mongoEndpoint); + + if (!client.ListDatabaseNames().ToList().Contains(mongoEndpoint.DatabaseName)) + _log.Warn($"Database {mongoEndpoint.DatabaseName} does not exist, it will be created."); + + var db = mongoEndpoint.GetDatabase(); + + var existingCollections = new HashSet(db.ListCollectionNames().ToEnumerable()); + + var serviceConfig = Config.GetConfig(); + + if (serviceConfig == null) + throw new ProcessFatalException("Could not retrieve the CxAnalytix service configuration from the CxAnalytix configuration file."); + + + typeof(CxAnalytixService).GetProperties().ActionForEach(p => + { + if (p.GetCustomAttribute() != null) + { + if (!existingCollections.Contains(p.GetValue(serviceConfig))) + { + var collectionName = p.GetValue(serviceConfig) as String; + if (collectionName.IsNullOrEmpty()) + _log.Info($"Skipping creation for collection for record type [{p.Name}] because the collection name is not defined."); + else + { + _log.Info($"Creating collection for record type [{p.Name}] with the name [{collectionName}]."); + db.CreateCollection(collectionName); + } + } + else + _log.Info($"Collection for record type [{p.Name}] named [{p.GetValue(serviceConfig)}] already exists."); + } + }); + } + } +} diff --git a/MongoTool/Extensions.cs b/MongoTool/Extensions.cs new file mode 100644 index 00000000..00fa4cd0 --- /dev/null +++ b/MongoTool/Extensions.cs @@ -0,0 +1,24 @@ +using MongoDB.Bson; +using MongoDB.Driver; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CxAnalytix.MongoTool +{ + internal static class Extensions + { + public static MongoUrl ChangeToDB(this MongoUrl src, String loginDBName) => + new MongoUrlBuilder(src.ToString()) { DatabaseName = loginDBName }.ToMongoUrl(); + + public static IMongoDatabase GetDatabase(this MongoUrl src) => new MongoClient(src).GetDatabase(src.DatabaseName); + + public static IMongoDatabase GetDatabase(this MongoUrl src, String dbName) => new MongoClient(src).GetDatabase(dbName); + + + public static bool CheckSuccess(this BsonDocument result) => result.First().Name.Equals("ok"); + + } +} diff --git a/MongoTool/MongoTool.csproj b/MongoTool/MongoTool.csproj new file mode 100644 index 00000000..65dc484f --- /dev/null +++ b/MongoTool/MongoTool.csproj @@ -0,0 +1,45 @@ + + + Exe + net6.0 + + CxAnalytix.MongoTool + Debug;Release;ReleaseWindows;ReleaseLinux + Checkmarx + Checkmarx + 0.0.0 + Tool to initialize or update the CxAnalytix MongoDB schema. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. + https://github.com/checkmarx-ts/CxAnalytix/wiki + $(SolutionDir)README.md + https://github.com/checkmarx-ts/CxAnalytix + + disable + disable + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/MongoTool/Program.cs b/MongoTool/Program.cs new file mode 100644 index 00000000..90afffec --- /dev/null +++ b/MongoTool/Program.cs @@ -0,0 +1,119 @@ +using System; +using CommandLine; +using CommandLine.Text; +using CxAnalytix.Configuration.Impls; +using CxAnalytix.MongoTool; +using CxAnalytix.Out.MongoDBOutput.Config.Impl; +using log4net; +using log4net.Core; +using MongoDB.Driver; + + +var parser = new Parser(settings => { settings.HelpWriter = null; }); +var parsedData = parser.ParseArguments(args); + + +var logPattern = new log4net.Layout.PatternLayout("[%-5level][%utcdate{yyyy-MM-ddTHH:mm:ss.fffZ}][%c] %message%newline"); +var repo = LogManager.GetRepository() as log4net.Repository.Hierarchy.Hierarchy; +var file = new log4net.Appender.RollingFileAppender() { Layout = logPattern, File = "MongoTool.log", AppendToFile = true }; +file.ActivateOptions(); +repo.Root.AddAppender(file); +repo.Root.AddAppender(new log4net.Appender.ConsoleAppender() { Layout = logPattern }); +repo.Configured = true; + +Console.WriteLine("CxAnalytix MongoDB Schema Tool"); +Console.WriteLine("Run with --help to see command line options."); + + +parsedData.WithParsed((opts) => +{ + if (opts.Debug) + LogManager.GetRepository().Threshold = Level.Debug; + else + LogManager.GetRepository().Threshold = Level.Info; + + var _log = LogManager.GetLogger(typeof(Program)); + + _log.Info("START"); + + if (opts.UseConfigConnectionString && opts.MongoConnectionString == null) + try + { + _log.Info("Loading MongoDB connection string from the CxAnalytix configuration file."); + + var mongo_config = Config.GetConfig(); + + if (mongo_config != null) + opts.MongoConnectionString = new MongoUrl(mongo_config.ConnectionString); + } + catch (Exception ex) + { + _log.Error("Failed to load the MongoDB connection URL from the CxAnalytix configuration file."); + + _log.Debug(ex.Message); + _log.Debug(ex.StackTrace); + if (ex.InnerException != null) + _log.Debug(ex.InnerException.Message); + } + + if (opts.MongoConnectionString == null) + { + _log.Error("A MongoDB connection URL is required."); + Environment.Exit(1); + } + else + { + try + { + _log.Info($"Operations are using [{opts.LoginDB}] as the login database."); + + UserCreator.CreateOrUpdateUser(opts.MongoConnectionString, opts.LoginDB); + + if (!String.IsNullOrEmpty(opts.MongoUser) && !String.IsNullOrEmpty(opts.MongoPassword)) + UserCreator.CreateOrUpdateUser(opts.MongoConnectionString, opts.LoginDB, opts.MongoUser, opts.MongoPassword); + + CollectionCreator.CreateCollections(opts.MongoConnectionString); + } + catch (Exception ex) + { + _log.Error(ex.Message); + _log.Debug(ex.StackTrace); + } + } + + _log.Info("END"); + + +}).WithNotParsed(errors => +{ + var helpText = HelpText.AutoBuild(parsedData, h => HelpText.DefaultParsingErrorsHandler(parsedData, h), e=> e); + Console.WriteLine(helpText); + Environment.Exit(1); +}); + + + +public class CommandLineOpts +{ + [Option('u', "url", SetName="connection", Default = null, Required = false, HelpText = "Use this MongoDB connection URL.")] + public MongoUrl MongoConnectionString { get; set; } + + [Option('c', SetName = "connection", Default = true, Required = false, HelpText = "Load the MongoDB connection URL from the CxAnalytix configuration file.")] + public bool UseConfigConnectionString { get; set; } + + [Option('d', "debug", Default = false, Required = false, HelpText = "Enable debug output.")] + public bool Debug { get; set; } + + [Option('l', "login-db", Default = "admin", Required = false, HelpText = "Specify the MongoDB database used for login.")] + public String LoginDB { get; set; } + + [Option("mongo-user", Default = null, Required = false, HelpText = "The name of a MongoDB user to create and assign to the CxAnalytix database with minimal privileges.")] + public String MongoUser { get; set; } + + [Option("mongo-password", Default = null, Required = false, HelpText = "The password for the MongoDB user that is created.")] + public String MongoPassword { get; set; } + +} + + + diff --git a/MongoTool/UserCreator.cs b/MongoTool/UserCreator.cs new file mode 100644 index 00000000..b440a19b --- /dev/null +++ b/MongoTool/UserCreator.cs @@ -0,0 +1,68 @@ +using CxAnalytix.Exceptions; +using log4net; +using MongoDB.Bson; +using MongoDB.Driver; +using MongoDB.Driver.Linq; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CxAnalytix.MongoTool +{ + internal static class UserCreator + { + private static ILog _log = LogManager.GetLogger(typeof(UserCreator)); + + public static void CreateOrUpdateUser(MongoUrl mongoEndpoint, String loginDBName) + { + try + { + CreateOrUpdateUser(mongoEndpoint, loginDBName, mongoEndpoint.Username, mongoEndpoint.Password); + } + catch (MongoCommandException ex) + { + _log.Warn($"Failed to set roles for database [{mongoEndpoint.DatabaseName}] user [{mongoEndpoint.Username}] Error [{ex.Message}]"); + } + } + + private static void CreateOrUpdateUser(IMongoDatabase inThisDB, String targetDBName, String targetUser, String targetPassword) + { + var getUsersCommand = new BsonDocument() { { "usersInfo", 1 } }; + var usersDB = inThisDB.RunCommand(getUsersCommand); + + var foundUser = usersDB.GetElement("users").Value.AsBsonArray.SingleOrDefault((bval) => bval["user"].AsString.ToLower().CompareTo(targetUser.ToLower()) == 0); + + BsonDocument command = null; + if (foundUser == null || foundUser.IsBsonNull) + { + _log.Info($"Creating user [{inThisDB.DatabaseNamespace}.{targetUser}] with [readWrite] role for database [{targetDBName}]"); + + command = new BsonDocument() { { "createUser", targetUser }, { "pwd", targetPassword}, + { "roles", new BsonArray() { new BsonDocument() { { "role", "readWrite" }, { "db", targetDBName } } } } }; + + } + else + { + _log.Info($"User [{inThisDB.DatabaseNamespace}.{targetUser}] exists, granting [readWrite] role for database [{targetDBName}]"); + + command = new BsonDocument() { { "grantRolesToUser", targetUser }, + { "roles", new BsonArray() { new BsonDocument() { { "role", "readWrite" }, { "db", targetDBName } } } } }; + } + + if (!inThisDB.RunCommand(command).CheckSuccess()) + throw new ProcessFatalException($"Error executing command for [{inThisDB.DatabaseNamespace}.{targetUser}]"); + } + + public static void CreateOrUpdateUser(MongoUrl mongoEndpoint, String loginDBName, String targetUser, String targetPassword) + { + var adminDB = mongoEndpoint.ChangeToDB(loginDBName).GetDatabase(); + var targetDB = adminDB.Client.GetDatabase(mongoEndpoint.DatabaseName); + + CreateOrUpdateUser(adminDB, mongoEndpoint.DatabaseName, targetUser, targetPassword); + CreateOrUpdateUser(targetDB, mongoEndpoint.DatabaseName, targetUser, targetPassword); + } + + } +} diff --git a/Output/AMQPOutput/AMQPOutput.csproj b/Output/AMQPOutput/AMQPOutput.csproj index 8014dc54..2c21f3a9 100644 --- a/Output/AMQPOutput/AMQPOutput.csproj +++ b/Output/AMQPOutput/AMQPOutput.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -21,8 +21,8 @@ - - + + diff --git a/Output/Log4NetOutput/Log4NetOutput.csproj b/Output/Log4NetOutput/Log4NetOutput.csproj index c94c1181..f2da9b8b 100644 --- a/Output/Log4NetOutput/Log4NetOutput.csproj +++ b/Output/Log4NetOutput/Log4NetOutput.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -17,8 +17,7 @@ - - + diff --git a/Output/MongoDBOutput/MongoDBOutFactory.cs b/Output/MongoDBOutput/MongoDBOutFactory.cs index 2ae07dd7..2f5d6e14 100644 --- a/Output/MongoDBOutput/MongoDBOutFactory.cs +++ b/Output/MongoDBOutput/MongoDBOutFactory.cs @@ -41,42 +41,15 @@ private static void Init() _client = new MongoClient(mu); if (!_client.ListDatabaseNames().ToList().Contains(mu.DatabaseName)) - _log.Warn($"Database {mu.DatabaseName} does not exist, it will be created."); + throw new ProcessFatalException($"Database {mu.DatabaseName} does not exist, did you forget to run the CxAnalytix MongoTool?"); _db = _client.GetDatabase(mu.DatabaseName); - - // It is a violation of OOP principles for this component to know about these records. At some point the schema - // creation may be moved to an installer that initializes the DB prior to running the application. - _schemas.Add(Service.SASTScanDetailRecordName, - MongoDBOut.CreateInstance(_db, Service.SASTScanDetailRecordName, OutConfig.ShardKeys[Service.SASTScanDetailRecordName])); - - _schemas.Add(Service.SASTScanSummaryRecordName, MongoDBOut.CreateInstance(_db, Service.SASTScanSummaryRecordName, - OutConfig.ShardKeys[Service.SASTScanSummaryRecordName])); - - _schemas.Add(Service.SCAScanSummaryRecordName, MongoDBOut.CreateInstance(_db, Service.SCAScanSummaryRecordName, - OutConfig.ShardKeys[Service.SCAScanSummaryRecordName])); - - _schemas.Add(Service.SCAScanDetailRecordName, MongoDBOut.CreateInstance(_db, Service.SCAScanDetailRecordName, - OutConfig.ShardKeys[Service.SCAScanDetailRecordName])); - - _schemas.Add(Service.ProjectInfoRecordName, MongoDBOut.CreateInstance(_db, Service.ProjectInfoRecordName, - OutConfig.ShardKeys[Service.ProjectInfoRecordName])); - - _schemas.Add(Service.PolicyViolationsRecordName, MongoDBOut.CreateInstance(_db, Service.PolicyViolationsRecordName, - OutConfig.ShardKeys[Service.PolicyViolationsRecordName])); - - if (Service.ScanStatisticsRecordName != null && !String.IsNullOrEmpty(Service.ScanStatisticsRecordName)) - _schemas.Add(Service.ScanStatisticsRecordName, MongoDBOut.CreateInstance(_db, Service.ScanStatisticsRecordName, - OutConfig.ShardKeys[Service.ScanStatisticsRecordName])); - } catch (Exception ex) { _log.Error("Error initializing MongoDB connectivity.", ex); _client = null; -#pragma warning disable CA2200 // Rethrow to preserve stack details - throw ex; -#pragma warning restore CA2200 // Rethrow to preserve stack details + throw; } } @@ -125,24 +98,12 @@ public override IRecordRef RegisterRecord(string recordName) throw new ProcessFatalException("The connection to MongoDB could not be established."); lock (_schemas) + { if (!_schemas.ContainsKey(recordName)) - { _schemas.Add(recordName, MongoDBOut.CreateInstance(_db, recordName, OutConfig.ShardKeys[recordName])); - - return _schemas[recordName]; - } - else - { - var dest = _schemas[recordName]; - - if (!dest.VerifyOrCreateSchema()) - { - _log.Warn($"Schema for {recordName} could not be verified or created, no data will be output for this record type."); - return new Dummy(); - } - - return dest; - } + + return _schemas[recordName]; + } } } } diff --git a/Output/MongoDBOutput/MongoDBOutput.csproj b/Output/MongoDBOutput/MongoDBOutput.csproj index cc92cf12..85cdd24b 100644 --- a/Output/MongoDBOutput/MongoDBOutput.csproj +++ b/Output/MongoDBOutput/MongoDBOutput.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -34,9 +34,9 @@ - - - + + + diff --git a/Output/MongoDBOutput/PolicyViolationsSchema.cs b/Output/MongoDBOutput/PolicyViolationsSchema.cs deleted file mode 100644 index d4ea34d6..00000000 --- a/Output/MongoDBOutput/PolicyViolationsSchema.cs +++ /dev/null @@ -1,45 +0,0 @@ -using log4net; -using MongoDB.Bson; -using MongoDB.Driver; -using System; -using System.Collections.Generic; -using System.Text; - -namespace CxAnalytix.Out.MongoDBOutput -{ - internal class PolicyViolationsSchema : MongoDBOut, ISchema - { - private static ILog _log = LogManager.GetLogger(typeof(PolicyViolationsSchema)); - - public override bool VerifyOrCreateSchema() - { - - var opts = new CreateIndexOptions() - { - Background = true - }; - - opts.Name = "ProjectName-A+ScanId-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ProjectName")) - .Ascending(new StringFieldDefinition("ScanId")) - , opts)); - - opts.Name = "FirstViolationDetectionDate-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("FirstViolationDetectionDate")) - , opts)); - - opts.Name = "ViolationOccurredDate-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ViolationOccurredDate")) - , opts)); - - return true; - } - - } -} diff --git a/Output/MongoDBOutput/ProjectInfoSchema.cs b/Output/MongoDBOutput/ProjectInfoSchema.cs deleted file mode 100644 index d909c390..00000000 --- a/Output/MongoDBOutput/ProjectInfoSchema.cs +++ /dev/null @@ -1,43 +0,0 @@ -using log4net; -using MongoDB.Bson; -using MongoDB.Driver; -using System; - -namespace CxAnalytix.Out.MongoDBOutput -{ - internal class ProjectInfoSchema : MongoDBOut, ISchema - { - private static ILog _log = LogManager.GetLogger(typeof(ProjectInfoSchema)); - - public override bool VerifyOrCreateSchema() - { - - var opts = new CreateIndexOptions() - { - Background = true - }; - - opts.Name = "ProjectName-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ProjectName")) - , opts)); - - opts.Name = "SAST_LastScanDate-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("SAST_LastScanDate")) - , opts)); - - opts.Name = "SCA_LastScanDate-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("SCA_LastScanDate")) - , opts)); - - - return true; - } - - } -} diff --git a/Output/MongoDBOutput/SCADetailSchema.cs b/Output/MongoDBOutput/SCADetailSchema.cs deleted file mode 100644 index d9b8a270..00000000 --- a/Output/MongoDBOutput/SCADetailSchema.cs +++ /dev/null @@ -1,45 +0,0 @@ -using log4net; -using MongoDB.Bson; -using MongoDB.Driver; -using System; - -namespace CxAnalytix.Out.MongoDBOutput -{ - internal class SCADetailSchema : MongoDBOut, ISchema - { - private static ILog _log = LogManager.GetLogger(typeof(SCADetailSchema)); - - public override bool VerifyOrCreateSchema() - { - - var opts = new CreateIndexOptions() - { - Background = true - }; - - opts.Name = "ProjectName-A+ScanId-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ProjectName")) - .Ascending(new StringFieldDefinition("ScanId")) - , opts)); - - opts.Name = "LibraryName-A+LibraryVersion-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("LibraryName")) - .Ascending(new StringFieldDefinition("LibraryVersion")) - , opts)); - - - opts.Name = "ScanFinished-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ScanFinished")) - , opts)); - - - return true; - } - } -} diff --git a/Output/MongoDBOutput/SCASummarySchema.cs b/Output/MongoDBOutput/SCASummarySchema.cs deleted file mode 100644 index 28bafdb4..00000000 --- a/Output/MongoDBOutput/SCASummarySchema.cs +++ /dev/null @@ -1,80 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; -using log4net; -using MongoDB.Bson; -using MongoDB.Driver; - -namespace CxAnalytix.Out.MongoDBOutput -{ - internal class SCASummarySchema : MongoDBOut, ISchema - { - private static ILog _log = LogManager.GetLogger(typeof(SastSummarySchema)); - - public override bool VerifyOrCreateSchema() - { - var opts = new CreateIndexOptions() - { - Background = true - }; - - opts.Name = "ProjectName-A+ScanId-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ProjectName")) - .Ascending(new StringFieldDefinition("ScanId")) - , opts)); - - - opts.Name = "ScanFinished-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ScanFinished")) - , opts)); - - opts.Name = "RulesViolated-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("RulesViolated")) - , opts)); - - opts.Name = "HighVulnerabilityLibraries-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("HighVulnerabilityLibraries")) - , opts)); - - opts.Name = "MediumVulnerabilityLibraries-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("MediumVulnerabilityLibraries")) - , opts)); - - opts.Name = "LowVulnerabilityLibraries-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("LowVulnerabilityLibraries")) - , opts)); - - opts.Name = "LegalHigh-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("LegalHigh")) - , opts)); - - opts.Name = "LegalMedium-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("LegalMedium")) - , opts)); - - opts.Name = "LegalLow-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("LegalLow")) - , opts)); - - return true; - } - } -} diff --git a/Output/MongoDBOutput/SastDetailSchema.cs b/Output/MongoDBOutput/SastDetailSchema.cs deleted file mode 100644 index 5496088a..00000000 --- a/Output/MongoDBOutput/SastDetailSchema.cs +++ /dev/null @@ -1,74 +0,0 @@ -using log4net; -using MongoDB.Bson; -using MongoDB.Driver; -using System; - -namespace CxAnalytix.Out.MongoDBOutput -{ - internal class SastDetailSchema : MongoDBOut, ISchema - { - private static ILog _log = LogManager.GetLogger(typeof(SastDetailSchema)); - - public override bool VerifyOrCreateSchema() - { - var opts = new CreateIndexOptions() - { - Background = true - }; - - opts.Name = "ProjectName-A+ScanId-D"; - if (!MongoUtil.IndexExists (Collection, opts.Name) ) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ProjectName")) - .Descending(new StringFieldDefinition("ScanId")) - , opts)); - - opts.Name = "ScanId-D+PathId-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Descending(new StringFieldDefinition("ScanId")) - .Ascending(new StringFieldDefinition("PathId")) - , opts)); - - opts.Name = "SimilarityId-D+ProjectName-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Descending(new StringFieldDefinition("SimilarityId")) - .Ascending(new StringFieldDefinition("ProjectName")) - , opts)); - - opts.Name = "SinkFileName-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("SinkFileName")) - , opts)); - - opts.Name = "QueryName-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("QueryName")) - , opts)); - - opts.Name = "ResultSeverity-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ResultSeverity")) - , opts)); - - opts.Name = "QueryLanguage-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("QueryLanguage")) - , opts)); - - opts.Name = "ScanFinished-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ScanFinished")) - , opts)); - - - return true; - } - } -} diff --git a/Output/MongoDBOutput/SastSummarySchema.cs b/Output/MongoDBOutput/SastSummarySchema.cs deleted file mode 100644 index 77d311d6..00000000 --- a/Output/MongoDBOutput/SastSummarySchema.cs +++ /dev/null @@ -1,61 +0,0 @@ -using System; -using log4net; -using MongoDB.Bson; -using MongoDB.Driver; - -namespace CxAnalytix.Out.MongoDBOutput -{ - internal class SastSummarySchema : MongoDBOut, ISchema - { - private static ILog _log = LogManager.GetLogger(typeof(SastSummarySchema)); - - public override bool VerifyOrCreateSchema() - { - - var opts = new CreateIndexOptions() - { - Background = true - }; - - opts.Name = "ProjectName-A+ScanId-D"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ProjectName")) - .Descending(new StringFieldDefinition("ScanId")) - , opts)); - - opts.Name = "High-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("High")) - , opts)); - - opts.Name = "Medium-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("Medium")) - , opts)); - - opts.Name = "Low-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("Low")) - , opts)); - - opts.Name = "Information-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("Information")) - , opts)); - - opts.Name = "ScanFinished-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("ScanFinished")) - , opts)); - - - return true; - } - } -} diff --git a/Output/MongoDBOutput/ScanStatisticsSchema.cs b/Output/MongoDBOutput/ScanStatisticsSchema.cs deleted file mode 100644 index 9adb5294..00000000 --- a/Output/MongoDBOutput/ScanStatisticsSchema.cs +++ /dev/null @@ -1,32 +0,0 @@ -using log4net; -using MongoDB.Bson; -using MongoDB.Driver; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace CxAnalytix.Out.MongoDBOutput -{ - internal class ScanStatisticsSchema : MongoDBOut, ISchema - { - private static ILog _log = LogManager.GetLogger(typeof(ProjectInfoSchema)); - - public override bool VerifyOrCreateSchema() - { - var opts = new CreateIndexOptions() - { - Background = true - }; - - opts.Name = "Language-A"; - if (!MongoUtil.IndexExists(Collection, opts.Name)) - Collection.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys - .Ascending(new StringFieldDefinition("Language")) - , opts)); - - return true; - } - } -} diff --git a/QA/RegressionTester/RegressionTester.csproj b/QA/RegressionTester/RegressionTester.csproj index d13a53d1..3935015d 100644 --- a/QA/RegressionTester/RegressionTester.csproj +++ b/QA/RegressionTester/RegressionTester.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -17,7 +17,7 @@ - + diff --git a/README.md b/README.md index a0825dc6..aaf77288 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,9 @@ CxAnalytix is a background process that crawls and extracts vulnerability data f * OSA (optional) * Management & Orchestration (optional) * Checkmarx SCA +* Checkmarx One + * SAST Scans + * SCA Scans The vulnerability data is then transformed into a flattened JSON document and stored for at-scale data analysis. Pluggable data transports are implemented for storage flexibility; data can be easily stored and accessed locally or transported to data lakes for more advanced analysis. diff --git a/SDK/SDK.Modules/SDK.Modules.csproj b/SDK/SDK.Modules/SDK.Modules.csproj index a301e9c6..fe3b37ca 100644 --- a/SDK/SDK.Modules/SDK.Modules.csproj +++ b/SDK/SDK.Modules/SDK.Modules.csproj @@ -10,7 +10,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -19,9 +19,9 @@ - + - + diff --git a/SDK/StringFormatExamples/StringFormatExamples.csproj b/SDK/StringFormatExamples/StringFormatExamples.csproj index f49fc33d..9275bd70 100644 --- a/SDK/StringFormatExamples/StringFormatExamples.csproj +++ b/SDK/StringFormatExamples/StringFormatExamples.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix diff --git a/Tests/Configuration_Tests/Configuration_Tests.csproj b/Tests/Configuration_Tests/Configuration_Tests.csproj index 418814a0..17d35a04 100644 --- a/Tests/Configuration_Tests/Configuration_Tests.csproj +++ b/Tests/Configuration_Tests/Configuration_Tests.csproj @@ -25,9 +25,9 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/CxAuditTrailsCrawler_Tests/CxAuditTrailsCrawler_Tests.csproj b/Tests/CxAuditTrailsCrawler_Tests/CxAuditTrailsCrawler_Tests.csproj index d90f5254..93335cce 100644 --- a/Tests/CxAuditTrailsCrawler_Tests/CxAuditTrailsCrawler_Tests.csproj +++ b/Tests/CxAuditTrailsCrawler_Tests/CxAuditTrailsCrawler_Tests.csproj @@ -9,13 +9,13 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/CxRestClient_Tests/CxRestClient_Tests.csproj b/Tests/CxRestClient_Tests/CxRestClient_Tests.csproj index ccb64554..d1ef0009 100644 --- a/Tests/CxRestClient_Tests/CxRestClient_Tests.csproj +++ b/Tests/CxRestClient_Tests/CxRestClient_Tests.csproj @@ -26,10 +26,10 @@ - + - - + + diff --git a/Tests/Extensions_Test/Extensions_Test.csproj b/Tests/Extensions_Test/Extensions_Test.csproj index 18d3c99f..2cd74fc0 100644 --- a/Tests/Extensions_Test/Extensions_Test.csproj +++ b/Tests/Extensions_Test/Extensions_Test.csproj @@ -27,8 +27,8 @@ - - + + diff --git a/Tests/LogCleaner_Tests/LogCleaner_Tests.csproj b/Tests/LogCleaner_Tests/LogCleaner_Tests.csproj index 5c582870..32fa1581 100644 --- a/Tests/LogCleaner_Tests/LogCleaner_Tests.csproj +++ b/Tests/LogCleaner_Tests/LogCleaner_Tests.csproj @@ -27,8 +27,8 @@ - - + + diff --git a/Tests/ProjectFilter_Tests/ProjectFilter_Tests.csproj b/Tests/ProjectFilter_Tests/ProjectFilter_Tests.csproj index 7cac98bd..aa4a41cc 100644 --- a/Tests/ProjectFilter_Tests/ProjectFilter_Tests.csproj +++ b/Tests/ProjectFilter_Tests/ProjectFilter_Tests.csproj @@ -9,13 +9,13 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Tests/TransformLogic_Tests/TransformLogic_Tests.csproj b/Tests/TransformLogic_Tests/TransformLogic_Tests.csproj index 9762c5c2..a4042835 100644 --- a/Tests/TransformLogic_Tests/TransformLogic_Tests.csproj +++ b/Tests/TransformLogic_Tests/TransformLogic_Tests.csproj @@ -26,12 +26,12 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Tests/Utilities_Tests/Utilities_Tests.csproj b/Tests/Utilities_Tests/Utilities_Tests.csproj index 727345e2..39a93d5b 100644 --- a/Tests/Utilities_Tests/Utilities_Tests.csproj +++ b/Tests/Utilities_Tests/Utilities_Tests.csproj @@ -9,13 +9,13 @@ - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/XForm/Common/BaseTransformer.cs b/XForm/Common/BaseTransformer.cs index e4662d8c..7bc175a2 100644 --- a/XForm/Common/BaseTransformer.cs +++ b/XForm/Common/BaseTransformer.cs @@ -52,11 +52,7 @@ public BaseTransformer(string moduleName, Type moduleImplType, String stateStora PolicyViolationDetailOut = Output.RegisterRecord(Service.PolicyViolationsRecordName); ScaScanSummaryOut = Output.RegisterRecord(Service.SCAScanSummaryRecordName); ScaScanDetailOut = Output.RegisterRecord(Service.SCAScanDetailRecordName); - - if (Service.ScanStatisticsRecordName != null && !String.IsNullOrEmpty(Service.ScanStatisticsRecordName)) - ScanStatisticsOut = Output.RegisterRecord(Service.ScanStatisticsRecordName); - else - ScanStatisticsOut = null; + ScanStatisticsOut = Output.RegisterRecord(Service.ScanStatisticsRecordName); ThreadOpts = new ParallelOptions() diff --git a/XForm/Common/Common.csproj b/XForm/Common/Common.csproj index 947fdaea..8843e1c6 100644 --- a/XForm/Common/Common.csproj +++ b/XForm/Common/Common.csproj @@ -10,7 +10,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix diff --git a/XForm/CxOneTransformer/CxOneTransformer.csproj b/XForm/CxOneTransformer/CxOneTransformer.csproj index dd067cff..61da0c0f 100644 --- a/XForm/CxOneTransformer/CxOneTransformer.csproj +++ b/XForm/CxOneTransformer/CxOneTransformer.csproj @@ -11,7 +11,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix diff --git a/XForm/SastTransformer/SastTransformer.csproj b/XForm/SastTransformer/SastTransformer.csproj index d8ab8f1c..f6893e2d 100644 --- a/XForm/SastTransformer/SastTransformer.csproj +++ b/XForm/SastTransformer/SastTransformer.csproj @@ -8,7 +8,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix @@ -34,7 +34,7 @@ - + diff --git a/XForm/ScaTransformer/ScaTransformer.csproj b/XForm/ScaTransformer/ScaTransformer.csproj index df885c4f..e0250adf 100644 --- a/XForm/ScaTransformer/ScaTransformer.csproj +++ b/XForm/ScaTransformer/ScaTransformer.csproj @@ -6,7 +6,7 @@ Checkmarx 0.0.0 CxAnalytix crawls and extracts vulnerability scan data from Checkmarx products. - Copyright (c) 2019-2022 Checkmarx. All rights reserved. + Copyright (c) 2019-2023 Checkmarx. All rights reserved. https://github.com/checkmarx-ts/CxAnalytix/wiki $(SolutionDir)README.md https://github.com/checkmarx-ts/CxAnalytix diff --git a/config_files/dotnet.exe.config b/config_files/dotnet.exe.config index 37896c9c..43f83602 100644 --- a/config_files/dotnet.exe.config +++ b/config_files/dotnet.exe.config @@ -20,8 +20,7 @@ @@ -34,7 +33,7 @@ diff --git a/manual/configuration-general.tex b/manual/configuration-general.tex index d7c4b4f7..69ae3cc4 100644 --- a/manual/configuration-general.tex +++ b/manual/configuration-general.tex @@ -281,17 +281,17 @@ \subsection{CxAnalytix Service and CLI Execution Configuration}\label{sec:runtim values can be found in\\ the \hyperref[lst:outmodules]{Available Output Modules} list.}\\ \midrule - \texttt{SASTScanSummaryRecordName}\\ + \makecell[tc]{\texttt{SASTScanSummaryRecordName}\\ \texttt{SASTScanDetailRecordName}\\ \texttt{SCAScanSummaryRecordName}\\ \texttt{SCAScanDetailRecordName}\\ \texttt{ProjectInfoRecordName}\\ - \texttt{PolicyViolationsRecordName} & N/A & Yes & \makecell[tl]{The name of the corresponding\\ - record collection configured\\ - in the output.}\\ - \midrule - \texttt{ScanStatisticsRecordName} & N/A & No & \makecell[tl]{The name of the record for storing\\ - scan statistics in the output.}\\ + \texttt{PolicyViolationsRecordName}\\ + \texttt{ScanStatisticsRecordName}} & N/A & No & \makecell[tl]{The name of the corresponding\\ + record collection configured in the \\ + output. If blank or omitted,\\ + the corresponding record type\\ + is not output.}\\ \bottomrule \end{tabularx} \end{table} diff --git a/manual/deploymentguide.tex b/manual/deploymentguide.tex index 86a6c42c..a092bd53 100644 --- a/manual/deploymentguide.tex +++ b/manual/deploymentguide.tex @@ -46,7 +46,7 @@ \section{MongoDB Sizing} cluster members' hardware specifics. The machine specification will also need to account for CPU required for any indexing and data ingestion. \end{itemize} -\subsection{Bring Your Own Indexing} +\subsection{Bring Your Own Indexing}\label{sec:bringyourownindex} The MongoDB implementation requires that the user account configured for CxAnalytix to interact with MongoDB has read/write access to tables and collections specified in the configuration. Upon the first run, tables, collections, and some default indexes will be created. diff --git a/manual/installing.tex b/manual/installing.tex index a4a5acc5..2e9f1958 100644 --- a/manual/installing.tex +++ b/manual/installing.tex @@ -176,11 +176,79 @@ \section{Logging Configuration} \texttt{cxanalytix.log4net} file is generally located in the same directory as the \texttt{cxanalytix.config} file, and will allow for customizing the logging output location and verbosity. +\section{MongoDB Schema Initialization}\label{sec:mongotool} +Versions of CxAnalytix prior to 2.1.1 initialized the MongoDB collection schema on the first run after the MongoDB output is configured. Prior +versions would also create several default indexes as part of the collection schema creation. This is no longer the case. + +\noindent\\As of CxAnalytix 2.1.1, it is now required to execute \texttt{MongoTool} prior to the first start after initial installtion, +upgrades or configuration changes that would change the MongoDB collection schema. Any indexes that were created by CxAnalytix in versions +prior to version 2.1.1 can be safely dropped if desired. See Section \ref{sec:bringyourownindex} for information regarding defining indexes +appropriate for your querying needs. + +\noindent\\The executable \texttt{MongoTool} command line parameters are described in Table \ref{tab:mongo_tool_opts}. + + +\subsection{MongoDB Secure Usage Pattern} + +Many applications that use databases deploy the runtime application with a user account that does +not have administrative privileges. Using a non-administrative user in the configured +\hyperref[sec:mongo_config]{MongoDB connection URL} may be desired to limit the capability +of the CxAnalytix MongoDB user if the credentials are somehow exposed. Using \texttt{MongoTool}, +these steps can be followed to configure CxAnalytix to access MongoDB with a minimally privileged +user account: + +\begin{enumerate} + \item Using the \texttt{-u} or \texttt{--url} option, provide the MongoDB connection + URL to \texttt{MongoTool} that contains the administrative account and password. + This allows \texttt{MongoTool} to create collections and users with the appropriate roles. + + \item Provide the \texttt{----mongo-user} and \texttt{--mongo-password} options + to \texttt{MongoTool}. These are the credentials for a minimally-privileged user + that will be created as part of the MongoDB schema creation. + + \item Configure the \hyperref[sec:mongo_config]{MongoDB connection URL} with the + user credentials for the minimally-privileged user created by \texttt{MongoTool}. +\end{enumerate} + + +\begin{table} + \centering + \begin{tabular}{|l|l|l|l|} + \toprule + \textbf{Option} & \textbf{Required} & \textbf{Default} & \textbf{Description}\\ + \midrule + \makecell[cl]{\texttt{-d}\\ + \texttt{---debug}} & No & False & \makecell[cl]{Turns on debug logging.}\\ + \midrule + \makecell[cl]{\texttt{-u}\\ + \texttt{---url}} & No & N/A & \makecell[cl]{Specify the URL to use for the MongoDB connection. This must\\ + be provided if the \texttt{-c} flag is not provided. This option and the \\ + \texttt{-c} option can not be used at the same time.}\\ + \midrule + \makecell[cl]{\texttt{-c}} & No & N/A & \makecell[cl]{Use the existing CxAnalytix configuration to connect to\\ + MongoDB if the \texttt{-u} or \texttt{---url} option is not provided. This option\\ + can not be used with the \texttt{-u} or \texttt{---url} options.}\\ + \midrule + \makecell[cl]{\texttt{-l}\\ + \texttt{---login-db}} & No & "admin" & \makecell[cl]{The MongoDB database used for login.}\\ + \midrule + \makecell[cl]{\texttt{---mongo-user}} & No & null & \makecell[cl]{If provided, the name of the MongoDB user to\\ + create and assign to the CxAnalytix database\\ + with minimal privileges.}\\ + \midrule + \makecell[cl]{\texttt{---mongo-password}} & No & null & \makecell[cl]{The password to set for the user name defined with\\ + the option \texttt{---mongo-user}.}\\ + \bottomrule + \end{tabular} + \caption{MongoTool Command Line Options} + \label{tab:mongo_tool_opts} +\end{table} -\section{Upgrading} +\section{Upgrading} + Configuration elements found in \texttt{cxanalytix.config} are generally backwards compatible for versions of CxAnalytix with the same major version\footnote{The major version is the first number in the version stamp. e.g. v2.0.0 has a major version of 2.}. New features may require additions @@ -193,5 +261,10 @@ \section{Upgrading} \noindent\\Note that if installing from the CxAnalytix zip binary that there is a default version of \texttt{cxanalytix.config} -and \texttt{cxanalytix.log4net} in the zip binary. These default configuration files can safely be removed upon -upgrading the CxAnalytix binaries. +and \texttt{cxanalytix.log4net} in the zip binary. These default configuration files are provided for first time installers to +use to get CxAnalytix running for the first time. These files should be removed upon upgrading the CxAnalytix binaries. If +CxAnalytix fails to start properly after upgrade, the example files may be loaded first according to the rules described in +Chapter \ref{chap:installation}: \textit{Configuration File Path Resolution}. + +\noindent\\If your output is MongoDB and the upgrade will change the collection schema, you should run \texttt{MongoTool} as +described in Section \ref{sec:mongotool} \ No newline at end of file diff --git a/manual/release_notes-content.tex b/manual/release_notes-content.tex index f47686f2..22abb5dc 100644 --- a/manual/release_notes-content.tex +++ b/manual/release_notes-content.tex @@ -1,3 +1,21 @@ +\section{2.1.1} + +\subsection*{FEATURES} + \begin{itemize} + \item Issue 54 - Explicit record suppression + \item Issue 55 - Externalize MongoDB schema creation + \end{itemize} + + \subsubsection*{Explicit record suppression} + All records can be suppressed by removing the associated record mapping attribute from the CxAnalytixService configuration element + or setting the name of the record to an empty string. + + \subsubsection*{Externalize MongoDB schema creation} + A new executable named MongoTool can be found in the source distribution. This tool will create the MongoDB database and collections + as well as assign the appropriate user permissions to the database. Default indexes (other than for the \_id field) are no longer created. You can drop + any previously created default indexes if desired. + + \section{2.1.0} \subsection*{UPDATES}