From d623beeeb0c81531bf87cf36eb44285792354c49 Mon Sep 17 00:00:00 2001 From: Nathan Leach Date: Tue, 28 Sep 2021 12:10:15 -0500 Subject: [PATCH] Feature/stability (#105) * Bump MongoDB.Bson from 2.13.0 to 2.13.1 Bumps MongoDB.Bson from 2.13.0 to 2.13.1. --- updated-dependencies: - dependency-name: MongoDB.Bson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Bump Microsoft.NET.Test.Sdk from 16.10.0 to 16.11.0 Bumps [Microsoft.NET.Test.Sdk](https://github.com/microsoft/vstest) from 16.10.0 to 16.11.0. - [Release notes](https://github.com/microsoft/vstest/releases) - [Commits](https://github.com/microsoft/vstest/compare/v16.10.0...v16.11.0) --- updated-dependencies: - dependency-name: Microsoft.NET.Test.Sdk dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * retry counts, fix the M&O abort delay * fix MNO error handling * Bump MongoDB.Driver from 2.13.0 to 2.13.1 (#96) * memory leaks no more * GC tuning * memory leak fixes * adding error verbosity * add OpTimer * use Trace method from extension * enable more trace options * more verbose error handling * forgot to actually log the message Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Configuration/CxConnection.cs | 7 + .../Configuration_Tests.csproj | 2 +- CxAnalytixCLI/Program.cs | 5 +- CxAnalytixCLI/runtimeconfig.template.json | 5 + CxAnalytixDaemon/Daemon.cs | 3 +- CxAnalytixDaemon/runtimeconfig.template.json | 5 + CxAnalytixService/ServiceLifecycleControl.cs | 3 +- CxAnalytixService/runtimeconfig.template.json | 5 + .../CxAuditTrailsCrawler_Tests.csproj | 2 +- CxRestClient/CxRestContext.cs | 25 +- CxRestClient/IO/CxClientFactory.cs | 7 +- CxRestClient/MNO/CxMnoPolicies.cs | 50 ++-- CxRestClient/OSA/CxOsaLibraries.cs | 17 +- CxRestClient/OSA/CxOsaLicenses.cs | 31 ++- CxRestClient/OSA/CxOsaScans.cs | 25 +- CxRestClient/OSA/CxOsaSummaryReport.cs | 9 +- CxRestClient/OSA/CxOsaVulnerabilities.cs | 13 +- CxRestClient/SAST/CxPresets.cs | 29 ++- CxRestClient/SAST/CxProjects.cs | 25 +- CxRestClient/SAST/CxSastScans.cs | 49 ++-- CxRestClient/SAST/CxTeams.cs | 25 +- CxRestClient/Utility/OpTimer.cs | 32 +++ CxRestClient/Utility/WebOperation.cs | 223 ++++++++++++------ CxRestClient_Tests/CxRestClient_Tests.csproj | 2 +- Extensions_Test/Extensions_Test.csproj | 2 +- LogCleaner_Tests/LogCleaner_Tests.csproj | 2 +- MongoDBOutput/MongoDBOutput.csproj | 6 +- .../ProjectFilter_Tests.csproj | 2 +- README.md | 8 + TransformLogic/Transformer.cs | 117 +++++---- .../TransformLogic_Tests.csproj | 2 +- Utilities_Tests/Utilities_Tests.csproj | 2 +- 32 files changed, 491 insertions(+), 249 deletions(-) create mode 100644 CxAnalytixCLI/runtimeconfig.template.json create mode 100644 CxAnalytixDaemon/runtimeconfig.template.json create mode 100644 CxAnalytixService/runtimeconfig.template.json create mode 100644 CxRestClient/Utility/OpTimer.cs diff --git a/Configuration/CxConnection.cs b/Configuration/CxConnection.cs index 965cf0b0..17b0a7a9 100644 --- a/Configuration/CxConnection.cs +++ b/Configuration/CxConnection.cs @@ -42,5 +42,12 @@ public bool ValidateCertificates set { this["ValidateCertificates"] = value; } } + [ConfigurationProperty("RetryLoop", IsRequired = false, DefaultValue = 0)] + public int RetryLoop + { + get => (int)this["RetryLoop"]; + set { this["RetryLoop"] = value; } + } + } } diff --git a/Configuration_Tests/Configuration_Tests.csproj b/Configuration_Tests/Configuration_Tests.csproj index dea55ea2..40c307c8 100644 --- a/Configuration_Tests/Configuration_Tests.csproj +++ b/Configuration_Tests/Configuration_Tests.csproj @@ -25,7 +25,7 @@ - + all diff --git a/CxAnalytixCLI/Program.cs b/CxAnalytixCLI/Program.cs index 8d9ca6f0..ce01b768 100644 --- a/CxAnalytixCLI/Program.cs +++ b/CxAnalytixCLI/Program.cs @@ -36,8 +36,9 @@ static void Main(string[] args) .WithOpTimeout(Config.Connection.TimeoutSeconds) .WithSSLValidate(Config.Connection.ValidateCertificates) .WithUsername(Config.Credentials.Username) - .WithPassword(Config.Credentials.Password). - WithMNOServiceURL (Config.Connection.MNOUrl); + .WithPassword(Config.Credentials.Password) + .WithMNOServiceURL (Config.Connection.MNOUrl) + .WithRetryLoop(Config.Connection.RetryLoop); using (CancellationTokenSource t = new CancellationTokenSource()) { diff --git a/CxAnalytixCLI/runtimeconfig.template.json b/CxAnalytixCLI/runtimeconfig.template.json new file mode 100644 index 00000000..12467a6d --- /dev/null +++ b/CxAnalytixCLI/runtimeconfig.template.json @@ -0,0 +1,5 @@ +{ + "configProperties": { + "System.GC.Server": true + } +} \ No newline at end of file diff --git a/CxAnalytixDaemon/Daemon.cs b/CxAnalytixDaemon/Daemon.cs index 3d846a7e..e454ba11 100644 --- a/CxAnalytixDaemon/Daemon.cs +++ b/CxAnalytixDaemon/Daemon.cs @@ -40,7 +40,8 @@ public Task StartAsync(CancellationToken cancellationToken) .WithOpTimeout(Config.Connection.TimeoutSeconds) .WithSSLValidate(Config.Connection.ValidateCertificates) .WithUsername(Config.Credentials.Username) - .WithPassword(Config.Credentials.Password); + .WithPassword(Config.Credentials.Password) + .WithRetryLoop(Config.Connection.RetryLoop); var restCtx = builder.Build(); diff --git a/CxAnalytixDaemon/runtimeconfig.template.json b/CxAnalytixDaemon/runtimeconfig.template.json new file mode 100644 index 00000000..12467a6d --- /dev/null +++ b/CxAnalytixDaemon/runtimeconfig.template.json @@ -0,0 +1,5 @@ +{ + "configProperties": { + "System.GC.Server": true + } +} \ No newline at end of file diff --git a/CxAnalytixService/ServiceLifecycleControl.cs b/CxAnalytixService/ServiceLifecycleControl.cs index d3583900..b793840d 100644 --- a/CxAnalytixService/ServiceLifecycleControl.cs +++ b/CxAnalytixService/ServiceLifecycleControl.cs @@ -86,7 +86,8 @@ protected override void OnStart(string[] args) .WithOpTimeout(Config.Connection.TimeoutSeconds) .WithSSLValidate(Config.Connection.ValidateCertificates) .WithUsername(Config.Credentials.Username) - .WithPassword(Config.Credentials.Password); + .WithPassword(Config.Credentials.Password) + .WithRetryLoop(Config.Connection.RetryLoop); var restCtx = builder.Build(); diff --git a/CxAnalytixService/runtimeconfig.template.json b/CxAnalytixService/runtimeconfig.template.json new file mode 100644 index 00000000..12467a6d --- /dev/null +++ b/CxAnalytixService/runtimeconfig.template.json @@ -0,0 +1,5 @@ +{ + "configProperties": { + "System.GC.Server": true + } +} \ No newline at end of file diff --git a/CxAuditTrailsCrawler_Tests/CxAuditTrailsCrawler_Tests.csproj b/CxAuditTrailsCrawler_Tests/CxAuditTrailsCrawler_Tests.csproj index ddeaf41b..bfe25554 100644 --- a/CxAuditTrailsCrawler_Tests/CxAuditTrailsCrawler_Tests.csproj +++ b/CxAuditTrailsCrawler_Tests/CxAuditTrailsCrawler_Tests.csproj @@ -9,7 +9,7 @@ - + all diff --git a/CxRestClient/CxRestContext.cs b/CxRestClient/CxRestContext.cs index 881b3c89..7d044685 100644 --- a/CxRestClient/CxRestContext.cs +++ b/CxRestClient/CxRestContext.cs @@ -34,6 +34,7 @@ internal CxRestContext() public TimeSpan Timeout { get; internal set; } public CxClientFactory Json { get; internal set; } public CxClientFactory Xml { get; internal set; } + public int RetryLoop { get; internal set; } private readonly Object _tokenLock = new object(); @@ -55,7 +56,7 @@ internal LoginToken SastToken } private LoginToken _mnoToken; - internal LoginToken MNOToken + internal LoginToken? MNOToken { get { @@ -65,7 +66,7 @@ internal LoginToken MNOToken set { - _mnoToken = value; + _mnoToken = value.GetValueOrDefault(); } } @@ -260,6 +261,13 @@ public CxRestContextBuilder WithPassword(String pass) return this; } + private int _retryLoop; + public CxRestContextBuilder WithRetryLoop(int loopCount) + { + _retryLoop = loopCount; + return this; + } + public CxRestContext Build() { @@ -272,6 +280,12 @@ public CxRestContext Build() if (_pass == null) throw new InvalidOperationException("Password was not specified."); + if (_retryLoop < 0) + throw new InvalidOperationException("Retry loop can't be < 0."); + + if (_timeout < 0) + throw new InvalidOperationException("Timeout can't be < 0."); + var timeoutSpan = new TimeSpan(0, 0, _timeout); HttpClientSingleton.Initialize(_validate, timeoutSpan); @@ -279,11 +293,12 @@ public CxRestContext Build() CxRestContext retVal = new CxRestContext() { SastToken = GetLoginToken(_url, _user, _pass, SAST_SCOPE), - MNOToken = GetLoginToken(_url, _user, _pass, $"{MNO_SCOPE} {SAST_SCOPE}"), + MNOToken = String.IsNullOrEmpty(_mnoUrl) ? new Nullable () : GetLoginToken(_url, _user, _pass, $"{MNO_SCOPE} {SAST_SCOPE}"), Url = _url, - MnoUrl = String.IsNullOrEmpty (_mnoUrl) ? _url : _mnoUrl, + MnoUrl = _mnoUrl, ValidateSSL = _validate, - Timeout = timeoutSpan + Timeout = timeoutSpan, + RetryLoop = _retryLoop }; retVal.Json = new CxClientFactory("application/json", retVal); diff --git a/CxRestClient/IO/CxClientFactory.cs b/CxRestClient/IO/CxClientFactory.cs index 19004e9e..e1736689 100644 --- a/CxRestClient/IO/CxClientFactory.cs +++ b/CxRestClient/IO/CxClientFactory.cs @@ -30,8 +30,11 @@ public CxRestClient CreateSastClient(String apiVersion) public CxRestClient CreateMnoClient(String apiVersion) { - return new CxRestClient(new System.Net.Http.Headers.AuthenticationHeaderValue(Context.MNOToken.TokenType, - Context.MNOToken.Token), MediaType, apiVersion); + if (!Context.MNOToken.HasValue) + return null; + + return new CxRestClient(new System.Net.Http.Headers.AuthenticationHeaderValue(Context.MNOToken.Value.TokenType, + Context.MNOToken.Value.Token), MediaType, apiVersion); } } diff --git a/CxRestClient/MNO/CxMnoPolicies.cs b/CxRestClient/MNO/CxMnoPolicies.cs index 334f25f1..cb866236 100644 --- a/CxRestClient/MNO/CxMnoPolicies.cs +++ b/CxRestClient/MNO/CxMnoPolicies.cs @@ -61,10 +61,13 @@ public static String GetProjectPoliciesSingleField(CxRestContext ctx, ctx.Json.CreateMnoClient , (response) => { - JToken jt = JToken.Load(new JsonTextReader(new StreamReader - (response.Content.ReadAsStreamAsync().Result))); + using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result)) + using (var jtr = new JsonTextReader(sr)) + { + JToken jt = JToken.Load(jtr); - return GetFlatPolicyNames(jt); + return GetFlatPolicyNames(jt); + } } , CxRestContext.MakeUrl(ctx.MnoUrl, String.Format(PROJECT_POLICY_URL_SUFFIX, projectId)) , ctx @@ -118,24 +121,18 @@ public static PolicyCollection GetAllPolicies(CxRestContext ctx, ctx.Json.CreateMnoClient , (response) => { - JToken jt = JToken.Load(new JsonTextReader(new StreamReader - (response.Content.ReadAsStreamAsync().Result))); + using (var sr = new StreamReader (response.Content.ReadAsStreamAsync().Result)) + using (var jtr = new JsonTextReader(sr)) + { + JToken jt = JToken.Load(jtr); - return ParsePolicies(ctx, token, jt); + return ParsePolicies(ctx, token, jt); + } } , CxRestContext.MakeUrl(ctx.MnoUrl, POLICY_LIST_URL_SUFFIX) , ctx , token - , exceptionErrorLogic: (ex) => - { - - if (ex is System.AggregateException) - foreach (var x in (ex as System.AggregateException).InnerExceptions) - if (x is System.Net.Http.HttpRequestException) - return false; - - return true; - }, apiVersion: null); + , apiVersion: null); } @@ -146,18 +143,21 @@ public static IEnumerable GetPolicyIdsForProject(CxRestContext ctx, ctx.Json.CreateMnoClient , (response) => { - JToken jt = JToken.Load(new JsonTextReader(new StreamReader - (response.Content.ReadAsStreamAsync().Result))); + using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result)) + using (var jtr = new JsonTextReader(sr)) + { + JToken jt = JToken.Load(jtr); - LinkedList policyIds = new LinkedList(); + LinkedList policyIds = new LinkedList(); - using (JTokenReader reader = new JTokenReader(jt)) - while (JsonUtils.MoveToNextProperty(reader, "id")) - { - policyIds.AddLast(Convert.ToInt32(((JProperty)reader.CurrentToken).Value)); - } + using (JTokenReader reader = new JTokenReader(jt)) + while (JsonUtils.MoveToNextProperty(reader, "id")) + { + policyIds.AddLast(Convert.ToInt32(((JProperty)reader.CurrentToken).Value)); + } - return policyIds; + return policyIds; + } } , CxRestContext.MakeUrl(ctx.MnoUrl, String.Format(PROJECT_POLICY_URL_SUFFIX, projectId)) diff --git a/CxRestClient/OSA/CxOsaLibraries.cs b/CxRestClient/OSA/CxOsaLibraries.cs index cfc61f89..713b84a6 100644 --- a/CxRestClient/OSA/CxOsaLibraries.cs +++ b/CxRestClient/OSA/CxOsaLibraries.cs @@ -64,7 +64,7 @@ public class Match } - private class LibrariesReader : IEnumerable, IEnumerator + private class LibrariesReader : IEnumerable, IEnumerator, IDisposable { private JToken _json; private JTokenReader _reader; @@ -83,7 +83,11 @@ internal LibrariesReader(JToken json) public void Dispose() { - _reader = null; + if (_reader != null) + { + _reader.Close(); + _reader = null; + } } public IEnumerator GetEnumerator() @@ -111,8 +115,8 @@ public bool MoveNext() if (!(_arrayPos < _libArray.Count)) return false; - _currentLibrary = (Library)new JsonSerializer(). - Deserialize(new JTokenReader(_libArray[_arrayPos]), typeof(Library)); + using (var jtr = new JTokenReader(_libArray[_arrayPos])) + _currentLibrary = (Library)new JsonSerializer().Deserialize(jtr, typeof(Library)); return true; } @@ -147,7 +151,7 @@ public static IEnumerable GetLibraries(CxRestContext ctx, CancellationT { var beforeCount = returnLibs.Count; - returnLibs.AddRange(WebOperation.ExecuteGet( + using (var libReader = WebOperation.ExecuteGet( ctx.Json.CreateSastClient , (response) => { @@ -160,7 +164,8 @@ public static IEnumerable GetLibraries(CxRestContext ctx, CancellationT } , url(curPage++) , ctx - , token)); + , token)) + returnLibs.AddRange(libReader); if (returnLibs.Count == beforeCount) break; diff --git a/CxRestClient/OSA/CxOsaLicenses.cs b/CxRestClient/OSA/CxOsaLicenses.cs index 398c045f..57e4c52c 100644 --- a/CxRestClient/OSA/CxOsaLicenses.cs +++ b/CxRestClient/OSA/CxOsaLicenses.cs @@ -54,7 +54,7 @@ public class License public String Url { get; internal set; } } - private class LicensesReader : IEnumerable, IEnumerator + private class LicensesReader : IEnumerable, IEnumerator, IDisposable { private JToken _json; private JTokenReader _reader; @@ -73,7 +73,11 @@ internal LicensesReader(JToken json) public void Dispose() { - _reader = null; + if (_reader != null) + { + _reader.Close(); + _reader = null; + } } public IEnumerator GetEnumerator() @@ -102,8 +106,8 @@ public bool MoveNext() if (!(_arrayPos < _licenseArray.Count)) return false; - _currentLicense = (License)new JsonSerializer(). - Deserialize(new JTokenReader(_licenseArray[_arrayPos]), typeof(License)); + using (var jtr = new JTokenReader(_licenseArray[_arrayPos])) + _currentLicense = (License)new JsonSerializer().Deserialize(jtr, typeof(License)); return true; } @@ -128,12 +132,13 @@ public static IEnumerable GetLicenses(CxRestContext ctx, CancellationTo {"scanId", Convert.ToString (scanId) } }); + List retVal = new List(); - return WebOperation.ExecuteGet>( - ctx.Json.CreateSastClient - , (response) => - { - using (var sr = new StreamReader (response.Content.ReadAsStreamAsync().Result)) + using (var licReader = WebOperation.ExecuteGet( + ctx.Json.CreateSastClient + , (response) => + { + using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result)) using (var jtr = new JsonTextReader(sr)) { JToken jt = JToken.Load(jtr); @@ -141,8 +146,12 @@ public static IEnumerable GetLicenses(CxRestContext ctx, CancellationTo } } , url - , ctx - , token); + , ctx + , token)) + { + retVal.AddRange(licReader); + return retVal; + } } } diff --git a/CxRestClient/OSA/CxOsaScans.cs b/CxRestClient/OSA/CxOsaScans.cs index 0b5b29cd..47258526 100644 --- a/CxRestClient/OSA/CxOsaScans.cs +++ b/CxRestClient/OSA/CxOsaScans.cs @@ -36,7 +36,7 @@ public override string ToString() } - private class ScansReader : IEnumerable, IEnumerator + private class ScansReader : IEnumerable, IEnumerator, IDisposable { private JToken _json; @@ -59,7 +59,11 @@ internal ScansReader() public void Dispose() { - _reader = null; + if (_reader != null) + { + _reader.Close(); + _reader = null; + } } public IEnumerator GetEnumerator() @@ -144,7 +148,7 @@ public static IEnumerable GetScans(CxRestContext ctx, CancellationToken to var beforeCount = osaScans.Count; - var scans = WebOperation.ExecuteGet>(ctx.Json.CreateSastClient + using (var scans = WebOperation.ExecuteGet(ctx.Json.CreateSastClient , (response) => { using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result)) @@ -167,15 +171,14 @@ public static IEnumerable GetScans(CxRestContext ctx, CancellationToken to } return true; - }); - - - if (scans != null) - osaScans.AddRange(scans); - + })) + { + if (scans != null) + osaScans.AddRange(scans); - if (osaScans.Count == beforeCount) - break; + if (osaScans.Count == beforeCount) + break; + } } diff --git a/CxRestClient/OSA/CxOsaSummaryReport.cs b/CxRestClient/OSA/CxOsaSummaryReport.cs index f8dfc56e..793003cb 100644 --- a/CxRestClient/OSA/CxOsaSummaryReport.cs +++ b/CxRestClient/OSA/CxOsaSummaryReport.cs @@ -50,10 +50,11 @@ public class ScanSummary private static ScanSummary ParseScanSummary(JToken jt) { - var reader = new JTokenReader(jt); - - JsonSerializer js = new JsonSerializer(); - return js.Deserialize(reader, typeof(ScanSummary)) as ScanSummary; + using (var reader = new JTokenReader(jt)) + { + JsonSerializer js = new JsonSerializer(); + return js.Deserialize(reader, typeof(ScanSummary)) as ScanSummary; + } } diff --git a/CxRestClient/OSA/CxOsaVulnerabilities.cs b/CxRestClient/OSA/CxOsaVulnerabilities.cs index 25674aef..fcc76f20 100644 --- a/CxRestClient/OSA/CxOsaVulnerabilities.cs +++ b/CxRestClient/OSA/CxOsaVulnerabilities.cs @@ -66,7 +66,7 @@ public class Vulnerability } - private class VulnerabilityReader : IEnumerable, IEnumerator + private class VulnerabilityReader : IEnumerable, IEnumerator, IDisposable { private JToken _json; private JTokenReader _reader; @@ -91,7 +91,11 @@ public IEnumerator GetEnumerator() void IDisposable.Dispose() { - _reader = null; + if (_reader != null) + { + _reader.Close(); + _reader = null; + } } IEnumerator IEnumerable.GetEnumerator() @@ -118,8 +122,9 @@ public bool MoveNext() if (!(_arrayPos < _vulnArray.Count)) return false; - _currentVuln = (Vulnerability)new JsonSerializer(). - Deserialize(new JTokenReader(_vulnArray[_arrayPos]), typeof(Vulnerability)); + using (var jtr = new JTokenReader(_vulnArray[_arrayPos])) + _currentVuln = (Vulnerability)new JsonSerializer(). + Deserialize(jtr, typeof(Vulnerability)); return true; } diff --git a/CxRestClient/SAST/CxPresets.cs b/CxRestClient/SAST/CxPresets.cs index 680e7b19..ff62b5ee 100644 --- a/CxRestClient/SAST/CxPresets.cs +++ b/CxRestClient/SAST/CxPresets.cs @@ -32,7 +32,7 @@ public override string ToString() } } - private class PresetReader : IEnumerable, IEnumerator + private class PresetReader : IEnumerable, IEnumerator, IDisposable { private JToken _json; @@ -54,7 +54,11 @@ internal PresetReader(JToken json) public void Dispose() { - _reader = null; + if (_reader != null) + { + _reader.Close(); + _reader = null; + } } public IEnumerator GetEnumerator() @@ -87,20 +91,21 @@ IEnumerator IEnumerable.GetEnumerator() public static IEnumerable GetPresets(CxRestContext ctx, CancellationToken token) { - return WebOperation.ExecuteGet( + using (var presetReader = WebOperation.ExecuteGet( ctx.Json.CreateSastClient , (response) => { - using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result)) - using (var jtr = new JsonTextReader(sr)) - { - JToken jt = JToken.Load(jtr); - return new PresetReader(jt); - } - } - , CxRestContext.MakeUrl(ctx.Url, URL_SUFFIX) + using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result)) + using (var jtr = new JsonTextReader(sr)) + { + JToken jt = JToken.Load(jtr); + return new PresetReader(jt); + } + } + , CxRestContext.MakeUrl(ctx.Url, URL_SUFFIX) , ctx - , token); + , token)) + return new List(presetReader); } } diff --git a/CxRestClient/SAST/CxProjects.cs b/CxRestClient/SAST/CxProjects.cs index 5503bec1..37dbbb4f 100644 --- a/CxRestClient/SAST/CxProjects.cs +++ b/CxRestClient/SAST/CxProjects.cs @@ -49,7 +49,7 @@ public override string ToString() => $"{ProjectId}:{ProjectName} [TeamId: {TeamId} Public: {IsPublic} CustomFields: {CustomFields.Count}]"; } - private class ProjectReader : IEnumerable, IEnumerator + private class ProjectReader : IEnumerable, IEnumerator, IDisposable { private JToken _json; @@ -70,7 +70,11 @@ internal ProjectReader(JToken json, CxRestContext ctx, CancellationToken token) public void Dispose() { - _reader = null; + if (_reader != null) + { + _reader.Close(); + _reader = null; + } _ctx = null; } @@ -104,10 +108,10 @@ public bool MoveNext() if (!(_arrayPos < _projectArray.Count)) return false; - _curProject = (Project)new JsonSerializer(). - Deserialize(new JTokenReader(_projectArray[_arrayPos]), typeof(Project)); + using (var jtr = new JTokenReader(_projectArray[_arrayPos])) + _curProject = (Project)new JsonSerializer().Deserialize(jtr, typeof(Project)); - if (!_curProject.IsPublic) + if (!_curProject.IsPublic) { _curProject = null; continue; @@ -139,10 +143,10 @@ IEnumerator IEnumerable.GetEnumerator() public static IEnumerable GetProjects(CxRestContext ctx, CancellationToken token) { - return WebOperation.ExecuteGet( - ctx.Json.CreateSastClient - , (response) => - { + using (var projectReader = WebOperation.ExecuteGet( + ctx.Json.CreateSastClient + , (response) => + { using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result)) using (var jtr = new JsonTextReader(sr)) { @@ -153,7 +157,8 @@ public static IEnumerable GetProjects(CxRestContext ctx, CancellationTo } , CxRestContext.MakeUrl(ctx.Url, URL_SUFFIX) , ctx - , token, apiVersion: "2.0"); + , token, apiVersion: "2.0")) + return new List(projectReader); } } } diff --git a/CxRestClient/SAST/CxSastScans.cs b/CxRestClient/SAST/CxSastScans.cs index 23002dfb..c4090aa5 100644 --- a/CxRestClient/SAST/CxSastScans.cs +++ b/CxRestClient/SAST/CxSastScans.cs @@ -64,13 +64,15 @@ public String Languages { LinkedList langs = new LinkedList(); - var langDicts = (List>)new JsonSerializer(). - Deserialize(new JTokenReader(scan_state["languageStateCollection"] as JArray), typeof(List>)); + using (var jtr = new JTokenReader(scan_state["languageStateCollection"] as JArray)) + { + var langDicts = (List>)new JsonSerializer().Deserialize(jtr, typeof(List>)); - foreach (var langDict in langDicts) - langs.AddLast(langDict["languageName"]); + foreach (var langDict in langDicts) + langs.AddLast(langDict["languageName"]); - return String.Join(';', langs); + return String.Join(';', langs); + } } } @@ -99,7 +101,7 @@ public override string ToString() } - private class ScansReader : IEnumerable, IEnumerator + private class ScansReader : IEnumerable, IEnumerator, IDisposable { private JToken _json; @@ -116,7 +118,11 @@ internal ScansReader(JToken json) public void Dispose() { - _reader = null; + if (_reader != null) + { + _reader.Close(); + _reader = null; + } } public IEnumerator GetEnumerator() @@ -177,26 +183,26 @@ public static IEnumerable GetScans(CxRestContext ctx, CancellationToken to return GetScans(ctx, token, ScanStatus.All); } - public static IEnumerable GetScans(CxRestContext ctx, CancellationToken token, - ScanStatus specificStatus, int? projectId = null) - { - String url = null; + public static IEnumerable GetScans(CxRestContext ctx, CancellationToken token, + ScanStatus specificStatus, int? projectId = null) + { + String url = null; - var args = new Dictionary(); + var args = new Dictionary(); - if (specificStatus != ScanStatus.All) - args.Add("scanStatus", specificStatus.ToString()); + if (specificStatus != ScanStatus.All) + args.Add("scanStatus", specificStatus.ToString()); - if (projectId != null && projectId.HasValue) - args.Add("projectId", Convert.ToString (projectId.Value)); + if (projectId != null && projectId.HasValue) + args.Add("projectId", Convert.ToString(projectId.Value)); url = CxRestContext.MakeUrl(ctx.Url, URL_SUFFIX, args); - return WebOperation.ExecuteGet( + using (var scansReader = WebOperation.ExecuteGet( ctx.Json.CreateSastClient , (response) => { - using (var sr = new StreamReader (response.Content.ReadAsStreamAsync().Result)) + using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result)) using (var jtr = new JsonTextReader(sr)) { jtr.DateParseHandling = DateParseHandling.None; @@ -206,7 +212,8 @@ public static IEnumerable GetScans(CxRestContext ctx, CancellationToken to } , url , ctx - , token); - } - } + , token)) + return new List(scansReader); + } + } } diff --git a/CxRestClient/SAST/CxTeams.cs b/CxRestClient/SAST/CxTeams.cs index 163b92a6..2fcecadd 100644 --- a/CxRestClient/SAST/CxTeams.cs +++ b/CxRestClient/SAST/CxTeams.cs @@ -21,7 +21,7 @@ public class CxTeams private CxTeams() { } - private class TeamReader : IEnumerable, IEnumerator + private class TeamReader : IEnumerable, IEnumerator, IDisposable { private JToken _json; @@ -40,7 +40,11 @@ internal TeamReader(JToken json) public void Dispose() { - _reader = null; + if (_reader != null) + { + _reader.Close(); + _reader = null; + } } public IEnumerator GetEnumerator() @@ -68,9 +72,8 @@ public bool MoveNext() if (!(_arrayPos < _teamArray.Count)) return false; - - _curTeam = (Team)new JsonSerializer(). - Deserialize(new JTokenReader(_teamArray[_arrayPos]), typeof(Team)); + using (var jtr = new JTokenReader(_teamArray[_arrayPos])) + _curTeam = (Team)new JsonSerializer().Deserialize(jtr, typeof(Team)); return true; } @@ -97,7 +100,10 @@ public class Team public static IEnumerable GetTeams(CxRestContext ctx, CancellationToken token) { - return WebOperation.ExecuteGet( + + List retVal = new List(); + + using (var teamReader = WebOperation.ExecuteGet( ctx.Json.CreateSastClient , (response) => { @@ -110,7 +116,12 @@ public static IEnumerable GetTeams(CxRestContext ctx, CancellationToken to } , CxRestContext.MakeUrl(ctx.Url, URL_SUFFIX) , ctx - , token); + , token)) + { + retVal.AddRange(teamReader); + + return retVal; + } } } diff --git a/CxRestClient/Utility/OpTimer.cs b/CxRestClient/Utility/OpTimer.cs new file mode 100644 index 00000000..694ddf4f --- /dev/null +++ b/CxRestClient/Utility/OpTimer.cs @@ -0,0 +1,32 @@ +using log4net; +using System; +using CxAnalytix.Extensions; +using System.Collections.Generic; +using System.Text; + +namespace CxRestClient.Utility +{ + internal class OpTimer : IDisposable + { + + private static ILog _log = LogManager.GetLogger(typeof(OpTimer)); + + private DateTime _start; + private String _op; + + internal OpTimer(String operationName) + { + _start = DateTime.Now; + _op = operationName; + + } + + public void Dispose() + { + if (_log.Logger.IsEnabledFor(log4net.Core.Level.Trace)) + { + _log.Trace($"Operation [{_op}] completed in [{DateTime.Now.Subtract(_start).TotalMilliseconds:0.##}ms]"); + } + } + } +} diff --git a/CxRestClient/Utility/WebOperation.cs b/CxRestClient/Utility/WebOperation.cs index 87369e71..cb769a06 100644 --- a/CxRestClient/Utility/WebOperation.cs +++ b/CxRestClient/Utility/WebOperation.cs @@ -8,6 +8,7 @@ using System.IO; using System.Net; using System.Net.Http; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -25,104 +26,138 @@ private static T ExecuteOperation(Func Func opExecutor, CxRestContext ctx, CancellationToken token, Func responseErrorLogic, Func exceptionErrorLogic, String apiVersion) { - var endRetryAt = DateTime.Now.Add(ctx.Timeout); + int loopsLeft = ctx.RetryLoop; - int delay = RETRY_DELAY_MS; + UnrecoverableOperationException nonRecoveryException = new UnrecoverableOperationException(); - DateTime? recoveryStartedAt = null; - bool inRecovery = false; - HttpStatusCode lastFailCode = HttpStatusCode.OK; + while (loopsLeft >= 0) + { + if (loopsLeft != ctx.RetryLoop) + _log.Warn($"Retry loop {ctx.RetryLoop - loopsLeft} of {ctx.RetryLoop}"); - UnrecoverableOperationException nonRecoveryException = new UnrecoverableOperationException(); + loopsLeft--; + nonRecoveryException = new UnrecoverableOperationException("Loop retries exhausted"); - while (DateTime.Now.CompareTo(endRetryAt) <= 0) - { - try - { - using (var client = clientFactory(apiVersion)) - using (var response = opExecutor(client)) - { - if (token.IsCancellationRequested) - { - _log.Warn($"Execution of operation has been cancelled."); - return default(T); - } + var endRetryAt = DateTime.Now.Add(ctx.Timeout); + int delay = RETRY_DELAY_MS; - if (response.IsSuccessStatusCode) - { - if (inRecovery) - _log.Info($"Operation successful after last error - " - + $"recovered after {DateTime.Now.Subtract(recoveryStartedAt.Value).TotalMilliseconds}ms "); + DateTime? recoveryStartedAt = null; + bool inRecovery = false; + HttpStatusCode lastFailCode = HttpStatusCode.OK; - return onSuccess(response); - } - else if (response.StatusCode == HttpStatusCode.NotFound) - { - _log.Warn($"{response.StatusCode} error with URI {response.RequestMessage.RequestUri}, not attempting retry."); - return default(T); - } - else - { - if (!recoveryStartedAt.HasValue) - recoveryStartedAt = DateTime.Now; - if (responseErrorLogic != null && !responseErrorLogic(response)) + while (DateTime.Now.CompareTo(endRetryAt) <= 0) + { + try + { + using (var client = clientFactory(apiVersion)) + using (var response = opExecutor(client)) + { + if (token.IsCancellationRequested) + { + _log.Warn($"Execution of operation has been cancelled."); return default(T); + } - if (!inRecovery) - _log.Warn($"Request failed with response {Convert.ToInt32(response.StatusCode)}({response.StatusCode})" + - $" - Retrying until {endRetryAt}"); - else if (lastFailCode != response.StatusCode) - _log.Warn($"Still in recovery, new failure status code: " - + $"{Convert.ToInt32(response.StatusCode)}({response.StatusCode})" + - $" - Retrying until {endRetryAt}"); - inRecovery = true; - lastFailCode = response.StatusCode; + if (response.IsSuccessStatusCode) + { + if (inRecovery) + _log.Info($"Operation successful after last error - " + + $"recovered after {DateTime.Now.Subtract(recoveryStartedAt.Value).TotalMilliseconds}ms "); - nonRecoveryException = new UnrecoverableOperationException(response.StatusCode, - response.RequestMessage.RequestUri); + return onSuccess(response); + } + else if (response.StatusCode == HttpStatusCode.NotFound) + { + _log.Warn($"{response.StatusCode} error with URI {response.RequestMessage.RequestUri}, not attempting retry."); - switch (response.StatusCode) + return default(T); + } + else { - case HttpStatusCode.Unauthorized: - case HttpStatusCode.Forbidden: - ctx.Reauthenticate(); - break; - default: - break; + if (!recoveryStartedAt.HasValue) + recoveryStartedAt = DateTime.Now; + + if (responseErrorLogic != null && !responseErrorLogic(response)) + return default(T); + + if (!inRecovery) + _log.Warn($"Request failed with response {Convert.ToInt32(response.StatusCode)}({response.StatusCode})" + + $" - Retrying until {endRetryAt}"); + else if (lastFailCode != response.StatusCode) + _log.Warn($"Still in recovery, new failure status code: " + + $"{Convert.ToInt32(response.StatusCode)}({response.StatusCode})" + + $" - Retrying until {endRetryAt}"); + + inRecovery = true; + lastFailCode = response.StatusCode; + + nonRecoveryException = new UnrecoverableOperationException(response.StatusCode, + response.RequestMessage.RequestUri); + + switch (response.StatusCode) + { + case HttpStatusCode.Unauthorized: + case HttpStatusCode.Forbidden: + ctx.Reauthenticate(); + break; + default: + break; + } } } } - } - catch (Exception ex) - { - if (!recoveryStartedAt.HasValue) - recoveryStartedAt = DateTime.Now; + catch (Exception ex) + { + if (!recoveryStartedAt.HasValue) + recoveryStartedAt = DateTime.Now; - if (!inRecovery) - _log.Error($"Exception: {ex.GetType()} Source: {ex.Source} Message: {ex.Message}" + - $" - Retrying until {endRetryAt}"); + if (!inRecovery) + _log.Error($"Exception: {ex.GetType()} Source: {ex.Source} Message: {ex.Message}" + + $" - Retrying until {endRetryAt}"); - inRecovery = true; + inRecovery = true; - nonRecoveryException = new UnrecoverableOperationException($"Last exception caught", ex); + nonRecoveryException = new UnrecoverableOperationException($"Last exception caught: {ex.GetType().Name}: {ex.Message}"); - if (exceptionErrorLogic != null && !exceptionErrorLogic(ex)) - throw ex; + if (exceptionErrorLogic != null && !exceptionErrorLogic(ex)) + throw ex; + } + + _log.Debug($"Waiting {delay}ms before retry."); + Task.Delay(delay, token).Wait(); + delay *= RETRY_DELAY_INCREASE_FACTOR; } - _log.Debug($"Waiting {delay}ms before retry."); - Task.Delay(delay, token).Wait(); - delay *= RETRY_DELAY_INCREASE_FACTOR; + if (inRecovery) + _log.Debug("Retry time exceeded, while loop exited during recovery."); } throw nonRecoveryException; } + private static void LogAggregateException(AggregateException aex) + { + StringBuilder sb = new StringBuilder(); + + aex.Handle((x) => { + + sb.AppendLine("----- EXCEPTION -----"); + if (x is UnrecoverableOperationException) + sb.AppendLine(x.Message); + else + sb.AppendLine($"Type: {x.GetType().FullName} Message: {x.Message}"); + + return true; + }); + + _log.Error($"Aggregate exception: {sb.ToString()}"); + } + public static T ExecuteGet(Func clientFactory, Func onSuccess, String url, CxRestContext ctx, CancellationToken token, Func responseErrorLogic = null, Func exceptionErrorLogic = null, String apiVersion = "1.0") @@ -133,7 +168,28 @@ public static T ExecuteGet(Func clientF , (client) => { _log.Trace($"Executing GET operation at {url}"); - return client.GetAsync(url, token).Result; + + try + { + using (new OpTimer($"GET {url}")) + { + var result = client.GetAsync(url, token).Result; + + _log.Trace($"GET operation at {url} status: {(int)result.StatusCode}:{result.ReasonPhrase}"); + + return result; + } + } + catch(AggregateException aex) + { + LogAggregateException(aex); + throw aex; + } + catch (Exception ex) + { + _log.Error($"GET operation failed for [{url}]", ex); + throw ex; + } } , ctx , token @@ -155,10 +211,31 @@ public static T ExecutePost(Func client { _log.Trace($"Executing POST operation at {url}"); - // HttpClient.SendAsync disposes of the payload on send - // this means if there is an error, a new instance is needed - // on retry. - return client.PostAsync(url, (contentFactory != null) ? contentFactory() : null, token).Result; + try + { + // HttpClient.SendAsync disposes of the payload on send + // this means if there is an error, a new instance is needed + // on retry. + using (new OpTimer($"POST {url}")) + { + var result = client.PostAsync(url, (contentFactory != null) ? contentFactory() : null, token).Result; + + _log.Trace($"POST operation at {url} status: {(int)result.StatusCode}:{result.ReasonPhrase}"); + + return result; + } + } + catch (AggregateException aex) + { + LogAggregateException(aex); + throw aex; + } + catch (Exception ex) + { + _log.Error($"POST operation failed for [{url}]", ex); + throw ex; + + } } , ctx , token diff --git a/CxRestClient_Tests/CxRestClient_Tests.csproj b/CxRestClient_Tests/CxRestClient_Tests.csproj index 0997a196..e84840d2 100644 --- a/CxRestClient_Tests/CxRestClient_Tests.csproj +++ b/CxRestClient_Tests/CxRestClient_Tests.csproj @@ -29,7 +29,7 @@ - + diff --git a/Extensions_Test/Extensions_Test.csproj b/Extensions_Test/Extensions_Test.csproj index b4cbaa55..d1dbbb01 100644 --- a/Extensions_Test/Extensions_Test.csproj +++ b/Extensions_Test/Extensions_Test.csproj @@ -28,7 +28,7 @@ - + diff --git a/LogCleaner_Tests/LogCleaner_Tests.csproj b/LogCleaner_Tests/LogCleaner_Tests.csproj index a7d725e3..e067ca25 100644 --- a/LogCleaner_Tests/LogCleaner_Tests.csproj +++ b/LogCleaner_Tests/LogCleaner_Tests.csproj @@ -28,7 +28,7 @@ - + diff --git a/MongoDBOutput/MongoDBOutput.csproj b/MongoDBOutput/MongoDBOutput.csproj index c828114c..757849f9 100644 --- a/MongoDBOutput/MongoDBOutput.csproj +++ b/MongoDBOutput/MongoDBOutput.csproj @@ -25,9 +25,9 @@ - - - + + + diff --git a/ProjectFilter_Tests/ProjectFilter_Tests.csproj b/ProjectFilter_Tests/ProjectFilter_Tests.csproj index c3692397..40f23e44 100644 --- a/ProjectFilter_Tests/ProjectFilter_Tests.csproj +++ b/ProjectFilter_Tests/ProjectFilter_Tests.csproj @@ -9,7 +9,7 @@ - + diff --git a/README.md b/README.md index b9340ab2..613910b0 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,14 @@ Please see the [CxAnalytix Wiki](https://github.com/checkmarx-ts/CxAnalytix/wiki ## Version History +* 1.3.2 + * BUG FIXES + * Memory leak in M&O client REST API code fixed. + * Added the `RetryLoop` configuration to allow retries after timeout. + * Stopped the attempt to load policies at startup if the M&O URL is not provided. + * Stability fixes for AMQP outputs. + * Dependency upgrades. + * Garbage collection tuning. * 1.3.1 * FEATURES * Platform-specific tarballs are now created. This is to address the dynamic loading of DPAPI that .Net Core apparently doesn't handle well in cross-platform builds. diff --git a/TransformLogic/Transformer.cs b/TransformLogic/Transformer.cs index d451726a..75968a3c 100644 --- a/TransformLogic/Transformer.cs +++ b/TransformLogic/Transformer.cs @@ -19,6 +19,7 @@ using CxAnalytix.Interfaces.Transform; using OutputBootstrapper; using CxAnalytix.Extensions; +using CxAnalytix.Exceptions; namespace CxAnalytix.TransformLogic { @@ -99,11 +100,14 @@ private void SastReportOutput(IOutputTransaction trx, ScanDescriptor scan) }); _log.Warn("END exception report"); + + throw aex; } catch (Exception ex) { _log.Warn($"Error attempting to retrieve the SAST XML report for {scan.ScanId}" + $" in project {scan.Project.ProjectId}: {scan.Project.ProjectName}. ", ex); + throw ex; } } @@ -129,6 +133,10 @@ private void ScaReportOutput(IOutputTransaction trx, ScanDescriptor sd) licenseCount.Add(l.RiskLevel, 1); } } + catch (UnrecoverableOperationException uopex) + { + throw uopex; + } catch (Exception ex) { _log.Warn($"Could not obtain license data for scan {sd.ScanId} in project " + @@ -147,6 +155,10 @@ private void ScaReportOutput(IOutputTransaction trx, ScanDescriptor sd) foreach (var lib in libraries) libraryIndex.Add(lib.LibraryId, lib); } + catch (UnrecoverableOperationException uopex) + { + throw uopex; + } catch (Exception ex) { _log.Warn($"Could not obtain library data for scan {sd.ScanId} in project " + @@ -219,6 +231,10 @@ private void OutputScaScanDetails(IOutputTransaction trx, ScanDescriptor sd, Dic trx.write(ScaScanDetailOut, flat); } } + catch (UnrecoverableOperationException uopex) + { + throw uopex; + } catch (Exception ex) { _log.Warn($"Could not obtain vulnerability data for scan {sd.ScanId} in project " + @@ -256,6 +272,10 @@ private void OutputScaScanSummary(IOutputTransaction trx, ScanDescriptor sd, Dic flat.Add("VulnerableAndOutdated", summary.VulnerableAndOutdated); flat.Add("VulnerableAndUpdated", summary.VulnerableAndUpdated); } + catch (UnrecoverableOperationException uopex) + { + throw uopex; + } catch (Exception ex) { _log.Warn($"Error obtaining summary report for SCA scan {sd.ScanId} " + @@ -286,16 +306,15 @@ private async Task ResolveScans() var teamsTask = PopulateTeams(); - _log.Debug("Resolving projects."); - - var projects = await Task.Run(() => CxProjects.GetProjects(RestContext, CancelToken), CancelToken); + var projectsTask = Task.Run(() => CxProjects.GetProjects(RestContext, CancelToken), CancelToken); Policies = await policyTask; Teams = await teamsTask; Presets = await presetsTask; - Parallel.ForEach(projects, new ParallelOptions { CancellationToken = CancelToken }, (p) => + _log.Debug("Resolving projects."); + Parallel.ForEach(await projectsTask, new ParallelOptions { CancellationToken = CancelToken }, (p) => { if (p.ProjectName == null) @@ -471,27 +490,31 @@ private async Task> PopulatePresets() private async Task PopulatePolicies() { - // Policies may not have data if M&O is not installed. - try - { - return await Task.Run( - () => - { - _log.Debug("Retrieving policies, if available."); - return new ProjectPolicyIndex(CxMnoPolicies.GetAllPolicies(RestContext, CancelToken)); + if (!String.IsNullOrEmpty(Configuration.Config.Connection.MNOUrl)) + // Policies will not have data if M&O is not installed. + try + { + return await Task.Run( + () => + { + _log.Debug("Retrieving policies, if available."); - }, CancelToken); - } - catch (Exception ex) - { - String msg = "Policy data is not available. M&O was unreachable. You can omit the M&O URL in the configuration if M&O is not installed."; + return new ProjectPolicyIndex(CxMnoPolicies.GetAllPolicies(RestContext, CancelToken)); - if (_log.IsDebugEnabled) - _log.Debug(msg, ex); - else - _log.Warn(msg); - } + }, CancelToken); + } + catch (Exception ex) + { + String msg = "Policy data is not available. M&O was unreachable. You can omit the M&O URL in the configuration if M&O is not installed."; + + if (_log.IsDebugEnabled) + _log.Debug(msg, ex); + else + _log.Warn(msg); + } + else + _log.Info("The M&O URL was not provided, policy data will not be available."); return null; @@ -558,35 +581,43 @@ private void ExecuteSweep() using (var scanTrx = Output.StartTransaction()) { - // Increment the policy violation stats for each scan. - scan.IncrementPolicyViolations(PolicyViolations[scan.Project.ProjectId].GetViolatedRulesByScanId(scan.ScanId)); + try + { + // Increment the policy violation stats for each scan. + scan.IncrementPolicyViolations(PolicyViolations[scan.Project.ProjectId].GetViolatedRulesByScanId(scan.ScanId)); - _log.Info($"Processing {scan.ScanProduct} scan {scan.ScanId}:{scan.Project.ProjectId}:{scan.Project.TeamName}:{scan.Project.ProjectName}[{scan.FinishedStamp}]"); + _log.Info($"Processing {scan.ScanProduct} scan {scan.ScanId}:{scan.Project.ProjectId}:{scan.Project.TeamName}:{scan.Project.ProjectName}[{scan.FinishedStamp}]"); - switch (scan.ScanProduct) - { - case ScanProductType.SAST: - SastReportOutput(scanTrx, scan); - break; + switch (scan.ScanProduct) + { + case ScanProductType.SAST: + SastReportOutput(scanTrx, scan); + break; - case ScanProductType.SCA: - ScaReportOutput(scanTrx, scan); - break; + case ScanProductType.SCA: + ScaReportOutput(scanTrx, scan); + break; - } + } - OutputPolicyViolationDetails(scanTrx, scan); + OutputPolicyViolationDetails(scanTrx, scan); - // Persist the date of this scan since it has been output. - if (!CancelToken.IsCancellationRequested && scanTrx.Commit()) - _state.ScanCompleted(scan); - else + // Persist the date of this scan since it has been output. + if (!CancelToken.IsCancellationRequested && scanTrx.Commit()) + { + _state.ScanCompleted(scan); + continue; + } + } + catch (Exception) { - // Stop processing further scans in this project if the commit - // for the scan information fails. - _log.Warn($"Stopped processing scans for project {project.ProjectId}:{project.TeamName}:{project.ProjectName} at {scan.ScanId}, will resume here next crawl."); - return; + _log.Debug("Exception caught during scan output. Exceptions should have already been logged."); } + + // Stop processing further scans in this project if the commit + // for the scan information fails. + _log.Warn($"Stopped processing scans for project {project.ProjectId}:{project.TeamName}:{project.ProjectName} at {scan.ScanId}, will resume here next crawl."); + return; } } diff --git a/TransformLogic_Tests/TransformLogic_Tests.csproj b/TransformLogic_Tests/TransformLogic_Tests.csproj index 1216a86c..d30f22ec 100644 --- a/TransformLogic_Tests/TransformLogic_Tests.csproj +++ b/TransformLogic_Tests/TransformLogic_Tests.csproj @@ -31,7 +31,7 @@ all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/Utilities_Tests/Utilities_Tests.csproj b/Utilities_Tests/Utilities_Tests.csproj index 8320ba4b..201f2979 100644 --- a/Utilities_Tests/Utilities_Tests.csproj +++ b/Utilities_Tests/Utilities_Tests.csproj @@ -9,7 +9,7 @@ - +