Feature/stability (#105)

* Bump MongoDB.Bson from 2.13.0 to 2.13.1 Bumps MongoDB.Bson from 2.13.0 to 2.13.1. --- updated-dependencies: - dependency-name: MongoDB.Bson dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <[email protected]> * Bump Microsoft.NET.Test.Sdk from 16.10.0 to 16.11.0 Bumps [Microsoft.NET.Test.Sdk](https://github.com/microsoft/vstest) from 16.10.0 to 16.11.0. - [Release notes](https://github.com/microsoft/vstest/releases) - [Commits](microsoft/vstest@v16.10.0...v16.11.0) --- updated-dependencies: - dependency-name: Microsoft.NET.Test.Sdk dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <[email protected]> * retry counts, fix the M&O abort delay * fix MNO error handling * Bump MongoDB.Driver from 2.13.0 to 2.13.1 (#96) * memory leaks no more * GC tuning * memory leak fixes * adding error verbosity * add OpTimer * use Trace method from extension * enable more trace options * more verbose error handling * forgot to actually log the message Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
checkmarx-ts · Sep 28, 2021 · d623bee · d623bee
1 parent caf1b24
commit d623bee
Show file tree

Hide file tree

Showing 32 changed files with 491 additions and 249 deletions.
diff --git a/Configuration/CxConnection.cs b/Configuration/CxConnection.cs
@@ -42,5 +42,12 @@ public bool ValidateCertificates
             set { this["ValidateCertificates"] = value; }
         }
 
+        [ConfigurationProperty("RetryLoop", IsRequired = false, DefaultValue = 0)]
+        public int RetryLoop
+        {
+            get => (int)this["RetryLoop"];
+            set { this["RetryLoop"] = value; }
+        }
+
     }
 }
diff --git a/Configuration_Tests/Configuration_Tests.csproj b/Configuration_Tests/Configuration_Tests.csproj
@@ -25,7 +25,7 @@
 	</PropertyGroup>
 
 	<ItemGroup>
-		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" />
+		<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
 		<PackageReference Include="xunit" Version="2.4.1" />
 		<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
 			<PrivateAssets>all</PrivateAssets>

diff --git a/CxAnalytixCLI/Program.cs b/CxAnalytixCLI/Program.cs
@@ -36,8 +36,9 @@ static void Main(string[] args)
             .WithOpTimeout(Config.Connection.TimeoutSeconds)
             .WithSSLValidate(Config.Connection.ValidateCertificates)
             .WithUsername(Config.Credentials.Username)
-            .WithPassword(Config.Credentials.Password).
-            WithMNOServiceURL (Config.Connection.MNOUrl);
+            .WithPassword(Config.Credentials.Password)
+            .WithMNOServiceURL (Config.Connection.MNOUrl)
+            .WithRetryLoop(Config.Connection.RetryLoop);
 
             using (CancellationTokenSource t = new CancellationTokenSource())
             {

diff --git a/CxAnalytixCLI/runtimeconfig.template.json b/CxAnalytixCLI/runtimeconfig.template.json
@@ -0,0 +1,5 @@
+{
+  "configProperties": {
+    "System.GC.Server": true
+  }
+}
diff --git a/CxAnalytixDaemon/Daemon.cs b/CxAnalytixDaemon/Daemon.cs
@@ -40,7 +40,8 @@ public Task StartAsync(CancellationToken cancellationToken)
 			.WithOpTimeout(Config.Connection.TimeoutSeconds)
 			.WithSSLValidate(Config.Connection.ValidateCertificates)
 			.WithUsername(Config.Credentials.Username)
-			.WithPassword(Config.Credentials.Password);
+			.WithPassword(Config.Credentials.Password)
+			.WithRetryLoop(Config.Connection.RetryLoop);
 
 			var restCtx = builder.Build();
 

diff --git a/CxAnalytixDaemon/runtimeconfig.template.json b/CxAnalytixDaemon/runtimeconfig.template.json
@@ -0,0 +1,5 @@
+{
+  "configProperties": {
+    "System.GC.Server": true
+  }
+}
diff --git a/CxAnalytixService/ServiceLifecycleControl.cs b/CxAnalytixService/ServiceLifecycleControl.cs
@@ -86,7 +86,8 @@ protected override void OnStart(string[] args)
             .WithOpTimeout(Config.Connection.TimeoutSeconds)
             .WithSSLValidate(Config.Connection.ValidateCertificates)
             .WithUsername(Config.Credentials.Username)
-            .WithPassword(Config.Credentials.Password);
+            .WithPassword(Config.Credentials.Password)
+            .WithRetryLoop(Config.Connection.RetryLoop);
 
             var restCtx = builder.Build();
 

diff --git a/CxAnalytixService/runtimeconfig.template.json b/CxAnalytixService/runtimeconfig.template.json
@@ -0,0 +1,5 @@
+{
+  "configProperties": {
+    "System.GC.Server": true
+  }
+}
diff --git a/CxAuditTrailsCrawler_Tests/CxAuditTrailsCrawler_Tests.csproj b/CxAuditTrailsCrawler_Tests/CxAuditTrailsCrawler_Tests.csproj
@@ -9,7 +9,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.10.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.11.0" />
     <PackageReference Include="xunit" Version="2.4.1" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
       <PrivateAssets>all</PrivateAssets>

diff --git a/CxRestClient/CxRestContext.cs b/CxRestClient/CxRestContext.cs
@@ -34,6 +34,7 @@ internal CxRestContext()
         public TimeSpan Timeout { get; internal set; }
         public CxClientFactory Json { get; internal set; }
         public CxClientFactory Xml { get; internal set; }
+        public int RetryLoop { get; internal set; }
 
 
         private readonly Object _tokenLock = new object();
@@ -55,7 +56,7 @@ internal LoginToken SastToken
         }
 
         private LoginToken _mnoToken;
-        internal LoginToken MNOToken
+        internal LoginToken? MNOToken
         {
             get
             {
@@ -65,7 +66,7 @@ internal LoginToken MNOToken
 
             set
             {
-                _mnoToken = value;
+                _mnoToken = value.GetValueOrDefault();
             }
         }
 
@@ -260,6 +261,13 @@ public CxRestContextBuilder WithPassword(String pass)
                 return this;
             }
 
+            private int _retryLoop;
+            public CxRestContextBuilder WithRetryLoop(int loopCount)
+			{
+                _retryLoop = loopCount;
+                return this;
+			}
+
 
             public CxRestContext Build()
             {
@@ -272,18 +280,25 @@ public CxRestContext Build()
                 if (_pass == null)
                     throw new InvalidOperationException("Password was not specified.");
 
+                if (_retryLoop < 0)
+                    throw new InvalidOperationException("Retry loop can't be < 0.");
+
+                if (_timeout < 0)
+                    throw new InvalidOperationException("Timeout can't be < 0.");
+
                 var timeoutSpan = new TimeSpan(0, 0, _timeout);
 
                 HttpClientSingleton.Initialize(_validate, timeoutSpan);
 
                 CxRestContext retVal = new CxRestContext()
                 {
                     SastToken = GetLoginToken(_url, _user, _pass, SAST_SCOPE),
-                    MNOToken =  GetLoginToken(_url, _user, _pass, $"{MNO_SCOPE} {SAST_SCOPE}"),
+                    MNOToken =  String.IsNullOrEmpty(_mnoUrl) ? new Nullable<LoginToken> () : GetLoginToken(_url, _user, _pass, $"{MNO_SCOPE} {SAST_SCOPE}"),
                     Url = _url,
-                    MnoUrl = String.IsNullOrEmpty (_mnoUrl) ? _url : _mnoUrl,
+                    MnoUrl = _mnoUrl,
                     ValidateSSL = _validate,
-                    Timeout = timeoutSpan
+                    Timeout = timeoutSpan,
+                    RetryLoop = _retryLoop
                 };
 
                 retVal.Json = new CxClientFactory("application/json", retVal);

diff --git a/CxRestClient/IO/CxClientFactory.cs b/CxRestClient/IO/CxClientFactory.cs
@@ -30,8 +30,11 @@ public CxRestClient CreateSastClient(String apiVersion)
 
         public CxRestClient CreateMnoClient(String apiVersion)
         {
-            return new CxRestClient(new System.Net.Http.Headers.AuthenticationHeaderValue(Context.MNOToken.TokenType,
-                Context.MNOToken.Token), MediaType, apiVersion);
+            if (!Context.MNOToken.HasValue)
+                return null;
+
+            return new CxRestClient(new System.Net.Http.Headers.AuthenticationHeaderValue(Context.MNOToken.Value.TokenType,
+                Context.MNOToken.Value.Token), MediaType, apiVersion);
         }
     }
 

diff --git a/CxRestClient/MNO/CxMnoPolicies.cs b/CxRestClient/MNO/CxMnoPolicies.cs
@@ -61,10 +61,13 @@ public static String GetProjectPoliciesSingleField(CxRestContext ctx,
                 ctx.Json.CreateMnoClient
                 , (response) =>
                 {
-                    JToken jt = JToken.Load(new JsonTextReader(new StreamReader
-                        (response.Content.ReadAsStreamAsync().Result)));
+                    using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result))
+                    using (var jtr = new JsonTextReader(sr))
+                    {
+                        JToken jt = JToken.Load(jtr);
 
-                    return GetFlatPolicyNames(jt);
+                        return GetFlatPolicyNames(jt);
+                    }
                 }
                 , CxRestContext.MakeUrl(ctx.MnoUrl, String.Format(PROJECT_POLICY_URL_SUFFIX, projectId))
                 , ctx
@@ -118,24 +121,18 @@ public static PolicyCollection GetAllPolicies(CxRestContext ctx,
 				ctx.Json.CreateMnoClient
 				, (response) =>
 				{
-					JToken jt = JToken.Load(new JsonTextReader(new StreamReader
-						(response.Content.ReadAsStreamAsync().Result)));
+                    using (var sr = new StreamReader (response.Content.ReadAsStreamAsync().Result))
+                    using (var jtr = new JsonTextReader(sr))
+                    {
+                        JToken jt = JToken.Load(jtr);
 
-					return ParsePolicies(ctx, token, jt);
+                        return ParsePolicies(ctx, token, jt);
+                    }
 				}
 				, CxRestContext.MakeUrl(ctx.MnoUrl, POLICY_LIST_URL_SUFFIX)
 				, ctx
 				, token
-				, exceptionErrorLogic: (ex) =>
-				{
-
-                    if (ex is System.AggregateException)
-                        foreach (var x in (ex as System.AggregateException).InnerExceptions)
-                            if (x is System.Net.Http.HttpRequestException)
-                                return false;
-
-					return true;
-				}, apiVersion: null);
+                , apiVersion: null);
 		}
 
 
@@ -146,18 +143,21 @@ public static IEnumerable<int> GetPolicyIdsForProject(CxRestContext ctx,
 			ctx.Json.CreateMnoClient
 			, (response) =>
 			{
-				JToken jt = JToken.Load(new JsonTextReader(new StreamReader
-	                (response.Content.ReadAsStreamAsync().Result)));
+                using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result))
+                using (var jtr = new JsonTextReader(sr))
+                {
+                    JToken jt = JToken.Load(jtr);
 
-				LinkedList<int> policyIds = new LinkedList<int>();
+                    LinkedList<int> policyIds = new LinkedList<int>();
 
-				using (JTokenReader reader = new JTokenReader(jt))
-					while (JsonUtils.MoveToNextProperty(reader, "id"))
-					{
-						policyIds.AddLast(Convert.ToInt32(((JProperty)reader.CurrentToken).Value));
-					}
+                    using (JTokenReader reader = new JTokenReader(jt))
+                        while (JsonUtils.MoveToNextProperty(reader, "id"))
+                        {
+                            policyIds.AddLast(Convert.ToInt32(((JProperty)reader.CurrentToken).Value));
+                        }
 
-				return policyIds;
+                    return policyIds;
+                }
 
 			}
 			, CxRestContext.MakeUrl(ctx.MnoUrl, String.Format(PROJECT_POLICY_URL_SUFFIX, projectId))

diff --git a/CxRestClient/OSA/CxOsaLibraries.cs b/CxRestClient/OSA/CxOsaLibraries.cs
@@ -64,7 +64,7 @@ public class Match
 		}
 
 
-		private class LibrariesReader : IEnumerable<Library>, IEnumerator<Library>
+		private class LibrariesReader : IEnumerable<Library>, IEnumerator<Library>, IDisposable
 		{
 			private JToken _json;
 			private JTokenReader _reader;
@@ -83,7 +83,11 @@ internal LibrariesReader(JToken json)
 
 			public void Dispose()
 			{
-				_reader = null;
+				if (_reader != null)
+				{
+					_reader.Close();
+					_reader = null;
+				}
 			}
 
 			public IEnumerator<Library> GetEnumerator()
@@ -111,8 +115,8 @@ public bool MoveNext()
 				if (!(_arrayPos < _libArray.Count))
 					return false;
 
-				_currentLibrary = (Library)new JsonSerializer().
-				Deserialize(new JTokenReader(_libArray[_arrayPos]), typeof(Library));
+				using (var jtr = new JTokenReader(_libArray[_arrayPos]))
+					_currentLibrary = (Library)new JsonSerializer().Deserialize(jtr, typeof(Library));
 
 				return true;
 			}
@@ -147,7 +151,7 @@ public static IEnumerable<Library> GetLibraries(CxRestContext ctx, CancellationT
 			{
 				var beforeCount = returnLibs.Count;
 
-				returnLibs.AddRange(WebOperation.ExecuteGet<LibrariesReader>(
+				using (var libReader = WebOperation.ExecuteGet<LibrariesReader>(
 				ctx.Json.CreateSastClient
 				, (response) =>
 				{
@@ -160,7 +164,8 @@ public static IEnumerable<Library> GetLibraries(CxRestContext ctx, CancellationT
 				}
 				, url(curPage++)
 				, ctx
-				, token));
+				, token))
+					returnLibs.AddRange(libReader);
 
 				if (returnLibs.Count == beforeCount)
 					break;

diff --git a/CxRestClient/OSA/CxOsaLicenses.cs b/CxRestClient/OSA/CxOsaLicenses.cs
@@ -54,7 +54,7 @@ public class License
             public String Url { get; internal set; }
         }
 
-        private class LicensesReader : IEnumerable<License>, IEnumerator<License>
+        private class LicensesReader : IEnumerable<License>, IEnumerator<License>, IDisposable
         {
             private JToken _json;
             private JTokenReader _reader;
@@ -73,7 +73,11 @@ internal LicensesReader(JToken json)
 
             public void Dispose()
             {
-                _reader = null;
+                if (_reader != null)
+                {
+                    _reader.Close();
+                    _reader = null;
+                }
             }
 
             public IEnumerator<License> GetEnumerator()
@@ -102,8 +106,8 @@ public bool MoveNext()
                 if (!(_arrayPos < _licenseArray.Count))
                     return false;
 
-                _currentLicense = (License)new JsonSerializer().
-                    Deserialize(new JTokenReader(_licenseArray[_arrayPos]), typeof(License));
+                using (var jtr = new JTokenReader(_licenseArray[_arrayPos]))
+                _currentLicense = (License)new JsonSerializer().Deserialize(jtr, typeof(License));
 
                 return true;
             }
@@ -128,21 +132,26 @@ public static IEnumerable<License> GetLicenses(CxRestContext ctx, CancellationTo
                     {"scanId", Convert.ToString (scanId)  }
                 });
 
+            List<License> retVal = new List<License>();
 
-            return WebOperation.ExecuteGet<IEnumerable<License>>(
-			ctx.Json.CreateSastClient
-			, (response) =>
-			{
-                using (var sr = new StreamReader (response.Content.ReadAsStreamAsync().Result))
+            using (var licReader = WebOperation.ExecuteGet<LicensesReader>(
+            ctx.Json.CreateSastClient
+            , (response) =>
+            {
+                using (var sr = new StreamReader(response.Content.ReadAsStreamAsync().Result))
                 using (var jtr = new JsonTextReader(sr))
                 {
                     JToken jt = JToken.Load(jtr);
                     return new LicensesReader(jt);
                 }
             }
             , url
-			, ctx
-			, token);
+            , ctx
+            , token))
+            {
+                retVal.AddRange(licReader);
+                return retVal;
+            }
 
         }
     }