Fix cache posioning

Chutzpah/BatchCompiler/BatchCompilerService.cs

@@ -23,7 +23,7 @@ public BatchCompilerService(IProcessHelper processHelper, IFileSystemWrapper fil
             this.sourceMapDiscoverer = sourceMapDiscoverer;
         }
 
-        public void Compile(IEnumerable<TestContext> testContexts)
+        public void Compile(IEnumerable<TestContext> testContexts, ITestMethodRunnerCallback callback = null)
         {
             // Group the test contexts by test settings to run batch aware settings like compile
             // For each test settings file that defines a compile step we will run it and update 
@@ -101,7 +101,8 @@ where testSettings.Compile.Extensions.Any(x => file.Path.EndsWith(x, StringCompa
                             if (!testSettings.Compile.IgnoreMissingFiles.GetValueOrDefault())
                             {
                                 // Throw and fail here since if we cant find the file we cannot be sure anything will run
-                                throw new FileNotFoundException(error, outputPath);
+                                var exception = new FileNotFoundException(error, outputPath);
+                                callback.ExceptionThrown(exception, outputPath);
                             }
                         }
                     }

Chutzpah/BatchCompiler/IBatchCompilerService.cs

@@ -6,6 +6,6 @@ namespace Chutzpah.BatchProcessor
 {
     public interface IBatchCompilerService
     {
-        void Compile(IEnumerable<TestContext> testContexts);
+        void Compile(IEnumerable<TestContext> testContexts, ITestMethodRunnerCallback callback = null);
     }
 }

Chutzpah/ChutzpahTestSettingsService.cs

@@ -75,30 +75,36 @@ public ChutzpahTestSettingsFile FindSettingsFileFromDirectory(string directory,
             ChutzpahTestSettingsFile settings;
             if (!ChutzpahSettingsFileCache.TryGetValue(directory, out settings))
             {
-                return ProcessSettingsFile(directory, environments, new List<ChutzpahSettingsFileEnvironment>()).InheritFromDefault();
+                ChutzpahSettingsFileEnvironment environment = null;
+                if (environments != null)
+                {
+                    environment = environments.GetSettingsFileEnvironment(directory);
+                }
+
+                return ProcessSettingsFile(directory, environment).InheritFromDefault();
             }
             else
             {
                 return settings;
             }
         }
 
-        private ChutzpahTestSettingsFile ProcessSettingsFile(string directory, ChutzpahSettingsFileEnvironments environments, IList<ChutzpahSettingsFileEnvironment> environmentChain)
+        private ChutzpahTestSettingsFile ProcessSettingsFile(string directory, ChutzpahSettingsFileEnvironment environment, bool forceFresh = false)
         {
             if (string.IsNullOrEmpty(directory)) return ChutzpahTestSettingsFile.Default;
 
             directory = directory.TrimEnd('/', '\\');
 
             ChutzpahTestSettingsFile settings;
-            if (!ChutzpahSettingsFileCache.TryGetValue(directory, out settings))
+            if (!ChutzpahSettingsFileCache.TryGetValue(directory, out settings) || forceFresh)
             {
                 var testSettingsFilePath = fileProbe.FindTestSettingsFile(directory);
                 if (string.IsNullOrEmpty(testSettingsFilePath))
                 {
                     ChutzpahTracer.TraceInformation("Chutzpah.json file not found given starting directory {0}", directory);
                     settings = ChutzpahTestSettingsFile.Default;
                 }
-                else if (!ChutzpahSettingsFileCache.TryGetValue(Path.GetDirectoryName(testSettingsFilePath), out settings))
+                else if (!ChutzpahSettingsFileCache.TryGetValue(Path.GetDirectoryName(testSettingsFilePath), out settings) || forceFresh)
                 {
                     ChutzpahTracer.TraceInformation("Chutzpah.json file found at {0} given starting directory {1}", testSettingsFilePath, directory);
                     settings = serializer.DeserializeFromFile<ChutzpahTestSettingsFile>(testSettingsFilePath);
@@ -111,25 +117,9 @@ private ChutzpahTestSettingsFile ProcessSettingsFile(string directory, ChutzpahS
                     {
                         settings.IsDefaultSettings = false;
                     }
-
-                    ChutzpahSettingsFileEnvironment environment = null;
-                    if (environments != null)
-                    {
-                        environment = environments.GetSettingsFileEnvironment(directory);
-
-                        if (environment != null)
-                        {
-                            // Add this environment to track the list
-                            // of environments we have as we walk up the chutzpah.json chain. 
-                            // We enumerate in order from child to parent letting the more specific environment win
-                            environmentChain.Add(environment);
-                        }
-                    }
-
-
                     settings.SettingsFileDirectory = Path.GetDirectoryName(testSettingsFilePath);
 
-                    var chutzpahVariables = BuildChutzpahReplacementVariables(testSettingsFilePath, environmentChain, settings);
+                    var chutzpahVariables = BuildChutzpahReplacementVariables(testSettingsFilePath, environment, settings);
 
                     ResolveTestHarnessDirectory(settings, chutzpahVariables);
 
@@ -141,18 +131,22 @@ private ChutzpahTestSettingsFile ProcessSettingsFile(string directory, ChutzpahS
 
                     ProcessTraceFilePath(settings, chutzpahVariables);
 
-                    ProcessInheritance(environments, environmentChain, settings, chutzpahVariables);
+                    ProcessInheritance(environment, settings, chutzpahVariables);
 
-                    // Add a mapping in the cache for the directory that contains the test settings file
-                    ChutzpahSettingsFileCache.TryAdd(settings.SettingsFileDirectory, settings);
+                    if (!forceFresh)
+                    {
+                        // Add a mapping in the cache for the directory that contains the test settings file
+                        ChutzpahSettingsFileCache.TryAdd(settings.SettingsFileDirectory, settings);
+                    }
                 }
 
-                // Add mapping in the cache for the original directory tried to skip needing to traverse the tree again
-                ChutzpahSettingsFileCache.TryAdd(directory, settings);
+                if (!forceFresh)
+                {
+                    // Add mapping in the cache for the original directory tried to skip needing to traverse the tree again
+                    ChutzpahSettingsFileCache.TryAdd(directory, settings);
+                }
             }
 
-
-
             return settings;
         }
 
@@ -165,7 +159,7 @@ private void ProcessTraceFilePath(ChutzpahTestSettingsFile settings, IDictionary
             }
         }
 
-        private void ProcessInheritance(ChutzpahSettingsFileEnvironments environments, IList<ChutzpahSettingsFileEnvironment> environmentChain, ChutzpahTestSettingsFile settings, IDictionary<string, string> chutzpahVariables)
+        private void ProcessInheritance(ChutzpahSettingsFileEnvironment environment, ChutzpahTestSettingsFile settings, IDictionary<string, string> chutzpahVariables)
         {
             if (settings.InheritFromParent || !string.IsNullOrEmpty(settings.InheritFromPath))
             {
@@ -190,7 +184,11 @@ private void ProcessInheritance(ChutzpahSettingsFileEnvironments environments, I
                     settings.InheritFromPath = settingsToInherit;
                 }
 
-                var parentSettingsFile = ProcessSettingsFile(settings.InheritFromPath, environments, environmentChain);
+                // If we have any environment properties do not use cached
+                // parents and re-evaluate using current environment
+                var forceFresh = environment != null && environment.Properties.Any();
+
+                var parentSettingsFile = ProcessSettingsFile(settings.InheritFromPath, environment, forceFresh);
 
                 if (!parentSettingsFile.IsDefaultSettings)
                 {
@@ -454,7 +452,7 @@ private string ExpandChutzpahVariables(IDictionary<string, string> chutzpahCompi
             return chutzpahCompileVariables.Aggregate(str, (current, pair) => current.Replace(pair.Key, pair.Value));
         }
 
-        private IDictionary<string, string> BuildChutzpahReplacementVariables(string settingsFilePath, IList<ChutzpahSettingsFileEnvironment> environmentChain, ChutzpahTestSettingsFile settings)
+        private IDictionary<string, string> BuildChutzpahReplacementVariables(string settingsFilePath, ChutzpahSettingsFileEnvironment environment, ChutzpahTestSettingsFile settings)
         {
             IDictionary<string, string> chutzpahVariables = new Dictionary<string, string>();
 
@@ -472,20 +470,16 @@ private IDictionary<string, string> BuildChutzpahReplacementVariables(string set
             // This is not needed but it is a nice alias
             AddChutzpahVariable(chutzpahVariables, "cmdexe", Environment.ExpandEnvironmentVariables("%comspec%"));
 
-
-            // See if we have a settingsfileenvironment set and if so add its properties as chutzpah settings file variables
-            foreach (var environment in environmentChain)
+            if (environment != null)
             {
+                // See if we have a settingsfileenvironment set and if so add its properties as chutzpah settings file variables
                 var props = environment.Properties;
                 foreach (var prop in props)
                 {
-                    if (!string.IsNullOrEmpty(prop.Value))
-                    {
-                        AddChutzpahVariable(chutzpahVariables, prop.Name, prop.Value);
-                    }
+                    AddChutzpahVariable(chutzpahVariables, prop.Name, prop.Value);
                 }
             }
-            
+
 
             return chutzpahVariables;
         }

Chutzpah/ITestRunner.cs

@@ -31,6 +31,7 @@ public interface ITestRunner
 
         IEnumerable<TestCase> DiscoverTests(string testPath);
         IEnumerable<TestCase> DiscoverTests(IEnumerable<string> testPaths);
+        IEnumerable<TestCase> DiscoverTests(IEnumerable<string> testPaths, TestOptions options, ITestMethodRunnerCallback callback);
         bool IsTestFile(string testFile, ChutzpahSettingsFileEnvironments envionrments);
         IEnumerable<TestCase> DiscoverTests(IEnumerable<string> testPaths, TestOptions options);
         void CleanTestContext(TestContext context);

Chutzpah/TestRunner.cs

@@ -113,6 +113,12 @@ public IEnumerable<TestCase> DiscoverTests(IEnumerable<string> testPaths, TestOp
             return summary.Tests;
         }
 
+        public IEnumerable<TestCase> DiscoverTests(IEnumerable<string> testPaths, TestOptions options, ITestMethodRunnerCallback callback)
+        {
+            var summary = ProcessTestPaths(testPaths, options, TestExecutionMode.Discovery, callback);
+            return summary.Tests;
+        }
+
 
         public TestCaseSummary RunTests(string testPath, ITestMethodRunnerCallback callback = null)
         {
@@ -298,7 +304,7 @@ private bool PerformBatchCompile(ITestMethodRunnerCallback callback, IEnumerable
         {
             try
             {
-                batchCompilerService.Compile(testContexts);
+                batchCompilerService.Compile(testContexts, callback);
             }
             catch (FileNotFoundException e)
             {

Facts/Library/BatchCompilerServiceFacts.cs

@@ -102,10 +102,11 @@ public void Will_throw_if_file_is_missing()
                 .Returns(false)
                 .Returns(true);
             service.Mock<IFileSystemWrapper>().Setup(x => x.FileExists(It.Is<string>(f => f.EndsWith(".ts")))).Returns(true);
+            var callback = service.Mock<ITestMethodRunnerCallback>();
 
-            var ex  = Record.Exception( () => service.ClassUnderTest.Compile(new[] { context }));
+            service.ClassUnderTest.Compile(new[] { context }, callback.Object);
 
-            Assert.IsType<FileNotFoundException>(ex);
+            callback.Verify(x => x.ExceptionThrown(It.IsAny<FileNotFoundException>(), It.IsAny<string>()));
             service.Mock<IProcessHelper>().Verify(x => x.RunBatchCompileProcess(It.IsAny<BatchCompileConfiguration>()), Times.Never());
         }
 

VS2012.TestAdapter/ChutzpahTestDiscoverer.cs

@@ -8,6 +8,7 @@
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
+using Chutzpah.Callbacks;
 
 namespace Chutzpah.VS2012.TestAdapter
 {
@@ -44,21 +45,12 @@ public void DiscoverTests(IEnumerable<string> sources, IDiscoveryContext discove
                 ChutzpahSettingsFileEnvironments = new ChutzpahSettingsFileEnvironments(settings.ChutzpahSettingsFileEnvironments)
             };
 
-            IList<TestError> errors;
-            var testCases = testRunner.DiscoverTests(sources, testOptions, out errors);
 
             ChutzpahTracer.TraceInformation("Sending discovered tests to test case discovery sink");
 
-            foreach (var testCase in testCases)
-            {
-                var vsTestCase = testCase.ToVsTestCase();
-                discoverySink.SendTestCase(vsTestCase);
-            }
+            var callback = new ParallelRunnerCallbackAdapter(new DiscoveryCallback(logger, discoverySink));
+            var testCases = testRunner.DiscoverTests(sources, testOptions, callback);
 
-            foreach (var error in errors)
-            {
-                logger.SendMessage(TestMessageLevel.Error, RunnerCallback.FormatFileErrorMessage(error));
-            }
 
             ChutzpahTracer.TraceInformation("End Test Adapter Discover Tests");
 

VS2012.TestAdapter/ChutzpahTestExecutor.cs

@@ -1,15 +1,10 @@
 using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
 using System.Linq;
 using Chutzpah.Callbacks;
-using Chutzpah.Coverage;
 using Chutzpah.Models;
 using Chutzpah.VS.Common;
-using Chutzpah.Wrappers;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
-using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
 
 namespace Chutzpah.VS2012.TestAdapter
 {

VS2012.TestAdapter/DiscoveryCallback.cs

@@ -0,0 +1,36 @@
+using System;
+using Chutzpah.Models;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
+
+namespace Chutzpah.VS2012.TestAdapter
+{
+    public class DiscoveryCallback : RunnerCallback
+    {
+        private readonly IMessageLogger logger;
+        private readonly ITestCaseDiscoverySink discoverySink;
+
+        public DiscoveryCallback(IMessageLogger logger, ITestCaseDiscoverySink discoverySink)
+        {
+            this.logger = logger;
+            this.discoverySink = discoverySink;
+        }
+
+        public override void FileError(TestError error)
+        {
+            logger.SendMessage(TestMessageLevel.Error, GetFileErrorMessage(error));
+        }
+
+        public override void ExceptionThrown(Exception exception, string fileName)
+        {
+            logger.SendMessage(TestMessageLevel.Error, GetExceptionThrownMessage(exception, fileName));
+        }
+
+        public override void TestFinished(TestCase test)
+        {
+            var testCase = test.ToVsTestCase();
+            discoverySink.SendTestCase(testCase);
+        }
+
+    }
+}

VS2012.TestAdapter/VS2012.TestAdapter.csproj

@@ -59,6 +59,7 @@
     <Compile Include="ChutzpahAdapterSettings.cs" />
     <Compile Include="ChutzpahAdapterSettingsProvider.cs" />
     <Compile Include="ChutzpahExtensionMethods.cs" />
+    <Compile Include="DiscoveryCallback.cs" />
     <Compile Include="ExecutionCallback.cs" />
     <Compile Include="ChutzpahTestDiscoverer.cs" />
     <Compile Include="IChutzpahSettingsMapper.cs" />

VS2012/Adapter/ChutzpahTestContainerDiscoverer.cs

@@ -424,22 +424,30 @@ private void UpdateChutzpahEnvironmentForProject(string projectPath)
                 var dirPath = Path.GetDirectoryName(projectPath);
                 foreach (var prop in ChutzpahMsBuildProps.GetProps())
                 {
-                    chutzpahEnvProps.Add(new ChutzpahSettingsFileEnvironmentProperty(prop, buildProject.GetPropertyValue(prop)));
+                    var value = buildProject.GetPropertyValue(prop);
+                    if (!string.IsNullOrEmpty(value))
+                    {
+                        chutzpahEnvProps.Add(new ChutzpahSettingsFileEnvironmentProperty(prop, value));
+                    }
                 }
 
                 lock (sync)
                 {
                     var envProps = settingsMapper.Settings.ChutzpahSettingsFileEnvironments
                                                           .FirstOrDefault(x => x.Path.TrimEnd('/', '\\').Equals(dirPath, StringComparison.OrdinalIgnoreCase));
-                    if (envProps == null)
+
+                    if (envProps == null && chutzpahEnvProps.Any())
                     {
                         envProps = new ChutzpahSettingsFileEnvironment(dirPath);
 
                         settingsMapper.Settings.ChutzpahSettingsFileEnvironments.Add(envProps);
 
                     }
 
-                    envProps.Properties = chutzpahEnvProps;
+                    if (envProps != null)
+                    {
+                        envProps.Properties = chutzpahEnvProps;
+                    }
                 }
             }
         }