Skip to content

Commit a20d77f

Browse files
nohwndnohwnd
committed
Disable DynamicNative instrumentation by default (#15298)
* Disable DynamicNative instrumentation by default * Also commit tests * Apply suggestion from @nohwnd
1 parent 9cbffb0 commit a20d77f

File tree

5 files changed

+237
-0
lines changed

5 files changed

+237
-0
lines changed

‎src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,10 @@ private FeatureFlag() { }
7575
// Disable setting DOTNET_ROOT environment variable on non-Windows platforms. We used to set it only only on Windows when we found testhost.exe, now we set it always to allow xunit v3 to run tests in child process.
7676
public const string VSTEST_DISABLE_DOTNET_ROOT_ON_NONWINDOWS = nameof(VSTEST_DISABLE_DOTNET_ROOT_ON_NONWINDOWS);
7777

78+
// Disable turning dynamic code coverage for native code to OFF by default. Setting this to 1 will skip adding the setting.
79+
public const string VSTEST_DISABLE_DYNAMICNATIVE_CODECOVERAGE_DEFAULT_SETTING = nameof(VSTEST_DISABLE_DYNAMICNATIVE_CODECOVERAGE_DEFAULT_SETTING);
80+
81+
7882

7983
[Obsolete("Only use this in tests.")]
8084
internal static void Reset()

‎src/Microsoft.TestPlatform.Utilities/InferRunSettingsHelper.cs‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -718,4 +718,84 @@ private static bool IsFrameworkIncompatible(Framework sourceFramework, Framework
718718
return !sourceFramework.Name.Equals(Framework.DefaultFramework.Name, StringComparison.OrdinalIgnoreCase)
719719
&& !sourceFramework.Name.Equals(targetFramework.Name, StringComparison.OrdinalIgnoreCase);
720720
}
721+
722+
public static bool UpdateCollectCoverageSettings(XmlDocument xmlDocument)
723+
{
724+
var root = xmlDocument.DocumentElement;
725+
726+
var dataCollectorNodes = root?.SelectNodes("DataCollectionRunSettings/DataCollectors/DataCollector");
727+
if (dataCollectorNodes == null)
728+
{
729+
return false;
730+
}
731+
foreach (XmlNode dataCollectorNode in dataCollectorNodes)
732+
{
733+
var dataCollectorFound = false;
734+
foreach (XmlAttribute attribute in dataCollectorNode.Attributes!)
735+
{
736+
if (attribute.Name.Equals("friendlyName", StringComparison.OrdinalIgnoreCase) &&
737+
attribute.Value.Equals("Code Coverage", StringComparison.OrdinalIgnoreCase))
738+
{
739+
dataCollectorFound = true;
740+
break;
741+
}
742+
743+
if (attribute.Name.Equals("uri", StringComparison.OrdinalIgnoreCase) &&
744+
attribute.Value.Equals(CodeCoverageCollectorUri, StringComparison.OrdinalIgnoreCase))
745+
{
746+
dataCollectorFound = true;
747+
break;
748+
}
749+
750+
if (attribute.Name.Equals("assemblyQualifiedName", StringComparison.OrdinalIgnoreCase) &&
751+
attribute.Value.IndexOf("Microsoft.VisualStudio.Coverage.DynamicCoverageDataCollector", StringComparison.OrdinalIgnoreCase) >= 0)
752+
{
753+
dataCollectorFound = true;
754+
break;
755+
}
756+
}
757+
758+
if (dataCollectorFound)
759+
{
760+
var coverageCollectorNode = dataCollectorNode;
761+
// Code coverage settings are present, we should update them.
762+
763+
var dynamicNativeInstrumentationNode = coverageCollectorNode.SelectSingleNode("Configuration/CodeCoverage/EnableDynamicNativeInstrumentation");
764+
765+
if (dynamicNativeInstrumentationNode == null)
766+
{
767+
// EnableDynamicNativeInstrumentation is not set explicitly, we should set it. Whole tree might not exist.
768+
769+
var currentNode = coverageCollectorNode;
770+
var paths = "Configuration/CodeCoverage/EnableDynamicNativeInstrumentation".Split('/');
771+
foreach (var nodeName in paths)
772+
{
773+
var found = false;
774+
foreach (XmlNode childNode in currentNode.ChildNodes)
775+
{
776+
if (childNode.Name == nodeName)
777+
{
778+
currentNode = childNode;
779+
found = true;
780+
break;
781+
}
782+
}
783+
784+
if (!found)
785+
{
786+
var newNode = xmlDocument.CreateElement(nodeName);
787+
currentNode.AppendChild(newNode);
788+
currentNode = newNode;
789+
}
790+
}
791+
792+
currentNode.InnerXml = "False";
793+
794+
return true;
795+
}
796+
}
797+
}
798+
799+
return false;
800+
}
721801
}

‎src/Microsoft.TestPlatform.Utilities/PublicAPI/PublicAPI.Shipped.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,4 +39,6 @@ static Microsoft.VisualStudio.TestPlatform.Utilities.MSTestSettingsUtilities.IsL
3939
static Microsoft.VisualStudio.TestPlatform.Utilities.ParallelRunSettingsUtilities.UpdateRunSettingsWithParallelSettingIfNotConfigured(System.Xml.XPath.XPathNavigator! navigator) -> void
4040
static Microsoft.VisualStudio.TestPlatform.Utilities.StringExtensions.Tokenize(this string? input, char separator, char escape) -> System.Collections.Generic.IEnumerable<string!>!
4141
static Microsoft.VisualStudio.TestPlatform.Utilities.InferRunSettingsHelper.UpdateBatchSize(System.Xml.XmlDocument! runSettingsDocument, long batchSizeValue) -> void
42+
static Microsoft.VisualStudio.TestPlatform.Utilities.InferRunSettingsHelper.UpdateCollectCoverageSettings(System.Xml.XmlDocument! xmlDocument) -> bool
43+
4244

‎src/vstest.console/TestPlatformHelpers/TestRequestManager.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -845,6 +845,11 @@ private bool UpdateRunSettingsIfRequired(
845845
settingsUpdated |= AddOrUpdateBuiltInLoggers(document, runConfiguration, loggerRunSettings);
846846
settingsUpdated |= AddOrUpdateBatchSize(document, runConfiguration, isDiscovery);
847847

848+
if (!FeatureFlag.Instance.IsSet(FeatureFlag.VSTEST_DISABLE_DYNAMICNATIVE_CODECOVERAGE_DEFAULT_SETTING))
849+
{
850+
settingsUpdated |= UpdateCollectCoverageSettings(document, runConfiguration);
851+
}
852+
848853
updatedRunSettingsXml = navigator.OuterXml;
849854

850855
return settingsUpdated;
@@ -1246,6 +1251,11 @@ private static bool UpdateMSBuildLoggerIfExists(
12461251
return false;
12471252
}
12481253

1254+
internal static bool UpdateCollectCoverageSettings(XmlDocument xmlDocument, RunConfiguration _)
1255+
{
1256+
return InferRunSettingsHelper.UpdateCollectCoverageSettings(xmlDocument);
1257+
}
1258+
12491259
private void RunTests(
12501260
IRequestData requestData,
12511261
TestRunCriteria testRunCriteria,

‎test/vstest.console.UnitTests/TestPlatformHelpers/TestRequestManagerTests.cs‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2663,6 +2663,147 @@ public void AddOrUpdateBatchSizeSetsRunConfigurationAndBatchSize()
26632663
Assert.AreEqual("<RunSettings><RunConfiguration><BatchSize>1000</BatchSize></RunConfiguration></RunSettings>", xmlDocument.OuterXml);
26642664
}
26652665

2666+
[TestMethod]
2667+
public void UpdateCodeCoverageSettings_SetEnableDynamicNativeInstrumentationToFalse_WhenNotPresent()
2668+
{
2669+
// Arrange
2670+
var xmlDocument = new XmlDocument();
2671+
xmlDocument.LoadXml("""
2672+
<?xml version="1.0" encoding="utf-8"?>
2673+
<RunSettings>
2674+
<DataCollectionRunSettings>
2675+
<DataCollectors>
2676+
<DataCollector friendlyName="Code Coverage" uri="datacollector://Microsoft/CodeCoverage/2.0" assemblyQualifiedName="Microsoft.VisualStudio.Coverage.DynamicCoverageDataCollector, Microsoft.VisualStudio.TraceCollector, Version=11.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
2677+
<Configuration>
2678+
<CoverageLogLevel>All</CoverageLogLevel>
2679+
<InstrumentationLogLevel>All</InstrumentationLogLevel>
2680+
<ManagedVanguardLogLevel>Verbose</ManagedVanguardLogLevel>
2681+
<CoverageFileLogPath>%LOGS_DIR%</CoverageFileLogPath>
2682+
<CodeCoverage>
2683+
<FileLogPath>%LOGS_DIR%</FileLogPath>
2684+
<LogLevel>All</LogLevel>
2685+
<UseVerifiableInstrumentation>False</UseVerifiableInstrumentation>
2686+
<EnableStaticNativeInstrumentation>False</EnableStaticNativeInstrumentation>
2687+
</CodeCoverage>
2688+
</Configuration>
2689+
</DataCollector>
2690+
</DataCollectors>
2691+
</DataCollectionRunSettings>
2692+
</RunSettings>
2693+
""");
2694+
var configuration = new RunConfiguration();
2695+
2696+
// Act
2697+
var result = TestRequestManager.UpdateCollectCoverageSettings(xmlDocument, configuration);
2698+
2699+
// Assert
2700+
Assert.IsTrue(result);
2701+
StringAssert.Contains(xmlDocument.OuterXml, "<EnableStaticNativeInstrumentation>False</EnableStaticNativeInstrumentation><EnableDynamicNativeInstrumentation>False</EnableDynamicNativeInstrumentation></CodeCoverage>");
2702+
}
2703+
2704+
[TestMethod]
2705+
public void UpdateCodeCoverageSettings_SetEnableDynamicNativeInstrumentationToFalse_WhenNotPresentAndParentDetailsOfConfigurationAreAlsoNotPresent()
2706+
{
2707+
// Arrange
2708+
var xmlDocument = new XmlDocument();
2709+
xmlDocument.LoadXml("""
2710+
<?xml version="1.0" encoding="utf-8"?>
2711+
<RunSettings>
2712+
<DataCollectionRunSettings>
2713+
<DataCollectors>
2714+
<DataCollector friendlyName="Code Coverage" uri="datacollector://Microsoft/CodeCoverage/2.0" assemblyQualifiedName="Microsoft.VisualStudio.Coverage.DynamicCoverageDataCollector, Microsoft.VisualStudio.TraceCollector, Version=11.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
2715+
</DataCollector>
2716+
</DataCollectors>
2717+
</DataCollectionRunSettings>
2718+
</RunSettings>
2719+
""");
2720+
var configuration = new RunConfiguration();
2721+
2722+
// Act
2723+
var result = TestRequestManager.UpdateCollectCoverageSettings(xmlDocument, configuration);
2724+
2725+
// Assert
2726+
Assert.IsTrue(result);
2727+
StringAssert.Contains(xmlDocument.OuterXml, $"<Configuration><CodeCoverage><EnableDynamicNativeInstrumentation>False</EnableDynamicNativeInstrumentation></CodeCoverage></Configuration></DataCollector>");
2728+
}
2729+
2730+
[TestMethod]
2731+
[DataRow("True")]
2732+
[DataRow("False")]
2733+
public void UpdateCodeCoverageSettings_DontSetEnableDynamicNativeInstrumentationToFalse_WhenAlreadyPresent(string setting)
2734+
{
2735+
// Arrange
2736+
var xmlDocument = new XmlDocument();
2737+
xmlDocument.LoadXml($"""
2738+
<?xml version="1.0" encoding="utf-8"?>
2739+
<RunSettings>
2740+
<DataCollectionRunSettings>
2741+
<DataCollectors>
2742+
<DataCollector friendlyName="Code Coverage" uri="datacollector://Microsoft/CodeCoverage/2.0" assemblyQualifiedName="Microsoft.VisualStudio.Coverage.DynamicCoverageDataCollector, Microsoft.VisualStudio.TraceCollector, Version=11.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a">
2743+
<Configuration>
2744+
<CoverageLogLevel>All</CoverageLogLevel>
2745+
<InstrumentationLogLevel>All</InstrumentationLogLevel>
2746+
<ManagedVanguardLogLevel>Verbose</ManagedVanguardLogLevel>
2747+
<CoverageFileLogPath>%LOGS_DIR%</CoverageFileLogPath>
2748+
<CodeCoverage>
2749+
<EnableDynamicNativeInstrumentation>{setting}</EnableDynamicNativeInstrumentation>
2750+
</CodeCoverage>
2751+
</Configuration>
2752+
</DataCollector>
2753+
</DataCollectors>
2754+
</DataCollectionRunSettings>
2755+
</RunSettings>
2756+
""");
2757+
var configuration = new RunConfiguration();
2758+
2759+
// Act
2760+
var result = TestRequestManager.UpdateCollectCoverageSettings(xmlDocument, configuration);
2761+
2762+
// Assert
2763+
// No matter what user has set, we don't override it.
2764+
Assert.IsFalse(result);
2765+
StringAssert.Contains(xmlDocument.OuterXml, $"<CodeCoverage><EnableDynamicNativeInstrumentation>{setting}</EnableDynamicNativeInstrumentation></CodeCoverage>");
2766+
}
2767+
2768+
[TestMethod]
2769+
[DataRow("friendlyName=\"Code Coverage\"")]
2770+
[DataRow("friendlyName=\"code coverage\"")]
2771+
[DataRow("uri=\"datacollector://Microsoft/CodeCoverage/2.0\"")]
2772+
[DataRow("uri=\"datacollector://microsoft/codecoverage/2.0\"")]
2773+
[DataRow("assemblyQualifiedName=\"Microsoft.VisualStudio.Coverage.DynamicCoverageDataCollector, Microsoft.VisualStudio.TraceCollector, Version=11.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a\"")]
2774+
public void UpdateCodeCoverageSettings_SetEnableDynamicNativeInstrumentationToFalse_WhenUserUsesImperfectNamesForCollector(string collector)
2775+
{
2776+
// Arrange
2777+
var xmlDocument = new XmlDocument();
2778+
xmlDocument.LoadXml($"""
2779+
<?xml version="1.0" encoding="utf-8"?>
2780+
<RunSettings>
2781+
<DataCollectionRunSettings>
2782+
<DataCollectors>
2783+
<DataCollector {collector}>
2784+
<Configuration>
2785+
<CoverageLogLevel>All</CoverageLogLevel>
2786+
<InstrumentationLogLevel>All</InstrumentationLogLevel>
2787+
<ManagedVanguardLogLevel>Verbose</ManagedVanguardLogLevel>
2788+
<CoverageFileLogPath>%LOGS_DIR%</CoverageFileLogPath>
2789+
<CodeCoverage>
2790+
</CodeCoverage>
2791+
</Configuration>
2792+
</DataCollector>
2793+
</DataCollectors>
2794+
</DataCollectionRunSettings>
2795+
</RunSettings>
2796+
""");
2797+
var configuration = new RunConfiguration();
2798+
2799+
// Act
2800+
var result = TestRequestManager.UpdateCollectCoverageSettings(xmlDocument, configuration);
2801+
2802+
// Assert
2803+
Assert.IsTrue(result);
2804+
StringAssert.Contains(xmlDocument.OuterXml, $"<CodeCoverage><EnableDynamicNativeInstrumentation>False</EnableDynamicNativeInstrumentation></CodeCoverage>");
2805+
}
2806+
26662807
private static DiscoveryRequestPayload CreateDiscoveryPayload(string runsettings)
26672808
{
26682809
var discoveryPayload = new DiscoveryRequestPayload

0 commit comments

Comments
 (0)