Produce binlogs from integration tests (#4761)

microsoft · Jan 23, 2025 · 1e3add2 · 1e3add2
1 parent 75047f0
commit 1e3add2
Show file tree

Hide file tree

Showing 4 changed files with 103 additions and 7 deletions.
diff --git a/azure-pipelines-official.yml b/azure-pipelines-official.yml
@@ -148,6 +148,22 @@ extends:
                 name: Test
                 displayName: Test
 
+              - task: CopyFiles@2
+                displayName: 'Copy binlogs'
+                inputs:
+                  SourceFolder: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/testsuite'
+                  Contents: |
+                    **/*.binlog
+                  TargetFolder: '$(Build.ArtifactStagingDirectory)/binlogs'
+                condition: always()
+
+              - task: PublishBuildArtifacts@1
+                displayName: 'Publish integration tests binlogs'
+                inputs:
+                  PathtoPublish: '$(Build.ArtifactStagingDirectory)/binlogs'
+                  ArtifactName: Integration_Tests_Windows_Binlogs_$(_BuildConfig)
+                condition: always()
+
               # Remove temporary artifacts to avoid finding binskim issues for exes we don't own.
               - pwsh: |
                   Remove-Item -Path $(Build.SourcesDirectory)/artifacts/tmp -Recurse -Force

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -112,6 +112,22 @@ stages:
               ArtifactName: TestResults_Windows_$(_BuildConfig)
             condition: failed()
 
+          - task: CopyFiles@2
+            displayName: 'Copy binlogs'
+            inputs:
+              SourceFolder: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/testsuite'
+              Contents: |
+                **/*.binlog
+              TargetFolder: '$(Build.ArtifactStagingDirectory)/binlogs'
+            condition: always()
+
+          - task: PublishBuildArtifacts@1
+            displayName: 'Publish integration tests binlogs'
+            inputs:
+              PathtoPublish: '$(Build.ArtifactStagingDirectory)/binlogs'
+              ArtifactName: Integration_Tests_Windows_Binlogs_$(_BuildConfig)
+            condition: always()
+
           # Upload code coverage to codecov.io
           - script: $(Build.SourcesDirectory)/.dotnet/dotnet msbuild -restore
               eng/CodeCoverage.proj
@@ -158,6 +174,22 @@ stages:
               ArtifactName: TestResults_Linux_$(_BuildConfig)
             condition: failed()
 
+          - task: CopyFiles@2
+            displayName: 'Copy binlogs'
+            inputs:
+              SourceFolder: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/testsuite'
+              Contents: |
+                **/*.binlog
+              TargetFolder: '$(Build.ArtifactStagingDirectory)/binlogs'
+            condition: always()
+
+          - task: PublishBuildArtifacts@1
+            displayName: 'Publish integration tests binlogs'
+            inputs:
+              PathtoPublish: '$(Build.ArtifactStagingDirectory)/binlogs'
+              ArtifactName: Integration_Tests_Linux_Binlogs_$(_BuildConfig)
+            condition: always()
+
       - job: MacOS
         timeoutInMinutes: 90
         pool:
@@ -196,3 +228,19 @@ stages:
               PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)'
               ArtifactName: TestResults_MacOs_$(_BuildConfig)
             condition: failed()
+
+          - task: CopyFiles@2
+            displayName: 'Copy binlogs'
+            inputs:
+              SourceFolder: '$(Build.SourcesDirectory)/artifacts/tmp/$(_BuildConfig)/testsuite'
+              Contents: |
+                **/*.binlog
+              TargetFolder: '$(Build.ArtifactStagingDirectory)/binlogs'
+            condition: always()
+
+          - task: PublishBuildArtifacts@1
+            displayName: 'Publish integration tests binlogs'
+            inputs:
+              PathtoPublish: '$(Build.ArtifactStagingDirectory)/binlogs'
+              ArtifactName: Integration_Tests_MacOS_Binlogs_$(_BuildConfig)
+            condition: always()
diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/DotnetCli.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/DotnetCli.cs
@@ -56,7 +56,8 @@ public static async Task<DotnetMuxerResult> RunAsync(
         int retryCount = 5,
         bool disableCodeCoverage = true,
         bool warnAsError = true,
-        bool suppressPreviewDotNetMessage = true)
+        bool suppressPreviewDotNetMessage = true,
+        [CallerMemberName] string callerMemberName = "")
     {
         await s_maxOutstandingCommands_semaphore.WaitAsync();
         try
@@ -107,15 +108,15 @@ public static async Task<DotnetMuxerResult> RunAsync(
 
             if (DoNotRetry)
             {
-                return await CallTheMuxerAsync(args, environmentVariables, workingDirectory, timeoutInSeconds, failIfReturnValueIsNotZero);
+                return await CallTheMuxerAsync(args, environmentVariables, workingDirectory, timeoutInSeconds, failIfReturnValueIsNotZero, callerMemberName);
             }
             else
             {
                 IEnumerable<TimeSpan> delay = Backoff.ExponentialBackoff(TimeSpan.FromSeconds(3), retryCount, factor: 1.5);
                 return await Policy
                     .Handle<Exception>()
                     .WaitAndRetryAsync(delay)
-                    .ExecuteAsync(async () => await CallTheMuxerAsync(args, environmentVariables, workingDirectory, timeoutInSeconds, failIfReturnValueIsNotZero));
+                    .ExecuteAsync(async () => await CallTheMuxerAsync(args, environmentVariables, workingDirectory, timeoutInSeconds, failIfReturnValueIsNotZero, callerMemberName));
             }
         }
         finally
@@ -124,20 +125,38 @@ public static async Task<DotnetMuxerResult> RunAsync(
         }
     }
 
+    private static bool IsDotNetTestWithExeOrDll(string args)
+        => args.StartsWith("test ", StringComparison.Ordinal) && (args.Contains(".dll") || args.Contains(".exe"));
+
     // Workaround NuGet issue https://github.com/NuGet/Home/issues/14064
-    private static async Task<DotnetMuxerResult> CallTheMuxerAsync(string args, Dictionary<string, string?> environmentVariables, string? workingDirectory, int timeoutInSeconds, bool failIfReturnValueIsNotZero)
+    private static async Task<DotnetMuxerResult> CallTheMuxerAsync(string args, Dictionary<string, string?> environmentVariables, string? workingDirectory, int timeoutInSeconds, bool failIfReturnValueIsNotZero, string binlogBaseFileName)
         => await Policy
             .Handle<InvalidOperationException>(ex => ex.Message.Contains("MSB4236"))
             .WaitAndRetryAsync(retryCount: 3, sleepDurationProvider: static _ => TimeSpan.FromSeconds(2))
-            .ExecuteAsync(async () => await CallTheMuxerCoreAsync(args, environmentVariables, workingDirectory, timeoutInSeconds, failIfReturnValueIsNotZero));
+            .ExecuteAsync(async () => await CallTheMuxerCoreAsync(args, environmentVariables, workingDirectory, timeoutInSeconds, failIfReturnValueIsNotZero, binlogBaseFileName));
 
-    private static async Task<DotnetMuxerResult> CallTheMuxerCoreAsync(string args, Dictionary<string, string?> environmentVariables, string? workingDirectory, int timeoutInSeconds, bool failIfReturnValueIsNotZero)
+    private static async Task<DotnetMuxerResult> CallTheMuxerCoreAsync(string args, Dictionary<string, string?> environmentVariables, string? workingDirectory, int timeoutInSeconds, bool failIfReturnValueIsNotZero, string binlogBaseFileName)
     {
         if (args.StartsWith("dotnet ", StringComparison.OrdinalIgnoreCase))
         {
             throw new InvalidOperationException("Command should not start with 'dotnet'");
         }
 
+        if (!args.Contains("-bl:") && !IsDotNetTestWithExeOrDll(args))
+        {
+            // We do this here rather than in the caller so that different retries produce different binlog file names.
+            string binlogFullPath = Path.Combine(TempDirectory.GetTestSuiteDirectory(), $"{binlogBaseFileName}-{DateTime.Now.Ticks}.binlog");
+            string binlogArg = $" -bl:\"{binlogFullPath}\"";
+            if (args.IndexOf("-- ", StringComparison.Ordinal) is int platformArgsIndex && platformArgsIndex > 0)
+            {
+                args = args.Insert(platformArgsIndex, binlogArg + " ");
+            }
+            else
+            {
+                args += binlogArg;
+            }
+        }
+
         using DotnetMuxer dotnet = new(environmentVariables);
         int exitCode = await dotnet.ExecuteAsync(args, workingDirectory, timeoutInSeconds);
 

diff --git a/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs b/test/Utilities/Microsoft.Testing.TestInfrastructure/TempDirectory.cs
@@ -141,6 +141,19 @@ public string CopyFile(string filePath)
         return destination;
     }
 
+    internal static string GetTestSuiteDirectory()
+    {
+        string currentDirectory = AppContext.BaseDirectory;
+        while (System.IO.Path.GetFileName(currentDirectory) != "artifacts" && currentDirectory is not null)
+        {
+            currentDirectory = System.IO.Path.GetDirectoryName(currentDirectory)!;
+        }
+
+        return currentDirectory is null
+            ? throw new InvalidOperationException("artifacts folder not found")
+            : System.IO.Path.Combine(currentDirectory, "tmp", Constants.BuildConfiguration, "testsuite");
+    }
+
     /// <summary>
     /// Creates an unique temporary directory.
     /// </summary>
@@ -162,7 +175,7 @@ internal static (string BaseDirectory, string FinalDirectory) CreateUniqueDirect
                 throw new InvalidOperationException("artifacts folder not found");
             }
 
-            string directoryPath = System.IO.Path.Combine(currentDirectory, "tmp", Constants.BuildConfiguration, "testsuite", RandomId.Next());
+            string directoryPath = System.IO.Path.Combine(GetTestSuiteDirectory(), RandomId.Next());
             Directory.CreateDirectory(directoryPath);
 
             string directoryBuildProps = System.IO.Path.Combine(directoryPath, "Directory.Build.props");