Core Tools OOP Host (#3802)

* this doesn't work

* this finally works now

* default should be oop host

* added edge cases

* fixing formatting

* using determineTargetFramework

* saving tests

* adding copy step for OOP

* fixing the webhost reference

* removing test

* setting host version in ps script

* update to validate worker versions script

* update versions for worker packages

* adding ToString

* validating worker version

* validate worker versions

* updating csproj to compile

* addressing pr feedback

* updating build steps

* adding build step

* fixing build step

* trying to get this working

* reverrting target runtimes

* updating tests

* adding dotnet info step

* adding changes

* removing extra test

* trying to specify architecture

* modifying tests to see if they work

* narrowing down to tests that are failing

* trying to see if it works with nobuild flag

* addressing pr feedback

* updating tests with latest logging

* addressing comments and marking flaky tests

* updating so that we are only using net8 framework

* pushing change for branch build

* adding single quotes

* reverting quotes

* adding code mirror fiile

* updating build step

* updatinng build steps

* updating build step

* adding step for dotnet publish

* set inprochost compilation system to diff value and skip flaky test

* updating public build pipeline to trigger

* adding extra changes for pipeline

* public build yml

* updating official build

* readd net8 build artifact step

* readding space back

* addressing initial comments

* adding explicit openTelemetry dlls

* simplifying logic of startHostAction

* addressing PR feedback

* start tests

* changing some of the tests back

* reverting test back to normal

* fixing spacing for csproj

* addressing PR feedback

* adding extra variable

* adding logic for edge case scenarios

* added edge cases tests

* removing extra line in node

* addressing comments

* moving validate host runtime to its own method

* forgot to add return statement

Author: aishwaryabh

build.ps1

@@ -23,11 +23,11 @@ if ($env:IntegrationBuildNumber)
         throw $errorMessage
     }
 
-    $buildCommand = { dotnet run --integrationTests }
+    $buildCommand = { dotnet run --integrationTests --skipArtifactGeneration}
 }
 else
 {
-    $buildCommand = { dotnet run --ci }
+    $buildCommand = { dotnet run --ci --skipArtifactGeneration}
 }
 
 Write-Host "Running $buildCommand"

build/BuildSteps.cs

@@ -102,7 +102,7 @@ public static void DotnetPack()
             Shell.Run("dotnet", $"pack {Settings.SrcProjectPath} " +
                                 $"/p:BuildNumber=\"{Settings.BuildNumber}\" " +
                                 $"/p:NoWorkers=\"true\" " +
-                                $"/p:TargetFramework=net6.0 " +  // without TargetFramework, the generated nuspec has incorrect path for the copy files operation.
+                                $"/p:TargetFramework=net8.0 " +  // without TargetFramework, the generated nuspec has incorrect path for the copy files operation.
                                 $"/p:CommitHash=\"{Settings.CommitId}\" " +
                                 (string.IsNullOrEmpty(Settings.IntegrationBuildNumber) ? string.Empty : $"/p:IntegrationBuildNumber=\"{Settings.IntegrationBuildNumber}\" ") +
                                 $"-o {outputPath} -c Release --no-build");
@@ -116,8 +116,7 @@ public static void DotnetPublishForZips()
                 var outputPath = Path.Combine(Settings.OutputDir, runtime);
                 var rid = GetRuntimeId(runtime);
 
-                ExecuteDotnetPublish(outputPath, rid, "net6.0", skipLaunchingNet8ChildProcess: isMinVersion);
-
+                ExecuteDotnetPublish(outputPath, rid, "net8.0", skipLaunchingNet8ChildProcess: isMinVersion);
                 if (isMinVersion)
                 {
                     RemoveLanguageWorkers(outputPath);
@@ -342,7 +341,7 @@ public static void Test()
 
             Environment.SetEnvironmentVariable("DURABLE_FUNCTION_PATH", Settings.DurableFolder);
 
-            Shell.Run("dotnet", $"test {Settings.TestProjectFile} -f net6.0 --logger trx");
+            Shell.Run("dotnet", $"test {Settings.TestProjectFile} -f net8.0 --logger trx");
         }
 
         public static void CopyBinariesToSign()
@@ -643,10 +642,10 @@ public static void DotnetPublishForNupkg()
             Shell.Run("dotnet", $"publish {Settings.ProjectFile} " +
                                 $"/p:BuildNumber=\"{Settings.BuildNumber}\" " +
                                 $"/p:NoWorkers=\"true\" " +
-                                $"/p:TargetFramework=net6.0 " +
+                                $"/p:TargetFramework=net8.0 " +
                                 $"/p:CommitHash=\"{Settings.CommitId}\" " +
                                 (string.IsNullOrEmpty(Settings.IntegrationBuildNumber) ? string.Empty : $"/p:IntegrationBuildNumber=\"{Settings.IntegrationBuildNumber}\" ") +
-                                $"-c Release -f net6.0");
+                                $"-c Release -f net8.0");
         }
 
         public static void GenerateSBOMManifestForNupkg()

build/Program.cs

@@ -29,11 +29,11 @@ static void Main(string[] args)
                 .Then(TestPreSignedArtifacts, skip: !args.Contains("--ci"))
                 .Then(CopyBinariesToSign, skip: !args.Contains("--ci"))
                 .Then(Test)
-                .Then(Zip)
+                .Then(Zip, skip: args.Contains("--skipArtifactGeneration"))
                 .Then(DotnetPublishForNupkg)
                 .Then(DotnetPack)
                 .Then(CreateIntegrationTestsBuildManifest, skip: !args.Contains("--integrationTests"))
-                .Then(UploadToStorage, skip: !args.Contains("--ci"))
+                .Then(UploadToStorage, skip: !args.Contains("--ci") || args.Contains("--skipArtifactGeneration"))
                 .Run();
         }
     }

build/Settings.cs

@@ -341,6 +341,16 @@ public class SignInfo
                 "Microsoft.OData.Edm.dll",
                 "Microsoft.Spatial.dll",
                 "Mono.Posix.NETStandard.dll",
+                "OpenTelemetry.Api.dll",
+                "OpenTelemetry.Api.ProviderBuilderExtensions.dll",
+                "OpenTelemetry.dll",
+                "OpenTelemetry.Exporter.Console.dll",
+                "OpenTelemetry.Exporter.OpenTelemetryProtocol.dll",
+                "OpenTelemetry.Extensions.Hosting.dll",
+                "OpenTelemetry.Instrumentation.AspNetCore.dll",
+                "OpenTelemetry.Instrumentation.Http.dll",
+                "OpenTelemetry.PersistentStorage.Abstractions.dll",
+                "OpenTelemetry.PersistentStorage.FileSystem.dll",
                 Path.Combine("tools", "python", "packapp", "distlib")
             };
         }

code-mirror.yml

@@ -8,6 +8,7 @@ trigger:
     - release_4.0
     - release_3.0
     - release_4.0_hotfix
+    - feature/*
 
 resources:
   repositories:

eng/ci/official-build.yml

@@ -15,6 +15,7 @@ trigger:
     include:
     - v4.x
     - release_4.0
+    - feature/*
 
 resources:
   repositories:

eng/ci/public-build.yml

@@ -13,13 +13,15 @@ pr:
     include:
     - v4.x
     - release_4.0
+    - feature/*
 
 trigger:
   batch: true
   branches:
     include:
     - v4.x
     - release_4.0
+    - feature/*
 
 resources:
   repositories:

pipelineUtilities.psm1

@@ -124,7 +124,7 @@ function Install-DotnetVersion($Version,$Channel) {
     if ($IsWindows) {
         & .\$installScript -InstallDir "$env:ProgramFiles/dotnet" -Channel $Channel -Version $Version
         # Installing .NET into x86 directory since the E2E App runs the tests on x86 and looks for the specified framework there
-        & .\$installScript -InstallDir "$env:ProgramFiles (x86)/dotnet" -Channel $Channel -Version $Version
+        & .\$installScript -InstallDir "$env:ProgramFiles (x86)/dotnet" -Channel $Channel -Version $Version -Architecture x86
     } else {
         bash ./$installScript --install-dir /usr/share/dotnet -c $Channel -v $Version
     }

src/Azure.Functions.Cli/Actions/HostActions/StartHostAction.cs

@@ -7,7 +7,6 @@
 using System.Net;
 using System.Runtime.InteropServices;
 using System.Security.Cryptography.X509Certificates;
-using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Azure.Functions.Cli.Common;
 using Azure.Functions.Cli.Diagnostics;
@@ -24,6 +23,7 @@
 using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.WebHost;
+using Microsoft.Azure.WebJobs.Script.Workers;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -40,10 +40,6 @@ internal class StartHostAction : BaseAction
     {
         private const int DefaultPort = 7071;
         private const int DefaultTimeout = 20;
-        private const string Net6FrameworkDescriptionPrefix = ".NET 6.0";
-        private const string WindowsExecutableName = "func.exe";
-        private const string LinuxExecutableName = "func";
-        private const string InProc8DirectoryName = "in-proc8";
         private readonly ISecretsManager _secretsManager;
         private readonly IProcessManager _processManager;
         private IConfigurationRoot _hostJsonConfig;
@@ -81,6 +77,8 @@ internal class StartHostAction : BaseAction
 
         public string JsonOutputFile { get; set; }
 
+        public string? HostRuntime { get; set; }
+
         public StartHostAction(ISecretsManager secretsManager, IProcessManager processManager)
         {
             _secretsManager = secretsManager;
@@ -176,6 +174,11 @@ public override ICommandLineParserResult ParseArgs(string[] args)
                .WithDescription("If provided, a path to the file that will be used to write the output when using --enable-json-output.")
                .Callback(jsonOutputFile => JsonOutputFile = jsonOutputFile);
 
+            Parser
+               .Setup<string>("runtime")
+               .WithDescription($"If provided, determines which version of the host to start. Allowed values are '{DotnetConstants.InProc6HostRuntime}', '{DotnetConstants.InProc8HostRuntime}', and 'default' (which runs the out-of-process host).")
+               .Callback(startHostFromRuntime => HostRuntime = startHostFromRuntime);
+
             var parserResult = base.ParseArgs(args);
             bool verboseLoggingArgExists = parserResult.UnMatchedOptions.Any(o => o.LongName.Equals("verbose", StringComparison.OrdinalIgnoreCase));
             // Input args do not contain --verbose flag
@@ -418,15 +421,14 @@ private async Task<JObject> GetLocalSettingsJsonAsJObjectAsync()
         public override async Task RunAsync()
         {
             await PreRunConditions();
-
-            var isCurrentProcessNet6Build = RuntimeInformation.FrameworkDescription.Contains(Net6FrameworkDescriptionPrefix);
-            if (isCurrentProcessNet6Build && ShouldLaunchInProcNet8AsChildProcess() && await IsInProcNet8Enabled())
+            var isVerbose = VerboseLogging.HasValue && VerboseLogging.Value;
+            
+            // Return if we have already started a child process
+            if (await ShouldExitAfterDeterminingHostRuntime(isVerbose))
             {
-                await StartInProc8AsChildProcessAsync();
                 return;
             }
 
-            var isVerbose = VerboseLogging.HasValue && VerboseLogging.Value;
             if (isVerbose || EnvironmentHelper.GetEnvironmentVariableAsBool(Constants.DisplayLogo))
             {
                 Utilities.PrintLogo();
@@ -480,31 +482,128 @@ public override async Task RunAsync()
             await runTask;
         }
 
-        private static string GetInProcNet8ExecutablePath()
+        private async Task<bool> ShouldExitAfterDeterminingHostRuntime(bool isVerbose)
+        {
+            var isInProc = GlobalCoreToolsSettings.CurrentWorkerRuntime == WorkerRuntime.dotnet;
+            
+            // If --runtime param is set, handle runtime param logic; otherwise we infer the host to launch on startup
+            if (HostRuntime is not null)
+            {
+                // Validate host runtime passed in
+                await ValidateHostRuntime(isInProc, isVerbose);
+
+                if (!string.Equals(HostRuntime, "default", StringComparison.OrdinalIgnoreCase))
+                {
+                    var isNet8InProcSpecified = string.Equals(HostRuntime, DotnetConstants.InProc8HostRuntime, StringComparison.OrdinalIgnoreCase);
+                    StartHostAsChildProcess(isNet8InProcSpecified);
+                    return true;
+                }
+            }
+            else if (isInProc)
+            {
+                // Infer host runtime by checking if .NET 8 is enabled
+                var shouldNet8InProcBeLaunched = await IsInProcNet8Enabled();
+
+                if (shouldNet8InProcBeLaunched)
+                {
+                    PrintVerboseForHostSelection(isVerbose, DotnetConstants.InProc8HostRuntime);
+                }
+                // Otherwise start .NET 6 child process since we are running an inproc app
+                else
+                {
+                    PrintVerboseForHostSelection(isVerbose, DotnetConstants.InProc6HostRuntime);
+                }
+
+                StartHostAsChildProcess(shouldNet8InProcBeLaunched);
+                return true;
+
+            }
+            // If the host runtime parameter is not set and the app is not in-proc, we should default to out-of-process host
+            PrintVerboseForHostSelection(isVerbose, "out-of-process");
+            return false;
+        }
+
+        private async Task ValidateHostRuntime(bool isInProc, bool isVerbose)
+        {
+            var isOutOfProc = GlobalCoreToolsSettings.CurrentWorkerRuntime == WorkerRuntime.dotnetIsolated;
+            var isInProc6 = string.Equals(HostRuntime, DotnetConstants.InProc6HostRuntime, StringComparison.OrdinalIgnoreCase);
+            var isInProc8 = string.Equals(HostRuntime, DotnetConstants.InProc8HostRuntime, StringComparison.OrdinalIgnoreCase);
+
+            // If we are running an .NET isolated app and the user specifies inproc6 or inproc8, throw an error
+            if (isOutOfProc && (isInProc8 || isInProc6))
+            {
+                throw new CliException($"The runtime host value passed in, {HostRuntime}, is not a valid host version for your project. The host runtime is only valid for the worker runtime {WorkerRuntime.dotnet}");
+            }
+            // If we are not running a .NET app and the user specifies inproc6 or inproc8, throw an error
+            if (!isOutOfProc && !isInProc && (isInProc8 || isInProc6))
+            {
+                throw new CliException($"The runtime host value passed in, {HostRuntime}, is not a valid host version for your project. The runtime is only valid for {WorkerRuntime.dotnetIsolated} and {WorkerRuntime.dotnet}");
+            }
+
+            if (string.Equals(HostRuntime, "default", StringComparison.OrdinalIgnoreCase))
+            {
+                if (isInProc)
+                {
+                    throw new CliException($"The runtime host value passed in, default, is not a valid host version for your project. For the default host runtime, the worker runtime must be set to {WorkerRuntime.dotnetIsolated}.");
+                }
+                PrintVerboseForHostSelection(isVerbose, "out-of-process");
+                return;
+            }
+            else if (isInProc8)
+            {
+                if (!await IsInProcNet8Enabled())
+                {
+                    throw new CliException($"The runtime host value passed in, {DotnetConstants.InProc8HostRuntime}, is not a valid host version for your project. For the {DotnetConstants.InProc8HostRuntime} runtime, the {Constants.FunctionsInProcNet8Enabled} variable must be set while running a .NET 8 in-proc app.");
+                }
+                PrintVerboseForHostSelection(isVerbose, DotnetConstants.InProc8HostRuntime);
+                return;
+            }
+            else if (isInProc6)
+            {
+                if (await IsInProcNet8Enabled())
+                {
+                    throw new CliException($"The runtime host value passed in, {DotnetConstants.InProc6HostRuntime}, is not a valid host version for your project. For the {DotnetConstants.InProc6HostRuntime} runtime, the {Constants.FunctionsInProcNet8Enabled} variable must not be set while running a .NET 6 in-proc app.");
+                }
+                PrintVerboseForHostSelection(isVerbose, DotnetConstants.InProc6HostRuntime);
+                return;
+            }
+            // Throw an exception if HostRuntime is set to none of the expected values
+            throw new CliException($"Invalid host runtime '{HostRuntime}'. Valid values are 'default', '{DotnetConstants.InProc8HostRuntime}', '{DotnetConstants.InProc6HostRuntime}'.");
+        }
+
+        private void PrintVerboseForHostSelection(bool isVerbose, string hostRuntime)
+        {
+            if (isVerbose)
+            {
+                ColoredConsole.WriteLine(VerboseColor($"Selected {hostRuntime} host."));
+            }
+        }
+
+        private static string GetInProcExecutablePath(bool isNet8)
         {
             var funcExecutableDirectory = Path.GetDirectoryName(typeof(StartHostAction).Assembly.Location)!;
-            var executableName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? WindowsExecutableName : LinuxExecutableName;
+            var executableName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? DotnetConstants.WindowsExecutableName : DotnetConstants.LinuxExecutableName;
 
-            return Path.Combine(funcExecutableDirectory, InProc8DirectoryName, executableName);
+            return Path.Combine(funcExecutableDirectory, (isNet8 ? DotnetConstants.InProc8DirectoryName: DotnetConstants.InProc6DirectoryName), executableName);
         }
 
-        private Task StartInProc8AsChildProcessAsync()
+        private void StartHostAsChildProcess(bool shouldStartNet8ChildProcess)
         {
             if (VerboseLogging == true)
             {
-                ColoredConsole.WriteLine(VerboseColor($"Starting child process for in-process model host."));
+                ColoredConsole.WriteLine(VerboseColor($"Starting child process for {(shouldStartNet8ChildProcess ? DotnetConstants.InProc8HostRuntime : DotnetConstants.InProc6HostRuntime)} model host."));
             }
 
             var commandLineArguments = string.Join(" ", Environment.GetCommandLineArgs().Skip(1));
             var tcs = new TaskCompletionSource();
 
-            var inProc8FuncExecutablePath = GetInProcNet8ExecutablePath();
+            var funcExecutablePath = GetInProcExecutablePath(isNet8: shouldStartNet8ChildProcess);
 
-            EnsureNet8FuncExecutablePresent(inProc8FuncExecutablePath);
+            EnsureFuncExecutablePresent(funcExecutablePath, shouldStartNet8ChildProcess);
 
-            var inprocNet8ChildProcessInfo = new ProcessStartInfo
+            var childProcessInfo = new ProcessStartInfo
             {
-                FileName = inProc8FuncExecutablePath,
+                FileName = funcExecutablePath,
                 Arguments = $"{commandLineArguments} --no-build",
                 WorkingDirectory = Environment.CurrentDirectory,
                 UseShellExecute = false,
@@ -515,7 +614,7 @@ private Task StartInProc8AsChildProcessAsync()
 
             try
             {
-                var childProcess = Process.Start(inprocNet8ChildProcessInfo);
+                var childProcess = Process.Start(childProcessInfo);
                 if (VerboseLogging == true)
                 {
                     ColoredConsole.WriteLine(VerboseColor($"Started child process with ID: {childProcess.Id}"));
@@ -546,23 +645,21 @@ private Task StartInProc8AsChildProcessAsync()
             }
             catch (Exception ex)
             {
-                throw new CliException($"Failed to start the in-process model host. {ex.Message}");
+                throw new CliException($"Failed to start the {(shouldStartNet8ChildProcess ? DotnetConstants.InProc8HostRuntime : DotnetConstants.InProc6HostRuntime)} model host. {ex.Message}");
             }
-
-            return tcs.Task;
         }
 
-        private void EnsureNet8FuncExecutablePresent(string inProc8FuncExecutablePath)
+        private void EnsureFuncExecutablePresent(string funcExecutablePath, bool isInProcNet8)
         {
-            bool net8ExeExist = File.Exists(inProc8FuncExecutablePath);
+            bool funcExeExist = File.Exists(funcExecutablePath);
             if (VerboseLogging == true)
             {
-                ColoredConsole.WriteLine(VerboseColor($"{inProc8FuncExecutablePath} {(net8ExeExist ? "present" : "not present")} "));
+                ColoredConsole.WriteLine(VerboseColor($"{funcExecutablePath} {(funcExeExist ? "present" : "not present")} "));
             }
 
-            if (!net8ExeExist)
+            if (!funcExeExist)
             {
-                throw new CliException($"Failed to locate the in-process model host at {inProc8FuncExecutablePath}");
+                throw new CliException($"Failed to locate the {(isInProcNet8 ? DotnetConstants.InProc8HostRuntime : DotnetConstants.InProc6HostRuntime)} model host at {funcExecutablePath}");
             }
         }
 

src/Azure.Functions.Cli/Actions/LocalActions/CreateFunctionAction.cs

@@ -13,7 +13,6 @@
 using Azure.Functions.Cli.Interfaces;
 using Colors.Net;
 using Fclp;
-using ImTools;
 using Microsoft.Azure.AppService.Proxy.Common.Context;
 using Microsoft.Azure.WebJobs.Extensions.Http;
 using Microsoft.Azure.WebJobs.Script;