Application Insights: CSP fixes for client-side and RBAC authentication for import web-job (#75)

Author: sambetts

reports/Copilot/Copilot Agent Interactions.pbit

@@ -74,11 +74,11 @@
   <ItemGroup>
     <Compile Include="BaseInstallProcessClasses.cs" />
     <Compile Include="InstallerLogs.cs" />
-    <Compile Include="InstallerTasks\JobTasks\CreateOrUpdateRunbookConfigureTask.cs" />
     <Compile Include="InstallerTasks\JobTasks\ProfilingScriptsUploadToBlobStorageTask.cs" />
     <Compile Include="InstallerTasks\JobTasks\AutomationAccountTask.cs" />
     <Compile Include="ConfigureAzureComponentsTasks.cs" />
     <Compile Include="InstallerTasks\JobTasks\RunbookCreateOrUpdateTasks.cs" />
+    <Compile Include="InstallerTasks\ResourceSecurityInstallJob.cs" />
     <Compile Include="InstallerTasks\RunbooksInstallJob.cs" />
     <Compile Include="Models\AzStorageConnectionInfo.cs" />
     <Compile Include="SolutionUninstaller.cs" />

src/AnalyticsEngine/App.ControlPanel.Engine/App.ControlPanel.Engine.csproj

@@ -38,7 +38,7 @@ public ConfigureAzureComponentsTasks(SolutionInstallConfig config, ILogger logge
         /// Install configure & software on App Service, update target DB. 
         /// </summary>
         public async Task RunPostCreatePaaSTasks(WebSiteResource webApp, DatabasePaaSInfo dbInfo, StorageAccountResource storage, AutomationAccountResource automationAccount,
-            AppInsightsInfoWithApiAccess appInsights,
+            AppInsightsInfo appInsights,
             RedisResource redis, CognitiveServicesInfo cognitiveServicesInfo,
             KeyVaultResource keyVault, string serviceBusConnectionString, SubscriptionResource subscription)
         {
@@ -130,7 +130,7 @@ async Task ConfigureWebApp(WebSiteResource webApp, DatabasePaaSInfo backendInfo,
             StorageAccountResource storage,
             RedisResource redis,
             CognitiveServicesInfo cognitiveServicesInfo,
-            AppInsightsInfoWithApiAccess appInsights, string serviceBusConnectionString, KeyVaultResource keyVault)
+            AppInsightsInfo appInsights, string serviceBusConnectionString, KeyVaultResource keyVault)
         {
             // App settings
             var url = $"https://{webApp.Data.HostNames.First()}/";
@@ -148,14 +148,6 @@ async Task ConfigureWebApp(WebSiteResource webApp, DatabasePaaSInfo backendInfo,
             {
                 appSettings.Properties.Add("AppInsightsConnectionString", appInsights.ConnectionString);
             }
-            if (!string.IsNullOrEmpty(appInsights?.ApiKey))
-            {
-                appSettings.Properties.Add("AppInsightsApiKey", appInsights.ApiKey);
-            }
-            if (!string.IsNullOrEmpty(appInsights?.AppId))
-            {
-                appSettings.Properties.Add("AppInsightsAppId", appInsights.AppId);
-            }
 
             if (this.Config.CognitiveServicesEnabled)
             {

src/AnalyticsEngine/App.ControlPanel.Engine/ConfigureAzureComponentsTasks.cs

@@ -21,6 +21,7 @@ namespace App.ControlPanel.Engine.InstallerTasks
     /// </summary>
     public class AzurePaaSInstallJob : BaseAnalyticsSolutionInstallJob
     {
+        private readonly GetOrCreateResourceGroupTask _rgCreateTask;
         private readonly AutomationAccountTask _automationAccountTask;
 
         private readonly SqlServerTask _sqlServerTask;
@@ -37,14 +38,19 @@ public class AzurePaaSInstallJob : BaseAnalyticsSolutionInstallJob
 
         private readonly LogAnalyticsInstallTask _logAnalyticsInstallTask;
         private readonly AppInsightsInstallTask _appInsightsInstallTask;
-        private readonly AppInsightsConfigureApiTask _appInsightsConfigureApiTask;
         private readonly TextAnalyticsInstallTask _cognitiveServicesInstallTask;
 
         /// <summary>
         /// Add tasks in order for execution, some being chained
         /// </summary>
         public AzurePaaSInstallJob(ILogger logger, SolutionInstallConfig config, SubscriptionResource subscription) : base(logger, config, subscription)
         {
+
+            var tagDic = config.Tags.ToDictionary();
+
+            _rgCreateTask = new GetOrCreateResourceGroupTask(TaskConfig.GetConfigForName(config.ResourceGroupName), logger, Location, tagDic, subscription);
+            this.AddTask(_rgCreateTask);
+
             // Performance levels
             var appPerfTier = AppServicePlanTask.PERF_TIER_BASIC1;
             var sqlPerfTier = SqlDatabaseTask.PERF_TIER_BASIC;
@@ -55,8 +61,6 @@ public AzurePaaSInstallJob(ILogger logger, SolutionInstallConfig config, Subscri
                 appPerfTier = AppServicePlanTask.PERF_TIER_BASIC2;
             }
 
-            var tagDic = config.Tags.ToDictionary();
-
             // Web 
             var appServicePlanConfig = TaskConfig.GetConfigForName(config.AppServiceWebAppName).AddSetting(AppServicePlanTask.CONFIG_KEY_PERF_TIER, appPerfTier);
             _appServicePlanTask = new AppServicePlanTask(appServicePlanConfig, logger, Location, tagDic);
@@ -132,8 +136,7 @@ public AzurePaaSInstallJob(ILogger logger, SolutionInstallConfig config, Subscri
             var creds = new ClientSecretCredential(config.InstallerAccount.DirectoryId, config.InstallerAccount.ClientId, config.InstallerAccount.Secret);
             var appInsightsConfig = TaskConfig.GetConfigForName(config.AppInsightsName);
             _appInsightsInstallTask = new AppInsightsInstallTask(appInsightsConfig, logger, Location, tagDic, ResourceGroupName, config.Subscription.SubId, creds);
-            _appInsightsConfigureApiTask = new AppInsightsConfigureApiTask(appInsightsConfig, logger, Location, creds, _config.Subscription.SubId, ResourceGroupName);
-            this.AddTask(_logAnalyticsInstallTask, _appInsightsInstallTask, _appInsightsConfigureApiTask);
+            this.AddTask(_logAnalyticsInstallTask, _appInsightsInstallTask);
 
             // Cognitive
             if (config.CognitiveServicesEnabled)
@@ -166,7 +169,7 @@ public AzurePaaSInstallJob(ILogger logger, SolutionInstallConfig config, Subscri
         public DatabasePaaSInfo DatabasePaaSInfo => new DatabasePaaSInfo(CreatedSqlServer, CreatedSqlDatabase, _config);
         public RedisResource Redis => GetTaskResult<RedisResource>(_redisTask);
         public StorageAccountResource Storage => GetTaskResult<StorageAccountResource>(_storageAccountInstallTask);
-        public AppInsightsInfoWithApiAccess AppInsights => GetTaskResult<AppInsightsInfoWithApiAccess>(_appInsightsConfigureApiTask);
+        public AppInsightsInfo AppInsights => GetTaskResult<AppInsightsInfo>(_appInsightsInstallTask);
         public CognitiveServicesInfo CognitiveServicesInfo => _cognitiveServicesInstallTask != null ? GetTaskResult<CognitiveServicesInfo>(_cognitiveServicesInstallTask) : new CognitiveServicesInfo();
         public ServiceBusQueueResourceWithConnectionString SBQueueWithConnectionString => GetTaskResult<ServiceBusQueueResourceWithConnectionString>(_serviceBusQueueWithPolicyInstallTask);
         public KeyVaultResource KeyVault => GetTaskResult<KeyVaultResource>(_keyVaultTask);

src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/AzurePaaSInstallJob.cs

@@ -0,0 +1,34 @@
+using Azure.ResourceManager.Authorization;
+using Azure.ResourceManager.Resources;
+using CloudInstallEngine;
+using CloudInstallEngine.Azure.InstallTasks;
+using Common.Entities.Installer;
+using Microsoft.Extensions.Logging;
+
+namespace App.ControlPanel.Engine.InstallerTasks
+{
+    /// <summary>
+    /// Secures resources in the resource group by assigning RBAC roles.
+    /// </summary>
+    public class ResourceSecurityInstallJob : BaseAnalyticsSolutionInstallJob
+    {
+        private readonly RoleAssignmentTask _appInsightsReaderRoleTask;
+
+        public ResourceSecurityInstallJob(ILogger logger, SolutionInstallConfig config, SubscriptionResource subscription) : base(logger, config, subscription)
+        {
+            var tagDic = config.Tags.ToDictionary();
+
+            // Assign Reader role to the runtime account on the resource group (covers App Insights and all resources)
+            var readerRoleConfig = TaskConfig.GetConfigForPropAndVal(RoleAssignmentTask.CONFIG_KEY_ROLE_NAME, "Reader")
+                .AddSetting(RoleAssignmentTask.CONFIG_KEY_CLIENT_ID, config.RuntimeAccountOffice365.ClientId)
+                .AddSetting(RoleAssignmentTask.CONFIG_KEY_CLIENT_SECRET, config.RuntimeAccountOffice365.Secret)
+                .AddSetting(RoleAssignmentTask.CONFIG_KEY_TENANT_ID, config.RuntimeAccountOffice365.DirectoryId)
+                .AddSetting(RoleAssignmentTask.CONFIG_KEY_PRINCIPAL_TYPE, "ServicePrincipal");
+
+            _appInsightsReaderRoleTask = new RoleAssignmentTask(readerRoleConfig, logger, Location, tagDic);
+            this.AddTask(_appInsightsReaderRoleTask);
+        }
+
+        public RoleAssignmentResource AppInsightsReaderRole => GetTaskResult<RoleAssignmentResource>(_appInsightsReaderRoleTask);
+    }
+}

src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/JobTasks/CreateOrUpdateRunbookConfigureTask.cs

@@ -48,6 +48,17 @@ public async Task InstallOrUpdate()
                 var azureBackeEndCreationJob = new AzurePaaSInstallJob(_logger, Config, azureSub);
                 await azureBackeEndCreationJob.Install();
 
+                // Secure resources with RBAC roles
+                try
+                {
+                    var resourceSecurityJob = new ResourceSecurityInstallJob(_logger, Config, azureSub);
+                    await resourceSecurityJob.Install();
+                }
+                catch (Exception ex)
+                {
+                    _logger.LogError($"Failed to assign RBAC roles: {ex.Message}. Continuing installation...");
+                }
+
                 // Run stuff now everything in Azure is created
                 var tasks = new ConfigureAzureComponentsTasks(Config, _logger, _ftpConfig, InstalledByUsername, _softwareConfig, _configPassword);
                 await tasks.RunPostCreatePaaSTasks(

src/AnalyticsEngine/App.ControlPanel.Engine/InstallerTasks/ResourceSecurityInstallJob.cs

@@ -10,3 +10,8 @@ The scripts are run after the database schema has been updated with Entity Frame
 
 Note: SQL server doesn't normally support "```GO```" statements in stored procedures ("GO" is a SQL Server Management Studio thing to separate scripts in a single file), so you should avoid using them in your scripts here. 
 You *can* use the "GO" statement and the installer will split each segment by "GO" and then execute each segment (bit of a hack), but you should put the scripts in separate files or remove "GO" from any script here.
+
+## Profiling Extensions
+These scripts setup the SQL Server profiling extensions but are run by PS scripts found in /src/AnalyticsEngine/WebJob.Office365ActivityImporter/AutomationPS/ProfilingJobs/.
+
+The main logic is in "usp_CompileWeekly" - a stored procedure that compiles weekly profiling data from daily profiling data.

src/AnalyticsEngine/App.ControlPanel.Engine/SolutionInstaller.cs

@@ -182,12 +182,6 @@
     <Compile Include="MainForm.Designer.cs">
       <DependentUpon>MainForm.cs</DependentUpon>
     </Compile>
-    <Compile Include="NewSaltForm.cs">
-      <SubType>Form</SubType>
-    </Compile>
-    <Compile Include="NewSaltForm.Designer.cs">
-      <DependentUpon>NewSaltForm.cs</DependentUpon>
-    </Compile>
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="ProxyConfigForm.cs">
@@ -248,9 +242,6 @@
     <EmbeddedResource Include="MainForm.resx">
       <DependentUpon>MainForm.cs</DependentUpon>
     </EmbeddedResource>
-    <EmbeddedResource Include="NewSaltForm.resx">
-      <DependentUpon>NewSaltForm.cs</DependentUpon>
-    </EmbeddedResource>
     <EmbeddedResource Include="Properties\Resources.resx">
       <Generator>ResXFileCodeGenerator</Generator>
       <LastGenOutput>Resources.Designer.cs</LastGenOutput>