deploy-container-agents.yml

# This pipeline is for CI/CD
parameters:
- name: workspace
  displayName: Terraform Workspace
  type: string
  default: ci
  values:
  - ci
  - ci1
  - ci2
  - ci3
  - cd
  - cd1
  - cd2
  - cd3
  - cd3
  - test
  - test1
  - test2
  - test3
- name: buildImage
  displayName: Build agent image
  type: boolean
  default: true
- name: asContainerJob
  displayName: Run as container job
  type: boolean
  default: true
- name: clear
  displayName: Clear Terraform state
  type: boolean
  default: false
- name: unpinTerraform
  displayName: Unpin Terraform version
  type: boolean
  default: false
- name: containerEngine
  displayName: Container Engine (ACA/AKS)
  type: string
  default: Both
  values:
  - None
  - Container App
  - Kubernetes 
  - Both
- name: networkType
  displayName: Network type
  type: string
  default: NATGateway
  values:
  - None
  - NoGateway
  - NATGateway
  - Firewall 
- name: deployBastion
  displayName: Deploy bastion host
  type: boolean
  default: false
- name: retryCount
  displayName: Task retry count
  type: number
  default: 3
- name: destroy
  displayName: Destroy Infrastructure
  type: string
  default: 'If no prior resources exist'
  values:
  - Always
  - 'If no prior resources exist'
  - Never 
  - 'On failure'
  - 'On success'

name: $(Date:yyyyMMdd)$(Rev:.r)-$(Build.DefinitionVersion)-$(SourceBranchName)-${{ parameters.workspace }}-$(Build.BuildId)

trigger: none

pr:
  autoCancel: false
  branches:
    include:
    - '*'
  drafts: false
  paths:
    exclude:
    - '.devcontainer/**'  
    - 'visuals/**'  
    - '*.md'  

schedules:
- cron: '0 0 * * Mon,Thu'
  displayName: 'Bi-weekly build (UTC)'
  # Run if there are no changes
  always: 'true'
  branches:
    include:
    - main

variables:
- name: AZURE_CORE_ONLY_SHOW_ERRORS
  value: 'true'
- name: AZURE_EXTENSION_USE_DYNAMIC_INSTALL
  value: 'yes_without_prompt'
- name: 'correlationCapability'
  value: '$(System.CollectionUri)$(System.TeamProject)#$(Build.DefinitionName)-$(Build.BuildId)#$(TF_WORKSPACE)'
- name: date
  value: $[format('{0:yyyyMMdd}', pipeline.startTime)]
- name: HELM_CONFIG_HOME
  value: $(Build.SourcesDirectory)/helm
- name: 'imageRepository'
  value: 'pipelineagent/ubuntu-agent'
- ${{ if and(parameters.buildImage,or(eq(parameters.destroy, 'Always'),eq(parameters.destroy, 'On success'),ne(variables['Build.Reason'], 'Manual'))) }}:
    # CI: Use image produced in same build
  - name: 'imageTag'
    value: $(Build.BuildId)
  - name: 'TF_VAR_container_image'
    value: 'pipelineagent/ubuntu-agent:$(Build.BuildId)'
- ${{ else }}:
    # CD: Use latest image
  - name: 'imageTag'
    value: 'latest'
  - name: 'TF_VAR_container_image'
    value: 'pipelineagent/ubuntu-agent:latest'
- name: 'jobTimeOutMinutes'
  value: 60
- name: 'pool'
  value: 'Azure Pipelines'
- name: 'identityScriptDirectory'
  value: '$(Build.SourcesDirectory)/scripts/identity/scripts/azure-devops'
- name: 'scriptDirectory'
  value: '$(Build.SourcesDirectory)/scripts'
- name: 'terraformArtifactName'
  value: 'terraform-$(System.PlanId)'
- name: 'terraformDirectory'
  value: '$(Build.SourcesDirectory)/terraform'
- name: 'TF_IN_AUTOMATION'
  value: 'true'
- name: 'TF_INPUT'
  value: 0
- name: 'TF_VAR_aca_agent_pool_name'
  value: 'aca-container-agents-${{ parameters.workspace }}'
- name: 'TF_VAR_aks_agent_pool_name'
  value: 'aks-container-agents-${{ parameters.workspace }}'
# - name: 'TF_VAR_aks_private_cluster_enabled'
#   value: ${{ lower(eq(parameters.networkType, 'Firewall')) }}
- name: 'TF_VAR_configure_access_control'
  value: 'false' # Assume Azure Service Connection identity is not in Azure Owner role
- name: 'TF_VAR_create_agent_pools'
  value: ${{ lower(ne(parameters.workspace, 'ci')) }}
- name: 'TF_VAR_deploy_aks'
  value: ${{ lower(or(eq(parameters.containerEngine, 'Kubernetes'),eq(parameters.containerEngine, 'Both'))) }}
- name: 'TF_VAR_deploy_bastion'
  value: '${{ lower(parameters.deployBastion) }}'
- name: 'TF_VAR_deploy_container_app'
  value: ${{ lower(or(eq(parameters.containerEngine, 'Container App'),eq(parameters.containerEngine, 'Both'))) }}
- name: 'TF_VAR_deploy_network'
  value: ${{ lower(ne(parameters.networkType, 'None')) }}
- name: 'TF_VAR_devops_project'
  value: '$(System.TeamProject)'
- name: 'TF_VAR_devops_url'
  value: '$(System.CollectionUri)'
- ${{ if ne(parameters.networkType, 'None') }}:
  - name: 'TF_VAR_gateway_type'
    value: '${{ parameters.networkType }}'
- name: 'TF_VAR_environment_variables'
  value: '{PIPELINE_DEMO_JOB_CAPABILITY_ACA="$(correlationCapability)"}'
- name: 'TF_VAR_repository'
  value: 'azure-pipelines-container-agent'
- name: 'TF_VAR_resource_suffix'
  value: '$(Build.BuildId)'
- name: 'TF_VAR_run_id'
  value: '$(Build.BuildId)'
- name: 'TF_WORKSPACE'
  value: '${{ parameters.workspace }}'
- name: 'toolsImageFQName'
  ${{ if parameters.buildImage }}:
    value: $(containerRegistry).azurecr.io/pipelineagent/ubuntu-agent-tools:$(Build.BuildId)
  ${{ else }}:
    value: $(containerRegistry).azurecr.io/pipelineagent/ubuntu-agent-tools:latest
- name: 'vmImage'
  value: 'ubuntu-latest'
- group: 'build-container-agent-image' # shared with build-image.yml
  # should contain:
  # containerRegistry -> name of Container Registry Service Connection
- group: 'pipeline-container-agents-cd' # shared with terraform-ci.yml
  # should contain:
  # subscriptionConnection -> name of Azure Registry Service Connection
  # TF_STATE_RESOURCE_GROUP_NAME, TF_STATE_STORAGE_ACCOUNT_NAME, TF_STATE_CONTAINER_NAME -> Terraform azurerm backend
  # (See https://developer.hashicorp.com/terraform/language/settings/backends/azurerm)
  # TF_VAR_agent_identity_resource_id -> Resource id of pre-created User-assigned Managed Identity used to access Container Registry
  # TF_VAR_container_registry_id -> Resource id of pre-created Container Registry
  # TF_VAR_devops_pat -> Personal Access Token used by agent to register itself, requires Agent Pools read & manage scope.

jobs:    
- ${{ if parameters.buildImage }}:
  - template: build-image-template.yml
    parameters:
      name: buildImage
      displayName: Build images (BuildId)
      imageTag: latest

- template: image-info-template.yml
  parameters:
    name: testImage
    ${{ if parameters.buildImage }}:
      dependsOn: 
      - buildImage
    ${{ if and(parameters.buildImage,or(eq(parameters.destroy, 'Always'),eq(parameters.destroy, 'On success'),ne(variables['Build.Reason'], 'Manual'))) }}:
      # CI: Use image produced in same build
      displayName: Test tools image (BuildId)
      imageTag: $(Build.BuildId)
    ${{ else }}:
      # CD: Use latest image
      displayName: Test tools image (latest)
      imageTag: latest

- job: provision
  ${{ if parameters.buildImage }}:
    dependsOn: 
    - buildImage
    - testImage
  ${{ if eq(parameters.containerEngine, 'Container App') }}:
    displayName: 'Provision Container App infrastructure'
  ${{ if eq(parameters.containerEngine, 'Kubernetes') }}:
    displayName: 'Provision Kubernetes (AKS) infrastructure'
  ${{ if eq(parameters.containerEngine, 'Both') }}:
    displayName: 'Provision Container App and Kubernetes (AKS) infrastructure'
  ${{ if eq(parameters.containerEngine, 'None') }}:
    displayName: 'Provision infrastructure'
  timeoutInMinutes: $[ variables['jobTimeOutMinutes'] ]

  pool:
    name: $[ coalesce(variables.pool,'Azure Pipelines') ]
    vmImage: $[ coalesce(variables.vmImage,'ubuntu-latest') ]
  ${{ if parameters.asContainerJob }}:
    container:
      image: $(toolsImageFQName)
      endpoint: containerEndpoint

  steps:
  - checkout: self
    submodules: true

  - ${{ if not(parameters.unpinTerraform) }}:
    - pwsh: |
        $terraformVersion = (Get-Content .terraform-version)
        Write-Host "##vso[task.setvariable variable=version;isOutput=true]${terraformVersion}"
        Copy-Item backend.tf.sample backend.tf
      name: terraformConfig
      displayName: 'Prepare Terraform config (pinned)'
      workingDirectory: '$(terraformDirectory)'

  - ${{ if parameters.unpinTerraform }}:
    - pwsh: |
        (Get-Content ./terraform.tf) -replace "required_version *= `" *(~>|=) +",'required_version = ">= ' | Out-File terraform.tf
        Get-Content ./terraform.tf
        Write-Host "##vso[task.setvariable variable=version;isOutput=true]latest"
        Copy-Item backend.tf.sample backend.tf
      name: terraformConfig
      displayName: 'Prepare Terraform config (unpinned/latest)'
      workingDirectory: '$(terraformDirectory)'

  - task: TerraformInstaller@0
    displayName: 'Install Terraform'
    inputs:
      terraformVersion: '$(terraformConfig.version)'

  - ${{ if parameters.unpinTerraform }}:
    # Unpin version e.g. "= 2.56" -> "~> 2.56"
    - pwsh: |
        (Get-Content ./terraform.tf) -replace " = `" *= +",' = "~> ' | Out-File terraform.tf
        Get-Content ./terraform.tf
        if (Test-Path .terraform.lock.hcl) {
          Remove-Item .terraform.lock.hcl -Force
        }
      displayName: 'Unpin Terraform provider versions'
      workingDirectory: '$(terraformDirectory)'

  - task: AzureCLI@2
    displayName: 'Terraform init'
    inputs:
      azureSubscription: '$(subscriptionConnection)'
      scriptType: pscore
      scriptLocation: inlineScript
      inlineScript: |
        $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1
        ./deploy.ps1 -Init
      addSpnToEnvironment: true
      useGlobalConfig: true
      failOnStandardError: true
      workingDirectory: '$(scriptDirectory)'

  - task: AzureCLI@2
    name: terraformVariables
    displayName: 'Prepare Terraform variables'
    inputs:
      azureSubscription: '$(subscriptionConnection)'
      scriptType: pscore
      scriptLocation: inlineScript
      inlineScript: |
        $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1

        if ($${{ not(parameters.clear) }} -and (!((terraform output resource_suffix 2>&1) -match "Warning"))) {
          $env:TF_VAR_RESOURCE_SUFFIX = $null
          $env:TF_VAR_resource_suffix = "$(terraform output -raw resource_suffix 2>$null)"
        }

        # Generate AzDO token if no PAT is set
        if (!$env:TF_VAR_devops_pat) {
          az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 `
                                      --output json `
                                      | ConvertFrom-Json `
                                      | Set-Variable bearerToken
          $env:TF_VAR_devops_pat = $bearerToken.accessToken
          $expiresOn = [DateTime]::Parse($bearerToken.expiresOn)
          $warningMessage = "AzDO token expires on $expiresOn (in $(($expiresOn - [DateTime]::Now).TotalMinutes) minutes). Agent creation will fail after that."
          Write-Warning $warningMessage
          Write-Host "##vso[task.logissue type=warning]${warningMessage}"
        }
        Write-Host "##vso[task.setvariable variable=azdoToken;isOutput=true;issecret=true]${env:TF_VAR_devops_pat}"
          
        # List environment variables
        Get-ChildItem -Path Env: `
                      -Recurse `
                      -Include ALTERNATE_*,ARM_*,AZURE_*,PIPELINE_DEMO_*,SYSTEM_*,TF_* `
                      | ForEach-Object { 
                          if ($_.Name -match 'SECRET|TOKEN|PAT') {
                            $_.Value = '***'
                          } 
                          $_
                        } `
                      | Sort-Object -Property Name `
                      | Format-Table -HideTableHeader

        # Convert uppercased Terraform environment variables to .auto.tfvars file
        foreach ($tfvar in $(Get-ChildItem -Path Env: -Recurse -Include TF_VAR_*)) {
          $terraformVariableName = $tfvar.Name.Substring(7).ToLowerInvariant()
          $terraformVariableValue = $tfVar.Value

          if ($terraformVariableValue -imatch "^\W*(true|false|\[[^\]]*\]|\{[^\}]*\})\W*$") {
            # Boolean or List, write as-is
            Write-Output "${terraformVariableName} = ${terraformVariableValue}" | Out-File ci.auto.tfvars -Append -Force
          } else {
            Write-Output "${terraformVariableName} = `"${terraformVariableValue}`"" | Out-File ci.auto.tfvars -Append -Force
          }
        } 
        Write-Host "Contents of ci.auto.tfvars:"
        Get-Content ci.auto.tfvars | Sort-Object `
                                   | ForEach-Object { 
                                       if ($_.Name -match 'SECRET|TOKEN|PAT') {
                                         $_.Value = '***'
                                       } 
                                       $_
                                     } 
      addSpnToEnvironment: true
      useGlobalConfig: true
      failOnStandardError: true
      workingDirectory: '$(terraformDirectory)'

  - publish: $(terraformDirectory)
    displayName: 'Publish Terraform directory'
    artifact: $(terraformArtifactName)

  - ${{ if eq(parameters.clear, true) }}:
    - task: AzureCLI@2
      name: clear
      displayName: 'Clear Terraform state'
      inputs:
        azureSubscription: '$(subscriptionConnection)'
        scriptType: pscore
        scriptLocation: inlineScript
        inlineScript: |
          $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1

          $terraformState = (terraform state pull | ConvertFrom-Json)
          if ($terraformState.resources) {
            Write-Host "Clearing Terraform state in workspace ${env:TF_WORKSPACE}..."
            $terraformState.outputs = New-Object PSObject # Empty output
            $terraformState.resources = @() # No resources
            $terraformState.serial++
            $terraformState | ConvertTo-Json | terraform state push -
          } else {
            Write-Host "No resources in Terraform state in workspace ${env:TF_WORKSPACE}..."
          }
          terraform state pull 
        addSpnToEnvironment: true
        useGlobalConfig: true
        failOnStandardError: true
        workingDirectory: '$(terraformDirectory)'

  - task: AzureCLI@2
    displayName: 'Terraform plan & apply'
    retryCountOnTaskFailure: ${{ parameters.retryCount }}
    inputs:
      azureSubscription: '$(subscriptionConnection)'
      scriptType: pscore
      scriptLocation: inlineScript
      inlineScript: |
        $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1

        terraform -chdir='$(terraformDirectory)' show -json | ConvertFrom-Json -AsHashtable | Set-Variable tfState 
        $priorResourcesExist = $tfState.ContainsKey('values')
        if ($priorResourcesExist) {
          Write-Warning "Prior resources exist in workspace '${env:TF_WORKSPACE}'"
          Write-Host "##vso[task.logissue type=warning]Prior resources exist in workspace '${env:TF_WORKSPACE}'"
        } else {
          Write-Host "No prior resources exist in workspace '${env:TF_WORKSPACE}'"
        }
        Write-Host "##vso[task.setvariable variable=priorResourcesExist;isOutput=true]$($priorResourcesExist.ToString().ToLower())"

        Write-Host ""
        ./deploy.ps1 -apply -force
      addSpnToEnvironment: true
      useGlobalConfig: true
      failOnStandardError: true
      workingDirectory: '$(scriptDirectory)'      

  - task: AzureCLI@2
    displayName: 'Terraform output'
    name: terraform
    retryCountOnTaskFailure: ${{ parameters.retryCount }}
    inputs:
      azureSubscription: '$(subscriptionConnection)'
      scriptType: pscore
      scriptLocation: inlineScript
      inlineScript: |
        $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1

        Write-Host ""
        ./deploy.ps1 -output -debug
      addSpnToEnvironment: true
      useGlobalConfig: true
      failOnStandardError: true
      workingDirectory: '$(scriptDirectory)'      

  - ${{ if or(eq(parameters.containerEngine, 'Kubernetes'),eq(parameters.containerEngine, 'Both')) }}:
    - task: AzureCLI@2
      displayName: 'Wait for AKS'
      retryCountOnTaskFailure: ${{ parameters.retryCount }}
      inputs:
        azureSubscription: '$(subscriptionConnection)'
        scriptType: pscore
        scriptLocation: inlineScript
        inlineScript: |
          az aks wait  -n $(terraform.aks_name) -g $(terraform.resource_group_name) --created
          az aks wait  -n $(terraform.aks_name) -g $(terraform.resource_group_name) --updated
        useGlobalConfig: true
        failOnStandardError: true
        workingDirectory: '$(scriptDirectory)'      

  # https://keda.sh/docs/deploy/#helm
  - ${{ if or(eq(parameters.containerEngine, 'Kubernetes'),eq(parameters.containerEngine, 'Both')) }}:
    - task: HelmDeploy@0
      displayName: "Add KEDA Helm repo"
      retryCountOnTaskFailure: ${{ parameters.retryCount }}
      continueOnError: true # Treat failure as warning during destroy, we will clean up anyway
      inputs:
        connectionType: 'Azure Resource Manager'
        azureSubscription: '$(subscriptionConnection)'
        azureResourceGroup: '$(terraform.resource_group_name)'
        kubernetesCluster: '$(terraform.aks_name)'
        useClusterAdmin: true
        command: 'repo'
        arguments: 'add kedacore https://kedacore.github.io/charts'

  - ${{ if or(eq(parameters.containerEngine, 'Kubernetes'),eq(parameters.containerEngine, 'Both')) }}:
    - task: HelmDeploy@0
      displayName: "Update KEDA Helm repo"
      retryCountOnTaskFailure: ${{ parameters.retryCount }}
      inputs:
        connectionType: 'Azure Resource Manager'
        azureSubscription: '$(subscriptionConnection)'
        azureResourceGroup: '$(terraform.resource_group_name)'
        kubernetesCluster: '$(terraform.aks_name)'
        useClusterAdmin: true
        command: 'repo'
        arguments: 'update'

  - ${{ if or(eq(parameters.containerEngine, 'Kubernetes'),eq(parameters.containerEngine, 'Both')) }}:
    - task: HelmDeploy@0
      displayName: "Install KEDA Helm chart"
      retryCountOnTaskFailure: ${{ parameters.retryCount }}
      inputs:
        connectionType: 'Azure Resource Manager'
        azureSubscription: '$(subscriptionConnection)'
        azureResourceGroup: '$(terraform.resource_group_name)'
        kubernetesCluster: '$(terraform.aks_name)'
        useClusterAdmin: true
        namespace: 'keda'
        command: 'upgrade'
        chartType: 'Name'
        chartName: 'kedacore/keda'
        releaseName: 'keda'
        arguments: '--create-namespace'

    - task: HelmDeploy@0
      displayName: "Deploy agent to AKS"
      retryCountOnTaskFailure: ${{ parameters.retryCount }}
      inputs:
        arguments: '--debug'
        azureSubscription: '$(subscriptionConnection)'
        azureResourceGroup: '$(terraform.resource_group_name)'
        kubernetesCluster: '$(terraform.aks_name)'
        useClusterAdmin: true
        command: upgrade
        chartType: FilePath
        chartPath: "helm/pipeline-keda-agents"
        releaseName: "pipeline-keda-agents"
        valueFile: "helm/pipeline-keda-agents/values.yaml,$(terraform.helm_environment_values_file_abs_path)"
        overrideValues: |
          image.repository=$(terraform.container_registry_name).azurecr.io/$(imageRepository)
          linux.azureDevOps.accessToken=$(terraformVariables.azdoToken)
          linux.azureDevOps.orgUrl=$(System.CollectionUri)
          linux.azureDevOps.capability=$(correlationCapability)
          linux.azureDevOps.poolName=$(terraform.aks_agent_pool_name)
          linux.image.acrLoginServer=$(terraform.container_registry_name)
          linux.image.tag=$(imageTag)
          linux.podPrefix=aks-$(TF_WORKSPACE)-$(TF_VAR_resource_suffix)
          linux.trigger.poolId=$(terraform.aks_agent_pool_id)
          storage.accountName=$(terraform.diagnostics_storage_account_name_string)
          storage.accountKey=$(terraform.diagnostics_storage_key)
          storage.resourceGroupName=$(terraform.resource_group_name)
        waitForExecution: true

  - ${{ if or(eq(parameters.containerEngine, 'Container App'),eq(parameters.containerEngine, 'Both')) }}:
    - task: AzureCLI@2
      displayName: 'Wait for Container App agents to come online'
      inputs:
        azureSubscription: '$(subscriptionConnection)'
        scriptType: pscore
        scriptLocation: inlineScript
        inlineScript: |
          ./wait_for_agents.ps1 -CapabilityName PIPELINE_DEMO_JOB_CAPABILITY_ACA `
                                -CapabilityValue $(correlationCapability) `
                                -OrganizationUrl $(System.CollectionUri) `
                                -PoolId $(terraform.aca_agent_pool_id)
        useGlobalConfig: true
        failOnStandardError: true
        workingDirectory: '$(scriptDirectory)'

  - ${{ if or(eq(parameters.containerEngine, 'Kubernetes'),eq(parameters.containerEngine, 'Both')) }}:
    - task: AzureCLI@2
      displayName: 'Wait for Kubernetes agents to come online'
      inputs:
        azureSubscription: '$(subscriptionConnection)'
        scriptType: pscore
        scriptLocation: inlineScript
        inlineScript: |
          ./wait_for_agents.ps1 -CapabilityName PIPELINE_DEMO_JOB_CAPABILITY_AKS `
                                -CapabilityValue $(correlationCapability) `
                                -OrganizationUrl $(System.CollectionUri) `
                                -PoolId $(terraform.aks_agent_pool_id)
        useGlobalConfig: true
        failOnStandardError: true
        workingDirectory: '$(scriptDirectory)'

  - ${{ if not(eq(parameters.destroy, 'Never')) }}:
    - pwsh: |
        Write-Host "##vso[task.setvariable variable=result;isOutput=true]success"
      name: provisioningResult
      displayName: 'Indicate provisioning success'
      condition: succeeded()

- ${{ if or(eq(parameters.containerEngine, 'Container App'),eq(parameters.containerEngine, 'Both')) }}:
  - job: acaAgent
    dependsOn: provision
    displayName: Test Container App Agent
    strategy:
      parallel: 3
    pool:
      name: '$(TF_VAR_aca_agent_pool_name)'
      demands:
      - agent.os -equals Linux
      - CONTAINER_APP_NAME
      # - CONTAINER_APP_NAME -equals $[ dependencies.provision.outputs['apply.container_app_name'] ]
      # - PIPELINE_DEMO_JOB_CAPABILITY_ACA -equals $(correlationCapability)
      # Repeat capability expression as there is no nested variable expansion inside demands
      - PIPELINE_DEMO_JOB_CAPABILITY_ACA -equals $(System.CollectionUri)$(System.TeamProject)#$(Build.DefinitionName)-$(Build.BuildId)#$(TF_WORKSPACE)

    steps:
    - checkout: self
      fetchDepth: 100
      retryCountOnTaskFailure: ${{ parameters.retryCount }}

    - pwsh: |
        Write-Host "`$PSVersionTable.OS"
        $PSVersionTable.OS

        # List environment variables
        Get-ChildItem Env: | Sort-Object -Property Name | Format-Table

        # List installed modules
        Get-InstalledModule | Sort-Object -Property Name
      displayName: 'Display agent info'

    - ${{ if not(eq(parameters.destroy, 'Never')) }}:
      - pwsh: |
          Write-Host "##vso[task.setvariable variable=result;isOutput=true]success"
        name: provisioningResult
        displayName: 'Indicate deployment success'
        condition: succeeded()

- ${{ if or(eq(parameters.containerEngine, 'Kubernetes'),eq(parameters.containerEngine, 'Both')) }}:
  - job: aksAgent
    dependsOn: provision
    displayName: Test Kubernetes Agent
    strategy:
      parallel: 3
    pool:
      name: '$(TF_VAR_aks_agent_pool_name)'
      demands:
      - agent.os -equals Linux
      # - PIPELINE_DEMO_JOB_CAPABILITY_AKS -equals $(correlationCapability)
      # Repeat capability expression as there is no nested variable expansion inside demands
      - PIPELINE_DEMO_JOB_CAPABILITY_AKS -equals $(System.CollectionUri)$(System.TeamProject)#$(Build.DefinitionName)-$(Build.BuildId)#$(TF_WORKSPACE)

    steps:
    - checkout: self
      fetchDepth: 100
      retryCountOnTaskFailure: ${{ parameters.retryCount }}

    - pwsh: |
        Write-Host "`$PSVersionTable.OS"
        $PSVersionTable.OS
        Get-InstalledModule | Sort-Object -Property Name

        # List environment variables
        Get-ChildItem Env: | Sort-Object -Property Name | Format-Table
      displayName: 'Display agent info'
    - ${{ if not(eq(parameters.destroy, 'Never')) }}:
      - pwsh: |
          Write-Host "##vso[task.setvariable variable=result;isOutput=true]success"
        name: provisioningResult
        displayName: 'Indicate deployment success'
        condition: succeeded()
   
- ${{ if ne(parameters.destroy, 'Never') }}:
  - job: destroy
    dependsOn: 
      - provision
      - ${{ if or(eq(parameters.containerEngine, 'Container App'),eq(parameters.containerEngine, 'Both')) }}:
        - acaAgent
      - ${{ if or(eq(parameters.containerEngine, 'Kubernetes'),eq(parameters.containerEngine, 'Both')) }}:
        - aksAgent
    ${{ if ne(variables['Build.Reason'],'Manual') }}:
      displayName: 'Destroy infrastructure (always on pr/schedule)'
      condition: succeededOrFailed()
    ${{ elseif eq(parameters.destroy, 'Always') }}:
      displayName: 'Destroy infrastructure'
      condition: succeededOrFailed()
    ${{ elseif eq(parameters.destroy, 'If no prior resources exist') }}:
      displayName: 'Destroy infrastructure (${{ lower(parameters.destroy) }})'
      condition: eq(dependencies.provision.outputs['apply.priorResourcesExist'],'false')
    ${{ elseif eq(parameters.destroy, 'On failure') }}:
      displayName: 'Destroy (${{ lower(parameters.destroy) }}) infrastructure'
      ${{ if eq(parameters.containerEngine, 'Container App') }}:
        condition: or(ne(dependencies.acaAgent.outputs['job1.provisioningResult.result'],'success'),ne(dependencies.acaAgent.outputs['job2.provisioningResult.result'],'success'),ne(dependencies.acaAgent.outputs['job3.provisioningResult.result'],'success'))
      ${{ if eq(parameters.containerEngine, 'Kubernetes') }}:
        condition: or(ne(dependencies.aksAgent.outputs['job1.provisioningResult.result'],'success'),ne(dependencies.aksAgent.outputs['job2.provisioningResult.result'],'success'),ne(dependencies.aksAgent.outputs['job3.provisioningResult.result'],'success'))
      ${{ if eq(parameters.containerEngine, 'Both') }}:
        condition: or(ne(dependencies.acaAgent.outputs['job1.provisioningResult.result'],'success'),ne(dependencies.acaAgent.outputs['job2.provisioningResult.result'],'success'),ne(dependencies.acaAgent.outputs['job3.provisioningResult.result'],'success'),ne(dependencies.aksAgent.outputs['job1.provisioningResult.result'],'success'),ne(dependencies.aksAgent.outputs['job2.provisioningResult.result'],'success'),ne(dependencies.aksAgent.outputs['job3.provisioningResult.result'],'success'))
    ${{ elseif eq(parameters.destroy, 'On success') }}:
      displayName: 'Destroy (${{ lower(parameters.destroy) }}) infrastructure'
      ${{ if eq(parameters.containerEngine, 'Container App') }}:
        condition: and(eq(dependencies.acaAgent.outputs['job1.provisioningResult.result'],'success'),eq(dependencies.acaAgent.outputs['job2.provisioningResult.result'],'success'),eq(dependencies.acaAgent.outputs['job3.provisioningResult.result'],'success'))
      ${{ if eq(parameters.containerEngine, 'Kubernetes') }}:
        condition: and(eq(dependencies.aksAgent.outputs['job1.provisioningResult.result'],'success'),eq(dependencies.aksAgent.outputs['job2.provisioningResult.result'],'success'),eq(dependencies.aksAgent.outputs['job3.provisioningResult.result'],'success'))
      ${{ if eq(parameters.containerEngine, 'Both') }}:
        condition: and(eq(dependencies.acaAgent.outputs['job1.provisioningResult.result'],'success'),eq(dependencies.acaAgent.outputs['job2.provisioningResult.result'],'success'),eq(dependencies.acaAgent.outputs['job3.provisioningResult.result'],'success'),eq(dependencies.aksAgent.outputs['job1.provisioningResult.result'],'success'),eq(dependencies.aksAgent.outputs['job2.provisioningResult.result'],'success'),eq(dependencies.aksAgent.outputs['job3.provisioningResult.result'],'success'))
    timeoutInMinutes: $[ variables['jobTimeOutMinutes'] ]

    pool:
      # name: $[ coalesce(variables.pool,'Azure Pipelines') ]
      vmImage: $[ coalesce(variables.vmImage,'ubuntu-latest') ]

    # ${{ if parameters.asContainerJob }}:
    #   container:
    #     image: $(toolsImageFQName)
    #     endpoint: containerEndpoint

    variables:
      terraformVersion: $[ coalesce(dependencies.provision.outputs['terraformConfig.version'],'latest') ]

    steps:
    - checkout: self
      submodules: true

    - task: DownloadPipelineArtifact@2
      inputs:
        artifact: $(terraformArtifactName)
        path: $(terraformDirectory)

    - bash: |
        echo Marking Terraform provider binaries as executable...
        find . | xargs chmod +x
      displayName: 'Make Terraform providers executable'
      workingDirectory: $(terraformDirectory)/.terraform/providers

    - task: TerraformInstaller@0
      displayName: 'Install Terraform'
      inputs:
        terraformVersion: '$(terraformVersion)'

    - task: AzureCLI@2
      displayName: 'Terraform destroy (${{ lower(parameters.destroy) }})'
      ${{ if eq(parameters.destroy, 'Always') }}:
        condition: succeededOrFailed()
      ${{ if eq(parameters.destroy, 'On failure') }}:
        condition: failed()
      ${{ if eq(parameters.destroy, 'On success') }}:
        condition: succeeded()
      continueOnError: true # Treat failure as warning during destroy, we will clean up anyway
      retryCountOnTaskFailure: ${{ parameters.retryCount }}
      inputs:
        azureSubscription: '$(subscriptionConnection)'
        scriptType: pscore
        scriptLocation: inlineScript
        inlineScript: |
          $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1
          ./deploy.ps1 -destroy -force
        addSpnToEnvironment: true
        useGlobalConfig: true
        failOnStandardError: true
        workingDirectory: '$(scriptDirectory)'

    - task: AzureCLI@2
      name: teardown
      displayName: 'Tear down remaining Azure resources'
      inputs:
        azureSubscription: '$(subscriptionConnection)'
        scriptType: pscore
        scriptLocation: inlineScript
        inlineScript: |
          $ErrorActionPreference = "Continue" # Continue to remove resources if remove by resource group fails
          # Build JMESPath expression
          $tagQuery = "[?tags.repository == '$(TF_VAR_repository)' && tags.workspace == '$(TF_WORKSPACE)' && tags.provisionerClientId == '${env:servicePrincipalId}' && properties.provisioningState != 'Deleting'].id"
          Write-Host "Removing resources identified by `"$tagQuery`"..."

          # Remove resource groups
          $resourceGroupIDs = $(az group list --query "${tagQuery}" -o tsv)
          if ($resourceGroupIDs) {
            Write-Host "Removing resource group(s) `"${resourceGroupIDs}`"..."
            &{ # az writes information to stderr
              $ErrorActionPreference = 'SilentlyContinue'
              az resource delete --ids $resourceGroupIDs 2>&1
            }
          } else {
            Write-Host "No resource groups to remove"
          }

          # Remove (remaining) resources
          $resourceIDs = $(az resource list --query "${tagQuery}" -o tsv)
          if ($resourceIDs) {
            Write-Host "Removing resources `"${resourceIDs}`"..."
            &{ # az writes information to stderr
              $ErrorActionPreference = 'SilentlyContinue'
              az resource delete --ids $resourceIDs 2>&1
            }
          } else {
            Write-Host "No resources to remove"
          }
        addSpnToEnvironment: true
        useGlobalConfig: true
        failOnStandardError: true
        workingDirectory: '$(scriptDirectory)'

    - task: AzureCLI@2
      displayName: 'Tear down agent pools'
      condition: and(succeededOrFailed(),eq(coalesce(variables['TF_VAR_create_agent_pools'],'true'),'true'))
      inputs:
        azureSubscription: '$(subscriptionConnection)'
        scriptType: pscore
        scriptLocation: inlineScript
        inlineScript: |
          ./delete_agent_pool.ps1 -OrganizationUrl $(System.CollectionUri) `
                                  -PoolName $(TF_VAR_aca_agent_pool_name), $(TF_VAR_aks_agent_pool_name)
        useGlobalConfig: true
        failOnStandardError: true
        workingDirectory: '$(scriptDirectory)'

    - task: AzureCLI@2
      name: cleanup
      displayName: 'Clean up Terraform state'
      inputs:
        azureSubscription: '$(subscriptionConnection)'
        scriptType: pscore
        scriptLocation: inlineScript
        inlineScript: |
          $(identityScriptDirectory)/set_terraform_azurerm_vars.ps1

          $terraformState = (terraform state pull | ConvertFrom-Json)
          if ($terraformState.resources) {
            Write-Host "Clearing Terraform state in workspace ${env:TF_WORKSPACE}..."
            $terraformState.check_results = $null
            $terraformState.outputs = New-Object PSObject # Empty output
            $terraformState.resources = @() # No resources
            $terraformState.serial++
            $terraformState | ConvertTo-Json | terraform state push -
          } else {
            Write-Host "No resources in Terraform state in workspace ${env:TF_WORKSPACE}..."
          }
          terraform state pull 
        addSpnToEnvironment: true
        useGlobalConfig: true
        failOnStandardError: true
        workingDirectory: '$(terraformDirectory)'