[ODS-5502] - Add Postgres Initdev workflows (#494)

* Add postgres workflows to mirror newly created sql server initdev workflows This initdev will not package since the packages from the sql server workflow are what will get published, like what is done on TeamCity * Remove smoke testing from build The second initdev build does smoke testing for all three versions so doing a smoke test here is just repeating the same work * Cleanup workflows Removed not needed env vars only needed for publishing and removed creating package step from postgres workflow since we will use the packages created in the other workflow * Rename workflow file to remove integration test from it Since they are not run during this workflow * Start existing Postgres service in runner * Add parser for parsing test report files * Add missing token * Add starting postgres to second build * Remove test reporting * Remove usage of variable not present in script and replace with setting to true This matches previous parts of build that set UsePlugins to true * Update engine to be postgres * Remove Write-Host statements not needed anymore * Remove package from step name since no packaging happens
Ed-Fi-Alliance-OSS · Aug 26, 2022 · 0c16eff · 0c16eff
1 parent cec4b4f
commit 0c16eff
Show file tree

Hide file tree

Showing 4 changed files with 314 additions and 10 deletions.
diff --git a/.../workflows/InitDev (Postgres), Tests using Sandbox, Year-Specific and Shared Instance.yml b/.../workflows/InitDev (Postgres), Tests using Sandbox, Year-Specific and Shared Instance.yml
@@ -0,0 +1,208 @@
+# SPDX-License-Identifier: Apache-2.0
+# Licensed to the Ed-Fi Alliance under one or more agreements.
+# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+# See the LICENSE and NOTICES files in the project root for more information.
+
+name: InitDev (Postgres), Smoke Tests Sandbox, Year-Specific, and Shared Instance
+
+on:
+  pull_request:
+   branches: [main]
+  push:
+   branches: [main]
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+  repository_dispatch:
+    types: [triggered-from-ods-repo]
+
+env:
+  INFORMATIONAL_VERSION: "6.0"
+  BUILD_INCREMENTER: "1118"
+  CONFIGURATION: "Release"
+  VERSION_MAJOR: "6"
+  VERSION_MINOR: "0"
+  CURRENT_BRANCH: ${{ GITHUB.HEAD_REF }}
+  ODS_CURRENT_BRANCH: ${{ github.event.client_payload.ref }}
+  EDFI_SUITE_NUMBER: "Suite3"
+
+jobs:
+  build:
+
+    runs-on: windows-latest
+
+    steps:
+    - name: Setup .NET
+      uses: actions/setup-dotnet@c0d4ad69d8bd405d234f1c9166d383b7a4f69ed8 # 2.1.0
+      with:
+        dotnet-version: 6.0.x
+    - name: Support longpaths
+      run: git config --system core.longpaths true
+    - name: Checkout Ed-Fi-ODS-Implementation
+      uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2
+      with:
+          repository: Ed-Fi-Alliance-OSS/Ed-Fi-ODS-Implementation
+          path: Ed-Fi-ODS-Implementation/
+    - name: Is pull request branch exists in Ed-Fi-ODS-Implementation
+      working-directory: ./Ed-Fi-ODS-Implementation/
+      shell: pwsh
+      run: |
+            $patternName = 'refs/heads/' +  '${{ env.CURRENT_BRANCH }}'
+            $is_pull_request_branch = 'False'
+            $is_pull_request_branch = git ls-remote --heads origin ${{ env.CURRENT_BRANCH }} | Select-String -Pattern $patternName -SimpleMatch -Quiet
+            if ($is_pull_request_branch -eq $true) {
+              git fetch origin ${{ env.CURRENT_BRANCH }}
+              git checkout ${{ env.CURRENT_BRANCH }}
+            }
+    - name: Checkout Ed-Fi-ODS
+      uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2
+      with:
+          repository: Ed-Fi-Alliance-OSS/Ed-Fi-ODS
+          path: Ed-Fi-ODS/
+    - name: Is pull request branch exists in Ed-Fi-ODS
+      working-directory: ./Ed-Fi-ODS/
+      shell: pwsh
+      run: |
+           $patternName = 'refs/heads/' +  '${{ env.CURRENT_BRANCH }}'
+           $is_pull_request_branch = 'False'
+           $is_pull_request_branch = git ls-remote --heads origin ${{ env.CURRENT_BRANCH }} | Select-String -Pattern $patternName -SimpleMatch -Quiet
+           $odsBranch = '${{ env.ODS_CURRENT_BRANCH }}'
+           if($odsBranch -ne ''){
+              git fetch origin $odsBranch
+              git checkout $odsBranch
+           } else {
+              if ($is_pull_request_branch -eq $true) {
+                git fetch origin ${{ env.CURRENT_BRANCH }}
+                git checkout ${{ env.CURRENT_BRANCH }}
+              }
+           }
+    - name: Start PostgreSQL
+      run: |
+        $pgService = Get-Service -Name postgresql*
+        Set-Service -InputObject $pgService -Status running -StartupType automatic
+        Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru
+    - name: ODS/API InitDev and SdkGen
+      working-directory: ./Ed-Fi-ODS-Implementation/
+      run: |
+        $ErrorActionPreference = 'Stop'
+        [int]$BuildCounter =  "${{ github.run_number }}"
+        [int]$BuildIncrementer = "${{ env.BUILD_INCREMENTER }}"
+        [int]$newRevision =  $BuildCounter + $BuildIncrementer
+        [string]$version = "${{ env.VERSION_MAJOR}}"+ "." +"${{ env.VERSION_MINOR}}" + "." + $newRevision.ToString()
+        .\build.github.ps1 -Engine PostgreSQL -PackageVersion $version -RepositoryRoot $env:GITHUB_WORKSPACE -UsePlugins -RunSdkGen -GenerateSdkPackages -NoPackaging
+      shell: pwsh
+    - name: Upload initdev logs
+      if: success() || failure()
+      uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+      with:
+        name: Initdev logs
+        path: |
+          ${{ github.workspace }}/Ed-Fi-ODS-Implementation/Application/EdFi.Ods.Api.IntegrationTestHarness/bin/**/PostmanIntegrationTestsLog.log
+          ${{ github.workspace }}/Ed-Fi-ODS-Implementation/Application/EdFi.Ods.Api.IntegrationTestHarness/bin/**/*TestHarnessLog.log
+    - name: Unit test EdFi.LoadTools 
+      working-directory: ./Ed-Fi-ODS-Implementation/
+      run: |
+           dotnet test "$env:GITHUB_WORKSPACE/Ed-Fi-ODS/Utilities/DataLoading/LoadTools.sln" --filter TestCategory!~'RunManually'
+      shell: pwsh
+    - name: ODS/API Smoke Test Sandbox
+      working-directory: ./Ed-Fi-ODS-Implementation/
+      run: |
+        $ErrorActionPreference = 'Stop'
+        . $env:GITHUB_WORKSPACE/Ed-Fi-ODS-Implementation/Initialize-PowershellForDevelopment.ps1
+
+        Reset-TestPopulatedTemplateDatabase
+        $params = @{
+                InstallType   = 'Sandbox'
+                Engine        = 'PostgreSQL'
+                DropDatabases = $true
+                NoDuration    = $true
+                UsePlugins    = $true
+        }
+        Initialize-DeploymentEnvironment @params
+
+        if(Test-Path $env:GITHUB_WORKSPACE/Ed-Fi-ODS/Utilities/DataLoading/EdFi.SmokeTest.Console/bin/Release/net6.0/publish) {
+          Remove-Item -Path $env:GITHUB_WORKSPACE/Ed-Fi-ODS/Utilities/DataLoading/EdFi.SmokeTest.Console/bin/Release/net6.0/publish -Recurse
+        }
+       
+        $params = @{
+                testSets                 = @("NonDestructiveApi", "NonDestructiveSdk", "DestructiveSdk")
+                smokeTestDll             = "$env:GITHUB_WORKSPACE/Ed-Fi-ODS/Utilities/SdkGen/EdFi.SdkGen.Console/csharp/src/EdFi.OdsApi.Sdk/bin/Release/net6.0/EdFi.OdsApi.Sdk.dll"
+                testHarnessLogNamePrefix = "SmokeTestSandbox"
+        }
+        . $env:GITHUB_WORKSPACE/Ed-Fi-ODS-Implementation/logistics/scripts/run-smoke-tests.ps1 @params
+      shell: pwsh
+    - name: Upload Smoke Test Sandbox Logs
+      if: success() || failure()
+      uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+      with:
+        name: Smoke Test Sandbox Logs
+        path: |
+          ${{ github.workspace }}/Ed-Fi-ODS/Utilities/DataLoading/EdFi.SmokeTest.Console/bin/**/EdFi.SmokeTest.log
+          ${{ github.workspace }}/Ed-Fi-ODS-Implementation/Application/EdFi.Ods.Api.IntegrationTestHarness/bin/**/SmokeTestSandbox.TestHarnessLog.log
+    - name: ODS/API Smoke Test YearSpecific
+      working-directory: ./Ed-Fi-ODS-Implementation/
+      run: |           
+        $ErrorActionPreference = 'Stop'
+        . $env:GITHUB_WORKSPACE/Ed-Fi-ODS-Implementation/Initialize-PowershellForDevelopment.ps1
+
+        Reset-TestPopulatedTemplateDatabase
+        $params = @{
+                InstallType   = 'YearSpecific'
+                Engine        = 'PostgreSQL'
+                OdsTokens     = '2022'
+                DropDatabases = $true
+                NoDuration    = $true
+                UsePlugins    = $true
+        }
+        Initialize-DeploymentEnvironment @params
+
+        $params = @{
+                schoolYear               = "2022"
+                testSets                 = @("NonDestructiveApi", "NonDestructiveSdk", "DestructiveSdk")
+                smokeTestDll             = "$env:GITHUB_WORKSPACE/Ed-Fi-ODS/Utilities/SdkGen/EdFi.SdkGen.Console/csharp/src/EdFi.OdsApi.Sdk/bin/Release/net6.0/EdFi.OdsApi.Sdk.dll"
+                testHarnessLogNamePrefix = "SmokeTestYearSpecific"
+        }
+        . $env:GITHUB_WORKSPACE/Ed-Fi-ODS-Implementation/logistics/scripts/run-smoke-tests.ps1 @params
+      shell: pwsh
+    - name: Upload Smoke Test YearSpecific Logs
+      if: success() || failure()
+      uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+      with:
+        name: Smoke Test YearSpecific Logs
+        path: |
+          ${{ github.workspace }}/Ed-Fi-ODS/Utilities/DataLoading/EdFi.SmokeTest.Console/bin/**/EdFi.SmokeTest.log
+          ${{ github.workspace }}/Ed-Fi-ODS-Implementation/Application/EdFi.Ods.Api.IntegrationTestHarness/bin/**/SmokeTestYearSpecific.TestHarnessLog.log
+    - name: ODS/API Smoke Test Shared Instance
+      working-directory: ./Ed-Fi-ODS-Implementation/
+      run: |           
+        $ErrorActionPreference = 'Stop'
+        . $env:GITHUB_WORKSPACE/Ed-Fi-ODS-Implementation/Initialize-PowershellForDevelopment.ps1
+
+        Reset-TestPopulatedTemplateDatabase
+        $params = @{
+                InstallType   = 'SharedInstance'
+                Engine        = 'PostgreSQL'
+                DropDatabases = $true
+                NoDuration    = $true
+                UsePlugins    = $true
+        }
+        Initialize-DeploymentEnvironment @params
+       
+        $params = @{
+                testSets                 = @("NonDestructiveApi", "NonDestructiveSdk", "DestructiveSdk")
+                smokeTestDll             = "$env:GITHUB_WORKSPACE/Ed-Fi-ODS/Utilities/SdkGen/EdFi.SdkGen.Console/csharp/src/EdFi.OdsApi.Sdk/bin/Release/net6.0/EdFi.OdsApi.Sdk.dll"
+                testHarnessLogNamePrefix = "SmokeTestSharedInstance"
+        }
+        . $env:GITHUB_WORKSPACE/Ed-Fi-ODS-Implementation/logistics/scripts/run-smoke-tests.ps1 @params
+      shell: pwsh
+    - name: Upload Smoke Test Shared Instance Logs
+      if: success() || failure()
+      uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+      with:
+        name: Smoke Test Shared Instance Logs
+        path: |
+          ${{ github.workspace }}/Ed-Fi-ODS/Utilities/DataLoading/EdFi.SmokeTest.Console/bin/**/EdFi.SmokeTest.log
+          ${{ github.workspace }}/Ed-Fi-ODS-Implementation/Application/EdFi.Ods.Api.IntegrationTestHarness/bin/**/SmokeTestSharedInstance.TestHarnessLog.log
diff --git a/.github/workflows/InitDev (Postgres),Unit tests,Integration tests.yml b/.github/workflows/InitDev (Postgres),Unit tests,Integration tests.yml
@@ -0,0 +1,102 @@
+# SPDX-License-Identifier: Apache-2.0
+# Licensed to the Ed-Fi Alliance under one or more agreements.
+# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+# See the LICENSE and NOTICES files in the project root for more information.
+
+name: InitDev (Postgres), Unit tests, Integration tests
+
+on:
+  pull_request:
+   branches: [main]
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+  repository_dispatch:
+    types: [triggered-from-ods-repo]
+
+env:
+  INFORMATIONAL_VERSION: "6.0"
+  BUILD_INCREMENTER: "1118"
+  CONFIGURATION: "Release"
+  VERSION_MAJOR: "6"
+  VERSION_MINOR: "0"
+  CURRENT_BRANCH: ${{ GITHUB.HEAD_REF }}
+  ODS_CURRENT_BRANCH: ${{ github.event.client_payload.ref }}
+  EDFI_SUITE_NUMBER: "Suite3"
+
+jobs:
+  build:
+
+    runs-on: windows-latest
+
+    steps:
+    - name: Setup .NET
+      uses: actions/setup-dotnet@c0d4ad69d8bd405d234f1c9166d383b7a4f69ed8 # 2.1.0
+      with:
+        dotnet-version: 6.0.x
+    - name: Support longpaths
+      run: git config --system core.longpaths true
+    - name: Checkout Ed-Fi-ODS-Implementation
+      uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2
+      with:
+          repository: Ed-Fi-Alliance-OSS/Ed-Fi-ODS-Implementation
+          path: Ed-Fi-ODS-Implementation/
+    - name: Is pull request branch exists in Ed-Fi-ODS-Implementation
+      working-directory: ./Ed-Fi-ODS-Implementation/
+      shell: pwsh
+      run: |
+            $patternName = 'refs/heads/' +  '${{ env.CURRENT_BRANCH }}'
+            $is_pull_request_branch = 'False'
+            $is_pull_request_branch = git ls-remote --heads origin ${{ env.CURRENT_BRANCH }} | Select-String -Pattern $patternName -SimpleMatch -Quiet
+            if ($is_pull_request_branch -eq $true) {
+              git fetch origin ${{ env.CURRENT_BRANCH }}
+              git checkout ${{ env.CURRENT_BRANCH }}
+            }
+    - name: Checkout Ed-Fi-ODS
+      uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # v2
+      with:
+          repository: Ed-Fi-Alliance-OSS/Ed-Fi-ODS
+          path: Ed-Fi-ODS/
+    - name: Is pull request branch exists in Ed-Fi-ODS
+      working-directory: ./Ed-Fi-ODS/
+      shell: pwsh
+      run: |
+           $patternName = 'refs/heads/' +  '${{ env.CURRENT_BRANCH }}'
+           $is_pull_request_branch = 'False'
+           $is_pull_request_branch = git ls-remote --heads origin ${{ env.CURRENT_BRANCH }} | Select-String -Pattern $patternName -SimpleMatch -Quiet
+           $odsBranch = '${{ env.ODS_CURRENT_BRANCH }}'
+           if($odsBranch -ne ''){
+              git fetch origin $odsBranch
+              git checkout $odsBranch
+           } else {
+              if ($is_pull_request_branch -eq $true) {
+                git fetch origin ${{ env.CURRENT_BRANCH }}
+                git checkout ${{ env.CURRENT_BRANCH }}
+              }
+           }
+    - name: Start PostgreSQL
+      run: |
+        $pgService = Get-Service -Name postgresql*
+        Set-Service -InputObject $pgService -Status running -StartupType automatic
+        Start-Process -FilePath "$env:PGBIN\pg_isready" -Wait -PassThru
+    - name: ODS/API InitDev, SdkGen, Integration Test, Unit Test
+      working-directory: ./Ed-Fi-ODS-Implementation/
+      run: |
+        $ErrorActionPreference = 'Stop'
+        [int]$BuildCounter =  "${{ github.run_number }}"
+        [int]$BuildIncrementer = "${{ env.BUILD_INCREMENTER }}"
+        [int]$newRevision =  $BuildCounter + $BuildIncrementer
+        [string]$version = "${{ env.VERSION_MAJOR}}"+ "." +"${{ env.VERSION_MINOR}}" + "." + $newRevision.ToString()
+        .\build.github.ps1 -Engine PostgreSQL -PackageVersion $version -RepositoryRoot $env:GITHUB_WORKSPACE -UsePlugins -RunSdkGen -GenerateSdkPackages -RunPostman -RunPester -RunDotnetTest -NoPackaging
+      shell: pwsh
+    - name: Upload initdev logs
+      if: success() || failure()
+      uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+      with:
+        name: Initdev logs
+        path: |
+          ${{ github.workspace }}/Ed-Fi-ODS-Implementation/Application/EdFi.Ods.Api.IntegrationTestHarness/bin/**/PostmanIntegrationTestsLog.log
+          ${{ github.workspace }}/Ed-Fi-ODS-Implementation/Application/EdFi.Ods.Api.IntegrationTestHarness/bin/**/*TestHarnessLog.log
diff --git a/.github/workflows/InitDev,Unit tests,Integration tests,Package.yml b/.github/workflows/InitDev,Unit tests,Integration tests,Package.yml
@@ -23,9 +23,6 @@ env:
   CONFIGURATION: "Release"
   VERSION_MAJOR: "6"
   VERSION_MINOR: "0"
-  AZURE_ARTIFACT_URL: "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/nuget/v3/index.json"
-  AZURE_ARTIFACT_NUGET_KEY: ${{ secrets.AZURE_ARTIFACTS_PERSONAL_ACCESS_TOKEN }}
-  VSS_NUGET_EXTERNAL_FEED_ENDPOINTS : '{"endpointCredentials": [{"endpoint": "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/nuget/v3/index.json","password": "${{ secrets.AZURE_ARTIFACTS_PERSONAL_ACCESS_TOKEN }}"}]}'
   CURRENT_BRANCH: ${{ GITHUB.HEAD_REF }}
   ODS_CURRENT_BRANCH: ${{ github.event.client_payload.ref }}
   EDFI_SUITE_NUMBER: "Suite3"
@@ -114,7 +111,7 @@ jobs:
         [int]$BuildIncrementer = "${{ env.BUILD_INCREMENTER }}"
         [int]$newRevision =  $BuildCounter + $BuildIncrementer
         [string]$version = "${{ env.VERSION_MAJOR}}"+ "." +"${{ env.VERSION_MINOR}}" + "." + $newRevision.ToString()
-        .\build.github.ps1 -PackageVersion $version -RepositoryRoot $env:GITHUB_WORKSPACE -UsePlugins -RunSdkGen -GenerateSdkPackages -RunPostman -RunPester -RunDotnetTest -RunSmokeTest -PackageOutput "$env:GITHUB_WORKSPACE/Ed-Fi-ODS-Implementation/NugetPackages"
+        .\build.github.ps1 -PackageVersion $version -RepositoryRoot $env:GITHUB_WORKSPACE -UsePlugins -RunSdkGen -GenerateSdkPackages -RunPostman -RunPester -RunDotnetTest -PackageOutput "$env:GITHUB_WORKSPACE/Ed-Fi-ODS-Implementation/NugetPackages"
       shell: pwsh
     - name: Upload initdev logs
       if: success() || failure()

diff --git a/... tests,Integration tests,Smoke Tests using Sandbox, Year-Specific and Shared Instance.yml b/... tests,Integration tests,Smoke Tests using Sandbox, Year-Specific and Shared Instance.yml
@@ -25,9 +25,6 @@ env:
   CONFIGURATION: "Release"
   VERSION_MAJOR: "6"
   VERSION_MINOR: "0"
-  AZURE_ARTIFACT_URL: "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/nuget/v3/index.json"
-  AZURE_ARTIFACT_NUGET_KEY: ${{ secrets.AZURE_ARTIFACTS_PERSONAL_ACCESS_TOKEN }}
-  VSS_NUGET_EXTERNAL_FEED_ENDPOINTS : '{"endpointCredentials": [{"endpoint": "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi/nuget/v3/index.json","password": "${{ secrets.AZURE_ARTIFACTS_PERSONAL_ACCESS_TOKEN }}"}]}'
   CURRENT_BRANCH: ${{ GITHUB.HEAD_REF }}
   ODS_CURRENT_BRANCH: ${{ github.event.client_payload.ref }}
   EDFI_SUITE_NUMBER: "Suite3"
@@ -143,7 +140,7 @@ jobs:
                 Engine        = 'SQLServer'
                 DropDatabases = $true
                 NoDuration    = $true
-                UsePlugins    = $UsePlugins.IsPresent
+                UsePlugins    = $true
         }
         Initialize-DeploymentEnvironment @params
 
@@ -179,7 +176,7 @@ jobs:
                 OdsTokens     = '2022'
                 DropDatabases = $true
                 NoDuration    = $true
-                UsePlugins    = $UsePlugins.IsPresent
+                UsePlugins    = $true
         }
         Initialize-DeploymentEnvironment @params
 
@@ -211,7 +208,7 @@ jobs:
                 Engine        = 'SQLServer'
                 DropDatabases = $true
                 NoDuration    = $true
-                UsePlugins    = $UsePlugins.IsPresent
+                UsePlugins    = $true
         }
         Initialize-DeploymentEnvironment @params