[DMS-353] Validate method allowed and build script fixes (#320)

Ed-Fi-Alliance-OSS · Oct 25, 2024 · 815a82d · 815a82d
1 parent f7e870e
commit 815a82d
Show file tree

Hide file tree

Showing 9 changed files with 288 additions and 99 deletions.
diff --git a/.github/workflows/on-dms-pullrequest.yml b/.github/workflows/on-dms-pullrequest.yml
@@ -179,9 +179,6 @@ jobs:
       - name: Checkout the Repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-      - name: Build
-        run: ./build-dms.ps1 Build -Configuration Debug
-
       - name: Run OpenSearch End to End Tests
         if: success()
         run: ./build-dms.ps1 E2ETest -EnableOpenSearch

diff --git a/build-dms.ps1 b/build-dms.ps1
@@ -242,17 +242,21 @@ function RunE2E {
 }
 
 function E2ETests {
+    Invoke-Step { DockerBuild }
+
     if ($EnableOpenSearch) {
-        try {
-            Push-Location eng/docker-compose/
-            ./start-local-dms.ps1 "./.env.e2e"
-        }
-        finally {
-            Pop-Location
+        Invoke-Execute {
+            try {
+                Push-Location eng/docker-compose/
+                ./start-local-dms.ps1 -EnvironmentFile "./.env.e2e"
+            }
+            finally {
+                Pop-Location
+            }
         }
     }
     else {
-        Invoke-Step { DockerBuild }
+        Invoke-Step { DockerRun }
     }
     Invoke-Step { RunE2E }
 }
@@ -273,7 +277,13 @@ function RunNuGetPack {
     # NU5100 is the warning about DLLs outside of a "lib" folder. We're
     # deliberately using that pattern, therefore we bypass the
     # warning.
-    dotnet pack $ProjectPath --no-build --no-restore --output $PSScriptRoot -p:NuspecFile=$nuspecPath -p:NuspecProperties="version=$PackageVersion;year=$copyrightYear" /p:NoWarn=NU5100
+    dotnet pack $ProjectPath `
+        --no-build `
+        --no-restore `
+        --output $PSScriptRoot `
+        -p:NuspecFile=$nuspecPath `
+        -p:NuspecProperties="version=$PackageVersion;year=$copyrightYear" `
+        /p:NoWarn=NU5100
 }
 
 function BuildPackage {
@@ -314,10 +324,13 @@ function Invoke-TestExecution {
             IgnoreCase = $true)]
         # File search filter
         [string]
-        $Filter
+        $Filter,
+
+        [switch]
+        $EnableOpenSearch
     )
     switch ($Filter) {
-        E2ETests { Invoke-Step { E2ETests } }
+        E2ETests { Invoke-Step { E2ETests -EnableOpenSearch:$EnableOpenSearch } }
         UnitTests { Invoke-Step { UnitTests } }
         IntegrationTests { Invoke-Step { IntegrationTests } }
         Default { "Unknow Test Type" }
@@ -398,7 +411,7 @@ Invoke-Main {
             Invoke-Publish
         }
         UnitTest { Invoke-TestExecution UnitTests }
-        E2ETest { Invoke-TestExecution E2ETests }
+        E2ETest { Invoke-TestExecution E2ETests -EnableOpenSearch:$EnableOpenSearch }
         IntegrationTest { Invoke-TestExecution IntegrationTests }
         Coverage { Invoke-Coverage }
         Package { Invoke-BuildPackage }

diff --git a/eng/docker-compose/setup-connectors.ps1 b/eng/docker-compose/setup-connectors.ps1
@@ -6,8 +6,7 @@ param (
     $EnvironmentFile = "./.env"
 )
 
-function IsReady([string] $Url)
-{
+function IsReady([string] $Url) {
     $maxAttempts = 6
     $attempt = 0
     $waitTime = 5
@@ -41,70 +40,73 @@ catch {
     Write-Error "Please provide valid .env file."
 }
 
-$sourcePort=$envFile["CONNECT_SOURCE_PORT"]
-$sinkPort=$envFile["CONNECT_SINK_PORT"]
+$sourcePort = $envFile["CONNECT_SOURCE_PORT"]
+$sinkPort = $envFile["CONNECT_SINK_PORT"]
 
 $sourceBase = "http://localhost:$sourcePort/connectors"
 $sinkBase = "http://localhost:$sinkPort/connectors"
 $sourceUrl = "$sourceBase/postgresql-source"
 $sinkUrl = "$sinkBase/opensearch-sink"
 
- # Source connector
- if(IsReady($sourceBase))
- {
-     try {
-         $sourceResponse = Invoke-RestMethod -Uri $sourceUrl -Method Get
-         if($sourceResponse)
-         {
-             Write-Output "Deleting existing source connector configuration."
-             Invoke-RestMethod -Method Delete -uri $sourceUrl
-         }
-     }
-     catch {
+# Source connector
+if (IsReady($sourceBase)) {
+    try {
+        $sourceResponse = Invoke-RestMethod -Uri $sourceUrl -Method Get -SkipHttpErrorCheck
+
+        # only true if the response was 200
+        if ($null -ne $sourceResponse.name) {
+            Write-Output "Deleting existing source connector configuration."
+            Invoke-RestMethod -Method Delete -uri $sourceUrl
+        }
+    }
+    catch {
+        Write-Output $_.Exception.Message
+    }
+
+    try {
+        $sourceBody = Get-Content "./postgresql_connector.json"
+        $sourceBody = $sourceBody.Replace("abcdefgh1!", $envFile["POSTGRES_PASSWORD"])
+
+        Write-Output "Installing source connector configuration"
+        Invoke-RestMethod -Method Post -uri $sourceBase -ContentType "application/json" -Body $sourceBody
+    }
+    catch {
         Write-Output $_.Exception.Message
-     }
+    }
+    Start-Sleep 2
+    Invoke-RestMethod -Method Get -uri $sourceUrl -SkipHttpErrorCheck
+}
+else {
+    Write-Output "Service at $sourceBase not available."
+}
 
-     try {
-         $sourceBody = Get-Content "./postgresql_connector.json"
-         $sourceBody = $sourceBody.Replace("abcdefgh1!", $envFile["POSTGRES_PASSWORD"])
-         Invoke-RestMethod -Method Post -uri $sourceBase -ContentType "application/json" -Body $sourceBody
-     }
-     catch {
-         Write-Output $_.Exception.Message
-     }
-     Start-Sleep 2
-     Invoke-RestMethod -Method Get -uri $sourceUrl
- }
- else {
-     Write-Output "Service at $sourceBase not available."
- }
+# Sink connector
+if (IsReady($sinkBase)) {
+    try {
+        $sinkResponse = Invoke-RestMethod -Uri $sinkUrl -Method Get -SkipHttpErrorCheck
 
- # Sink connector
- if(IsReady($sinkBase))
- {
-     try {
-         $sinkResponse = Invoke-RestMethod -Uri $sinkUrl -Method Get
-         if($sinkResponse)
-         {
-             Write-Output "Deleting existing sink connector configuration."
-             Invoke-RestMethod -Method Delete -uri $sinkUrl
-         }
-     }
-     catch {
+        if ($null -ne $sinkResponse.name) {
+            Write-Output "Deleting existing sink connector configuration."
+            Invoke-RestMethod -Method Delete -uri $sinkUrl
+        }
+    }
+    catch {
         Write-Output $_.Exception.Message
-     }
+    }
+
+    try {
+        $sinkBody = Get-Content "./opensearch_connector.json"
+        $sinkBody = $sinkBody.Replace("abcdefgh1!", $envFile["OPENSEARCH_ADMIN_PASSWORD"])
 
-     try {
-         $sinkBody = Get-Content "./opensearch_connector.json"
-         $sinkBody = $sinkBody.Replace("abcdefgh1!", $envFile["OPENSEARCH_ADMIN_PASSWORD"])
-         Invoke-RestMethod -Method Post -uri $sinkBase -ContentType "application/json" -Body $sinkBody
-     }
-     catch {
-         Write-Output $_.Exception.Message
-     }
-     Start-Sleep 2
-     Invoke-RestMethod -Method Get -uri $sinkUrl
- }
- else {
-     Write-Output "Service at $sinkBase not available."
- }
+        Write-Output "Installing sink connector configuration"
+        Invoke-RestMethod -Method Post -uri $sinkBase -ContentType "application/json" -Body $sinkBody
+    }
+    catch {
+        Write-Output $_.Exception.Message
+    }
+    Start-Sleep 2
+    Invoke-RestMethod -Method Get -uri $sinkUrl -SkipHttpErrorCheck
+}
+else {
+    Write-Output "Service at $sinkBase not available."
+}
diff --git a/eng/docker-compose/start-local-dms.ps1 b/eng/docker-compose/start-local-dms.ps1
@@ -39,13 +39,11 @@ $files = @(
     "local-dms.yml"
 )
 
-if($EnforceAuthorization)
-{
+if ($EnforceAuthorization) {
     $files += @("-f", "keycloak.yml")
 }
 
-if($EnableOpenSearchUI)
-{
+if ($EnableOpenSearchUI) {
     $files += @("-f", "kafka-opensearch-ui.yml")
 }
 
@@ -60,18 +58,24 @@ if ($d) {
     }
 }
 else {
-    $upArgs = @()
-    if ($r) { $upArgs += "--build" }
+    $upArgs = @(
+        "--detach"
+    )
+    if ($r) { $upArgs += @("--build") }
 
     Write-Output "Starting locally-built DMS"
-    if($EnforceAuthorization)
-    {
-        $env:IDENTITY_ENFORCE_AUTHORIZATION=$true
+    if ($EnforceAuthorization) {
+        $env:IDENTITY_ENFORCE_AUTHORIZATION = $true
     }
     else {
-        $env:IDENTITY_ENFORCE_AUTHORIZATION=$false
+        $env:IDENTITY_ENFORCE_AUTHORIZATION = $false
+    }
+
+    docker compose $files --env-file $EnvironmentFile up $upArgs
+
+    if ($LASTEXITCODE -ne 0) {
+        throw "Unable to start local Docker environment, with exit code $LASTEXITCODE."
     }
-    docker compose $files --env-file $EnvironmentFile up -d $upArg
 
     Start-Sleep 20
     ./setup-connectors.ps1 $EnvironmentFile