Skip to content

Commit

Permalink
[DMS-380] DMS-337 DMS-380 E2E Testing Validation for Vendors (#368)
Browse files Browse the repository at this point in the history
* Add script for starting the config service

* Add required e2e project files

* Add vendors feature file

* Update build script for including the docker containers for E2E

* update scope

* Include client role

* create network

* Fix config docker image name
  • Loading branch information
CSR2017 authored Dec 11, 2024
1 parent e988ef3 commit d3af85e
Show file tree
Hide file tree
Showing 13 changed files with 945 additions and 6 deletions.
9 changes: 5 additions & 4 deletions .github/workflows/on-prerelease.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ env:
DMS_PACKAGE_NAME: "EdFi.DataManagementService"
CS_PACKAGE_NAME: "EdFi.DmsConfigurationService"
IMAGE_NAME: ${{ vars.IMAGE_NAME }}
CONFIG_IMAGE_NAME: "edfialliance/dms-configuration-service"
DOCKER_USERNAME: ${{ vars.DOCKER_USERNAME }}
DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }}
REF: ${{ github.ref_name }}
Expand Down Expand Up @@ -508,13 +509,13 @@ jobs:
if [[ $PACKAGEVERSION =~ "alpha" ]]
then
# Pre-releases get the tag "pre"
DMSTAGS="${{ env.IMAGE_NAME }}:pre"
DMSTAGS="${{ env.CONFIG_IMAGE_NAME }}:pre"
else
# Releases get the version, plus shortened form for minor release.
# We are not using shortened form for major or using "latest"
# because they are too imprecise.
MINOR=`echo ${PACKAGEVERSION} | awk -F"." '{print $1"."$2}'`
DMSTAGS="${{ env.IMAGE_NAME }}:${PACKAGEVERSION},${{ env.IMAGE_NAME }}:${MINOR}"
DMSTAGS="${{ env.CONFIG_IMAGE_NAME }}:${PACKAGEVERSION},${{ env.CONFIG_IMAGE_NAME }}:${MINOR}"
fi
SEMVERSION=${PACKAGEVERSION:7} # strip off the leading 'cs-pre-'
Expand All @@ -534,13 +535,13 @@ jobs:
id: metadatamanagementservice
uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
with:
images: ${{ env.IMAGE_NAME }}
images: ${{ env.CONFIG_IMAGE_NAME }}

- name: Build and push Config image
uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
with:
context: "{{defaultContext}}:src/config"
cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:pre
cache-from: type=registry,ref=${{ env.CONFIG_IMAGE_NAME }}:pre
cache-to: type=inline
build-args: VERSION=${{ steps.prepare-tags.outputs.VERSION }}
file: Nuget.Dockerfile
Expand Down
10 changes: 9 additions & 1 deletion build-config.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,15 @@ function RunE2E {
}

function E2ETests {
Invoke-Step { DockerBuild }
Invoke-Execute {
try {
Push-Location eng/docker-compose/
./start-local-config.ps1 -EnvironmentFile "./.env.config.e2e"
}
finally {
Pop-Location
}
}
Invoke-Step { RunE2E }
}

Expand Down
36 changes: 36 additions & 0 deletions eng/docker-compose/.env.config.e2e
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# -----------------
# Postgres database
# -----------------
POSTGRES_PASSWORD=abcdefgh1!
POSTGRES_DB_NAME=edfi_configurationservice
POSTGRES_PORT=5435

# --------
# Keycloak
# --------

KEYCLOAK_ADMIN=admin
KEYCLOAK_ADMIN_PASSWORD=admin
KEYCLOAK_PORT=8045

# --------------
# Config Service
# --------------

DMS_CONFIG_ASPNETCORE_HTTP_PORTS=8081
DMS_CONFIG_DATASTORE=postgresql
DMS_CONFIG_DATABASE_CONNECTION_STRING=host=dms-postgresql;port=5432;username=postgres;password=${POSTGRES_PASSWORD};database=${POSTGRES_DB_NAME};
DMS_CONFIG_IDENTITY_ALLOW_REGISTRATION=true
DMS_CONFIG_IDENTITY_SERVICE_ROLE=config-service-app
DMS_CONFIG_IDENTITY_CLIENT_ROLE=dms-client
DMS_CONFIG_IDENTITY_AUTHORITY=http://dms-keycloak:8080/realms/edfi
DMS_CONFIG_IDENTITY_AUDIENCE=account
KEYCLOAK_URL=http://dms-keycloak:8080
KEYCLOAK_REALM=edfi
DMS_CONFIG_IDENTITY_CLIENT_ID=DmsConfigurationService
DMS_CONFIG_IDENTITY_CLIENT_SECRET=s3creT@09
DMS_CONFIG_IDENTITY_REQUIRE_HTTPS=false
DMS_CONFIG_IDENTITY_ROLE_CLAIM_TYPE=http://schemas\\.microsoft\\.com/ws/2008/06/identity/claims/role
DMS_CONFIG_IDENTITY_SCOPE=edfi_admin_api/full_access
DMS_CONFIG_LOG_LEVEL=Information
DMS_CONFIG_DEPLOY_DATABASE=true
2 changes: 1 addition & 1 deletion eng/docker-compose/local-config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ services:
environment:
ASPNETCORE_HTTP_PORTS: ${DMS_CONFIG_ASPNETCORE_HTTP_PORTS:-8081}
DMS_CONFIG_DATASTORE: ${DMS_CONFIG_DATASTORE:-postgresql}
DMS_CONFIG_DATABASE_CONNECTION_STRING: ${DMS_CONFIG_DATABASE_CONNECTION_STRING:-host=localhost;port=5432;username=postgres;database=edfi_configurationservice;}
DMS_CONFIG_DATABASE_CONNECTION_STRING: ${DMS_CONFIG_DATABASE_CONNECTION_STRING:-host=dms-postgresql;port=5432;username=postgres;password=${POSTGRES_PASSWORD};database=edfi_configurationservice;}
DMS_CONFIG_IDENTITY_ALLOW_REGISTRATION: ${DMS_CONFIG_IDENTITY_ALLOW_REGISTRATION:-false}
DMS_CONFIG_IDENTITY_SERVICE_ROLE: ${DMS_CONFIG_IDENTITY_SERVICE_ROLE:-config-service-app}
DMS_CONFIG_IDENTITY_CLIENT_ROLE: ${DMS_CONFIG_IDENTITY_CLIENT_ROLE:-dms-client}
Expand Down
66 changes: 66 additions & 0 deletions eng/docker-compose/start-local-config.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# SPDX-License-Identifier: Apache-2.0
# Licensed to the Ed-Fi Alliance under one or more agreements.
# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
# See the LICENSE and NOTICES files in the project root for more information.

[CmdletBinding()]
param (
# Stop services instead of starting them
[Switch]
$d,

# Delete volumes after stopping services
[Switch]
$v,

# Environment file
[string]
$EnvironmentFile = "./.env",

# Force a rebuild
[Switch]
$r
)

$files = @(
"-f",
"postgresql.yml",
"-f",
"local-config.yml",
"-f",
"keycloak.yml"
)

if ($d) {
if ($v) {
Write-Output "Shutting down with volume delete"
docker compose $files down -v
}
else {
Write-Output "Shutting down"
docker compose $files down
}
}
else {

$existingNetwork = docker network ls --filter name="dms" -q
if (! $existingNetwork) {
docker network create dms
}

$upArgs = @(
"--detach"
)
if ($r) { $upArgs += @("--build") }

Write-Output "Starting locally-built DMS config service"

docker compose $files --env-file $EnvironmentFile up $upArgs

if ($LASTEXITCODE -ne 0) {
throw "Unable to start local Docker environment, with exit code $LASTEXITCODE."
}

Start-Sleep 20
./setup-keycloak.ps1
}
9 changes: 9 additions & 0 deletions src/config/EdFi.DmsConfigurationService.sln
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EdFi.DmsConfigurationServic
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EdFi.DmsConfigurationService.Backend.Postgresql.Test.Integration", "backend\EdFi.DmsConfigurationService.Backend.Postgresql.Test.Integration\EdFi.DmsConfigurationService.Backend.Postgresql.Test.Integration.csproj", "{EE23DFFB-C61D-48FE-B821-2442D67D7AF9}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{C3055527-CED8-4831-8ED4-CCB03E353281}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "EdFi.DmsConfigurationService.Tests.E2E", "tests\EdFi.DmsConfigurationService.Tests.E2E\EdFi.DmsConfigurationService.Tests.E2E.csproj", "{334D8F2D-4D9D-4B8B-8F56-3F8EA7FDC218}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -68,6 +72,10 @@ Global
{EE23DFFB-C61D-48FE-B821-2442D67D7AF9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{EE23DFFB-C61D-48FE-B821-2442D67D7AF9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{EE23DFFB-C61D-48FE-B821-2442D67D7AF9}.Release|Any CPU.Build.0 = Release|Any CPU
{334D8F2D-4D9D-4B8B-8F56-3F8EA7FDC218}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{334D8F2D-4D9D-4B8B-8F56-3F8EA7FDC218}.Debug|Any CPU.Build.0 = Debug|Any CPU
{334D8F2D-4D9D-4B8B-8F56-3F8EA7FDC218}.Release|Any CPU.ActiveCfg = Release|Any CPU
{334D8F2D-4D9D-4B8B-8F56-3F8EA7FDC218}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand All @@ -82,6 +90,7 @@ Global
{5F62ACE6-B44A-40AE-B5EC-AF2BBA9FF1E3} = {82B01A63-4B8A-4965-9806-001128050FB5}
{E97DCC9F-C27F-497D-94FE-3DD39038C38B} = {F90FF019-91D3-462E-94A4-96B52681DB16}
{EE23DFFB-C61D-48FE-B821-2442D67D7AF9} = {82B01A63-4B8A-4965-9806-001128050FB5}
{334D8F2D-4D9D-4B8B-8F56-3F8EA7FDC218} = {C3055527-CED8-4831-8ED4-CCB03E353281}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {BB22426B-6AA8-4C58-9557-376F107B547B}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="coverlet.collector">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="FluentAssertions" />
<PackageReference Include="Microsoft.NET.Test.Sdk" />
<PackageReference Include="Microsoft.Playwright.NUnit" />
<PackageReference Include="Npgsql" />
<PackageReference Include="NUnit" />
<PackageReference Include="NUnit.Analyzers" />
<PackageReference Include="NUnit3TestAdapter" />
<PackageReference Include="Reqnroll" />
<PackageReference Include="Reqnroll.NUnit" />
</ItemGroup>

<ItemGroup>
<Using Include="NUnit.Framework" />
</ItemGroup>

<ItemGroup>
<Folder Include="Extensions\" />
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// SPDX-License-Identifier: Apache-2.0
// Licensed to the Ed-Fi Alliance under one or more agreements.
// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
// See the LICENSE and NOTICES files in the project root for more information.

using System.Globalization;
using System.Text.Encodings.Web;
using System.Text.Json;
using Reqnroll;

namespace EdFi.DmsConfigurationService.Tests.E2E.Extensions;

internal static class DataTableRowExtensions
{
public static string Parse(this DataTableRow dataRow)
{
var options = new JsonSerializerOptions
{
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
WriteIndented = true
};

var rowDict = new Dictionary<string, object>();
foreach (var column in dataRow.Keys)
{
rowDict[column] = ConvertValueToCorrectType(dataRow[column]);
}

return JsonSerializer.Serialize(rowDict, options);
}

private static object ConvertValueToCorrectType(string value)
{
// When other data type treated as string (ex: CalenderCode: "255901107")
if (value.StartsWith('"') && value.EndsWith('"'))
{
return value.Trim('"');
}

if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var intValue))
{
return intValue;
}

if (
decimal.TryParse(
value,
NumberStyles.Number,
CultureInfo.InvariantCulture,
out var decimalValue
)
)
{
return decimalValue;
}

if (
DateTime.TryParse(
value,
CultureInfo.InvariantCulture,
DateTimeStyles.None,
out var dateTimeValue
)
)
{
return dateTimeValue.Date.ToString("yyyy-MM-dd");
}

if (bool.TryParse(value, out var boolValue))
{
return boolValue;
}

if (value.StartsWith('[') && value.EndsWith(']') || value.StartsWith('{') && value.EndsWith('}'))
{
using var document =
JsonDocument.Parse(value) ?? throw new InvalidOperationException($"Error while parsing {value}");

return document.RootElement.Clone();
}

return value;
}
}
Loading

0 comments on commit d3af85e

Please sign in to comment.