[PERF-331] ODS/API docker startup

eng/ods-api/.gitignore

@@ -0,0 +1 @@
+.logs/

eng/ods-api/README.md

@@ -0,0 +1,20 @@
+# ODS-API Startup Scripts for Docker
+
+This directory contains startup scripts for the ODS/API running in Docker, using
+versions 5.3, 5.4, 6.1, and 6.2; and another that supports versions 7.1, 7.2,
+Docker Compose for orchestration. There are two scripts: one that supports
+and 7.3 (prerelease).
+
+> [!WARNING]
+> These are poorly secured environments and should not be exposed over the open
+> Internet: using default passwords available in open source code, and does not
+> use TLS encryption.
+
+In each respective directory, copy the `.env.example` file to `.env`. Review the
+`TAG` variable to determine which version you wish to run. Then run `start.ps1`
+to startup the containers and run a bootstrap SQL script that will inject the
+sample `client_id` and `client_secret` into the database. Note that the
+`client_id` is given access to education organization id 255901.
+
+To stop the services, run `start.ps1 -d` (for "down") or `start.ps1 -d -v` to
+stop and remove volumes.

eng/ods-api/v5-v6/.env.example

@@ -0,0 +1,55 @@
+# Options: v2.3.3 (for 6.2), v2.3.2 (for 6.1), v2.1.7 (for 5.4), v2.1.6 (for 5.3)
+TAG=v2.3.3
+
+# Local disk folder where log file volumns are mapped
+LOGS_FOLDER=.logs
+
+# Credentials used to authenticate to Postgres DB,  only needed if using Postgres DB
+# Both are used to enable auth_file security in PGBouncer
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=980jlej.23kd
+POSTGRES_DB=postgres
+
+# PostgreSQL client-side pooling. Consider only if not using PgBouncer (see repository README file for additional information)
+# NPG_POOLING_ENABLED=<Enables or disables client-side pooling (default: false)>
+# NPG_API_MAX_POOL_SIZE_ODS=<The maximum number of connections for each distinct ODS database from each Ed-Fi ODS API container.>
+# NPG_API_MAX_POOL_SIZE_ADMIN=<The maximum number of connections for the EdFi_Admin database from each Ed-Fi ODS API container.>
+# NPG_API_MAX_POOL_SIZE_SECURITY=<The maximum number of connections for the EdFi_Security database from each Ed-Fi ODS API container.>
+# NPG_API_MAX_POOL_SIZE_MASTER=<The maximum number of connections for the 'postgres' default database from each Ed-Fi ODS API container.>
+
+# The hostname of the main API, defaults to 'localhost', used to define the api's url
+API_HOSTNAME=localhost
+
+# The url path to the api, the default is 'api', used to define the api's url
+ODS_VIRTUAL_NAME=api
+
+# Enables or disables the TPDM module: https://www.ed-fi.org/teacher-prep-data-model/
+TPDM_ENABLED=true
+
+# The following needs to be set to specify a healthcheck test for the API
+# RECOMMENDED: To use the default internal API healthcheck endpoint, set the variable as follows:
+API_HEALTHCHECK_TEST="wget --no-verbose --tries=1 --output-document=/dev/null http://localhost/health || exit 1"
+#  To disable the healthcheck, remove the above and instead set the variable as follows:
+# API_HEALTHCHECK_TEST=/bin/true
+
+API_PORT=8001
+
+#
+# Swagger Settings
+#
+
+SWAGGER_PORT=8002
+
+CLIENT_ID=minimalKey
+CLIENT_SECRET=minimalSecret
+
+# The url path to swagger, the default is 'swagger'
+SWAGGER_VIRTUAL_NAME=swagger
+
+# The following needs to be set to specify a healthcheck test for Swagger
+# RECOMMENDED: To use the default internal Swagger healthcheck endpoint, set the variable as follows:
+SWAGGER_HEALTHCHECK_TEST="curl -f http://localhost/health"
+#  To disable the healthcheck, remove the above and instead set the variable as follows:
+# SWAGGER_HEALTHCHECK_TEST=/bin/true
+
+WebApiVersionUrl="http://localhost:${API_PORT}"

eng/ods-api/v5-v6/bootstrap.sql

@@ -0,0 +1,36 @@
+-- SPDX-License-Identifier: Apache-2.0
+-- Licensed to the Ed-Fi Alliance under one or more agreements.
+-- The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+-- See the LICENSE and NOTICES files in the project root for more information.
+
+insert into dbo.Vendors (VendorName)
+select 'Bootstrap Vendor'
+where not exists (select 1 from dbo.Vendors where VendorName = 'Bootstrap Vendor');
+
+insert into dbo.VendorNamespacePrefixes (NamespacePrefix, Vendor_VendorId)
+select 'uri://ed-fi.org', VendorId from dbo.Vendors
+where not exists (select 1 from dbo.VendorNamespacePrefixes where NamespacePrefix = 'uri://ed-fi.org');
+
+insert into dbo.Applications (ApplicationName, OperationalContextUri, Vendor_VendorId, ClaimSetName)
+select 'Bootstrap Application', 'uri://ed-fi.org', VendorId, 'Ed-Fi Sandbox' from dbo.Vendors
+where not exists (select 1 from dbo.Applications where ApplicationName = 'Bootstrap Application');
+
+insert into dbo.ApplicationEducationOrganizations (EducationOrganizationId, Application_ApplicationId)
+select 255901, ApplicationId
+from dbo.Applications
+where ApplicationName = 'Bootstrap Application'
+    and not exists (select 1 from dbo.ApplicationEducationOrganizations where EducationOrganizationId = 255901);
+
+insert into dbo.ApiClients (Key, Secret, Name, IsApproved, UseSandbox, SandboxType, SecretIsHashed, Application_ApplicationId)
+select 'minimalKey', 'minimalSecret', 'Bootstrap', true, false, 0, false, ApplicationId from dbo.Applications
+where not exists (select 1 from dbo.ApiClients where Name = 'Bootstrap');
+
+insert into dbo.ApiClientApplicationEducationOrganizations (ApiClient_ApiClientId, ApplicationEdOrg_ApplicationEdOrgId )
+select ApiClients.ApiClientId, ApplicationEducationOrganizations.ApplicationEducationOrganizationId
+from dbo.ApiClients
+cross join dbo.Applications
+inner join dbo.ApplicationEducationOrganizations on Applications.ApplicationId = ApplicationEducationOrganizations.Application_ApplicationId
+where ApiClients.Name = 'Bootstrap' and Applications.ApplicationName = 'Bootstrap Application'
+and not exists (select 1 from dbo.ApiClientApplicationEducationOrganizations
+    where ApiClient_ApiClientId = ApiClients.ApiClientId
+    and ApplicationEdOrg_ApplicationEdOrgId = ApplicationEducationOrganizations.ApplicationEducationOrganizationId);

eng/ods-api/v5-v6/docker-compose.yml

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+# Licensed to the Ed-Fi Alliance under one or more agreements.
+# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+# See the LICENSE and NOTICES files in the project root for more information.
+
+services:
+  db-ods:
+    image: edfialliance/ods-api-db-ods:${TAG:-v2.3.3}
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      TPDM_ENABLED: ${TPDM_ENABLED:-true}
+    volumes:
+      - vol-db-ods:/var/lib/postgresql/data
+    restart: always
+    container_name: ed-fi-db-ods
+    hostname: ed-fi-db-ods
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready  -U ${POSTGRES_USER}"]
+      start_period: "60s"
+      retries: 3
+
+  db-admin:
+    image: edfialliance/ods-api-db-admin:${TAG:-v2.3.3}
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      API_MODE: SharedInstsance
+    volumes:
+      - vol-db-admin:/var/lib/postgresql/data
+    restart: always
+    container_name: ed-fi-db-admin
+    hostname: ed-fi-db-admin
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready  -U ${POSTGRES_USER}"]
+      start_period: "60s"
+      retries: 3
+
+  api:
+    image: edfialliance/ods-api-web-api:${TAG:-v2.3.3}
+    environment:
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+      POSTGRES_PORT: 5432
+      API_MODE: sharedinstance
+      ODS_POSTGRES_HOST: ed-fi-db-ods
+      ADMIN_POSTGRES_HOST: ed-fi-db-admin
+      PATH_BASE: ""
+      TPDM_ENABLED: ${TPDM_ENABLED:-true}
+      API_HEALTHCHECK_TEST: ${API_HEALTHCHECK_TEST?Please consult env.example to set the API healthcheck test}
+      NPG_POOLING_ENABLED: ${NPG_POOLING_ENABLED:-false}
+      NPG_API_MAX_POOL_SIZE_ODS: ${NPG_API_MAX_POOL_SIZE_ODS}
+      NPG_API_MAX_POOL_SIZE_ADMIN: ${NPG_API_MAX_POOL_SIZE_ADMIN}
+      NPG_API_MAX_POOL_SIZE_SECURITY: ${NPG_API_MAX_POOL_SIZE_SECURITY}
+      NPG_API_MAX_POOL_SIZE_MASTER: ${NPG_API_MAX_POOL_SIZE_MASTER}
+      ODS_CONNECTION_STRING_ENCRYPTION_KEY: ${ODS_CONNECTION_STRING_ENCRYPTION_KEY}
+    volumes:
+      - ${LOGS_FOLDER}:/app/logs
+    ports:
+      - ${API_PORT:-8001}:80
+    depends_on:
+      - db-ods
+      - db-admin
+    restart: always
+    container_name: ed-fi-ods-api
+    hostname: ed-fi-ods-api
+    healthcheck:
+      test: ${API_HEALTHCHECK_TEST}
+      start_period: "60s"
+      retries: 3
+
+  swagger:
+    image: edfialliance/ods-api-web-swaggerui:${TAG:-v2.3.3}
+    environment:
+      POPULATED_KEY: ${CLIENT_ID}
+      POPULATED_SECRET: ${CLIENT_SECRET}
+      WebApiVersionUrl:  ${WebApiVersionUrl}
+      SWAGGER_HEALTHCHECK_TEST: ${SWAGGER_HEALTHCHECK_TEST?Please consult env.example to set the SWAGGER healthcheck test}
+    ports:
+      - ${SWAGGER_PORT:-8002}:80
+    depends_on:
+      - api
+    restart: always
+    container_name: ed-fi-swagger
+    hostname: ed-fi-swagger
+    healthcheck:
+      test: ${SWAGGER_HEALTHCHECK_TEST}
+      start_period: "60s"
+      retries: 3
+
+volumes:
+  vol-db-admin:
+    driver: local
+    name: perf_5_6_ods-api_admin-db
+  vol-db-ods:
+    driver: local
+    name: perf_5_6_ods-api_ods-db

eng/ods-api/v5-v6/start.ps1

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+# Licensed to the Ed-Fi Alliance under one or more agreements.
+# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+# See the LICENSE and NOTICES files in the project root for more information.
+
+[CmdletBinding()]
+param (
+    # Stop services instead of starting them
+    [Switch]
+    $d,
+
+    # Delete volumes after stopping services
+    [Switch]
+    $v
+)
+
+if ($d) {
+    if ($v) {
+        Write-Output "Shutting down services and deleting volumes"
+        docker compose down -v
+    }
+    else {
+        Write-Output "Shutting down services"
+        docker compose down
+    }
+}
+else {
+    $pull = "never"
+    if ($p) {
+        $pull = "always"
+    }
+
+    Write-Output "Starting services"
+    docker compose up -d
+
+    Start-Sleep 15
+
+    docker cp ./bootstrap.sql ed-fi-db-admin:/tmp/bootstrap.sql
+    docker exec -i ed-fi-db-admin sh -c "psql -U postgres -d EdFi_Admin -f /tmp/bootstrap.sql"
+    Start-Sleep 1
+    docker exec -i ed-fi-db-admin sh -c "rm /tmp/bootstrap.sql"
+}

eng/ods-api/v7/.env.example

@@ -0,0 +1,58 @@
+# Options: pre (7.3), 7.2, 7.1
+TAG=pre
+
+# Local disk folder where log file volumns are mapped
+LOGS_FOLDER=.logs
+
+# Credentials used to authenticate to Postgres DB,  only needed if using Postgres DB
+# Both are used to enable auth_file security in PGBouncer
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=980jlej.23kd
+POSTGRES_DB=postgres
+
+# PostgreSQL client-side pooling. Consider only if not using PgBouncer (see repository README file for additional information)
+# NPG_POOLING_ENABLED=<Enables or disables client-side pooling (default: false)>
+# NPG_API_MAX_POOL_SIZE_ODS=<The maximum number of connections for each distinct ODS database from each Ed-Fi ODS API container.>
+# NPG_API_MAX_POOL_SIZE_ADMIN=<The maximum number of connections for the EdFi_Admin database from each Ed-Fi ODS API container.>
+# NPG_API_MAX_POOL_SIZE_SECURITY=<The maximum number of connections for the EdFi_Security database from each Ed-Fi ODS API container.>
+# NPG_API_MAX_POOL_SIZE_MASTER=<The maximum number of connections for the 'postgres' default database from each Ed-Fi ODS API container.>
+
+# The hostname of the main API, defaults to 'localhost', used to define the api's url
+API_HOSTNAME=localhost
+
+# The url path to the api, the default is 'api', used to define the api's url
+ODS_VIRTUAL_NAME=api
+
+# Be sure to change this to a new value - do not use this key in a real deployment.
+ODS_CONNECTION_STRING_ENCRYPTION_KEY=+MkpJfdOoBs2W+UCibqwMcjAF5rUUk6AxPiOAIFNEWQ=
+
+# Enables or disables the TPDM module: https://www.ed-fi.org/teacher-prep-data-model/
+TPDM_ENABLED=true
+
+# The following needs to be set to specify a healthcheck test for the API
+# RECOMMENDED: To use the default internal API healthcheck endpoint, set the variable as follows:
+API_HEALTHCHECK_TEST="wget --no-verbose --tries=1 --output-document=/dev/null http://localhost/health || exit 1"
+#  To disable the healthcheck, remove the above and instead set the variable as follows:
+# API_HEALTHCHECK_TEST=/bin/true
+
+API_PORT=8001
+
+#
+# Swagger Settings
+#
+
+SWAGGER_PORT=8002
+
+CLIENT_ID=minimalKey
+CLIENT_SECRET=minimalSecret
+
+# The url path to swagger, the default is 'swagger'
+SWAGGER_VIRTUAL_NAME=swagger
+
+# The following needs to be set to specify a healthcheck test for Swagger
+# RECOMMENDED: To use the default internal Swagger healthcheck endpoint, set the variable as follows:
+SWAGGER_HEALTHCHECK_TEST="curl -f http://localhost/health"
+#  To disable the healthcheck, remove the above and instead set the variable as follows:
+# SWAGGER_HEALTHCHECK_TEST=/bin/true
+
+VERSION_URL=http://localhost:${API_PORT}

eng/ods-api/v7/bootstrap.sql

@@ -0,0 +1,49 @@
+-- SPDX-License-Identifier: Apache-2.0
+-- Licensed to the Ed-Fi Alliance under one or more agreements.
+-- The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+-- See the LICENSE and NOTICES files in the project root for more information.
+
+insert into dbo.OdsInstances (Name, InstanceType, ConnectionString)
+select 'default', 'ODS', 'host=ed-fi-db-ods;port=5432;username=postgres;password=980jlej.23kd;database=EdFi_Ods;application name=EdFi.Ods.WebApi'
+where not exists (select 1 from dbo.OdsInstances where Name = 'default');
+
+insert into dbo.Vendors (VendorName)
+select 'Bootstrap Vendor'
+where not exists (select 1 from dbo.Vendors where VendorName = 'Bootstrap Vendor');
+
+insert into dbo.VendorNamespacePrefixes (NamespacePrefix, Vendor_VendorId)
+select 'uri://ed-fi.org', VendorId from dbo.Vendors
+where not exists (select 1 from dbo.VendorNamespacePrefixes where NamespacePrefix = 'uri://ed-fi.org');
+
+insert into dbo.Applications (ApplicationName, OperationalContextUri, Vendor_VendorId, ClaimSetName)
+select 'Bootstrap Application', 'uri://ed-fi.org', VendorId, 'Ed-Fi Sandbox' from dbo.Vendors
+where not exists (select 1 from dbo.Applications where ApplicationName = 'Bootstrap Application');
+
+insert into dbo.ApplicationEducationOrganizations (EducationOrganizationId, Application_ApplicationId)
+select 255901, ApplicationId
+from dbo.Applications
+where ApplicationName = 'Bootstrap Application'
+    and not exists (select 1 from dbo.ApplicationEducationOrganizations where EducationOrganizationId = 255901);
+
+insert into dbo.ApiClients (Key, Secret, Name, IsApproved, UseSandbox, SandboxType, SecretIsHashed, Application_ApplicationId)
+select 'minimalKey', 'minimalSecret', 'Bootstrap', true, false, 0, false, ApplicationId from dbo.Applications
+where not exists (select 1 from dbo.ApiClients where Name = 'Bootstrap');
+
+insert into dbo.ApiClientApplicationEducationOrganizations (ApiClient_ApiClientId, ApplicationEdOrg_ApplicationEdOrgId )
+select ApiClients.ApiClientId, ApplicationEducationOrganizations.ApplicationEducationOrganizationId
+from dbo.ApiClients
+cross join dbo.Applications
+inner join dbo.ApplicationEducationOrganizations on Applications.ApplicationId = ApplicationEducationOrganizations.Application_ApplicationId
+where ApiClients.Name = 'Bootstrap' and Applications.ApplicationName = 'Bootstrap Application'
+and not exists (select 1 from dbo.ApiClientApplicationEducationOrganizations
+    where ApiClient_ApiClientId = ApiClients.ApiClientId
+    and ApplicationEdOrg_ApplicationEdOrgId = ApplicationEducationOrganizations.ApplicationEducationOrganizationId);
+
+insert into dbo.ApiClientOdsInstances (apiclient_apiclientid, odsinstance_odsinstanceid)
+select ApiClients.ApiClientId, OdsInstances.OdsInstanceId
+from dbo.ApiClients
+cross join dbo.OdsInstances
+where ApiClients.Name = 'Bootstrap' and OdsInstances.Name = 'default'
+and not exists (select 1 from dbo.ApiClientOdsInstances
+    where apiclient_apiclientid = ApiClients.ApiClientId and odsinstance_odsinstanceid = OdsInstances.odsinstanceid
+);