[RND 538] Load balancing API scale out (#288)

* Load balancing API scale out

* Some progress.

* working load balancer

* case correction

* Improve example env file

* Load balancer demonstration

* Add license header
Remove port 80

* Remove the temporary RND358 directory

* Undoing changes on local.authz.http

* Changes for metadata url.

* results - first draft.

* Trying to fix unit tests

* Documentation

* Replacing png images with jpeg format.

* Trying to fix docker build on on-pullrequest-dockerfile workflow.

* Fixing screenshots references.

* Documentation

---------

Co-authored-by: Stephen A. Fuqua <[email protected]>

.github/workflows/on-pullrequest-dockerfile.yml

@@ -37,5 +37,5 @@ jobs:
       - name: Build the NPM Docker image
         # To confirm no errors occur. The "local" one will be built by the e2e
         # testing process.
-        run: docker build --build-arg ML_VERSION=pre .
+        run: docker build --build-arg ML_VERSION=latest .
         working-directory: docker

Meadowlark-js/.env.example

@@ -1,32 +1,47 @@
-# Required settings for integration tests
-POSTGRES_USER=<postgres user>
-POSTGRES_PASSWORD=<postgres password>
-
-# Required settings to satisfy the configuration process - although not used by the
-# integration tests, failure to set these values will cause the configuration process
-# to fail.
-DOCUMENT_STORE_PLUGIN=@edfi/meadowlark-mongodb-backend
+# The OAUTH_SIGNING_KEY may need quotation marks around the value, unlike other keys.
+OAUTH_SIGNING_KEY="<run `openssl rand -base64 256` to create a key>"
+
+# With the exception of MONGODB_MAX_NUMBER_OF_RETRIES, do not override MongoDB settings here, 
+# because the integration tests use an "in memory" # version of MongoDB that is configured automatically. 
+# Any settings applied here will cause the connection to fail.
+
+MONGODB_MAX_NUMBER_OF_RETRIES=1
+
+
+#
+# The settings below are typically good enough to get started
+#
+
+#### Environment variables for security
 OWN_OAUTH_CLIENT_ID_FOR_CLIENT_AUTH=meadowlark_verify-only_key_1
 OWN_OAUTH_CLIENT_SECRET_FOR_CLIENT_AUTH=meadowlark_verify-only_secret_1
 OAUTH_SERVER_ENDPOINT_FOR_OWN_TOKEN_REQUEST=http://localhost:3000/local/oauth/token
 OAUTH_SERVER_ENDPOINT_FOR_TOKEN_VERIFICATION=http://localhost:3000/local/oauth/verify
 LOG_PRETTY_PRINT=true
 
+
 #### Authorization Server options
 # Configurable authorization store plugin - set to an npm package name
 AUTHORIZATION_STORE_PLUGIN=@edfi/meadowlark-mongodb-backend
 
-# The OAUTH_SIGNING_KEY may need quotation marks around the value, unlike other keys.
-OAUTH_SIGNING_KEY="<run `openssl rand -base64 256` to create a key>"
+### School year validation
+# Although there should only be one year of data in an Ed-Fi API instance,
+# there are cases where future years need to be recorded - for example, a 
+# student's expected (future) graduation year. Rather than listing these
+# out individually, provide a range of valid years. Any POST/PUT with
+# a school year outside this range will be rejected with a 400 response.
+BEGIN_ALLOWED_SCHOOL_YEAR=2022
+END_ALLOWED_SCHOOL_YEAR=2035
 
-# Optional settings. These are the default values, so they only need to be set in your
-# .env file if you need to override the defaults.
-MEADOWLARK_DATABASE_NAME=meadowlark
-POSTGRES_HOST=localhost
-POSTGRES_PORT=5432
+#### Experimental features
+
+# Allow the API to accept "1" for 1 and "true" for true, etc. Must be true when
+# using the client-side bulk loader.
+ALLOW_TYPE_COERCION=true
+# Allow the API to accept an extra { _ext: {} } property in a payload. Must be
+# true when bulk uploading the ODS/API "populated template".
+ALLOW__EXT_PROPERTY=true
 
-# With the exception of MONGODB_MAX_NUMBER_OF_RETRIES, do not override MongoDB settings here, 
-# because the integration tests use an "in memory" # version of MongoDB that is configured automatically. 
-# Any settings applied here will cause the connection to fail.
 
-MONGODB_MAX_NUMBER_OF_RETRIES=1
+SAVE_LOG_TO_FILE=false
+# LOG_FILE_LOCATION=c:/temp/

Meadowlark-js/Dockerfile

@@ -18,7 +18,7 @@ RUN apt-get update && \
   chmod 600 /var/log/meadowlark.log
 
 # Start a new layer so that we don't need to rebuild everything above
-FROM base as build
+FROM base AS build
 LABEL maintainer="Ed-Fi Alliance, LLC and Contributors <[email protected]>"
 WORKDIR /app
 
@@ -30,7 +30,7 @@ RUN npm run docker:install && \
   npm run build
 
 # Start from the node base image, copy all js files, and run Fastify
-FROM base as prod
+FROM base AS prod
 LABEL maintainer="Ed-Fi Alliance, LLC and Contributors <[email protected]>"
 WORKDIR /app
 

Meadowlark-js/docker-compose.yml

@@ -1,15 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+# Licensed to the Ed-Fi Alliance under one or more agreements.
+# The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+# See the LICENSE and NOTICES files in the project root for more information.
+
 version: '3.8'
 
 services:
   # Meadowlark API
-  meadowlark-api-ml-local:
-    # requires a local build, `npm run docker:build`.
-    image: meadowlark:latest
-    container_name: meadowlark-api-ml-local
-    hostname: api
+  meadowlark-api-ml-local-1:
+    image: meadowlark-api
+    build: ./
+    container_name: meadowlark-api-ml-local-1
+    hostname: meadowlark-api-ml-local-1
     ports:
-      - 3000:3000
-    environment:
+      - 3001:3000
+    environment: &ML_ENV
       OAUTH_SIGNING_KEY: ${OAUTH_SIGNING_KEY}
       # Next three settings should not be used as such in production environment
       OAUTH_HARD_CODED_CREDENTIALS_ENABLED: ${OAUTH_HARD_CODED_CREDENTIALS_ENABLED:-true}
@@ -41,12 +46,56 @@ services:
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-abcdefgh1!}
       LOG_TO_FILE: true
     restart: unless-stopped
-    healthcheck:
+    healthcheck: &ML_HEALTH
       test: curl -s http://localhost:3000/local >/dev/null || exit 1
       interval: 30s
       timeout: 10s
       retries: 50
 
+  meadowlark-api-ml-local-2:
+    image: meadowlark-api
+    build: ./
+    container_name: meadowlark-api-ml-local-2
+    hostname: meadowlark-api-ml-local-2
+    environment:
+      *ML_ENV
+    restart: unless-stopped
+    healthcheck:
+      *ML_HEALTH
+
+  meadowlark-api-ml-local-3:
+    image: meadowlark-api
+    build: ./
+    container_name: meadowlark-api-ml-local-3
+    hostname: meadowlark-api-ml-local-3
+    environment:
+      *ML_ENV
+    restart: unless-stopped
+    healthcheck:
+      *ML_HEALTH
+
+  meadowlark-oauth-ml-local-1:
+    image: meadowlark-api
+    build: ./
+    container_name: meadowlark-oauth-ml-local-1
+    hostname: meadowlark-oauth-ml-local-1
+    environment:
+      *ML_ENV
+    restart: unless-stopped
+    healthcheck:
+      *ML_HEALTH
+
+  meadowlark-oauth-ml-local-2:
+    image: meadowlark-api
+    build: ./
+    container_name: meadowlark-oauth-ml-local-2
+    hostname: meadowlark-oauth-ml-local-2
+    environment:
+      *ML_ENV
+    restart: unless-stopped
+    healthcheck:
+      *ML_HEALTH
+
   # OpenSearch backend
   opensearch-ml-local:
     image: opensearchproject/opensearch:2.7.0@sha256:55f1f67e7d3645aa838b63a589bce5645154ba275814e52d4638d371ca0f8cb5
@@ -159,6 +208,19 @@ services:
   #   volumes:
   #     - pgsql-ml-data:/var/lib/postgresql/data
 
+  lb:
+    image: nginx:alpine
+    volumes:
+      - ./sample-load-balancer.conf:/etc/nginx/conf.d/default.conf
+    ports:
+      - 3000:3000
+    depends_on:
+      - meadowlark-api-ml-local-1
+      - meadowlark-api-ml-local-2
+      - meadowlark-api-ml-local-3
+      - meadowlark-oauth-ml-local-1
+      - meadowlark-oauth-ml-local-2
+
 volumes:
   mongo-ml-local-data1:
     labels:

Meadowlark-js/packages/meadowlark-core/src/handler/MetadataHandler.ts

@@ -136,9 +136,13 @@ async function getFileFromBlobStorage(
  * Retrieves and caches a swagger specification from an external URL.
  */
 async function getSwaggerSpecification(url: string, frontendRequest: FrontendRequest): Promise<FrontendResponse> {
-  return getFileFromBlobStorage(url, frontendRequest, (data: any) =>
-    useTemplate(data, frontendRequest.headers?.Host || '', frontendRequest.stage),
-  );
+  return getFileFromBlobStorage(url, frontendRequest, (data: any) => {
+    const host =
+      frontendRequest.headers['x-forwarded-host'] && frontendRequest.headers['x-forwarded-port']
+        ? `${frontendRequest.headers['x-forwarded-host']}:${frontendRequest.headers['x-forwarded-port']}`
+        : frontendRequest.headers?.Host || '';
+    return useTemplate(data, host, frontendRequest.stage);
+  });
 }
 
 /**

Meadowlark-js/packages/meadowlark-core/src/handler/UrlBuilder.ts

@@ -13,5 +13,9 @@ const getHttpProtocol = (stage: string): string => (stage === 'local' ? 'http' :
  */
 export const buildBaseUrlFromRequest = (frontendRequest: FrontendRequest): string => {
   const protocol = getHttpProtocol(frontendRequest.stage);
-  return `${protocol}://${frontendRequest.headers.host}/${frontendRequest.stage}`;
+  const host =
+    frontendRequest.headers['x-forwarded-host'] && frontendRequest.headers['x-forwarded-port']
+      ? `${frontendRequest.headers['x-forwarded-host']}:${frontendRequest.headers['x-forwarded-port']}`
+      : frontendRequest.headers.host;
+  return `${protocol}://${host}/${frontendRequest.stage}`;
 };

Meadowlark-js/sample-load-balancer.conf

@@ -0,0 +1,45 @@
+upstream meadowlarkoauth {
+    server meadowlark-oauth-ml-local-1:3000;
+    server meadowlark-oauth-ml-local-2:3000;
+}
+
+upstream meadowlarkresources {
+    server meadowlark-api-ml-local-1:3000;
+    server meadowlark-api-ml-local-2:3000;
+    server meadowlark-api-ml-local-3:3000;
+}
+
+## use as reverse proxy
+# proxy_cache_path cache levels=1:2 keys_zone=STATIC:10m inactive=24h max_size=1g;
+
+server {
+    listen       3000;
+    server_name  localhost;
+
+    #access_log  /var/log/nginx/host.access.log  main;
+
+    location /local {
+        proxy_pass  http://meadowlarkresources;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host;
+        proxy_set_header X-Forwarded-Port 3000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location /local/oauth {
+        proxy_pass http://meadowlarkoauth;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host;
+        proxy_set_header X-Forwarded-Port 3000;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    #error_page  404              /404.html;
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+}

docs/performance-testing/RND-538-Raw-Results.txt

@@ -0,0 +1,48 @@
+
+## RND-538 - Raw Results
+
+WITH LOADBALANCING:
+
+FASTIFY_NUM_THREADS=4
+	Total Time: 00:02:47.7064319
+	Total Time: 00:02:37.4898545
+	Total Time: 00:02:40.2312613
+	Total Time: 00:02:35.9549423
+	Total Time: 00:02:43.1832261
+
+FASTIFY_NUM_THREADS=2
+	Total Time: 00:02:30.5095058
+	Total Time: 00:02:18.5902512
+	Total Time: 00:02:26.0740558
+	Total Time: 00:02:25.2780308
+	Total Time: 00:02:44.5890730
+
+FASTIFY_NUM_THREADS=1
+	Total Time: 00:02:33.4093732
+	Total Time: 00:02:44.8779734
+	Total Time: 00:02:43.5545080
+	Total Time: 00:02:41.5097791
+	Total Time: 00:02:38.8979786
+
+WITHOUT LOADBALANCING:
+
+FASTIFY_NUM_THREADS=4
+	Total Time: 00:02:26.6134977
+	Total Time: 00:02:28.2016753
+	Total Time: 00:02:36.1254584
+	Total Time: 00:02:29.2328461
+	Total Time: 00:02:21.2826979
+
+FASTIFY_NUM_THREADS=2
+	Total Time: 00:02:29.0765482
+	Total Time: 00:02:45.2344755
+	Total Time: 00:02:26.9476904
+	Total Time: 00:02:30.5897358
+	Total Time: 00:02:39.6380510
+
+FASTIFY_NUM_THREADS=1
+	Total Time: 00:03:09.6049885
+	Total Time: 00:03:01.4044423
+	Total Time: 00:02:59.3843134
+	Total Time: 00:02:58.3939055
+	Total Time: 00:03:13.0434055