feat: implement graceful shutdowns

This patch introduces graceful shutdowns in the form of a ready flag
that the health endpoints utilizes when determining whether the
readiness probe returns healthy or unhealthy.

When a shutdown signal is detected the ready flag is set to false and a
timeout for actual shutdown starts.

The shutdown process will wait until either all active requests have finished
processing OR the specified timeout is met. Whichever happens first.

Author: jnatten

src/config.ts

@@ -72,3 +72,4 @@ export const uptimeRepo = getEnvironmentVariabel("UPTIME_REPO", "oppetid");
 export const uptimeToken = getEnvironmentVariabel("UPTIME_API_TOKEN", undefined);
 export const googleApiKey = getEnvironmentVariabel("NDLA_GOOGLE_API_KEY", undefined);
 export const slowLogTimeout = getEnvironmentVariabel("SLOW_LOG_TIMEOUT", "500");
+export const gracePeriodSeconds = parseInt(getEnvironmentVariabel("READINESS_PROBE_DETECTION_SECONDS", "7"));

src/server.ts

@@ -8,23 +8,30 @@
 
 import compression from "compression";
 import cors from "cors";
-import express, { json, Request, Response } from "express";
+import express, { json } from "express";
 import promBundle from "express-prom-bundle";
 import { ApolloServer } from "@apollo/server";
 import { expressMiddleware } from "@apollo/server/express4";
 import { port } from "./config";
 import { resolvers } from "./resolvers";
 import { typeDefs } from "./schema";
 import correlationIdMiddleware from "./utils/correlationIdMiddleware";
-import { logError } from "./utils/logger";
+import { getLogger, logError } from "./utils/logger";
 import loggerMiddleware from "./utils/loggerMiddleware";
 import { contextExpressMiddleware } from "./utils/context/contextMiddleware";
 import { getContextOrThrow } from "./utils/context/contextStore";
+import { Server } from "http";
+import { healthRouter } from "./utils/healthRouter";
+import { activeRequestsMiddleware } from "./utils/activeRequestsMiddleware";
+import { gracefulShutdown } from "./utils/gracefulShutdown";
 
 const GRAPHQL_PORT = port;
 
 const app = express();
 
+let server: Server;
+let apolloServer: ApolloServer<ContextWithLoaders>;
+
 const metricsMiddleware = promBundle({
   includeMethod: true,
   includePath: false,
@@ -37,12 +44,11 @@ app.use(metricsMiddleware);
 app.use(compression());
 app.use(express.json({ limit: "1mb" }));
 
-app.get("/health", (_: Request, res: Response) => {
-  res.status(200).json({ status: 200, text: "Health check ok" });
-});
+app.use(healthRouter);
+app.use(activeRequestsMiddleware);
 
-async function startApolloServer() {
-  const server = new ApolloServer({
+async function startApolloServer(): Promise<void> {
+  apolloServer = new ApolloServer({
     typeDefs,
     resolvers,
     introspection: true,
@@ -60,21 +66,21 @@ async function startApolloServer() {
       };
     },
   });
-  await server.start();
+  await apolloServer.start();
   app.use(
     "/graphql-api/graphql",
     cors(),
     json(),
     correlationIdMiddleware,
     contextExpressMiddleware,
     loggerMiddleware,
-    expressMiddleware(server, { context: async () => getContextOrThrow() }),
+    expressMiddleware(apolloServer, { context: async () => getContextOrThrow() }),
+  );
+  server = app.listen(GRAPHQL_PORT, () =>
+    getLogger().info(`GraphQL Playground is now running on http://localhost:${GRAPHQL_PORT}/graphql-api/graphql`),
   );
 }
 
-startApolloServer();
+process.on("SIGTERM", () => gracefulShutdown(server, apolloServer));
 
-app.listen(GRAPHQL_PORT, () =>
-  // eslint-disable-next-line no-console
-  console.log(`GraphQL Playground is now running on http://localhost:${GRAPHQL_PORT}/graphql-api/graphql`),
-);
+startApolloServer();

src/utils/activeRequestsMiddleware.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) 2025-present, NDLA.
+ *
+ * This source code is licensed under the GPLv3 license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+
+import { NextFunction, Request, Response } from "express";
+
+let activeRequests = 0;
+
+export const activeRequestsMiddleware = (_: Request, res: Response, next: NextFunction): void => {
+  activeRequests++;
+  res.on("finish", () => {
+    activeRequests--;
+  });
+  next();
+};
+
+export const getActiveRequests = (): number => {
+  return activeRequests;
+};

src/utils/gracefulShutdown.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) 2025-present, NDLA.
+ *
+ * This source code is licensed under the GPLv3 license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+
+import { ApolloServer } from "@apollo/server";
+import { gracePeriodSeconds } from "../config";
+import { getActiveRequests } from "./activeRequestsMiddleware";
+import { getIsShuttingDown, setIsShuttingDown } from "./healthRouter";
+import { Server } from "http";
+import getLogger from "./logger";
+
+async function waitForActiveRequests() {
+  const timeout = 30000;
+  const pollInterval = 250;
+  const start = Date.now();
+
+  const activeRequests = getActiveRequests();
+  getLogger().info(`Waiting for ${activeRequests} active requests to finish...`);
+  while (getActiveRequests() > 0 && Date.now() - start < timeout) {
+    await new Promise((resolve) => setTimeout(resolve, pollInterval));
+  }
+
+  if (getActiveRequests() > 0) {
+    getLogger().warn(
+      `Timeout reached while waiting for active requests to finish. Active requests: ${getActiveRequests()}`,
+    );
+  } else {
+    getLogger().info("All active requests have finished processing.");
+  }
+}
+
+export async function gracefulShutdown(server: Server, apolloServer: ApolloServer<ContextWithLoaders>) {
+  const logger = getLogger();
+  if (getIsShuttingDown()) return;
+  setIsShuttingDown();
+  logger.info(
+    `Recieved shutdown signal, waiting ${gracePeriodSeconds} seconds for shutdown to be detected before stopping...`,
+  );
+  setTimeout(async () => {
+    logger.info("Shutting down gracefully...");
+    await waitForActiveRequests();
+    if (server) server.close();
+    if (apolloServer) await apolloServer.stop();
+    process.exit(0);
+  }, gracePeriodSeconds * 1000);
+}

src/utils/healthRouter.ts

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) 2025-present, NDLA.
+ *
+ * This source code is licensed under the GPLv3 license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ */
+
+import { Router, Request, Response } from "express";
+
+let isShuttingDown = false;
+
+export function getIsShuttingDown(): boolean {
+  return isShuttingDown;
+}
+
+export function setIsShuttingDown() {
+  isShuttingDown = true;
+}
+
+export const healthRouter = Router();
+const livenessHandler = (_: Request, res: Response) => {
+  res.status(200).json({ status: 200, text: "Health check ok" });
+};
+
+healthRouter.get("/health", livenessHandler);
+healthRouter.get("/health/liveness", livenessHandler);
+healthRouter.get("/health/readiness", (_: Request, res: Response) => {
+  if (!isShuttingDown) {
+    res.status(200).json({ status: 200, text: "Health check ok" });
+  } else {
+    res.status(500).json({ status: 500, text: "Service shutting down" });
+  }
+});