[RND-643] Investigate Serializable Snapshot Isolation (SSI) in Postgr…

…eSQL Add retry logic to upsert, update and delete. Update transactions to use SSI. Add validation to retry, when error code: 40001. Add the POSTGRES_MAX_NUMBER_OF_RETRIES to env file. Add tests to validate changes. Remove FOR SHARE NOWAIT from select
Ed-Fi-Exchange-OSS · Oct 17, 2023 · 92b2880 · 92b2880
1 parent f339095
commit 92b2880
Show file tree

Hide file tree

Showing 11 changed files with 771 additions and 310 deletions.
diff --git a/Meadowlark-js/.env.example b/Meadowlark-js/.env.example
@@ -7,7 +7,8 @@ OAUTH_SIGNING_KEY="<run `openssl rand -base64 256` to create a key>"
 
 MONGODB_MAX_NUMBER_OF_RETRIES=1
 
-
+# Set the maximum number of retries for PostgreSql.
+POSTGRES_MAX_NUMBER_OF_RETRIES=1
 #
 # The settings below are typically good enough to get started
 #

diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json b/Meadowlark-js/backends/meadowlark-postgresql-backend/package.json
@@ -22,6 +22,7 @@
     "@edfi/meadowlark-authz-server": "^0.3.6-pre-36",
     "@edfi/meadowlark-core": "^v0.3.6-pre-36",
     "@edfi/meadowlark-utilities": "^v0.3.6-pre-36",
+    "async-retry": "^1.3.3",
     "pg": "^8.11.1",
     "pg-format": "^1.0.4",
     "ramda": "0.29.0",

diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Delete.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/Delete.ts
@@ -3,7 +3,8 @@
 // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
 // See the LICENSE and NOTICES files in the project root for more information.
 
-import { Logger } from '@edfi/meadowlark-utilities';
+import retry from 'async-retry';
+import { Logger, Config } from '@edfi/meadowlark-utilities';
 import type { DeleteResult, DeleteRequest, ReferringDocumentInfo, MeadowlarkId } from '@edfi/meadowlark-core';
 import type { PoolClient } from 'pg';
 import {
@@ -26,91 +27,128 @@ export async function deleteDocumentByDocumentUuid(
   client: PoolClient,
 ): Promise<DeleteResult> {
   Logger.debug(`${moduleName}.deleteDocumentByDocumentUuid ${documentUuid}`, traceId);
-
-  let deleteResult: DeleteResult = { response: 'UNKNOWN_FAILURE', failureMessage: '' };
-  let meadowlarkId: MeadowlarkId = '' as MeadowlarkId;
+  let retryCount = 0;
 
   try {
-    await beginTransaction(client);
-    // Find the alias meadowlarkIds for the document to be deleted
-    const documentAliasIdsResult: MeadowlarkDocumentIdAndAliasId[] = await findAliasMeadowlarkIdsForDocumentByDocumentUuid(
-      client,
-      documentUuid,
-    );
-    // Each row contains documentUuid and corresponding meadowlarkId (meadowlark_id),
-    // we just need the first row to return the meadowlark_id
-    meadowlarkId = documentAliasIdsResult.length > 0 ? documentAliasIdsResult[0].meadowlark_id : ('' as MeadowlarkId);
-    if (validateNoReferencesToDocument) {
-      // All documents have alias meadowlarkIds. If no alias meadowlarkIds were found, the document doesn't exist
-      if (meadowlarkId === '') {
-        await rollbackTransaction(client);
-        Logger.debug(`${moduleName}.deleteDocumentByDocumentUuid: DocumentUuid ${documentUuid} does not exist`, traceId);
-        deleteResult = { response: 'DELETE_FAILURE_NOT_EXISTS' };
-        return deleteResult;
-      }
-
-      // Extract from the query result
-      const documentAliasMeadowlarkIds: MeadowlarkId[] = documentAliasIdsResult.map((ref) => ref.alias_meadowlark_id);
-
-      // Find any documents that reference this document, either it's own meadowlarkId or an alias
-      const referenceResult: MeadowlarkId[] = await findReferencingMeadowlarkIds(client, documentAliasMeadowlarkIds);
-
-      const references = referenceResult.filter((ref) => ref !== meadowlarkId);
-
-      // If this document is referenced, it's a validation failure
-      if (references.length > 0) {
-        Logger.debug(
-          `${moduleName}.deleteDocumentByDocumentUuid: Deleting document meadowlarkId ${meadowlarkId} failed due to existing references`,
-          traceId,
-        );
-
-        // Get the information of up to five referring documents for failure message purposes
-        const referenceIds = references.map((ref) => ref);
-        const referringDocumentInfo: ReferringDocumentInfo[] = await findReferringDocumentInfoForErrorReporting(
-          client,
-          referenceIds,
-        );
-
-        if (referringDocumentInfo.length === 0) {
+    const numberOfRetries: number = Config.get('POSTGRES_MAX_NUMBER_OF_RETRIES');
+    const deleteProcess = await retry(
+      async (bail) => {
+        let deleteResult: DeleteResult = { response: 'UNKNOWN_FAILURE', failureMessage: '' };
+        let meadowlarkId: MeadowlarkId = '' as MeadowlarkId;
+        try {
+          await beginTransaction(client);
+          // Find the alias meadowlarkIds for the document to be deleted
+          const documentAliasIdsResult: MeadowlarkDocumentIdAndAliasId[] =
+            await findAliasMeadowlarkIdsForDocumentByDocumentUuid(client, documentUuid);
+          // Each row contains documentUuid and corresponding meadowlarkId (meadowlark_id),
+          // we just need the first row to return the meadowlark_id
+          meadowlarkId = documentAliasIdsResult.length > 0 ? documentAliasIdsResult[0].meadowlark_id : ('' as MeadowlarkId);
+          if (validateNoReferencesToDocument) {
+            // All documents have alias meadowlarkIds. If no alias meadowlarkIds were found, the document doesn't exist
+            if (meadowlarkId === '') {
+              await rollbackTransaction(client);
+              Logger.debug(
+                `${moduleName}.deleteDocumentByDocumentUuid: DocumentUuid ${documentUuid} does not exist`,
+                traceId,
+              );
+              deleteResult = { response: 'DELETE_FAILURE_NOT_EXISTS' };
+              return deleteResult;
+            }
+
+            // Extract from the query result
+            const documentAliasMeadowlarkIds: MeadowlarkId[] = documentAliasIdsResult.map((ref) => ref.alias_meadowlark_id);
+
+            // Find any documents that reference this document, either it's own meadowlarkId or an alias
+            const referenceResult: MeadowlarkId[] = await findReferencingMeadowlarkIds(client, documentAliasMeadowlarkIds);
+
+            const references = referenceResult.filter((ref) => ref !== meadowlarkId);
+
+            // If this document is referenced, it's a validation failure
+            if (references.length > 0) {
+              Logger.debug(
+                `${moduleName}.deleteDocumentByDocumentUuid: Deleting document meadowlarkId ${meadowlarkId} failed due to existing references`,
+                traceId,
+              );
+
+              // Get the information of up to five referring documents for failure message purposes
+              const referenceIds = references.map((ref) => ref);
+              const referringDocumentInfo: ReferringDocumentInfo[] = await findReferringDocumentInfoForErrorReporting(
+                client,
+                referenceIds,
+              );
+
+              if (referringDocumentInfo.length === 0) {
+                await rollbackTransaction(client);
+                const errorMessage = `${moduleName}.deleteDocumentByDocumentUuid Error retrieving documents referenced by ${meadowlarkId}, a null result set was returned`;
+                deleteResult.failureMessage = errorMessage;
+                Logger.error(errorMessage, traceId);
+                return deleteResult;
+              }
+
+              deleteResult = {
+                response: 'DELETE_FAILURE_REFERENCE',
+                referringDocumentInfo,
+              };
+              await rollbackTransaction(client);
+              return deleteResult;
+            }
+          }
+
+          // Perform the document delete
+          Logger.debug(
+            `${moduleName}.deleteDocumentByDocumentUuid: Deleting document documentUuid ${documentUuid}`,
+            traceId,
+          );
+          deleteResult = await deleteDocumentRowByDocumentUuid(client, documentUuid);
+
+          // Delete references where this is the parent document
+          Logger.debug(
+            `${moduleName}.deleteDocumentByDocumentUuid Deleting references with documentUuid ${documentUuid} as parent meadowlarkId`,
+            traceId,
+          );
+          await deleteOutboundReferencesOfDocumentByMeadowlarkId(client, meadowlarkId);
+
+          // Delete this document from the aliases table
+          Logger.debug(
+            `${moduleName}.deleteDocumentByDocumentUuid Deleting alias entries with meadowlarkId ${meadowlarkId}`,
+            traceId,
+          );
+          await deleteAliasesForDocumentByMeadowlarkId(client, meadowlarkId);
+
+          await commitTransaction(client);
+        } catch (error) {
+          retryCount += 1;
           await rollbackTransaction(client);
-          const errorMessage = `${moduleName}.deleteDocumentByDocumentUuid Error retrieving documents referenced by ${meadowlarkId}, a null result set was returned`;
-          deleteResult.failureMessage = errorMessage;
-          Logger.error(errorMessage, traceId);
-          return deleteResult;
+          // If there's a serialization failure, rollback the transaction
+          if (error.code === '40001') {
+            if (retryCount >= numberOfRetries) {
+              throw Error('Error after maximum retries');
+            }
+            // Throws the error to be handled by async-retry
+            throw error;
+          } else {
+            // If it's not a serialization failure, don't retry and rethrow the error
+            Logger.error(`${moduleName}.deleteDocumentByDocumentUuid`, traceId, error);
+            // Throws the error to be handled by the caller
+            bail(error);
+          }
         }
-
-        deleteResult = {
-          response: 'DELETE_FAILURE_REFERENCE',
-          referringDocumentInfo,
-        };
-        await rollbackTransaction(client);
         return deleteResult;
-      }
-    }
-
-    // Perform the document delete
-    Logger.debug(`${moduleName}.deleteDocumentByDocumentUuid: Deleting document documentUuid ${documentUuid}`, traceId);
-    deleteResult = await deleteDocumentRowByDocumentUuid(client, documentUuid);
-
-    // Delete references where this is the parent document
-    Logger.debug(
-      `${moduleName}.deleteDocumentByDocumentUuid Deleting references with documentUuid ${documentUuid} as parent meadowlarkId`,
-      traceId,
+      },
+      {
+        retries: numberOfRetries,
+        onRetry: (error, attempt) => {
+          if (attempt === numberOfRetries) {
+            Logger.error('Error after maximum retries', error);
+          } else {
+            Logger.error('Retrying transaction due to error:', error);
+          }
+        },
+      },
     );
-    await deleteOutboundReferencesOfDocumentByMeadowlarkId(client, meadowlarkId);
-
-    // Delete this document from the aliases table
-    Logger.debug(
-      `${moduleName}.deleteDocumentByDocumentUuid Deleting alias entries with meadowlarkId ${meadowlarkId}`,
-      traceId,
-    );
-    await deleteAliasesForDocumentByMeadowlarkId(client, meadowlarkId);
-
-    await commitTransaction(client);
-  } catch (e) {
-    Logger.error(`${moduleName}.deleteDocumentByDocumentUuid`, traceId, e);
-    await rollbackTransaction(client);
-    return { response: 'UNKNOWN_FAILURE', failureMessage: '' };
+    return deleteProcess;
+  } catch (error) {
+    Logger.error(`${moduleName}.deleteDocumentByDocumentUuid`, traceId, error);
+    return { response: 'UNKNOWN_FAILURE', failureMessage: error.message ?? '' };
   }
-  return deleteResult;
 }
diff --git a/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/SqlHelper.ts b/Meadowlark-js/backends/meadowlark-postgresql-backend/src/repository/SqlHelper.ts
@@ -91,10 +91,7 @@ export async function findAliasMeadowlarkIdsForDocumentByMeadowlarkId(
   client: PoolClient,
   meadowlarkId: MeadowlarkId,
 ): Promise<MeadowlarkId[]> {
-  const querySelect = format(
-    `SELECT alias_meadowlark_id FROM meadowlark.aliases WHERE meadowlark_id = %L FOR SHARE NOWAIT`,
-    meadowlarkId,
-  );
+  const querySelect = format(`SELECT alias_meadowlark_id FROM meadowlark.aliases WHERE meadowlark_id = %L`, meadowlarkId);
   const queryResult: QueryResult<any> = await client.query(querySelect);
   return (hasResults(queryResult) ? queryResult.rows.map((ref) => ref.alias_meadowlark_id) : []) as MeadowlarkId[];
 }
@@ -115,7 +112,7 @@ export async function findAliasMeadowlarkIdsForDocumentByDocumentUuid(
   documentUuid: DocumentUuid,
 ): Promise<MeadowlarkDocumentIdAndAliasId[]> {
   const querySelect = format(
-    `SELECT alias_meadowlark_id, meadowlark_id FROM meadowlark.aliases WHERE document_uuid = %L FOR SHARE NOWAIT`,
+    `SELECT alias_meadowlark_id, meadowlark_id FROM meadowlark.aliases WHERE document_uuid = %L`,
     documentUuid,
   );
   const queryResult: QueryResult<any> = await client.query(querySelect);